Skip to content

fix: # 0.5.9 #133

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mikemiles-dev merged 9 commits into from
Jul 3, 2025
Merged

fix: # 0.5.9 #133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
c46c0f3
fix: # 0.5.9
Jul 1, 2025
739bf33
fix: IPFix added no template option
Jul 2, 2025
1b7d4b6
fix: added same for options
Jul 2, 2025
e34c28c
fix: Releases
Jul 2, 2025
83fad2b
fix: Rework to parsing ipfix remaining
Jul 2, 2025
52fdcc6
fix: releases
Jul 2, 2025
9c9dc33
fix: Yaf PEN types
Jul 2, 2025
44ff05b
fix: Added VMWare PEN types
Jul 2, 2025
cc7568f
V9 & Ipfix: Parse every flow record into a separate structure instead…
moritz-2001 Jul 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[package]
name = "netflow_parser"
description = "Parser for Netflow Cisco V5, V7, V9, IPFIX"
version = "0.5.8"
version = "0.5.9"
edition = "2024"
authors = ["michael.mileusnich@gmail.com"]
license = "MIT OR Apache-2.0"
Expand Down
12 changes: 12 additions & 0 deletions RELEASES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,15 @@
# 0.5.9
* IPFIX now supports multiple Templates in a flow
* Fixed bug with parsing IPFix fields that would omit some data.
* New IPFix FlowSetBody type added called NoTemplate and Empty.
* NoTemplate returns data that allows you to cache flows that do not have a template for later parsing.
* Correctly handling different Enterprise Field Types.
* Added Netscaler PEN Types.
* Added NAT PEN Types.
* Added YAF PEN Types.
* Added VMWARE PEN Types.
* Re-added Enterprise Field Type for Unknown Enterprise Types.

# 0.5.8
* V9 Found and fixed divide by 0 issue.
* IPFix Protocol Identifier now parsers as ProtocolIdentifier Field Type and not UnsignedDataNumber.
Expand Down
176 changes: 63 additions & 113 deletions src/netflow_common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use crate::NetflowPacket;
use crate::protocol::ProtocolTypes;
use crate::static_versions::{v5::V5, v7::V7};
use crate::variable_versions::data_number::FieldValue;
use crate::variable_versions::ipfix_lookup::IPFixField;
use crate::variable_versions::ipfix_lookup::{IANAIPFixField, IPFixField};
use crate::variable_versions::v9_lookup::V9Field;
use crate::variable_versions::{
ipfix::{FlowSetBody as IPFixFlowSetBody, IPFix},
Expand Down Expand Up @@ -125,7 +125,7 @@ impl From<&V9> for NetflowCommon {
if let V9FlowSetBody::Data(data) = &flowset.body {
for data_field in &data.fields {
let value_map: BTreeMap<V9Field, FieldValue> =
data_field.values().cloned().collect();
data_field.clone().into_iter().collect();
flowsets.push(NetflowCommonFlowSet {
src_addr: value_map
.get(&V9Field::Ipv4SrcAddr)
Expand Down Expand Up @@ -184,43 +184,50 @@ impl From<&IPFix> for NetflowCommon {
if let IPFixFlowSetBody::Data(data) = &flowset.body {
for data_field in &data.fields {
let value_map: BTreeMap<IPFixField, FieldValue> =
data_field.values().cloned().collect();
data_field.clone().into_iter().collect();
flowsets.push(NetflowCommonFlowSet {
src_addr: value_map
.get(&IPFixField::SourceIpv4address)
.or_else(|| value_map.get(&IPFixField::SourceIpv6address))
.get(&IPFixField::IANA(IANAIPFixField::SourceIpv4address))
.or_else(|| {
value_map
.get(&IPFixField::IANA(IANAIPFixField::SourceIpv6address))
})
.and_then(|v| v.try_into().ok()),
dst_addr: value_map
.get(&IPFixField::DestinationIpv4address)
.or_else(|| value_map.get(&IPFixField::DestinationIpv6address))
.get(&IPFixField::IANA(IANAIPFixField::DestinationIpv4address))
.or_else(|| {
value_map.get(&IPFixField::IANA(
IANAIPFixField::DestinationIpv6address,
))
})
.and_then(|v| v.try_into().ok()),
src_port: value_map
.get(&IPFixField::SourceTransportPort)
.get(&IPFixField::IANA(IANAIPFixField::SourceTransportPort))
.and_then(|v| v.try_into().ok()),
dst_port: value_map
.get(&IPFixField::DestinationTransportPort)
.get(&IPFixField::IANA(IANAIPFixField::DestinationTransportPort))
.and_then(|v| v.try_into().ok()),
protocol_number: value_map
.get(&IPFixField::ProtocolIdentifier)
.get(&IPFixField::IANA(IANAIPFixField::ProtocolIdentifier))
.and_then(|v| v.try_into().ok()),
protocol_type: value_map.get(&IPFixField::ProtocolIdentifier).and_then(
|v| {
protocol_type: value_map
.get(&IPFixField::IANA(IANAIPFixField::ProtocolIdentifier))
.and_then(|v| {
v.try_into()
.ok()
.map(|proto: u8| ProtocolTypes::from(proto))
},
),
}),
first_seen: value_map
.get(&IPFixField::FlowStartSysUpTime)
.get(&IPFixField::IANA(IANAIPFixField::FlowStartSysUpTime))
.and_then(|v| v.try_into().ok()),
last_seen: value_map
.get(&IPFixField::FlowEndSysUpTime)
.get(&IPFixField::IANA(IANAIPFixField::FlowEndSysUpTime))
.and_then(|v| v.try_into().ok()),
src_mac: value_map
.get(&IPFixField::SourceMacaddress)
.get(&IPFixField::IANA(IANAIPFixField::SourceMacaddress))
.and_then(|v| v.try_into().ok()),
dst_mac: value_map
.get(&IPFixField::DestinationMacaddress)
.get(&IPFixField::IANA(IANAIPFixField::DestinationMacaddress))
.and_then(|v| v.try_into().ok()),
});
}
Expand All @@ -237,8 +244,6 @@ impl From<&IPFix> for NetflowCommon {

#[cfg(test)]
mod common_tests {

use std::collections::BTreeMap;
use std::net::{IpAddr, Ipv4Addr};

use crate::netflow_common::NetflowCommon;
Expand All @@ -249,7 +254,7 @@ mod common_tests {
Data as IPFixData, FlowSet as IPFixFlowSet, FlowSetBody as IPFixFlowSetBody,
FlowSetHeader as IPFixFlowSetHeader, Header as IPFixHeader, IPFix,
};
use crate::variable_versions::ipfix_lookup::IPFixField;
use crate::variable_versions::ipfix_lookup::{IANAIPFixField, IPFixField};
use crate::variable_versions::v9::{
Data as V9Data, FlowSet as V9FlowSet, FlowSetBody as V9FlowSetBody,
FlowSetHeader as V9FlowSetHeader, Header as V9Header, V9,
Expand Down Expand Up @@ -402,66 +407,39 @@ mod common_tests {
},
body: V9FlowSetBody::Data(V9Data {
padding: vec![],
fields: vec![BTreeMap::from([
fields: vec![Vec::from([
(
0,
(
V9Field::Ipv4SrcAddr,
FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 1)),
),
V9Field::Ipv4SrcAddr,
FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 1)),
),
(
1,
(
V9Field::Ipv4DstAddr,
FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 2)),
),
V9Field::Ipv4DstAddr,
FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 2)),
),
(
2,
(
V9Field::L4SrcPort,
FieldValue::DataNumber(DataNumber::U16(1234)),
),
V9Field::L4SrcPort,
FieldValue::DataNumber(DataNumber::U16(1234)),
),
(
3,
(
V9Field::L4DstPort,
FieldValue::DataNumber(DataNumber::U16(80)),
),
V9Field::L4DstPort,
FieldValue::DataNumber(DataNumber::U16(80)),
),
(V9Field::Protocol, FieldValue::DataNumber(DataNumber::U8(6))),
(
4,
(V9Field::Protocol, FieldValue::DataNumber(DataNumber::U8(6))),
V9Field::FirstSwitched,
FieldValue::DataNumber(DataNumber::U32(100)),
),
(
5,
(
V9Field::FirstSwitched,
FieldValue::DataNumber(DataNumber::U32(100)),
),
V9Field::LastSwitched,
FieldValue::DataNumber(DataNumber::U32(200)),
),
(
6,
(
V9Field::LastSwitched,
FieldValue::DataNumber(DataNumber::U32(200)),
),
V9Field::InSrcMac,
FieldValue::MacAddr("00:00:00:00:00:01".to_string()),
),
(
7,
(
V9Field::InSrcMac,
FieldValue::MacAddr("00:00:00:00:00:01".to_string()),
),
),
(
8,
(
V9Field::InDstMac,
FieldValue::MacAddr("00:00:00:00:00:02".to_string()),
),
V9Field::InDstMac,
FieldValue::MacAddr("00:00:00:00:00:02".to_string()),
),
])],
}),
Expand Down Expand Up @@ -511,70 +489,42 @@ mod common_tests {
length: 0,
},
body: IPFixFlowSetBody::Data(IPFixData {
padding: vec![],
fields: vec![BTreeMap::from([
fields: vec![Vec::from([
(
0,
(
IPFixField::SourceIpv4address,
FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 1)),
),
IPFixField::IANA(IANAIPFixField::SourceIpv4address),
FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 1)),
),
(
1,
(
IPFixField::DestinationIpv4address,
FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 2)),
),
IPFixField::IANA(IANAIPFixField::DestinationIpv4address),
FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 2)),
),
(
2,
(
IPFixField::SourceTransportPort,
FieldValue::DataNumber(DataNumber::U16(1234)),
),
IPFixField::IANA(IANAIPFixField::SourceTransportPort),
FieldValue::DataNumber(DataNumber::U16(1234)),
),
(
3,
(
IPFixField::DestinationTransportPort,
FieldValue::DataNumber(DataNumber::U16(80)),
),
IPFixField::IANA(IANAIPFixField::DestinationTransportPort),
FieldValue::DataNumber(DataNumber::U16(80)),
),
(
4,
(
IPFixField::ProtocolIdentifier,
FieldValue::DataNumber(DataNumber::U8(6)),
),
IPFixField::IANA(IANAIPFixField::ProtocolIdentifier),
FieldValue::DataNumber(DataNumber::U8(6)),
),
(
5,
(
IPFixField::FlowStartSysUpTime,
FieldValue::DataNumber(DataNumber::U32(100)),
),
IPFixField::IANA(IANAIPFixField::FlowStartSysUpTime),
FieldValue::DataNumber(DataNumber::U32(100)),
),
(
6,
(
IPFixField::FlowEndSysUpTime,
FieldValue::DataNumber(DataNumber::U32(200)),
),
IPFixField::IANA(IANAIPFixField::FlowEndSysUpTime),
FieldValue::DataNumber(DataNumber::U32(200)),
),
(
7,
(
IPFixField::SourceMacaddress,
FieldValue::MacAddr("00:00:00:00:00:01".to_string()),
),
IPFixField::IANA(IANAIPFixField::SourceMacaddress),
FieldValue::MacAddr("00:00:00:00:00:01".to_string()),
),
(
8,
(
IPFixField::DestinationMacaddress,
FieldValue::MacAddr("00:00:00:00:00:02".to_string()),
),
IPFixField::IANA(IANAIPFixField::DestinationMacaddress),
FieldValue::MacAddr("00:00:00:00:00:02".to_string()),
),
])],
}),
Expand Down
18 changes: 9 additions & 9 deletions src/snapshots/netflow_parser__tests__base_tests__it_parses_0_length_fields_ipfix.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,26 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
field_count: 3
fields:
- field_type_number: 8
field_type: SourceIpv4address
field_length: 4
field_type:
IANA: SourceIpv4address
- field_type_number: 12
field_type: DestinationIpv4address
field_length: 4
field_type:
IANA: DestinationIpv4address
- field_type_number: 65
field_type: AssignedforNetFlowv9compatibility
field_length: 0
field_type:
IANA: AssignedforNetFlowv9compatibility
- header:
header_id: 256
length: 12
body:
Data:
fields:
- 0:
- SourceIpv4address
- - - IANA: SourceIpv4address
- Ip4Addr: 1.2.3.4
- 1:
- DestinationIpv4address
- - IANA: DestinationIpv4address
- Ip4Addr: 1.2.3.4
- 2:
- AssignedforNetFlowv9compatibility
- - IANA: AssignedforNetFlowv9compatibility
- Vec: []
27 changes: 12 additions & 15 deletions src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,35 +19,32 @@ expression: "NetflowParser::default().parse_bytes(&packet)"
field_count: 3
fields:
- field_type_number: 8
field_type: SourceIpv4address
field_length: 4
field_type:
IANA: SourceIpv4address
- field_type_number: 12
field_type: DestinationIpv4address
field_length: 4
field_type:
IANA: DestinationIpv4address
- field_type_number: 2
field_type: PacketDeltaCount
field_length: 4
field_type:
IANA: PacketDeltaCount
- header:
header_id: 256
length: 28
body:
Data:
fields:
- 0:
- SourceIpv4address
- - - IANA: SourceIpv4address
- Ip4Addr: 1.2.3.4
- 1:
- DestinationIpv4address
- - IANA: DestinationIpv4address
- Ip4Addr: 1.2.3.3
- 2:
- PacketDeltaCount
- - IANA: PacketDeltaCount
- DataNumber: 16909058
- 0:
- SourceIpv4address
- - - IANA: SourceIpv4address
- Ip4Addr: 0.2.0.2
- 1:
- DestinationIpv4address
- - IANA: DestinationIpv4address
- Ip4Addr: 0.1.2.3
- 2:
- PacketDeltaCount
- - IANA: PacketDeltaCount
- DataNumber: 67438087
6 changes: 2 additions & 4 deletions src/snapshots/netflow_parser__tests__base_tests__it_parses_ipfix_data_cached_template.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,7 @@ expression: parser.parse_bytes(&packet)
body:
Data:
fields:
- 0:
- PacketDeltaCount
- - - IANA: PacketDeltaCount
- DataNumber: 8
- 1:
- SourceIpv4address
- - IANA: SourceIpv4address
- Ip4Addr: 0.0.1.1
Loading