diff --git a/README.md b/README.md index 32ac2ee9..3765b72e 100644 --- a/README.md +++ b/README.md @@ -374,7 +374,25 @@ Each field mapping has a `primary` field (always checked first) and an optional Parsed V5, V7, V9, and IPFIX packets can be re-exported back into bytes. -**Note:** For V9/IPFIX, we only export the original padding we dissected and do not calculate/align the flowset padding ourselves. If you modify an existing V9/IPFIX flow or create your own, you must manually adjust the padding. +**V9/IPFIX Padding Behavior:** +- For **parsed packets**: Original padding is preserved exactly for byte-perfect round-trips +- For **manually created packets**: Padding is automatically calculated to align FlowSets to 4-byte boundaries + +**Creating Data Structs:** +For convenience, use `Data::new()` and `OptionsData::new()` to create data structures without manually specifying padding: + +```rust +use netflow_parser::variable_versions::ipfix::Data; + +// Padding is automatically set to empty vec and calculated during export +let data = Data::new(vec![vec![ + (field1, value1), + (field2, value2), +]]); +``` + +See `examples/manual_ipfix_creation.rs` for a complete example of creating IPFIX packets from scratch. + ```rust // 0000 00 05 00 01 03 00 04 00 05 00 06 07 08 09 00 01 ................ // 0010 02 03 04 05 06 07 08 09 00 01 02 03 04 05 06 07 ................ @@ -450,18 +468,14 @@ To run: ```cargo run --example netflow_udp_listener_multi_threaded``` -or - ```cargo run --example netflow_udp_listener_single_threaded``` -or - ```cargo run --example netflow_udp_listener_tokio``` -or - ```cargo run --example netflow_pcap``` +```cargo run --example manual_ipfix_creation``` + The pcap example also shows how to cache flows that have not yet discovered a template. ## Support My Work diff --git a/RELEASES.md b/RELEASES.md index f2e2b1a4..f54bfa3d 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -8,6 +8,12 @@ * Enhanced validation for malformed packets * Improved IPFIX error handling - parse errors now properly propagate * Added thread safety documentation and performance tuning guide +* **Fixed V9/IPFIX padding handling:** + * Fixed missing padding export for V9 Data FlowSets + * Added padding fields to IPFIX Data and OptionsData structures + * Auto-calculate padding for manually created packets (when padding field is empty) + * Preserve original padding for parsed packets (byte-perfect round-trips) + * Added `examples/manual_ipfix_creation.rs` demonstrating manual packet creation # 0.6.6 * Added configurable field mappings for V9 and IPFIX in NetflowCommon. diff --git a/examples/manual_ipfix_creation.rs b/examples/manual_ipfix_creation.rs new file mode 100644 index 00000000..82388f08 --- /dev/null +++ b/examples/manual_ipfix_creation.rs @@ -0,0 +1,117 @@ +//! Example demonstrating manual creation of IPFIX packets. +//! +//! This example shows how to create IPFIX packets from scratch. The library provides +//! the `calculate_padding` function to help calculate the correct padding bytes needed +//! to align FlowSets to 4-byte boundaries. + +use netflow_parser::variable_versions::data_number::FieldValue; +use netflow_parser::variable_versions::ipfix::{ + Data, FlowSet, FlowSetBody, FlowSetHeader, Header, IPFix, Template, TemplateField, +}; +use netflow_parser::variable_versions::ipfix_lookup::{IANAIPFixField, IPFixField}; +use std::net::Ipv4Addr; + +fn main() -> Result<(), Box> { + // Create a simple IPFIX header + let header = Header { + version: 10, + length: 0, // Will be calculated based on actual size + export_time: 1234567890, + sequence_number: 1, + observation_domain_id: 0, + }; + + // Create a template with 3 fields (which will need 1 byte of padding) + // Template structure: template_id (2) + field_count (2) + 3 fields * 4 bytes = 16 bytes + // 16 bytes is already aligned to 4, but let's create one that needs padding + + // Actually, let's create a template with 1 field to demonstrate padding + // Template: template_id (2) + field_count (2) + 1 field * 4 bytes = 8 bytes (no padding needed) + // Let's add a field with enterprise bit set: + 4 bytes enterprise = 12 bytes (no padding) + + // For demonstration, let's create a simple template that shows the concept + let template = Template { + template_id: 256, + field_count: 2, + fields: vec![ + TemplateField { + field_type_number: 8, // sourceIPv4Address + field_length: 4, + enterprise_number: None, + field_type: IPFixField::IANA(IANAIPFixField::SourceIpv4address), + }, + TemplateField { + field_type_number: 12, // destinationIPv4Address + field_length: 4, + enterprise_number: None, + field_type: IPFixField::IANA(IANAIPFixField::DestinationIpv4address), + }, + ], + }; + + // Calculate the template flowset length + // Header: 4 bytes (flowset_id + length) + // Template: 2 (template_id) + 2 (field_count) + 2 fields * 4 bytes = 12 bytes + // Total: 16 bytes (perfectly aligned, no padding needed) + let template_flowset = FlowSet { + header: FlowSetHeader { + header_id: 2, // Template FlowSet ID + length: 16, + }, + body: FlowSetBody::Template(template.clone()), + }; + + // Create data flowset with actual flow records + let data = Data::new(vec![vec![ + ( + IPFixField::IANA(IANAIPFixField::SourceIpv4address), + FieldValue::Ip4Addr(Ipv4Addr::new(192, 168, 1, 1)), + ), + ( + IPFixField::IANA(IANAIPFixField::DestinationIpv4address), + FieldValue::Ip4Addr(Ipv4Addr::new(10, 0, 0, 1)), + ), + ]]); + + // Data content: 2 fields * 4 bytes = 8 bytes (already aligned, no padding needed in this case) + // If we had unaligned data, we could use calculate_padding() to determine padding bytes needed + let data_flowset = FlowSet { + header: FlowSetHeader { + header_id: 256, // Matches template_id + length: 12, // Header (4) + Data (8) = 12 bytes + }, + body: FlowSetBody::Data(data), + }; + + // Create the IPFIX packet + let mut ipfix = IPFix { + header, + flowsets: vec![template_flowset, data_flowset], + }; + + // Update the total length in the header + ipfix.header.length = 16 + 16 + 12; // Header + Template FlowSet + Data FlowSet = 44 bytes + + // Export to bytes + let exported_bytes = ipfix.to_be_bytes()?; + + println!("Created IPFIX packet with {} bytes", exported_bytes.len()); + println!("Header length field: {}", ipfix.header.length); + println!( + "Actual exported length: {} (should match header.length)", + exported_bytes.len() + ); + + // Verify the packet structure + println!("\nPacket structure:"); + println!("- IPFIX Header: 16 bytes"); + println!( + "- Template FlowSet: {} bytes", + ipfix.flowsets[0].header.length + ); + println!("- Data FlowSet: {} bytes", ipfix.flowsets[1].header.length); + + println!("\n✓ Use calculate_padding() to calculate alignment when needed!"); + + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index 15a192e0..397628af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -345,7 +345,12 @@ //! //! Parsed V5, V7, V9, and IPFIX packets can be re-exported back into bytes. //! -//! **Note:** For V9/IPFIX, we only export the original padding we dissected and do not calculate/align the flowset padding ourselves. If you modify an existing V9/IPFIX flow or create your own, you must manually adjust the padding. +//! **V9/IPFIX Padding Behavior:** +//! - For **parsed packets**: Original padding is preserved exactly for byte-perfect round-trips +//! - For **manually created packets**: Padding is automatically calculated to align FlowSets to 4-byte boundaries - simply leave the `padding` field empty (`vec![]`) +//! +//! See `examples/manual_ipfix_creation.rs` for a complete example of creating IPFIX packets from scratch. +//! //! ```rust //! use netflow_parser::{NetflowParser, NetflowPacket}; //! @@ -417,18 +422,14 @@ //! //! ```cargo run --example netflow_udp_listener_multi_threaded``` //! -//! or -//! //! ```cargo run --example netflow_udp_listener_single_threaded``` //! -//! or -//! //! ```cargo run --example netflow_udp_listener_tokio``` //! -//! or -//! //! ```cargo run --example netflow_pcap``` //! +//! ```cargo run --example manual_ipfix_creation``` +//! //! The pcap example also shows how to cache flows that have not yet discovered a template. //! //! ## Support My Work diff --git a/src/variable_versions/ipfix.rs b/src/variable_versions/ipfix.rs index af912594..0975b9de 100644 --- a/src/variable_versions/ipfix.rs +++ b/src/variable_versions/ipfix.rs @@ -34,6 +34,14 @@ type TemplateId = u16; pub type IPFixFieldPair = (IPFixField, FieldValue); pub type IpFixFlowRecord = Vec; +/// Calculate padding needed to align to 4-byte boundary. +/// Returns a Vec of zero bytes with the appropriate length. +fn calculate_padding(content_size: usize) -> Vec { + const PADDING_SIZES: [usize; 4] = [0, 3, 2, 1]; + let padding_len = PADDING_SIZES[content_size % 4]; + vec![0u8; padding_len] +} + #[derive(Debug, PartialEq, Clone, Serialize)] pub struct IPFixParser { pub templates: HashMap, @@ -358,6 +366,20 @@ pub struct Data { Parse = "{ |i| FieldParser::parse::