From 77b632f5ab7d6ef0d59635e0aa74c93f449f72b7 Mon Sep 17 00:00:00 2001 From: mikemiles-dev Date: Fri, 19 Dec 2025 17:59:15 -0600 Subject: [PATCH 1/3] feat: Added LRU for Templates --- Cargo.toml | 3 +- README.md | 63 ++++++++++++++++++++++-- RELEASES.md | 9 ++++ src/tests.rs | 8 ++-- src/variable_versions/ipfix.rs | 76 ++++++++++++++++++++++------- src/variable_versions/v9.rs | 87 ++++++++++++++++++++++++---------- 6 files changed, 195 insertions(+), 51 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f1fc79e9..9c73df3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "netflow_parser" description = "Parser for Netflow Cisco V5, V7, V9, IPFIX" -version = "0.6.7" +version = "0.6.8" edition = "2024" authors = ["michael.mileusnich@gmail.com"] license = "MIT OR Apache-2.0" @@ -15,6 +15,7 @@ nom = "7.1.3" nom-derive = "0.10.1" mac_address = "1.1.5" serde = { version = "1.0.166", features = ["derive"] } +lru = "0.12" [features] default = ["parse_unknown_fields"] diff --git a/README.md b/README.md index 3765b72e..2d4ee083 100644 --- a/README.md +++ b/README.md @@ -218,6 +218,51 @@ let parsed = parser.parse_bytes(&v5_packet); This code will return an empty Vec as version 5 is not allowed. +## Template Cache Configuration + +V9 and IPFIX parsers use LRU (Least Recently Used) caching to store templates with a configurable size limit. This prevents memory exhaustion from template flooding attacks while maintaining good performance for legitimate traffic. + +### Default Behavior + +By default, parsers cache up to 1000 templates: + +```rust +use netflow_parser::NetflowParser; + +// Uses default cache size of 1000 templates per parser +let parser = NetflowParser::default(); +``` + +### Custom Cache Size + +You can configure the template cache size when creating parsers: + +```rust +use netflow_parser::{NetflowParser, variable_versions::v9::V9Parser, variable_versions::ipfix::IPFixParser}; + +// Create V9 parser with custom cache size +let v9_parser = V9Parser::try_new(5000)?; // Cache up to 5000 templates + +// Create IPFix parser with custom cache size +let ipfix_parser = IPFixParser::try_new(2000)?; // Cache up to 2000 templates + +// Note: try_new() returns Result and will fail if cache_size is 0 +``` + +### Cache Behavior + +- When the cache is full, the least recently used template is evicted +- Templates are keyed by template ID (per source) +- Each parser instance maintains its own template cache +- For multi-source deployments, create separate parser instances per source + +### Recommended Cache Sizes + +- **Single source**: 100-1000 templates (most exporters use 1-10 templates) +- **Multiple sources**: 1000-10000 templates per parser +- **Memory-constrained**: 100-500 templates +- **High-security environments**: 500-1000 templates (prevents DoS via template flooding) + ## Error Handling Configuration To prevent memory exhaustion from malformed packets, the parser limits the size of error buffer samples. By default, only the first 256 bytes of unparseable data are stored in error messages. You can customize this limit for all parsers: @@ -415,17 +460,27 @@ if let NetflowPacket::V5(v5) = NetflowParser::default() ## V9/IPFIX Notes -Parse the data (`&[u8]`) like any other version. The parser (`NetflowParser`) caches parsed templates, so you can send header/data flowset combos and it will use the cached templates. To see cached templates, use the parser for the correct version (`v9_parser` for V9, `ipfix_parser` for IPFIX). +Parse the data (`&[u8]`) like any other version. The parser (`NetflowParser`) caches parsed templates using LRU eviction, so you can send header/data flowset combos and it will use the cached templates. Templates are automatically cached and evicted when the cache limit is reached. -**IPFIX Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it. +**Template Cache Access:** +Template caches use `LruCache` internally. You can inspect cached templates, but note that accessing them may affect LRU ordering: ```rust use netflow_parser::NetflowParser; let parser = NetflowParser::default(); -dbg!(parser.v9_parser.templates); -dbg!(parser.v9_parser.options_templates); + +// Check if a template exists +if parser.v9_parser.templates.contains(&template_id) { + // Template is cached +} + +// Get cache stats +println!("V9 template cache size: {}", parser.v9_parser.templates.len()); +println!("V9 max cache size: {}", parser.v9_parser.max_template_cache_size); ``` +**IPFIX Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it. + To access templates flowset of a processed V9/IPFix flowset you can find the `flowsets` attribute on the Parsed Record. In there you can find `Templates`, `Option Templates`, and `Data` Flowsets. ## Performance & Thread Safety diff --git a/RELEASES.md b/RELEASES.md index f54bfa3d..1093379c 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,3 +1,12 @@ +# 0.6.8 + * Added LRU-based template caching for V9Parser and IPFixParser to prevent memory exhaustion + * Default template cache size: 1000 templates per parser (configurable) + * New `V9Parser::try_new(cache_size)` and `IPFixParser::try_new(cache_size)` constructors for custom cache sizes + * Added `V9ParserError` and `IPFixParserError` error types for proper error handling + * Template cache is automatically evicted using LRU policy when limit is reached + * Provides protection against DoS attacks via template flooding + * Removed `PartialEq`, `Clone`, and `Serialize` derives from parser structs (due to LruCache) + # 0.6.7 * Optimized NetflowCommon conversion with single-pass field lookups (reduced O(n*m) to O(n)) * Added V5/V7/DataNumber capacity pre-allocation diff --git a/src/tests.rs b/src/tests.rs index 8fe00346..2e0fe21a 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -265,7 +265,7 @@ mod base_tests { fields, }; let mut parser = NetflowParser::default(); - parser.v9_parser.templates.insert(258, template); + parser.v9_parser.templates.put(258, template); assert_yaml_snapshot!(parser.parse_bytes(&packet)); } @@ -352,7 +352,7 @@ mod base_tests { ..Default::default() }; let mut parser = NetflowParser::default(); - parser.ipfix_parser.templates.insert(258, template); + parser.ipfix_parser.templates.put(258, template); assert_yaml_snapshot!(parser.parse_bytes(&packet)); } @@ -367,7 +367,7 @@ mod base_tests { ..Default::default() }; let mut parser = NetflowParser::default(); - parser.ipfix_parser.templates.insert(258, template); + parser.ipfix_parser.templates.put(258, template); assert_yaml_snapshot!(parser.parse_bytes(&packet)); } @@ -382,7 +382,7 @@ mod base_tests { fields: vec![], }; let mut parser = NetflowParser::default(); - parser.v9_parser.templates.insert(258, template); + parser.v9_parser.templates.put(258, template); assert_yaml_snapshot!(parser.parse_bytes(&packet)); } diff --git a/src/variable_versions/ipfix.rs b/src/variable_versions/ipfix.rs index 0975b9de..15955d63 100644 --- a/src/variable_versions/ipfix.rs +++ b/src/variable_versions/ipfix.rs @@ -25,11 +25,34 @@ use crate::variable_versions::v9::{ Template as V9Template, }; -use std::collections::HashMap; +use lru::LruCache; +use std::num::NonZeroUsize; const DATA_TEMPLATE_IPFIX_ID: u16 = 2; const OPTIONS_TEMPLATE_IPFIX_ID: u16 = 3; +/// Default maximum number of templates to cache per parser +pub const DEFAULT_MAX_TEMPLATE_CACHE_SIZE: usize = 1000; + +/// Error type for IPFixParser creation +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IPFixParserError { + /// Template cache size must be greater than 0 + InvalidCacheSize, +} + +impl std::fmt::Display for IPFixParserError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + IPFixParserError::InvalidCacheSize => { + write!(f, "max_template_cache_size must be greater than 0") + } + } + } +} + +impl std::error::Error for IPFixParserError {} + type TemplateId = u16; pub type IPFixFieldPair = (IPFixField, FieldValue); pub type IpFixFlowRecord = Vec; @@ -42,12 +65,14 @@ fn calculate_padding(content_size: usize) -> Vec { vec![0u8; padding_len] } -#[derive(Debug, PartialEq, Clone, Serialize)] +#[derive(Debug)] pub struct IPFixParser { - pub templates: HashMap, - pub v9_templates: HashMap, - pub ipfix_options_templates: HashMap, - pub v9_options_templates: HashMap, + pub templates: LruCache, + pub v9_templates: LruCache, + pub ipfix_options_templates: LruCache, + pub v9_options_templates: LruCache, + /// Maximum number of templates to cache. Defaults to 1000. + pub max_template_cache_size: usize, /// Maximum number of bytes to include in error samples to prevent memory exhaustion. /// Defaults to 256 bytes. pub max_error_sample_size: usize, @@ -55,17 +80,33 @@ pub struct IPFixParser { impl Default for IPFixParser { fn default() -> Self { - Self { - templates: HashMap::default(), - v9_templates: HashMap::default(), - ipfix_options_templates: HashMap::default(), - v9_options_templates: HashMap::default(), - max_error_sample_size: 256, - } + // Safe to unwrap because DEFAULT_MAX_TEMPLATE_CACHE_SIZE is non-zero + Self::try_new(DEFAULT_MAX_TEMPLATE_CACHE_SIZE).unwrap() } } impl IPFixParser { + /// Create a new IPFixParser with a custom template cache size. + /// + /// # Arguments + /// * `max_template_cache_size` - Maximum number of templates to cache (must be > 0) + /// + /// # Errors + /// Returns `IPFixParserError::InvalidCacheSize` if `max_template_cache_size` is 0 + pub fn try_new(max_template_cache_size: usize) -> Result { + let cache_size = NonZeroUsize::new(max_template_cache_size) + .ok_or(IPFixParserError::InvalidCacheSize)?; + + Ok(Self { + templates: LruCache::new(cache_size), + v9_templates: LruCache::new(cache_size), + ipfix_options_templates: LruCache::new(cache_size), + v9_options_templates: LruCache::new(cache_size), + max_template_cache_size, + max_error_sample_size: 256, + }) + } + pub fn parse<'a>(&mut self, packet: &'a [u8]) -> ParsedNetflow<'a> { match IPFix::parse(packet, self) { Ok((remaining, ipfix)) => ParsedNetflow::Success { @@ -93,26 +134,25 @@ impl IPFixParser { /// Add templates to the parser by cloning from slice. fn add_ipfix_templates(&mut self, templates: &[Template]) { for t in templates { - self.templates.insert(t.template_id, t.clone()); + self.templates.put(t.template_id, t.clone()); } } fn add_ipfix_options_templates(&mut self, templates: &[OptionsTemplate]) { for t in templates { - self.ipfix_options_templates - .insert(t.template_id, t.clone()); + self.ipfix_options_templates.put(t.template_id, t.clone()); } } fn add_v9_templates(&mut self, templates: &[V9Template]) { for t in templates { - self.v9_templates.insert(t.template_id, t.clone()); + self.v9_templates.put(t.template_id, t.clone()); } } fn add_v9_options_templates(&mut self, templates: &[V9OptionsTemplate]) { for t in templates { - self.v9_options_templates.insert(t.template_id, t.clone()); + self.v9_options_templates.put(t.template_id, t.clone()); } } } diff --git a/src/variable_versions/v9.rs b/src/variable_versions/v9.rs index 8463a155..59430a99 100644 --- a/src/variable_versions/v9.rs +++ b/src/variable_versions/v9.rs @@ -17,11 +17,34 @@ use nom::multi::many0; use nom_derive::{Nom, Parse}; use serde::Serialize; -use std::collections::HashMap; +use lru::LruCache; +use std::num::NonZeroUsize; pub const DATA_TEMPLATE_V9_ID: u16 = 0; pub const OPTIONS_TEMPLATE_V9_ID: u16 = 1; +/// Default maximum number of templates to cache per parser +pub const DEFAULT_MAX_TEMPLATE_CACHE_SIZE: usize = 1000; + +/// Error type for V9Parser creation +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum V9ParserError { + /// Template cache size must be greater than 0 + InvalidCacheSize, +} + +impl std::fmt::Display for V9ParserError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + V9ParserError::InvalidCacheSize => { + write!(f, "max_template_cache_size must be greater than 0") + } + } + } +} + +impl std::error::Error for V9ParserError {} + type TemplateId = u16; pub type V9FieldPair = (V9Field, FieldValue); pub type V9FlowRecord = Vec; @@ -34,7 +57,44 @@ fn calculate_padding(content_size: usize) -> Vec { vec![0u8; padding_len] } +#[derive(Debug)] +pub struct V9Parser { + pub templates: LruCache, + pub options_templates: LruCache, + /// Maximum number of templates to cache. Defaults to 1000. + pub max_template_cache_size: usize, + /// Maximum number of bytes to include in error samples to prevent memory exhaustion. + /// Defaults to 256 bytes. + pub max_error_sample_size: usize, +} + +impl Default for V9Parser { + fn default() -> Self { + // Safe to unwrap because DEFAULT_MAX_TEMPLATE_CACHE_SIZE is non-zero + Self::try_new(DEFAULT_MAX_TEMPLATE_CACHE_SIZE).unwrap() + } +} + impl V9Parser { + /// Create a new V9Parser with a custom template cache size. + /// + /// # Arguments + /// * `max_template_cache_size` - Maximum number of templates to cache (must be > 0) + /// + /// # Errors + /// Returns `V9ParserError::InvalidCacheSize` if `max_template_cache_size` is 0 + pub fn try_new(max_template_cache_size: usize) -> Result { + let cache_size = NonZeroUsize::new(max_template_cache_size) + .ok_or(V9ParserError::InvalidCacheSize)?; + + Ok(Self { + templates: LruCache::new(cache_size), + options_templates: LruCache::new(cache_size), + max_template_cache_size, + max_error_sample_size: 256, + }) + } + pub fn parse<'a>(&mut self, packet: &'a [u8]) -> ParsedNetflow<'a> { match V9::parse(packet, self) { Ok((remaining, v9)) => ParsedNetflow::Success { @@ -60,25 +120,6 @@ impl V9Parser { } } -#[derive(Debug)] -pub struct V9Parser { - pub templates: HashMap, - pub options_templates: HashMap, - /// Maximum number of bytes to include in error samples to prevent memory exhaustion. - /// Defaults to 256 bytes. - pub max_error_sample_size: usize, -} - -impl Default for V9Parser { - fn default() -> Self { - Self { - templates: HashMap::default(), - options_templates: HashMap::default(), - max_error_sample_size: 256, - } - } -} - #[derive(Debug, PartialEq, Clone, Serialize, Nom)] #[nom(ExtraArgs(parser: &mut V9Parser))] pub struct V9 { @@ -214,9 +255,7 @@ impl FlowSetBody { let (i, templates) = Templates::parse(i)?; // Store templates efficiently - clone only what we need to cache for template in &templates.templates { - parser - .templates - .insert(template.template_id, template.clone()); + parser.templates.put(template.template_id, template.clone()); } Ok((i, FlowSetBody::Template(templates))) } @@ -226,7 +265,7 @@ impl FlowSetBody { for template in &options_templates.templates { parser .options_templates - .insert(template.template_id, template.clone()); + .put(template.template_id, template.clone()); } Ok((i, FlowSetBody::OptionsTemplate(options_templates))) } From 579b276663da16d8381124fd32e3651eb3df5709 Mon Sep 17 00:00:00 2001 From: mikemiles-dev Date: Sat, 20 Dec 2025 10:51:04 -0600 Subject: [PATCH 2/3] fix: Updated readme and lib --- README.md | 7 ------- src/lib.rs | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2d4ee083..8b40f026 100644 --- a/README.md +++ b/README.md @@ -256,13 +256,6 @@ let ipfix_parser = IPFixParser::try_new(2000)?; // Cache up to 2000 templates - Each parser instance maintains its own template cache - For multi-source deployments, create separate parser instances per source -### Recommended Cache Sizes - -- **Single source**: 100-1000 templates (most exporters use 1-10 templates) -- **Multiple sources**: 1000-10000 templates per parser -- **Memory-constrained**: 100-500 templates -- **High-security environments**: 500-1000 templates (prevents DoS via template flooding) - ## Error Handling Configuration To prevent memory exhaustion from malformed packets, the parser limits the size of error buffer samples. By default, only the first 256 bytes of unparseable data are stored in error messages. You can customize this limit for all parsers: diff --git a/src/lib.rs b/src/lib.rs index 397628af..fd8cc243 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -368,18 +368,68 @@ //! } //! ``` //! +//! ## Template Cache Configuration +//! +//! V9 and IPFIX parsers use LRU (Least Recently Used) caching to store templates with a configurable size limit. This prevents memory exhaustion from template flooding attacks while maintaining good performance for legitimate traffic. +//! +//! ### Default Behavior +//! +//! By default, parsers cache up to 1000 templates: +//! +//! ```rust +//! use netflow_parser::NetflowParser; +//! +//! // Uses default cache size of 1000 templates per parser +//! let parser = NetflowParser::default(); +//! ``` +//! +//! ### Custom Cache Size +//! +//! You can configure the template cache size when creating parsers: +//! +//! ```rust,ignore +//! use netflow_parser::{NetflowParser, variable_versions::v9::V9Parser, variable_versions::ipfix::IPFixParser}; +//! +//! // Create V9 parser with custom cache size +//! let v9_parser = V9Parser::try_new(5000)?; // Cache up to 5000 templates +//! +//! // Create IPFix parser with custom cache size +//! let ipfix_parser = IPFixParser::try_new(2000)?; // Cache up to 2000 templates +//! +//! // Note: try_new() returns Result and will fail if cache_size is 0 +//! ``` +//! +//! ### Cache Behavior +//! +//! - When the cache is full, the least recently used template is evicted +//! - Templates are keyed by template ID (per source) +//! - Each parser instance maintains its own template cache +//! - For multi-source deployments, create separate parser instances per source +//! //! ## V9/IPFIX Notes //! -//! Parse the data (`&[u8]`) like any other version. The parser (`NetflowParser`) caches parsed templates, so you can send header/data flowset combos and it will use the cached templates. To see cached templates, use the parser for the correct version (`v9_parser` for V9, `ipfix_parser` for IPFIX). +//! Parse the data (`&[u8]`) like any other version. The parser (`NetflowParser`) caches parsed templates using LRU eviction, so you can send header/data flowset combos and it will use the cached templates. Templates are automatically cached and evicted when the cache limit is reached. //! -//! **IPFIX Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it. +//! **Template Cache Access:** +//! Template caches use `LruCache` internally. You can inspect cached templates, but note that accessing them may affect LRU ordering: //! //! ```rust //! use netflow_parser::NetflowParser; //! let parser = NetflowParser::default(); -//! dbg!(parser.v9_parser.templates); -//! dbg!(parser.v9_parser.options_templates); +//! +//! // Check if a template exists +//! if parser.v9_parser.templates.contains(&template_id) { +//! // Template is cached +//! } +//! +//! // Get cache stats +//! println!("V9 template cache size: {}", parser.v9_parser.templates.len()); +//! println!("V9 max cache size: {}", parser.v9_parser.max_template_cache_size); +//! # let template_id = 256; //! ``` +//! +//! **IPFIX Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it. +//! //! To access templates flowset of a processed V9/IPFix flowset you can find the `flowsets` attribute on the Parsed Record. In there you can find `Templates`, `Option Templates`, and `Data` Flowsets. //! //! ## Performance & Thread Safety From 22a9f8632f5d15deeab84ca948f4d1a6a01d86a1 Mon Sep 17 00:00:00 2001 From: mikemiles-dev Date: Sat, 20 Dec 2025 10:58:20 -0600 Subject: [PATCH 3/3] fix: Fixed tests --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index fd8cc243..1168ef18 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -416,6 +416,7 @@ //! ```rust //! use netflow_parser::NetflowParser; //! let parser = NetflowParser::default(); +//! # let template_id = 256; //! //! // Check if a template exists //! if parser.v9_parser.templates.contains(&template_id) { @@ -425,7 +426,6 @@ //! // Get cache stats //! println!("V9 template cache size: {}", parser.v9_parser.templates.len()); //! println!("V9 max cache size: {}", parser.v9_parser.max_template_cache_size); -//! # let template_id = 256; //! ``` //! //! **IPFIX Note:** We only parse sequence number and domain id, it is up to you if you wish to validate it.