From 5330fba0d3c851135483f7137e0e07aa185686c4 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:42:07 +0000 Subject: [PATCH 01/23] feat(datadog_agent): support v3 series metrics intake --- build.rs | 2 + .../25790_datadog_agent_v3_series.feature.md | 3 + proto/vector/dd_metric_v3.proto | 74 ++++ src/sources/datadog_agent/metrics.rs | 411 +++++++++++++++++- src/sources/datadog_agent/mod.rs | 5 + src/sources/datadog_agent/tests.rs | 143 +++++- tests/e2e/datadog-metrics/config/test.yaml | 8 +- 7 files changed, 637 insertions(+), 9 deletions(-) create mode 100644 changelog.d/25790_datadog_agent_v3_series.feature.md create mode 100644 proto/vector/dd_metric_v3.proto diff --git a/build.rs b/build.rs index 01cd4bea5e8ef..ff7aa95c007bc 100644 --- a/build.rs +++ b/build.rs @@ -134,6 +134,7 @@ fn main() { println!("cargo:rerun-if-changed=proto/third-party/google/pubsub/v1/pubsub.proto"); println!("cargo:rerun-if-changed=proto/third-party/google/rpc/status.proto"); println!("cargo:rerun-if-changed=proto/vector/dd_metric.proto"); + println!("cargo:rerun-if-changed=proto/vector/dd_metric_v3.proto"); println!("cargo:rerun-if-changed=proto/vector/dd_trace.proto"); println!("cargo:rerun-if-changed=proto/vector/ddsketch_full.proto"); println!("cargo:rerun-if-changed=proto/vector/vector.proto"); @@ -162,6 +163,7 @@ fn main() { "lib/vector-core/proto/event.proto", "proto/vector/ddsketch_full.proto", "proto/vector/dd_metric.proto", + "proto/vector/dd_metric_v3.proto", "proto/vector/dd_trace.proto", "proto/third-party/google/pubsub/v1/pubsub.proto", "proto/third-party/google/rpc/status.proto", diff --git a/changelog.d/25790_datadog_agent_v3_series.feature.md b/changelog.d/25790_datadog_agent_v3_series.feature.md new file mode 100644 index 0000000000000..3c0d435711ba9 --- /dev/null +++ b/changelog.d/25790_datadog_agent_v3_series.feature.md @@ -0,0 +1,3 @@ +The `datadog_agent` source now accepts the v3 series metrics intake format used by default by Datadog Agent 7.81.0 and newer. + +authors: arfa79 diff --git a/proto/vector/dd_metric_v3.proto b/proto/vector/dd_metric_v3.proto new file mode 100644 index 0000000000000..18c901189d2bc --- /dev/null +++ b/proto/vector/dd_metric_v3.proto @@ -0,0 +1,74 @@ +// Extracted from https://github.com/DataDog/agent-payload/blob/master/proto/metrics/intake_v3.proto + +syntax = "proto3"; + +package datadoghq.api.metrics.v3; + +message Payload { + reserved 1; + Metadata metadata = 2; + MetricData metricData = 3; +} + +message Metadata { + repeated string tags = 1; + repeated string resources = 2; +} + +message MetricData { + bytes dictNameStr = 1; + bytes dictTagStr = 2; + repeated sint64 dictTagsets = 3; + + bytes dictResourceStr = 4; + repeated int64 dictResourceLen = 5; + repeated sint64 dictResourceType = 6; + repeated sint64 dictResourceName = 7; + + bytes dictSourceTypeName = 8; + repeated int32 dictOriginInfo = 9; + bytes dictUnitStr = 25; + + repeated uint64 types = 10; + repeated sint64 nameRefs = 11; + repeated sint64 tagsetRefs = 12; + repeated sint64 resourcesRefs = 13; + repeated uint64 intervals = 14; + repeated uint64 numPoints = 15; + repeated sint64 sourceTypeNameRefs = 23; + repeated sint64 originInfoRefs = 24; + repeated sint64 unitRefs = 26; + + repeated sint64 timestamps = 16; + repeated sint64 valsSint64 = 17; + repeated float valsFloat32 = 18; + repeated double valsFloat64 = 19; + repeated uint64 sketchNumBins = 20; + repeated sint32 sketchBinKeys = 21; + repeated uint32 sketchBinCnts = 22; +} + +enum metricType { + UNUSED = 0; + Count = 1; + Rate = 2; + Gauge = 3; + Sketch = 4; +} + +enum valueType { + Zero = 0x00; + Sint64 = 0x10; + Float32 = 0x20; + Float64 = 0x30; +} + +enum metricFlags { + flagNone = 0; + flagNoIndex = 0x100; + flagHasUnit = 0x200; +} + +message Response { + string error = 1; +} diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index a92bb5214bfce..ea09101b9d3a4 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -14,6 +14,7 @@ use vector_lib::{ use warp::{Filter, filters::BoxedFilter, path, path::FullPath, reply::Response}; use super::ddmetric_proto::{Metadata, MetricPayload, SketchPayload, metric_payload}; +use super::ddmetric_v3_proto::Payload as MetricPayloadV3; use super::{ApiKeyQueryParams, DatadogAgentSource, RequestHandler}; use crate::{ common::{ @@ -41,12 +42,15 @@ pub(super) fn build_warp_filter( ) -> BoxedFilter<(Response,)> { let sketches_service = sketches_service(handler.clone(), source.clone()); let series_v1_service = series_v1_service(handler.clone(), source.clone()); - let series_v2_service = series_v2_service(handler, source); + let series_v2_service = series_v2_service(handler.clone(), source.clone()); + let series_v3_service = series_v3_service(handler, source); sketches_service .or(series_v1_service) .unify() .or(series_v2_service) .unify() + .or(series_v3_service) + .unify() .boxed() } @@ -164,6 +168,49 @@ fn series_v2_service( .boxed() } +fn series_v3_service( + handler: RequestHandler, + source: DatadogAgentSource, +) -> BoxedFilter<(Response,)> { + warp::post() + .and( + path!("api" / "intake" / "metrics" / "v3" / "series" / ..) + .or(path!( + "api" / "intake" / "metrics" / "v3beta" / "series" / .. + )) + .unify() + .and(warp::path::full()), + ) + .and(warp::header::optional::("content-encoding")) + .and(warp::header::optional::("dd-api-key")) + .and(warp::query::()) + .and(capped_body()) + .and_then({ + move |path: FullPath, + encoding_header: Option, + api_token: Option, + query_params: ApiKeyQueryParams, + body: Bytes| { + let events = source + .decode(&encoding_header, body, path.as_str()) + .and_then(|body| { + decode_datadog_series_v3( + body, + source.api_key_extractor.extract( + path.as_str(), + api_token, + query_params.dd_api_key, + ), + source.split_metric_namespace, + &source.events_received, + ) + }); + handler.clone().handle_request(events, super::METRICS) + } + }) + .boxed() +} + fn decode_datadog_sketches( body: Bytes, api_key: Option>, @@ -218,6 +265,368 @@ fn decode_datadog_series_v2( Ok(metrics) } +fn decode_datadog_series_v3( + body: Bytes, + api_key: Option>, + split_metric_namespace: bool, + events_received: &Registered, +) -> Result, ErrorMessage> { + if body.is_empty() { + debug!(message = "Empty payload ignored."); + return Ok(Vec::new()); + } + + let metrics = decode_ddseries_v3(body, &api_key, split_metric_namespace).map_err(|error| { + ErrorMessage::new( + StatusCode::UNPROCESSABLE_ENTITY, + format!("Error decoding Datadog v3 series: {error:?}"), + ) + })?; + + events_received.emit(CountByteSize( + metrics.len(), + metrics.estimated_json_encoded_size_of(), + )); + + Ok(metrics) +} + +pub(crate) fn decode_ddseries_v3( + frame: Bytes, + api_key: &Option>, + split_metric_namespace: bool, +) -> crate::Result> { + let payload = MetricPayloadV3::decode(frame)?; + let series = payload.metric_data.map_or(Ok(Vec::new()), |metric_data| { + decode_v3_metric_data(&metric_data, payload.metadata.as_ref()) + })?; + let mut frame = Vec::new(); + MetricPayload { series }.encode(&mut frame)?; + decode_ddseries_v2(Bytes::from(frame), api_key, split_metric_namespace) +} + +fn decode_v3_metric_data( + data: &super::ddmetric_v3_proto::MetricData, + metadata: Option<&super::ddmetric_v3_proto::Metadata>, +) -> crate::Result> { + let names = decode_v3_strings(&data.dict_name_str, false)?; + let tag_strings = decode_v3_strings(&data.dict_tag_str, true)?; + let units = decode_v3_strings(&data.dict_unit_str, false)?; + let tagsets = decode_v3_tagsets(&data.dict_tagsets, &tag_strings, metadata)?; + let resources = decode_v3_resources(data, metadata)?; + let source_types = decode_v3_strings(&data.dict_source_type_name, false)?; + let origin_count = data.dict_origin_info.len() / 3 + 1; + + if !data.dict_origin_info.len().is_multiple_of(3) + || data.name_refs.len() < data.types.len() + || data.tagset_refs.len() < data.types.len() + || data.resources_refs.len() < data.types.len() + || data.intervals.len() < data.types.len() + || data.num_points.len() < data.types.len() + || data.source_type_name_refs.len() < data.types.len() + || data.origin_info_refs.len() < data.types.len() + { + return Err("invalid Datadog v3 metric columns".into()); + } + + let mut name_ref = 0; + let mut tagset_ref = 0; + let mut resources_ref = 0; + let mut source_type_ref = 0; + let mut origin_ref = 0; + let mut unit_ref = 0; + let mut unit_idx = 0; + let mut timestamp: i64 = 0; + let mut timestamp_idx = 0; + let mut sint64_idx = 0; + let mut float32_idx = 0; + let mut float64_idx = 0; + let mut series = Vec::with_capacity(data.types.len()); + + for index in 0..data.types.len() { + name_ref += data.name_refs[index]; + tagset_ref += data.tagset_refs[index]; + resources_ref += data.resources_refs[index]; + source_type_ref += data.source_type_name_refs[index]; + origin_ref += data.origin_info_refs[index]; + if name_ref < 0 + || name_ref as usize >= names.len() + || tagset_ref < 0 + || tagset_ref as usize >= tagsets.len() + || resources_ref < 0 + || resources_ref as usize >= resources.len() + || source_type_ref < 0 + || source_type_ref as usize >= source_types.len() + || origin_ref < 0 + || origin_ref as usize >= origin_count + { + return Err("invalid Datadog v3 dictionary reference".into()); + } + + let packed_type = data.types[index]; + let metric_type = packed_type & 0x0f; + let value_type = packed_type & 0xf0; + if metric_type == 4 { + return Err("Datadog v3 series payload contains a sketch".into()); + } + let unit = if packed_type & 0x200 != 0 { + if unit_idx >= data.unit_refs.len() { + return Err("invalid Datadog v3 unit column".into()); + } + unit_ref += data.unit_refs[unit_idx]; + unit_idx += 1; + if unit_ref < 0 || unit_ref as usize >= units.len() { + return Err("invalid Datadog v3 unit reference".into()); + } + units[unit_ref as usize].clone() + } else { + String::new() + }; + + let point_count = usize::try_from(data.num_points[index]) + .map_err(|_| "invalid Datadog v3 point count")?; + let mut points = Vec::with_capacity(point_count); + for _ in 0..point_count { + if timestamp_idx >= data.timestamps.len() { + return Err("invalid Datadog v3 timestamp column".into()); + } + timestamp = timestamp + .checked_add(data.timestamps[timestamp_idx]) + .ok_or("Datadog v3 timestamp overflow")?; + timestamp_idx += 1; + let value = match value_type { + 0 => 0.0, + 0x10 => { + let value = data + .vals_sint64 + .get(sint64_idx) + .ok_or("invalid Datadog v3 value column")?; + sint64_idx += 1; + *value as f64 + } + 0x20 => { + let value = data + .vals_float32 + .get(float32_idx) + .ok_or("invalid Datadog v3 value column")?; + float32_idx += 1; + *value as f64 + } + 0x30 => { + let value = data + .vals_float64 + .get(float64_idx) + .ok_or("invalid Datadog v3 value column")?; + float64_idx += 1; + *value + } + _ => return Err("invalid Datadog v3 value type".into()), + }; + points.push(super::ddmetric_proto::metric_payload::MetricPoint { value, timestamp }); + } + + let metric_type = match metric_type { + 1 => metric_payload::MetricType::Count, + 2 => metric_payload::MetricType::Rate, + 3 => metric_payload::MetricType::Gauge, + _ => metric_payload::MetricType::Unspecified, + }; + let metadata = if origin_ref == 0 { + None + } else { + let offset = (origin_ref as usize - 1) * 3; + Some(Metadata { + origin: Some(super::ddmetric_proto::Origin { + origin_product: data.dict_origin_info[offset] as u32, + origin_category: data.dict_origin_info[offset + 1] as u32, + origin_service: data.dict_origin_info[offset + 2] as u32, + }), + }) + }; + let resources = resources[resources_ref as usize] + .iter() + .map( + |(resource_type, name)| super::ddmetric_proto::metric_payload::Resource { + r#type: resource_type.clone(), + name: name.clone(), + }, + ) + .collect(); + series.push(super::ddmetric_proto::metric_payload::MetricSeries { + resources, + metric: names[name_ref as usize].clone(), + tags: tagsets[tagset_ref as usize].clone(), + points, + r#type: metric_type as i32, + unit, + source_type_name: source_types[source_type_ref as usize].clone(), + interval: i64::from( + u32::try_from(data.intervals[index]).map_err(|_| "invalid Datadog v3 interval")?, + ), + metadata, + }); + } + Ok(series) +} + +fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { + let mut values = vec![String::new()]; + let mut offset = 0; + while offset < raw.len() { + let (length, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let length = usize::try_from(length).map_err(|_| "Datadog v3 string length overflow")?; + let end = offset + .checked_add(length) + .ok_or("Datadog v3 string length overflow")?; + if end > raw.len() { + return Err("truncated Datadog v3 string dictionary".into()); + } + let bytes = &raw[offset..end]; + let value = match std::str::from_utf8(bytes) { + Ok(value) => value.to_owned(), + Err(_) if sanitize => String::from_utf8_lossy(bytes).into_owned(), + Err(error) => return Err(error.into()), + }; + values.push(value); + offset = end; + } + Ok(values) +} + +fn decode_v3_varint(raw: &[u8]) -> crate::Result<(u64, usize)> { + let mut value = 0; + for (index, byte) in raw.iter().copied().enumerate() { + if index == 10 { + return Err("Datadog v3 varint overflow".into()); + } + if index == 9 && byte > 1 { + return Err("Datadog v3 varint overflow".into()); + } + value |= u64::from(byte & 0x7f) << (index * 7); + if byte & 0x80 == 0 { + return Ok((value, index + 1)); + } + } + Err("truncated Datadog v3 varint".into()) +} + +fn decode_v3_tagsets( + packed: &[i64], + dictionary: &[String], + metadata: Option<&super::ddmetric_v3_proto::Metadata>, +) -> crate::Result>> { + let mut tagsets = vec![Vec::new()]; + let mut offset = 0; + while offset < packed.len() { + let size = usize::try_from(packed[offset]).map_err(|_| "invalid Datadog v3 tagset size")?; + offset += 1; + let end = offset + .checked_add(size) + .ok_or("Datadog v3 tagset length overflow")?; + if end > packed.len() { + return Err("truncated Datadog v3 tagset dictionary".into()); + } + let mut tags = Vec::new(); + let mut reference: i64 = 0; + for delta in &packed[offset..end] { + reference = reference + .checked_add(*delta) + .ok_or("Datadog v3 tagset reference overflow")?; + if reference < 0 { + let index = usize::try_from( + reference + .checked_neg() + .ok_or("Datadog v3 tagset reference overflow")?, + ) + .map_err(|_| "invalid Datadog v3 tagset reference")?; + tags.extend( + tagsets + .get(index) + .ok_or("invalid Datadog v3 tagset reference")? + .iter() + .cloned(), + ); + } else { + tags.push( + dictionary + .get(reference as usize) + .ok_or("invalid Datadog v3 tag reference")? + .clone(), + ); + } + } + tagsets.push(tags); + offset = end; + } + if let Some(metadata) = metadata { + for tagset in &mut tagsets { + for tag in &metadata.tags { + if !tagset.contains(tag) { + tagset.push(tag.clone()); + } + } + } + } + Ok(tagsets) +} + +fn decode_v3_resources( + data: &super::ddmetric_v3_proto::MetricData, + metadata: Option<&super::ddmetric_v3_proto::Metadata>, +) -> crate::Result>> { + let dictionary = decode_v3_strings(&data.dict_resource_str, false)?; + let mut resources = vec![Vec::new()]; + let mut offset = 0; + while offset < data.dict_resource_len.len() { + let size = usize::try_from(data.dict_resource_len[offset]) + .map_err(|_| "invalid Datadog v3 resource size")?; + let end = offset + .checked_add(size) + .ok_or("Datadog v3 resource length overflow")?; + if end > data.dict_resource_type.len() || end > data.dict_resource_name.len() { + return Err("truncated Datadog v3 resource dictionary".into()); + } + let mut set = Vec::with_capacity(size); + let mut type_ref: i64 = 0; + let mut name_ref: i64 = 0; + for index in offset..end { + type_ref = type_ref + .checked_add(data.dict_resource_type[index]) + .ok_or("Datadog v3 resource reference overflow")?; + name_ref = name_ref + .checked_add(data.dict_resource_name[index]) + .ok_or("Datadog v3 resource reference overflow")?; + if type_ref < 0 || name_ref < 0 { + return Err("invalid Datadog v3 resource reference".into()); + } + set.push(( + dictionary + .get(type_ref as usize) + .ok_or("invalid Datadog v3 resource reference")? + .clone(), + dictionary + .get(name_ref as usize) + .ok_or("invalid Datadog v3 resource reference")? + .clone(), + )); + } + resources.push(set); + offset += size; + } + if let Some(metadata) = metadata { + if metadata.resources.len() % 2 != 0 { + return Err("Datadog v3 metadata resources must be pairs".into()); + } + for set in &mut resources { + for pair in metadata.resources.chunks_exact(2) { + set.push((pair[0].clone(), pair[1].clone())); + } + } + } + Ok(resources) +} + /// Builds Vector's `EventMetadata` from the received metadata. Currently this is only /// utilized for passing through origin metadata set by the Agent. fn get_event_metadata(metadata: Option<&Metadata>) -> EventMetadata { diff --git a/src/sources/datadog_agent/mod.rs b/src/sources/datadog_agent/mod.rs index faa1226a19e45..d69980fe6fffd 100644 --- a/src/sources/datadog_agent/mod.rs +++ b/src/sources/datadog_agent/mod.rs @@ -12,6 +12,11 @@ pub(crate) mod ddmetric_proto { include!(concat!(env!("OUT_DIR"), "/datadog.agentpayload.rs")); } +#[allow(warnings, clippy::pedantic, clippy::nursery)] +pub(crate) mod ddmetric_v3_proto { + include!(concat!(env!("OUT_DIR"), "/datadoghq.api.metrics.v3.rs")); +} + #[allow(warnings)] pub(crate) mod ddtrace_proto { include!(concat!(env!("OUT_DIR"), "/dd_trace.rs")); diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 1dff2a7f96953..29404951a1398 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -49,7 +49,7 @@ use crate::{ serde::{default_decoding, default_framing_message_based}, sources::datadog_agent::{ DatadogAgentConfig, DatadogAgentSource, LOGS, LogMsg, METRICS, TRACES, ddmetric_proto, - ddtrace_proto, logs::decode_log_body, metrics::DatadogSeriesRequest, + ddmetric_v3_proto, ddtrace_proto, logs::decode_log_body, metrics::DatadogSeriesRequest, }, test_util::{ addr::{PortGuard, next_addr}, @@ -63,6 +63,7 @@ const DD_API_LOGS_V1_PATH: &str = "/v1/input/"; const DD_API_LOGS_V2_PATH: &str = "/api/v2/logs"; const DD_API_SERIES_V1_PATH: &str = "/api/v1/series"; const DD_API_SERIES_V2_PATH: &str = "/api/v2/series"; +const DD_API_SERIES_V3_PATH: &str = "/api/intake/metrics/v3/series"; const DD_API_SKETCHES_PATH: &str = "/api/beta/sketches"; const DD_API_TRACES_PATH: &str = "/api/v0.2/traces"; const HTTP_REQUEST_TIMEOUT: Duration = Duration::from_secs(5); @@ -357,6 +358,35 @@ async fn send_and_collect( .await } +async fn send_bytes_and_collect( + address: SocketAddr, + body: Vec, + headers: HeaderMap, + path: &'static str, + rx: impl Stream + Unpin, + expected_count: usize, +) -> Vec { + spawn_collect_n( + async move { + let response = timeout( + HTTP_REQUEST_TIMEOUT, + reqwest::Client::new() + .post(format!("http://{address}{path}")) + .headers(headers) + .body(body) + .send(), + ) + .await + .expect("send_bytes_and_collect request timed out") + .unwrap(); + assert_eq!(200, response.status().as_u16()); + }, + rx, + expected_count, + ) + .await +} + fn dd_api_key_headers() -> HeaderMap { let mut headers = HeaderMap::new(); headers.insert("dd-api-key", DD_API_KEY.parse().unwrap()); @@ -2332,6 +2362,117 @@ async fn decode_series_endpoint_v2() { .await; } +fn v3_string_dictionary(values: &[&str]) -> Vec { + values + .iter() + .flat_map(|value| { + let mut length = value.len() as u64; + let mut encoded = Vec::new(); + while length >= 0x80 { + encoded.push((length as u8 & 0x7f) | 0x80); + length >>= 7; + } + encoded.push(length as u8); + encoded.extend_from_slice(value.as_bytes()); + encoded + }) + .collect() +} + +#[tokio::test] +async fn decode_series_endpoint_v3() { + assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { + let (rx, _, _, addr, _guard) = + source(EventStatus::Delivered, true, true, false, true).await; + + let payload = ddmetric_v3_proto::Payload { + metadata: Some(ddmetric_v3_proto::Metadata { + tags: vec!["team:core".to_string()], + resources: Vec::new(), + }), + metric_data: Some(ddmetric_v3_proto::MetricData { + dict_name_str: v3_string_dictionary(&["namespace.dd_gauge"]), + dict_tag_str: v3_string_dictionary(&["foo:bar"]), + dict_tagsets: vec![1, 1], + dict_resource_str: v3_string_dictionary(&["host", "random_host"]), + dict_resource_len: vec![1], + dict_resource_type: vec![1], + dict_resource_name: vec![2], + dict_source_type_name: v3_string_dictionary(&["system"]), + dict_origin_info: vec![10, 10, 42], + dict_unit_str: Vec::new(), + types: vec![3 | 0x30], + name_refs: vec![1], + tagset_refs: vec![1], + resources_refs: vec![1], + intervals: vec![10], + num_points: vec![2], + source_type_name_refs: vec![1], + origin_info_refs: vec![1], + unit_refs: Vec::new(), + timestamps: vec![1542182950, 1], + vals_sint64: Vec::new(), + vals_float32: Vec::new(), + vals_float64: vec![3.14, 3.1415], + sketch_num_bins: Vec::new(), + sketch_bin_keys: Vec::new(), + sketch_bin_cnts: Vec::new(), + }), + }; + let mut body = Vec::new(); + payload.encode(&mut body).unwrap(); + let events = send_bytes_and_collect( + addr, + body, + dd_api_key_headers(), + DD_API_SERIES_V3_PATH, + rx, + 2, + ) + .await; + + let metric = events[0].as_metric(); + assert_eq!(metric.name(), "dd_gauge"); + assert_eq!(metric.namespace(), Some("namespace")); + assert_eq!(metric.kind(), MetricKind::Absolute); + assert_eq!(*metric.value(), MetricValue::Gauge { value: 3.14 }); + assert_tags( + metric, + metric_tags!( + "foo" => "bar", + "team" => "core", + "host" => "random_host", + "source_type_name" => "system" + ), + ); + assert_eq!( + metric + .metadata() + .datadog_origin_metadata() + .unwrap() + .product(), + Some(10) + ); + assert_eq!( + metric + .metadata() + .datadog_origin_metadata() + .unwrap() + .category(), + Some(10) + ); + assert_eq!( + metric + .metadata() + .datadog_origin_metadata() + .unwrap() + .service(), + Some(42) + ); + }) + .await; +} + #[test] fn test_output_schema_definition_json_vector_namespace() { let definition = serde_yaml::from_str::(indoc! { r#" diff --git a/tests/e2e/datadog-metrics/config/test.yaml b/tests/e2e/datadog-metrics/config/test.yaml index 933c644685198..b46170124fed9 100644 --- a/tests/e2e/datadog-metrics/config/test.yaml +++ b/tests/e2e/datadog-metrics/config/test.yaml @@ -12,13 +12,7 @@ runner: FAKE_INTAKE_VECTOR_ENDPOINT: "http://fakeintake-vector:80" matrix: - # Pinned to the last version before the Agent defaulted series metrics to - # the v3 intake API (7.81.0), which our datadog_agent source doesn't yet - # support. "7" is a floating tag that already resolves to >=7.81.0, so - # it's dropped in favor of the explicit 7.80.4 pin. Re-add an unpinned v7 - # leg once v3 series support lands - # (see https://github.com/vectordotdev/vector/issues/25790). - agent_version: ["7.80.4", "6"] + agent_version: ["7", "6"] # validate both series API versions series_api_version: ["v1", "v2"] From 8be2d900a465254683270fb75fc5d48b230a13af Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Mon, 31 Aug 2026 09:30:43 +0000 Subject: [PATCH 02/23] fix(datadog_agent): address v3 series review feedback --- lib/vector-core/proto/event.proto | 2 + lib/vector-core/src/event/metadata.rs | 43 +++++++++++ lib/vector-core/src/event/proto.rs | 6 ++ proto/vector/dd_metric.proto | 3 +- src/common/datadog.rs | 3 + src/sinks/datadog/metrics/encoder.rs | 34 ++++++--- src/sources/datadog_agent/metrics.rs | 104 +++++++++++++++++++------- src/sources/datadog_agent/tests.rs | 100 +++++++++++++++++++++---- 8 files changed, 241 insertions(+), 54 deletions(-) diff --git a/lib/vector-core/proto/event.proto b/lib/vector-core/proto/event.proto index 303aa6c2fe168..85157d45c24c9 100644 --- a/lib/vector-core/proto/event.proto +++ b/lib/vector-core/proto/event.proto @@ -75,6 +75,7 @@ message DatadogOriginMetadata { optional uint32 origin_product = 1; optional uint32 origin_category = 2; optional uint32 origin_service = 3; + optional int32 metric_type = 4; } message Secrets { @@ -94,6 +95,7 @@ message Metadata { OutputId upstream_id = 5; Secrets secrets = 6; bytes source_event_id = 7; + optional string datadog_metric_unit = 8; } message Metric { diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index 114c1c1e22308..b473c3ec5565f 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -84,6 +84,10 @@ pub(super) struct Inner { #[serde(default)] pub(crate) datadog_origin_metadata: Option, + /// Datadog metric unit received from an upstream Agent payload. + #[serde(default)] + pub(crate) datadog_metric_unit: Option, + /// An internal vector id that can be used to identify this event across all components. #[derivative(PartialEq = "ignore")] pub(crate) source_event_id: Option, @@ -92,6 +96,8 @@ pub(super) struct Inner { /// Metric Origin metadata for submission to Datadog. #[derive(Clone, Default, Debug, Deserialize, PartialEq, Serialize)] pub struct DatadogMetricOriginMetadata { + /// Datadog origin metric type. + metric_type: Option, /// `OriginProduct` product: Option, /// `OriginCategory` @@ -109,12 +115,25 @@ impl DatadogMetricOriginMetadata { #[must_use] pub fn new(product: Option, category: Option, service: Option) -> Self { Self { + metric_type: None, product, category, service, } } + /// Returns the Datadog origin metric type. + pub fn metric_type(&self) -> Option { + self.metric_type + } + + /// Sets the Datadog origin metric type. + #[must_use] + pub fn with_metric_type(mut self, metric_type: Option) -> Self { + self.metric_type = metric_type; + self + } + /// Returns a reference to the `OriginProduct`. pub fn product(&self) -> Option { self.product @@ -247,6 +266,16 @@ impl EventMetadata { self.inner.datadog_origin_metadata.as_ref() } + /// Returns the Datadog metric unit. + pub fn datadog_metric_unit(&self) -> Option<&str> { + self.inner.datadog_metric_unit.as_deref() + } + + /// Sets the Datadog metric unit. + pub fn set_datadog_metric_unit(&mut self, unit: String) { + self.get_mut().datadog_metric_unit = Some(unit); + } + /// Returns a reference to the event id. pub fn source_event_id(&self) -> Option { self.inner.source_event_id @@ -276,6 +305,7 @@ impl Default for Inner { upstream_id: None, dropped_fields: ObjectMap::new(), datadog_origin_metadata: None, + datadog_metric_unit: None, source_event_id: Some(Uuid::new_v4()), } } @@ -357,6 +387,13 @@ impl EventMetadata { self } + /// Sets the Datadog metric unit. + #[must_use] + pub fn with_datadog_metric_unit(mut self, unit: String) -> Self { + self.get_mut().datadog_metric_unit = Some(unit); + self + } + /// Replaces the existing `source_event_id` with the given one. #[must_use] pub fn with_source_event_id(mut self, source_event_id: Option) -> Self { @@ -373,6 +410,12 @@ impl EventMetadata { let other = other.into_owned(); inner.finalizers.merge(other.finalizers); inner.secrets.merge(other.secrets); + if inner.datadog_origin_metadata.is_none() { + inner.datadog_origin_metadata = other.datadog_origin_metadata; + } + if inner.datadog_metric_unit.is_none() { + inner.datadog_metric_unit = other.datadog_metric_unit; + } // Update `source_event_id` if necessary. if inner.source_event_id.is_none() { diff --git a/lib/vector-core/src/event/proto.rs b/lib/vector-core/src/event/proto.rs index 73878c9a68c5c..a840b19e03067 100644 --- a/lib/vector-core/src/event/proto.rs +++ b/lib/vector-core/src/event/proto.rs @@ -590,6 +590,7 @@ impl From for super::metadata::Secrets { impl From for DatadogOriginMetadata { fn from(value: super::DatadogMetricOriginMetadata) -> Self { Self { + metric_type: value.metric_type(), origin_product: value.product(), origin_category: value.category(), origin_service: value.service(), @@ -604,6 +605,7 @@ impl From for super::DatadogMetricOriginMetadata { value.origin_category, value.origin_service, ) + .with_metric_type(value.metric_type) } } @@ -631,6 +633,7 @@ impl From for Metadata { source_type, upstream_id, datadog_origin_metadata, + datadog_metric_unit, source_event_id, .. } = value.into_owned(); @@ -645,6 +648,7 @@ impl From for Metadata { upstream_id: upstream_id.map(|id| id.as_ref().clone()).map(Into::into), secrets, source_event_id: source_event_id.map_or(vec![], std::convert::Into::into), + datadog_metric_unit, } } } @@ -658,6 +662,7 @@ impl From for EventMetadata { upstream_id, secrets, datadog_origin_metadata, + datadog_metric_unit, source_event_id, } = value; @@ -694,6 +699,7 @@ impl From for EventMetadata { schema_definition: default_schema_definition(), dropped_fields: ObjectMap::new(), datadog_origin_metadata, + datadog_metric_unit, source_event_id, }), last_transform_timestamp: None, diff --git a/proto/vector/dd_metric.proto b/proto/vector/dd_metric.proto index 0bf7856eb6956..f953e55ec03a2 100644 --- a/proto/vector/dd_metric.proto +++ b/proto/vector/dd_metric.proto @@ -14,7 +14,8 @@ message CommonMetadata { } message Origin { - reserved 1,2,3; + reserved 1,2; + int32 metric_type = 3; uint32 origin_product = 4; uint32 origin_category = 5; uint32 origin_service = 6; diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 61b37b5ef8316..a51014af737cb 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -62,6 +62,9 @@ pub struct DatadogSeriesMetric { /// device #[serde(skip_serializing_if = "Option::is_none")] pub device: Option, + /// unit + #[serde(skip_serializing_if = "Option::is_none")] + pub unit: Option, /// metadata #[serde(skip_serializing_if = "Option::is_none")] pub metadata: Option, diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index c43c472034d73..bf40d5841bed1 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -491,6 +491,7 @@ fn generate_proto_metadata( } ddmetric_proto::Metadata { origin: Some(ddmetric_proto::Origin { + metric_type: origin.metric_type().unwrap_or_default(), origin_product: origin.product().unwrap_or_default(), origin_category: origin.category().unwrap_or_default(), origin_service: origin.service().unwrap_or_default(), @@ -718,8 +719,10 @@ fn series_to_proto_message( tags, points, r#type: metric_type.into(), - // unit is omitted - unit: "".to_string(), + unit: event_metadata + .datadog_metric_unit() + .unwrap_or_default() + .to_string(), source_type_name, interval: maybe_interval.unwrap_or(0) as i64, metadata, @@ -829,11 +832,14 @@ fn generate_origin_metadata( // - `log_to_metric` transform set the OriginService in the EventMetadata when it creates // the new metric. if let Some(pass_through) = maybe_pass_through { - Some(DatadogMetricOriginMetadata::new( - pass_through.product().or(Some(origin_product_value)), - pass_through.category().or(Some(ORIGIN_CATEGORY_VALUE)), - pass_through.service().or(Some(no_value)), - )) + Some( + DatadogMetricOriginMetadata::new( + pass_through.product().or(Some(origin_product_value)), + pass_through.category().or(Some(ORIGIN_CATEGORY_VALUE)), + pass_through.service().or(Some(no_value)), + ) + .with_metric_type(pass_through.metric_type()), + ) // No metadata has been set upstream } else { @@ -929,6 +935,7 @@ fn generate_series_metrics( host, source_type_name, device, + unit: event_metadata.datadog_metric_unit().map(ToOwned::to_owned), metadata, }]) } @@ -1529,9 +1536,12 @@ mod tests { let category = 11; let service = 9; - let event_metadata = EventMetadata::default().with_origin_metadata( - DatadogMetricOriginMetadata::new(Some(product), Some(category), Some(service)), - ); + let event_metadata = EventMetadata::default() + .with_origin_metadata( + DatadogMetricOriginMetadata::new(Some(product), Some(category), Some(service)) + .with_metric_type(Some(9)), + ) + .with_datadog_metric_unit("byte".to_string()); let counter = get_simple_counter_with_metadata(event_metadata); // series v1 @@ -1553,6 +1563,8 @@ mod tests { assert_eq!(generated_origin.product().unwrap(), product); assert_eq!(generated_origin.category().unwrap(), category); assert_eq!(generated_origin.service().unwrap(), service); + assert_eq!(generated_origin.metric_type(), Some(9)); + assert_eq!(actual.unit.as_deref(), Some("byte")); } // series v2 { @@ -1568,6 +1580,8 @@ mod tests { assert_eq!(generated_origin.origin_product, product); assert_eq!(generated_origin.origin_category, category); assert_eq!(generated_origin.origin_service, service); + assert_eq!(generated_origin.metric_type, 9); + assert_eq!(series_proto.unit, "byte"); } } diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index ea09101b9d3a4..2eff594f12e4b 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -300,12 +300,10 @@ pub(crate) fn decode_ddseries_v3( let series = payload.metric_data.map_or(Ok(Vec::new()), |metric_data| { decode_v3_metric_data(&metric_data, payload.metadata.as_ref()) })?; - let mut frame = Vec::new(); - MetricPayload { series }.encode(&mut frame)?; - decode_ddseries_v2(Bytes::from(frame), api_key, split_metric_namespace) + decode_ddseries(series, api_key, split_metric_namespace) } -fn decode_v3_metric_data( +pub(super) fn decode_v3_metric_data( data: &super::ddmetric_v3_proto::MetricData, metadata: Option<&super::ddmetric_v3_proto::Metadata>, ) -> crate::Result> { @@ -385,6 +383,17 @@ fn decode_v3_metric_data( let point_count = usize::try_from(data.num_points[index]) .map_err(|_| "invalid Datadog v3 point count")?; + let remaining_timestamps = data.timestamps.len().saturating_sub(timestamp_idx); + let remaining_values = match value_type { + 0 => remaining_timestamps, + 0x10 => data.vals_sint64.len().saturating_sub(sint64_idx), + 0x20 => data.vals_float32.len().saturating_sub(float32_idx), + 0x30 => data.vals_float64.len().saturating_sub(float64_idx), + _ => return Err("invalid Datadog v3 value type".into()), + }; + if point_count > remaining_timestamps || point_count > remaining_values { + return Err("invalid Datadog v3 point count".into()); + } let mut points = Vec::with_capacity(point_count); for _ in 0..point_count { if timestamp_idx >= data.timestamps.len() { @@ -431,15 +440,29 @@ fn decode_v3_metric_data( 3 => metric_payload::MetricType::Gauge, _ => metric_payload::MetricType::Unspecified, }; - let metadata = if origin_ref == 0 { + let no_index = packed_type & 0x100 != 0; + let metadata = if origin_ref == 0 && !no_index { None } else { - let offset = (origin_ref as usize - 1) * 3; + let (origin_product, origin_category, origin_service) = if origin_ref == 0 { + (0, 0, 0) + } else { + let offset = (origin_ref as usize - 1) * 3; + ( + u32::try_from(data.dict_origin_info[offset]) + .map_err(|_| "invalid Datadog v3 origin product")?, + u32::try_from(data.dict_origin_info[offset + 1]) + .map_err(|_| "invalid Datadog v3 origin category")?, + u32::try_from(data.dict_origin_info[offset + 2]) + .map_err(|_| "invalid Datadog v3 origin service")?, + ) + }; Some(Metadata { origin: Some(super::ddmetric_proto::Origin { - origin_product: data.dict_origin_info[offset] as u32, - origin_category: data.dict_origin_info[offset + 1] as u32, - origin_service: data.dict_origin_info[offset + 2] as u32, + metric_type: if no_index { 9 } else { 0 }, + origin_product, + origin_category, + origin_service, }), }) }; @@ -577,11 +600,10 @@ fn decode_v3_resources( ) -> crate::Result>> { let dictionary = decode_v3_strings(&data.dict_resource_str, false)?; let mut resources = vec![Vec::new()]; - let mut offset = 0; - while offset < data.dict_resource_len.len() { - let size = usize::try_from(data.dict_resource_len[offset]) - .map_err(|_| "invalid Datadog v3 resource size")?; - let end = offset + let mut entry_offset: usize = 0; + for &resource_len in &data.dict_resource_len { + let size = usize::try_from(resource_len).map_err(|_| "invalid Datadog v3 resource size")?; + let end = entry_offset .checked_add(size) .ok_or("Datadog v3 resource length overflow")?; if end > data.dict_resource_type.len() || end > data.dict_resource_name.len() { @@ -590,7 +612,7 @@ fn decode_v3_resources( let mut set = Vec::with_capacity(size); let mut type_ref: i64 = 0; let mut name_ref: i64 = 0; - for index in offset..end { + for index in entry_offset..end { type_ref = type_ref .checked_add(data.dict_resource_type[index]) .ok_or("Datadog v3 resource reference overflow")?; @@ -612,7 +634,7 @@ fn decode_v3_resources( )); } resources.push(set); - offset += size; + entry_offset = end; } if let Some(metadata) = metadata { if metadata.resources.len() % 2 != 0 { @@ -637,11 +659,14 @@ fn get_event_metadata(metadata: Option<&Metadata>) -> EventMetadata { "Deserialized origin_product: `{}` origin_category: `{}` origin_service: `{}`.", origin.origin_product, origin.origin_category, origin.origin_service, ); - EventMetadata::default().with_origin_metadata(DatadogMetricOriginMetadata::new( - Some(origin.origin_product), - Some(origin.origin_category), - Some(origin.origin_service), - )) + EventMetadata::default().with_origin_metadata( + DatadogMetricOriginMetadata::new( + Some(origin.origin_product), + Some(origin.origin_category), + Some(origin.origin_service), + ) + .with_metric_type((origin.metric_type != 0).then_some(origin.metric_type)), + ) }) } @@ -651,8 +676,15 @@ pub(crate) fn decode_ddseries_v2( split_metric_namespace: bool, ) -> crate::Result> { let payload = MetricPayload::decode(frame)?; - let decoded_metrics: Vec = payload - .series + decode_ddseries(payload.series, api_key, split_metric_namespace) +} + +fn decode_ddseries( + series: Vec, + api_key: &Option>, + split_metric_namespace: bool, +) -> crate::Result> { + let decoded_metrics: Vec = series .into_iter() .flat_map(|serie| { let (namespace, name) = if split_metric_namespace { @@ -662,7 +694,10 @@ pub(crate) fn decode_ddseries_v2( }; let mut tags = into_metric_tags(serie.tags); - let event_metadata = get_event_metadata(serie.metadata.as_ref()); + let mut event_metadata = get_event_metadata(serie.metadata.as_ref()); + if !serie.unit.is_empty() { + event_metadata.set_datadog_metric_unit(serie.unit.clone()); + } // It is possible to receive non-rate metrics from the Agent with an interval set. // That interval can be applied with the `as_rate` function in the Datadog UI. @@ -712,8 +747,6 @@ pub(crate) fn decode_ddseries_v2( }); (!serie.source_type_name.is_empty()) .then(|| tags.replace("source_type_name".into(), serie.source_type_name)); - // As per https://github.com/DataDog/datadog-agent/blob/a62ac9fb13e1e5060b89e731b8355b2b20a07c5b/pkg/serializer/internal/metrics/iterable_series.go#L224 - // serie.unit is omitted match metric_payload::MetricType::try_from(serie.r#type) { Ok(metric_payload::MetricType::Count) => serie .points @@ -854,6 +887,16 @@ fn into_vector_metric( split_metric_namespace: bool, ) -> Vec { let mut tags = into_metric_tags(dd_metric.tags.unwrap_or_default()); + let mut event_metadata = dd_metric + .metadata + .as_ref() + .and_then(|metadata| metadata.origin.as_ref()) + .map_or_else(EventMetadata::default, |origin| { + EventMetadata::default().with_origin_metadata(origin.clone()) + }); + if let Some(unit) = dd_metric.unit.filter(|unit| !unit.is_empty()) { + event_metadata.set_datadog_metric_unit(unit); + } if let Some(key) = log_schema().host_key() { dd_metric @@ -879,10 +922,11 @@ fn into_vector_metric( .points .iter() .map(|dd_point| { - Metric::new( + Metric::new_with_metadata( name.to_string(), MetricKind::Incremental, MetricValue::Counter { value: dd_point.1 }, + event_metadata.clone(), ) .with_timestamp(Some( Utc.timestamp_opt(dd_point.0, 0) @@ -897,10 +941,11 @@ fn into_vector_metric( .points .iter() .map(|dd_point| { - Metric::new( + Metric::new_with_metadata( name.to_string(), MetricKind::Absolute, MetricValue::Gauge { value: dd_point.1 }, + event_metadata.clone(), ) .with_timestamp(Some( Utc.timestamp_opt(dd_point.0, 0) @@ -918,12 +963,13 @@ fn into_vector_metric( .iter() .map(|dd_point| { let i = dd_metric.interval.filter(|v| *v != 0).unwrap_or(1); - Metric::new( + Metric::new_with_metadata( name.to_string(), MetricKind::Incremental, MetricValue::Counter { value: dd_point.1 * (i as f64), }, + event_metadata.clone(), ) .with_timestamp(Some( Utc.timestamp_opt(dd_point.0, 0) diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 29404951a1398..373be2ec6ed2f 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -938,6 +938,7 @@ async fn decode_series_endpoint_v1() { host: Some("random_host".to_string()), source_type_name: None, device: None, + unit: None, metadata: None, }, DatadogSeriesMetric { @@ -949,6 +950,7 @@ async fn decode_series_endpoint_v1() { host: Some("another_random_host".to_string()), source_type_name: None, device: None, + unit: None, metadata: None, }, DatadogSeriesMetric { @@ -960,6 +962,7 @@ async fn decode_series_endpoint_v1() { host: Some("a_host".to_string()), source_type_name: None, device: None, + unit: None, metadata: None, }, DatadogSeriesMetric { @@ -971,6 +974,7 @@ async fn decode_series_endpoint_v1() { host: None, source_type_name: None, device: None, + unit: None, metadata: None, }, DatadogSeriesMetric { @@ -982,6 +986,7 @@ async fn decode_series_endpoint_v1() { host: None, source_type_name: None, device: None, + unit: None, metadata: None, }, ], @@ -1150,6 +1155,7 @@ async fn decode_sketches() { }], metadata: Some(ddmetric_proto::Metadata { origin: Some(ddmetric_proto::Origin { + metric_type: 0, origin_product: 10, origin_category: 11, origin_service: 9, @@ -1507,6 +1513,7 @@ async fn split_outputs() { host: Some("random_host".to_string()), source_type_name: None, device: None, + unit: None, metadata: None, }], }; @@ -2177,6 +2184,7 @@ async fn decode_series_endpoint_v2() { interval: 0, metadata: Some(ddmetric_proto::Metadata { origin: Some(ddmetric_proto::Origin { + metric_type: 0, origin_product: 10, origin_category: 10, origin_service: 42, @@ -2379,6 +2387,54 @@ fn v3_string_dictionary(values: &[&str]) -> Vec { .collect() } +fn minimal_v3_metric_data() -> ddmetric_v3_proto::MetricData { + ddmetric_v3_proto::MetricData { + dict_name_str: v3_string_dictionary(&["metric"]), + dict_tag_str: Vec::new(), + dict_tagsets: Vec::new(), + dict_resource_str: Vec::new(), + dict_resource_len: Vec::new(), + dict_resource_type: Vec::new(), + dict_resource_name: Vec::new(), + dict_source_type_name: Vec::new(), + dict_origin_info: Vec::new(), + dict_unit_str: Vec::new(), + types: vec![3 | 0x30], + name_refs: vec![1], + tagset_refs: vec![0], + resources_refs: vec![0], + intervals: vec![0], + num_points: vec![1], + source_type_name_refs: vec![0], + origin_info_refs: vec![0], + unit_refs: Vec::new(), + timestamps: vec![1], + vals_sint64: Vec::new(), + vals_float32: Vec::new(), + vals_float64: vec![1.0], + sketch_num_bins: Vec::new(), + sketch_bin_keys: Vec::new(), + sketch_bin_cnts: Vec::new(), + } +} + +#[test] +fn decode_v3_rejects_point_count_larger_than_columns() { + let mut data = minimal_v3_metric_data(); + data.num_points[0] = u64::MAX; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_rejects_negative_origin_values() { + let mut data = minimal_v3_metric_data(); + data.dict_origin_info = vec![-1, 0, 0]; + data.origin_info_refs[0] = 1; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + #[tokio::test] async fn decode_series_endpoint_v3() { assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { @@ -2394,22 +2450,27 @@ async fn decode_series_endpoint_v3() { dict_name_str: v3_string_dictionary(&["namespace.dd_gauge"]), dict_tag_str: v3_string_dictionary(&["foo:bar"]), dict_tagsets: vec![1, 1], - dict_resource_str: v3_string_dictionary(&["host", "random_host"]), - dict_resource_len: vec![1], - dict_resource_type: vec![1], - dict_resource_name: vec![2], + dict_resource_str: v3_string_dictionary(&[ + "host", + "random_host", + "container", + "container_id", + ]), + dict_resource_len: vec![2, 1], + dict_resource_type: vec![1, 2, 3], + dict_resource_name: vec![2, 2, 4], dict_source_type_name: v3_string_dictionary(&["system"]), dict_origin_info: vec![10, 10, 42], - dict_unit_str: Vec::new(), - types: vec![3 | 0x30], - name_refs: vec![1], - tagset_refs: vec![1], - resources_refs: vec![1], - intervals: vec![10], - num_points: vec![2], - source_type_name_refs: vec![1], - origin_info_refs: vec![1], - unit_refs: Vec::new(), + dict_unit_str: v3_string_dictionary(&["byte"]), + types: vec![3 | 0x30 | 0x100 | 0x200, 3 | 0x30], + name_refs: vec![1, 0], + tagset_refs: vec![1, 0], + resources_refs: vec![1, 1], + intervals: vec![10, 10], + num_points: vec![1, 1], + source_type_name_refs: vec![1, 0], + origin_info_refs: vec![1, 0], + unit_refs: vec![1], timestamps: vec![1542182950, 1], vals_sint64: Vec::new(), vals_float32: Vec::new(), @@ -2442,6 +2503,7 @@ async fn decode_series_endpoint_v3() { "foo" => "bar", "team" => "core", "host" => "random_host", + "resource.container" => "container_id", "source_type_name" => "system" ), ); @@ -2469,6 +2531,15 @@ async fn decode_series_endpoint_v3() { .service(), Some(42) ); + assert_eq!( + metric + .metadata() + .datadog_origin_metadata() + .unwrap() + .metric_type(), + Some(9) + ); + assert_eq!(metric.metadata().datadog_metric_unit(), Some("byte")); }) .await; } @@ -2697,6 +2768,7 @@ async fn test_series_v1_split_metric_namespace_impl( host: Some("test_host".to_string()), source_type_name: None, device: None, + unit: None, metadata: None, }], }; From 584d5bcbee0dc9d0a1644345e5c768be62c7a3e2 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 1 Sep 2026 06:09:31 +0000 Subject: [PATCH 03/23] fix(datadog_agent): harden v3 series decoding --- src/sources/datadog_agent/metrics.rs | 64 ++++++++++++++++++++-------- src/sources/datadog_agent/tests.rs | 30 +++++++++++++ 2 files changed, 77 insertions(+), 17 deletions(-) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 2eff594f12e4b..3c6d13e1ed2c6 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -31,6 +31,8 @@ use crate::{ sources::util::{extract_tag_key_and_value, http::capped_body}, }; +const MAX_V3_EXPANDED_TAGSET_BYTES: usize = 16 * 1024 * 1024; + #[derive(Deserialize, Serialize)] pub(crate) struct DatadogSeriesRequest { pub(crate) series: Vec, @@ -402,6 +404,9 @@ pub(super) fn decode_v3_metric_data( timestamp = timestamp .checked_add(data.timestamps[timestamp_idx]) .ok_or("Datadog v3 timestamp overflow")?; + if Utc.timestamp_opt(timestamp, 0).single().is_none() { + return Err("invalid Datadog v3 timestamp".into()); + } timestamp_idx += 1; let value = match value_type { 0 => 0.0, @@ -475,6 +480,11 @@ pub(super) fn decode_v3_metric_data( }, ) .collect(); + let interval = + u32::try_from(data.intervals[index]).map_err(|_| "invalid Datadog v3 interval")?; + interval + .checked_mul(1000) + .ok_or("Datadog v3 interval milliseconds overflow")?; series.push(super::ddmetric_proto::metric_payload::MetricSeries { resources, metric: names[name_ref as usize].clone(), @@ -483,9 +493,7 @@ pub(super) fn decode_v3_metric_data( r#type: metric_type as i32, unit, source_type_name: source_types[source_type_ref as usize].clone(), - interval: i64::from( - u32::try_from(data.intervals[index]).map_err(|_| "invalid Datadog v3 interval")?, - ), + interval: i64::from(interval), metadata, }); } @@ -539,7 +547,8 @@ fn decode_v3_tagsets( dictionary: &[String], metadata: Option<&super::ddmetric_v3_proto::Metadata>, ) -> crate::Result>> { - let mut tagsets = vec![Vec::new()]; + let mut tagsets: Vec> = vec![Vec::new()]; + let mut expanded_bytes: usize = 0; let mut offset = 0; while offset < packed.len() { let size = usize::try_from(packed[offset]).map_err(|_| "invalid Datadog v3 tagset size")?; @@ -563,20 +572,34 @@ fn decode_v3_tagsets( .ok_or("Datadog v3 tagset reference overflow")?, ) .map_err(|_| "invalid Datadog v3 tagset reference")?; - tags.extend( - tagsets - .get(index) - .ok_or("invalid Datadog v3 tagset reference")? - .iter() - .cloned(), - ); + let referenced = tagsets + .get(index) + .ok_or("invalid Datadog v3 tagset reference")?; + let referenced_bytes = referenced.iter().try_fold(0usize, |total, tag| { + total + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(tag.len())) + .ok_or("Datadog v3 expanded tagset size overflow") + })?; + expanded_bytes = expanded_bytes + .checked_add(referenced_bytes) + .ok_or("Datadog v3 expanded tagset size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { + return Err("Datadog v3 expanded tagsets exceed size limit".into()); + } + tags.extend(referenced.iter().cloned()); } else { - tags.push( - dictionary - .get(reference as usize) - .ok_or("invalid Datadog v3 tag reference")? - .clone(), - ); + let tag = dictionary + .get(reference as usize) + .ok_or("invalid Datadog v3 tag reference")?; + expanded_bytes = expanded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(tag.len())) + .ok_or("Datadog v3 expanded tagset size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { + return Err("Datadog v3 expanded tagsets exceed size limit".into()); + } + tags.push(tag.clone()); } } tagsets.push(tags); @@ -586,6 +609,13 @@ fn decode_v3_tagsets( for tagset in &mut tagsets { for tag in &metadata.tags { if !tagset.contains(tag) { + expanded_bytes = expanded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(tag.len())) + .ok_or("Datadog v3 expanded tagset size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { + return Err("Datadog v3 expanded tagsets exceed size limit".into()); + } tagset.push(tag.clone()); } } diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 373be2ec6ed2f..d8439f09f10df 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2435,6 +2435,36 @@ fn decode_v3_rejects_negative_origin_values() { assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); } +#[test] +fn decode_v3_rejects_timestamp_outside_chrono_range() { + let mut data = minimal_v3_metric_data(); + data.timestamps[0] = i64::MAX; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_rejects_interval_millisecond_overflow() { + let mut data = minimal_v3_metric_data(); + data.intervals[0] = u64::from(u32::MAX / 1000 + 1); + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_rejects_excessive_tagset_expansion() { + let mut data = minimal_v3_metric_data(); + data.dict_tag_str = v3_string_dictionary(&[&"x".repeat(1024 * 1024)]); + data.dict_tagsets = vec![1, 1]; + for previous_tagset in 1..=5 { + data.dict_tagsets + .extend([2, -i64::from(previous_tagset), 0]); + } + data.tagset_refs[0] = 6; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + #[tokio::test] async fn decode_series_endpoint_v3() { assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { From e2463c4444a868792c7eeb623b17b9f7b6cceba0 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:27:50 +0000 Subject: [PATCH 04/23] fix(datadog_agent): bound v3 payload expansion --- src/sources/datadog_agent/metrics.rs | 121 ++++++++++++++++++++------- src/sources/datadog_agent/tests.rs | 53 ++++++++++++ 2 files changed, 146 insertions(+), 28 deletions(-) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 3c6d13e1ed2c6..8b1a81ce1ccfa 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -32,6 +32,7 @@ use crate::{ }; const MAX_V3_EXPANDED_TAGSET_BYTES: usize = 16 * 1024 * 1024; +const MAX_V3_EXPANDED_SERIES_BYTES: usize = 16 * 1024 * 1024; #[derive(Deserialize, Serialize)] pub(crate) struct DatadogSeriesRequest { @@ -313,10 +314,14 @@ pub(super) fn decode_v3_metric_data( let tag_strings = decode_v3_strings(&data.dict_tag_str, true)?; let units = decode_v3_strings(&data.dict_unit_str, false)?; let tagsets = decode_v3_tagsets(&data.dict_tagsets, &tag_strings, metadata)?; - let resources = decode_v3_resources(data, metadata)?; + let resources = decode_v3_resources(data)?; let source_types = decode_v3_strings(&data.dict_source_type_name, false)?; let origin_count = data.dict_origin_info.len() / 3 + 1; + if metadata.is_some_and(|metadata| metadata.resources.len() % 2 != 0) { + return Err("Datadog v3 metadata resources must be pairs".into()); + } + if !data.dict_origin_info.len().is_multiple_of(3) || data.name_refs.len() < data.types.len() || data.tagset_refs.len() < data.types.len() @@ -329,26 +334,37 @@ pub(super) fn decode_v3_metric_data( return Err("invalid Datadog v3 metric columns".into()); } - let mut name_ref = 0; - let mut tagset_ref = 0; - let mut resources_ref = 0; - let mut source_type_ref = 0; - let mut origin_ref = 0; - let mut unit_ref = 0; + let mut name_ref: i64 = 0; + let mut tagset_ref: i64 = 0; + let mut resources_ref: i64 = 0; + let mut source_type_ref: i64 = 0; + let mut origin_ref: i64 = 0; + let mut unit_ref: i64 = 0; let mut unit_idx = 0; let mut timestamp: i64 = 0; let mut timestamp_idx = 0; let mut sint64_idx = 0; let mut float32_idx = 0; let mut float64_idx = 0; + let mut expanded_series_bytes: usize = 0; let mut series = Vec::with_capacity(data.types.len()); for index in 0..data.types.len() { - name_ref += data.name_refs[index]; - tagset_ref += data.tagset_refs[index]; - resources_ref += data.resources_refs[index]; - source_type_ref += data.source_type_name_refs[index]; - origin_ref += data.origin_info_refs[index]; + name_ref = name_ref + .checked_add(data.name_refs[index]) + .ok_or("Datadog v3 name reference overflow")?; + tagset_ref = tagset_ref + .checked_add(data.tagset_refs[index]) + .ok_or("Datadog v3 tagset reference overflow")?; + resources_ref = resources_ref + .checked_add(data.resources_refs[index]) + .ok_or("Datadog v3 resource reference overflow")?; + source_type_ref = source_type_ref + .checked_add(data.source_type_name_refs[index]) + .ok_or("Datadog v3 source type reference overflow")?; + origin_ref = origin_ref + .checked_add(data.origin_info_refs[index]) + .ok_or("Datadog v3 origin reference overflow")?; if name_ref < 0 || name_ref as usize >= names.len() || tagset_ref < 0 @@ -373,7 +389,9 @@ pub(super) fn decode_v3_metric_data( if unit_idx >= data.unit_refs.len() { return Err("invalid Datadog v3 unit column".into()); } - unit_ref += data.unit_refs[unit_idx]; + unit_ref = unit_ref + .checked_add(data.unit_refs[unit_idx]) + .ok_or("Datadog v3 unit reference overflow")?; unit_idx += 1; if unit_ref < 0 || unit_ref as usize >= units.len() { return Err("invalid Datadog v3 unit reference".into()); @@ -396,6 +414,48 @@ pub(super) fn decode_v3_metric_data( if point_count > remaining_timestamps || point_count > remaining_values { return Err("invalid Datadog v3 point count".into()); } + + let tag_bytes = tagsets[tagset_ref as usize] + .iter() + .try_fold(0usize, |total, tag| checked_v3_string_bytes(total, tag))?; + let resource_bytes = resources[resources_ref as usize].iter().try_fold( + 0usize, + |total, (resource_type, name)| { + let total = checked_v3_string_bytes(total, resource_type)?; + checked_v3_string_bytes(total, name) + }, + )?; + let resource_bytes = metadata + .map(|metadata| { + metadata + .resources + .chunks_exact(2) + .try_fold(resource_bytes, |total, pair| { + let total = checked_v3_string_bytes(total, &pair[0])?; + checked_v3_string_bytes(total, &pair[1]) + }) + }) + .transpose()? + .unwrap_or(resource_bytes); + let series_bytes = + std::mem::size_of::() + .checked_add( + point_count + .checked_mul(std::mem::size_of::< + super::ddmetric_proto::metric_payload::MetricPoint, + >()) + .ok_or("Datadog v3 expanded series size overflow")?, + ) + .and_then(|total| total.checked_add(tag_bytes)) + .and_then(|total| total.checked_add(resource_bytes)) + .ok_or("Datadog v3 expanded series size overflow")?; + expanded_series_bytes = expanded_series_bytes + .checked_add(series_bytes) + .ok_or("Datadog v3 expanded series size overflow")?; + if expanded_series_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded series exceed size limit".into()); + } + let mut points = Vec::with_capacity(point_count); for _ in 0..point_count { if timestamp_idx >= data.timestamps.len() { @@ -446,7 +506,7 @@ pub(super) fn decode_v3_metric_data( _ => metric_payload::MetricType::Unspecified, }; let no_index = packed_type & 0x100 != 0; - let metadata = if origin_ref == 0 && !no_index { + let series_metadata = if origin_ref == 0 && !no_index { None } else { let (origin_product, origin_category, origin_service) = if origin_ref == 0 { @@ -471,7 +531,8 @@ pub(super) fn decode_v3_metric_data( }), }) }; - let resources = resources[resources_ref as usize] + let mut decoded_resources: Vec = resources + [resources_ref as usize] .iter() .map( |(resource_type, name)| super::ddmetric_proto::metric_payload::Resource { @@ -480,13 +541,21 @@ pub(super) fn decode_v3_metric_data( }, ) .collect(); + if let Some(payload_metadata) = metadata { + decoded_resources.extend(payload_metadata.resources.chunks_exact(2).map(|pair| { + super::ddmetric_proto::metric_payload::Resource { + r#type: pair[0].clone(), + name: pair[1].clone(), + } + })); + } let interval = u32::try_from(data.intervals[index]).map_err(|_| "invalid Datadog v3 interval")?; interval .checked_mul(1000) .ok_or("Datadog v3 interval milliseconds overflow")?; series.push(super::ddmetric_proto::metric_payload::MetricSeries { - resources, + resources: decoded_resources, metric: names[name_ref as usize].clone(), tags: tagsets[tagset_ref as usize].clone(), points, @@ -494,7 +563,7 @@ pub(super) fn decode_v3_metric_data( unit, source_type_name: source_types[source_type_ref as usize].clone(), interval: i64::from(interval), - metadata, + metadata: series_metadata, }); } Ok(series) @@ -626,7 +695,6 @@ fn decode_v3_tagsets( fn decode_v3_resources( data: &super::ddmetric_v3_proto::MetricData, - metadata: Option<&super::ddmetric_v3_proto::Metadata>, ) -> crate::Result>> { let dictionary = decode_v3_strings(&data.dict_resource_str, false)?; let mut resources = vec![Vec::new()]; @@ -666,19 +734,16 @@ fn decode_v3_resources( resources.push(set); entry_offset = end; } - if let Some(metadata) = metadata { - if metadata.resources.len() % 2 != 0 { - return Err("Datadog v3 metadata resources must be pairs".into()); - } - for set in &mut resources { - for pair in metadata.resources.chunks_exact(2) { - set.push((pair[0].clone(), pair[1].clone())); - } - } - } Ok(resources) } +fn checked_v3_string_bytes(total: usize, value: &str) -> crate::Result { + total + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(value.len())) + .ok_or_else(|| "Datadog v3 expanded string size overflow".into()) +} + /// Builds Vector's `EventMetadata` from the received metadata. Currently this is only /// utilized for passing through origin metadata set by the Agent. fn get_event_metadata(metadata: Option<&Metadata>) -> EventMetadata { diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index d8439f09f10df..976afc1af6af0 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2465,6 +2465,59 @@ fn decode_v3_rejects_excessive_tagset_expansion() { assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); } +#[test] +fn decode_v3_rejects_excessive_series_tag_cloning() { + let mut data = minimal_v3_metric_data(); + data.dict_tag_str = v3_string_dictionary(&[&"x".repeat(1024 * 1024)]); + data.dict_tagsets = vec![1, 1]; + data.types = vec![3 | 0x30; 17]; + data.name_refs = std::iter::once(1) + .chain(std::iter::repeat_n(0, 16)) + .collect(); + data.tagset_refs = std::iter::once(1) + .chain(std::iter::repeat_n(0, 16)) + .collect(); + data.resources_refs = vec![0; 17]; + data.intervals = vec![0; 17]; + data.num_points = vec![0; 17]; + data.source_type_name_refs = vec![0; 17]; + data.origin_info_refs = vec![0; 17]; + data.timestamps.clear(); + data.vals_float64.clear(); + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_rejects_reference_accumulator_overflow() { + let mut data = minimal_v3_metric_data(); + data.types = vec![3 | 0x30; 2]; + data.name_refs = vec![1, i64::MAX]; + data.tagset_refs = vec![0; 2]; + data.resources_refs = vec![0; 2]; + data.intervals = vec![0; 2]; + data.num_points = vec![0; 2]; + data.source_type_name_refs = vec![0; 2]; + data.origin_info_refs = vec![0; 2]; + data.timestamps.clear(); + data.vals_float64.clear(); + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_does_not_expand_unused_global_resources() { + let mut data = minimal_v3_metric_data(); + data.dict_resource_len = vec![0; 100_000]; + let metadata = ddmetric_v3_proto::Metadata { + tags: Vec::new(), + resources: vec!["container".to_string(), "x".repeat(1024)], + }; + + let series = super::metrics::decode_v3_metric_data(&data, Some(&metadata)).unwrap(); + assert_eq!(series[0].resources.len(), 1); +} + #[tokio::test] async fn decode_series_endpoint_v3() { assert_source_compliance(&HTTP_PUSH_SOURCE_TAGS, async { From 42d4e8684fa3027b408eb89b628a949611190745 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:36:18 +0000 Subject: [PATCH 05/23] fix(datadog_agent): tighten v3 expansion bounds --- src/sources/datadog_agent/metrics.rs | 61 +++++++++++++++++++--------- src/sources/datadog_agent/tests.rs | 36 ++++++++++++++++ 2 files changed, 77 insertions(+), 20 deletions(-) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 8b1a81ce1ccfa..4ae3dcfec6b9c 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -1,4 +1,4 @@ -use std::{num::NonZeroU32, sync::Arc}; +use std::{collections::HashSet, num::NonZeroU32, sync::Arc}; use bytes::Bytes; use chrono::{TimeZone, Utc}; @@ -32,6 +32,7 @@ use crate::{ }; const MAX_V3_EXPANDED_TAGSET_BYTES: usize = 16 * 1024 * 1024; +const MAX_V3_EXPANDED_RESOURCE_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_SERIES_BYTES: usize = 16 * 1024 * 1024; #[derive(Deserialize, Serialize)] @@ -347,7 +348,7 @@ pub(super) fn decode_v3_metric_data( let mut float32_idx = 0; let mut float64_idx = 0; let mut expanded_series_bytes: usize = 0; - let mut series = Vec::with_capacity(data.types.len()); + let mut series = Vec::new(); for index in 0..data.types.len() { name_ref = name_ref @@ -437,6 +438,15 @@ pub(super) fn decode_v3_metric_data( }) .transpose()? .unwrap_or(resource_bytes); + let tag_resource_bytes = tag_bytes + .checked_add(resource_bytes) + .ok_or("Datadog v3 expanded series size overflow")?; + let tag_resource_copies = point_count + .checked_add(1) + .ok_or("Datadog v3 expanded series size overflow")?; + let expanded_tag_resource_bytes = tag_resource_bytes + .checked_mul(tag_resource_copies) + .ok_or("Datadog v3 expanded series size overflow")?; let series_bytes = std::mem::size_of::() .checked_add( @@ -446,8 +456,7 @@ pub(super) fn decode_v3_metric_data( >()) .ok_or("Datadog v3 expanded series size overflow")?, ) - .and_then(|total| total.checked_add(tag_bytes)) - .and_then(|total| total.checked_add(resource_bytes)) + .and_then(|total| total.checked_add(expanded_tag_resource_bytes)) .ok_or("Datadog v3 expanded series size overflow")?; expanded_series_bytes = expanded_series_bytes .checked_add(series_bytes) @@ -675,13 +684,16 @@ fn decode_v3_tagsets( offset = end; } if let Some(metadata) = metadata { + let mut seen = HashSet::new(); + let metadata_tags = metadata + .tags + .iter() + .filter(|tag| seen.insert(tag.as_str())) + .collect::>(); for tagset in &mut tagsets { - for tag in &metadata.tags { + for &tag in &metadata_tags { if !tagset.contains(tag) { - expanded_bytes = expanded_bytes - .checked_add(std::mem::size_of::()) - .and_then(|total| total.checked_add(tag.len())) - .ok_or("Datadog v3 expanded tagset size overflow")?; + expanded_bytes = checked_v3_string_bytes(expanded_bytes, tag)?; if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { return Err("Datadog v3 expanded tagsets exceed size limit".into()); } @@ -698,8 +710,15 @@ fn decode_v3_resources( ) -> crate::Result>> { let dictionary = decode_v3_strings(&data.dict_resource_str, false)?; let mut resources = vec![Vec::new()]; + let mut expanded_bytes = std::mem::size_of::>(); let mut entry_offset: usize = 0; for &resource_len in &data.dict_resource_len { + expanded_bytes = expanded_bytes + .checked_add(std::mem::size_of::>()) + .ok_or("Datadog v3 expanded resource size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_RESOURCE_BYTES { + return Err("Datadog v3 expanded resources exceed size limit".into()); + } let size = usize::try_from(resource_len).map_err(|_| "invalid Datadog v3 resource size")?; let end = entry_offset .checked_add(size) @@ -707,7 +726,7 @@ fn decode_v3_resources( if end > data.dict_resource_type.len() || end > data.dict_resource_name.len() { return Err("truncated Datadog v3 resource dictionary".into()); } - let mut set = Vec::with_capacity(size); + let mut set = Vec::new(); let mut type_ref: i64 = 0; let mut name_ref: i64 = 0; for index in entry_offset..end { @@ -720,16 +739,18 @@ fn decode_v3_resources( if type_ref < 0 || name_ref < 0 { return Err("invalid Datadog v3 resource reference".into()); } - set.push(( - dictionary - .get(type_ref as usize) - .ok_or("invalid Datadog v3 resource reference")? - .clone(), - dictionary - .get(name_ref as usize) - .ok_or("invalid Datadog v3 resource reference")? - .clone(), - )); + let resource_type = dictionary + .get(type_ref as usize) + .ok_or("invalid Datadog v3 resource reference")?; + let resource_name = dictionary + .get(name_ref as usize) + .ok_or("invalid Datadog v3 resource reference")?; + expanded_bytes = checked_v3_string_bytes(expanded_bytes, resource_type)?; + expanded_bytes = checked_v3_string_bytes(expanded_bytes, resource_name)?; + if expanded_bytes > MAX_V3_EXPANDED_RESOURCE_BYTES { + return Err("Datadog v3 expanded resources exceed size limit".into()); + } + set.push((resource_type.clone(), resource_name.clone())); } resources.push(set); entry_offset = end; diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 976afc1af6af0..af61071d5522c 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2488,6 +2488,42 @@ fn decode_v3_rejects_excessive_series_tag_cloning() { assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); } +#[test] +fn decode_v3_rejects_excessive_point_tag_cloning() { + let mut data = minimal_v3_metric_data(); + data.dict_tag_str = v3_string_dictionary(&[&"x".repeat(1024 * 1024)]); + data.dict_tagsets = vec![1, 1]; + data.tagset_refs[0] = 1; + data.num_points[0] = 17; + data.timestamps = vec![1; 17]; + data.vals_float64 = vec![1.0; 17]; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_rejects_excessive_resource_expansion() { + let mut data = minimal_v3_metric_data(); + data.dict_resource_str = v3_string_dictionary(&[&"x".repeat(1024 * 1024)]); + data.dict_resource_len = vec![1; 17]; + data.dict_resource_type = vec![1; 17]; + data.dict_resource_name = vec![0; 17]; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_deduplicates_metadata_tags() { + let data = minimal_v3_metric_data(); + let metadata = ddmetric_v3_proto::Metadata { + tags: vec!["team:core".to_string(); 10_000], + resources: Vec::new(), + }; + + let series = super::metrics::decode_v3_metric_data(&data, Some(&metadata)).unwrap(); + assert_eq!(series[0].tags, ["team:core"]); +} + #[test] fn decode_v3_rejects_reference_accumulator_overflow() { let mut data = minimal_v3_metric_data(); From 068c2391cbe459986963f9aec275762651ffb6a7 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 1 Sep 2026 11:52:41 +0000 Subject: [PATCH 06/23] fix(datadog_agent): cap v3 decoded allocation --- src/sources/datadog_agent/metrics.rs | 42 +++++++++++++++++++++++-- src/sources/datadog_agent/tests.rs | 46 ++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 4ae3dcfec6b9c..722b4135e2c7c 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -34,6 +34,7 @@ use crate::{ const MAX_V3_EXPANDED_TAGSET_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_RESOURCE_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_SERIES_BYTES: usize = 16 * 1024 * 1024; +const MAX_V3_EXPANDED_EVENT_BYTES: usize = 16 * 1024 * 1024; #[derive(Deserialize, Serialize)] pub(crate) struct DatadogSeriesRequest { @@ -348,6 +349,7 @@ pub(super) fn decode_v3_metric_data( let mut float32_idx = 0; let mut float64_idx = 0; let mut expanded_series_bytes: usize = 0; + let mut expanded_event_bytes: usize = 0; let mut series = Vec::new(); for index in 0..data.types.len() { @@ -447,6 +449,12 @@ pub(super) fn decode_v3_metric_data( let expanded_tag_resource_bytes = tag_resource_bytes .checked_mul(tag_resource_copies) .ok_or("Datadog v3 expanded series size overflow")?; + let name = &names[name_ref as usize]; + let source_type = &source_types[source_type_ref as usize]; + let dictionary_string_bytes = checked_v3_string_bytes(0, name)?; + let dictionary_string_bytes = checked_v3_string_bytes(dictionary_string_bytes, &unit)?; + let dictionary_string_bytes = + checked_v3_string_bytes(dictionary_string_bytes, source_type)?; let series_bytes = std::mem::size_of::() .checked_add( @@ -457,6 +465,7 @@ pub(super) fn decode_v3_metric_data( .ok_or("Datadog v3 expanded series size overflow")?, ) .and_then(|total| total.checked_add(expanded_tag_resource_bytes)) + .and_then(|total| total.checked_add(dictionary_string_bytes)) .ok_or("Datadog v3 expanded series size overflow")?; expanded_series_bytes = expanded_series_bytes .checked_add(series_bytes) @@ -464,6 +473,22 @@ pub(super) fn decode_v3_metric_data( if expanded_series_bytes > MAX_V3_EXPANDED_SERIES_BYTES { return Err("Datadog v3 expanded series exceed size limit".into()); } + let event_bytes = std::mem::size_of::() + .checked_add(name.len()) + .and_then(|total| total.checked_add(tag_bytes)) + .and_then(|total| total.checked_add(resource_bytes)) + .and_then(|total| total.checked_add(unit.len())) + .ok_or("Datadog v3 expanded event size overflow")?; + expanded_event_bytes = expanded_event_bytes + .checked_add( + event_bytes + .checked_mul(point_count) + .ok_or("Datadog v3 expanded event size overflow")?, + ) + .ok_or("Datadog v3 expanded event size overflow")?; + if expanded_event_bytes > MAX_V3_EXPANDED_EVENT_BYTES { + return Err("Datadog v3 expanded events exceed size limit".into()); + } let mut points = Vec::with_capacity(point_count); for _ in 0..point_count { @@ -565,12 +590,12 @@ pub(super) fn decode_v3_metric_data( .ok_or("Datadog v3 interval milliseconds overflow")?; series.push(super::ddmetric_proto::metric_payload::MetricSeries { resources: decoded_resources, - metric: names[name_ref as usize].clone(), + metric: name.clone(), tags: tagsets[tagset_ref as usize].clone(), points, r#type: metric_type as i32, unit, - source_type_name: source_types[source_type_ref as usize].clone(), + source_type_name: source_type.clone(), interval: i64::from(interval), metadata: series_metadata, }); @@ -580,6 +605,7 @@ pub(super) fn decode_v3_metric_data( fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { let mut values = vec![String::new()]; + let mut expanded_bytes = std::mem::size_of::(); let mut offset = 0; while offset < raw.len() { let (length, consumed) = decode_v3_varint(&raw[offset..])?; @@ -597,6 +623,10 @@ fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { Err(_) if sanitize => String::from_utf8_lossy(bytes).into_owned(), Err(error) => return Err(error.into()), }; + expanded_bytes = checked_v3_string_bytes(expanded_bytes, &value)?; + if expanded_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); + } values.push(value); offset = end; } @@ -626,9 +656,15 @@ fn decode_v3_tagsets( metadata: Option<&super::ddmetric_v3_proto::Metadata>, ) -> crate::Result>> { let mut tagsets: Vec> = vec![Vec::new()]; - let mut expanded_bytes: usize = 0; + let mut expanded_bytes = std::mem::size_of::>(); let mut offset = 0; while offset < packed.len() { + expanded_bytes = expanded_bytes + .checked_add(std::mem::size_of::>()) + .ok_or("Datadog v3 expanded tagset size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { + return Err("Datadog v3 expanded tagsets exceed size limit".into()); + } let size = usize::try_from(packed[offset]).map_err(|_| "invalid Datadog v3 tagset size")?; offset += 1; let end = offset diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index af61071d5522c..e65dd0a5b4a1f 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2501,6 +2501,52 @@ fn decode_v3_rejects_excessive_point_tag_cloning() { assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); } +#[test] +fn decode_v3_rejects_excessive_event_expansion() { + let mut data = minimal_v3_metric_data(); + data.num_points[0] = 1_000_000; + data.timestamps = vec![0; 1_000_000]; + data.vals_float64 = vec![0.0; 1_000_000]; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_rejects_excessive_dictionary_string_cloning() { + let mut data = minimal_v3_metric_data(); + data.dict_name_str = v3_string_dictionary(&[&"x".repeat(1024 * 1024)]); + data.types = vec![3 | 0x30; 17]; + data.name_refs = std::iter::once(1) + .chain(std::iter::repeat_n(0, 16)) + .collect(); + data.tagset_refs = vec![0; 17]; + data.resources_refs = vec![0; 17]; + data.intervals = vec![0; 17]; + data.num_points = vec![0; 17]; + data.source_type_name_refs = vec![0; 17]; + data.origin_info_refs = vec![0; 17]; + data.timestamps.clear(); + data.vals_float64.clear(); + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_rejects_excessive_empty_string_entries() { + let mut data = minimal_v3_metric_data(); + data.dict_name_str = vec![0; 700_000]; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + +#[test] +fn decode_v3_rejects_excessive_empty_tagsets() { + let mut data = minimal_v3_metric_data(); + data.dict_tagsets = vec![0; 700_000]; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + #[test] fn decode_v3_rejects_excessive_resource_expansion() { let mut data = minimal_v3_metric_data(); From 40003463243d3f4203945c5eab0d16fed66d80d2 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:58:22 +0000 Subject: [PATCH 07/23] fix(datadog_agent): address remaining v3 review feedback --- src/sources/datadog_agent/metrics.rs | 94 +++++++++++++++++++++++++++- src/sources/datadog_agent/tests.rs | 44 +++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 722b4135e2c7c..45a133f2ad30d 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -35,6 +35,8 @@ const MAX_V3_EXPANDED_TAGSET_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_RESOURCE_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_SERIES_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_EVENT_BYTES: usize = 16 * 1024 * 1024; +const MAX_V3_METADATA_STRING_ENTRIES: usize = + MAX_V3_EXPANDED_TAGSET_BYTES / std::mem::size_of::(); #[derive(Deserialize, Serialize)] pub(crate) struct DatadogSeriesRequest { @@ -301,6 +303,7 @@ pub(crate) fn decode_ddseries_v3( api_key: &Option>, split_metric_namespace: bool, ) -> crate::Result> { + validate_v3_metadata_string_entries(&frame)?; let payload = MetricPayloadV3::decode(frame)?; let series = payload.metric_data.map_or(Ok(Vec::new()), |metric_data| { decode_v3_metric_data(&metric_data, payload.metadata.as_ref()) @@ -477,6 +480,7 @@ pub(super) fn decode_v3_metric_data( .checked_add(name.len()) .and_then(|total| total.checked_add(tag_bytes)) .and_then(|total| total.checked_add(resource_bytes)) + .and_then(|total| total.checked_add(source_type.len())) .and_then(|total| total.checked_add(unit.len())) .ok_or("Datadog v3 expanded event size overflow")?; expanded_event_bytes = expanded_event_bytes @@ -650,6 +654,89 @@ fn decode_v3_varint(raw: &[u8]) -> crate::Result<(u64, usize)> { Err("truncated Datadog v3 varint".into()) } +fn validate_v3_metadata_string_entries(raw: &[u8]) -> crate::Result<()> { + let mut offset = 0; + while offset < raw.len() { + let (key, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let wire_type = key & 0x07; + if key >> 3 == 2 && wire_type == 2 { + let (length, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let length = + usize::try_from(length).map_err(|_| "Datadog v3 metadata length overflow")?; + let end = offset + .checked_add(length) + .ok_or("Datadog v3 metadata length overflow")?; + let metadata = raw + .get(offset..end) + .ok_or("truncated Datadog v3 metadata")?; + validate_v3_repeated_string_entries(metadata)?; + offset = end; + } else { + offset = skip_v3_field(raw, offset, wire_type)?; + } + } + Ok(()) +} + +fn validate_v3_repeated_string_entries(raw: &[u8]) -> crate::Result<()> { + let mut offset = 0; + let mut entries = 0usize; + while offset < raw.len() { + let (key, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let field = key >> 3; + let wire_type = key & 0x07; + if matches!(field, 1 | 2) && wire_type == 2 { + entries = entries + .checked_add(1) + .ok_or("Datadog v3 metadata string count overflow")?; + if entries > MAX_V3_METADATA_STRING_ENTRIES { + return Err("Datadog v3 metadata contains too many strings".into()); + } + } + offset = skip_v3_field(raw, offset, wire_type)?; + } + Ok(()) +} + +fn skip_v3_field(raw: &[u8], offset: usize, wire_type: u64) -> crate::Result { + match wire_type { + 0 => { + let (_, consumed) = decode_v3_varint( + raw.get(offset..) + .ok_or("truncated Datadog v3 protobuf field")?, + )?; + offset + .checked_add(consumed) + .ok_or_else(|| "Datadog v3 protobuf field length overflow".into()) + } + 1 => offset + .checked_add(8) + .filter(|&end| end <= raw.len()) + .ok_or_else(|| "truncated Datadog v3 protobuf field".into()), + 2 => { + let (length, consumed) = decode_v3_varint( + raw.get(offset..) + .ok_or("truncated Datadog v3 protobuf field")?, + )?; + let length = + usize::try_from(length).map_err(|_| "Datadog v3 protobuf field length overflow")?; + offset + .checked_add(consumed) + .and_then(|start| start.checked_add(length)) + .filter(|&end| end <= raw.len()) + .ok_or_else(|| "truncated Datadog v3 protobuf field".into()) + } + 5 => offset + .checked_add(4) + .filter(|&end| end <= raw.len()) + .ok_or_else(|| "truncated Datadog v3 protobuf field".into()), + _ => Err("invalid Datadog v3 protobuf wire type".into()), + } +} + fn decode_v3_tagsets( packed: &[i64], dictionary: &[String], @@ -727,15 +814,18 @@ fn decode_v3_tagsets( .filter(|tag| seen.insert(tag.as_str())) .collect::>(); for tagset in &mut tagsets { + let mut tagset_members = tagset.iter().map(String::as_str).collect::>(); + let mut additional_tags = Vec::new(); for &tag in &metadata_tags { - if !tagset.contains(tag) { + if tagset_members.insert(tag) { expanded_bytes = checked_v3_string_bytes(expanded_bytes, tag)?; if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { return Err("Datadog v3 expanded tagsets exceed size limit".into()); } - tagset.push(tag.clone()); + additional_tags.push(tag.clone()); } } + tagset.extend(additional_tags); } } Ok(tagsets) diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index e65dd0a5b4a1f..3447976c55118 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2511,6 +2511,18 @@ fn decode_v3_rejects_excessive_event_expansion() { assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); } +#[test] +fn decode_v3_rejects_excessive_source_type_event_expansion() { + let mut data = minimal_v3_metric_data(); + data.dict_source_type_name = v3_string_dictionary(&[&"x".repeat(1024 * 1024)]); + data.source_type_name_refs[0] = 1; + data.num_points[0] = 17; + data.timestamps = vec![1; 17]; + data.vals_float64 = vec![1.0; 17]; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + #[test] fn decode_v3_rejects_excessive_dictionary_string_cloning() { let mut data = minimal_v3_metric_data(); @@ -2570,6 +2582,38 @@ fn decode_v3_deduplicates_metadata_tags() { assert_eq!(series[0].tags, ["team:core"]); } +#[test] +fn decode_v3_accepts_many_distinct_metadata_tags() { + let data = minimal_v3_metric_data(); + let metadata = ddmetric_v3_proto::Metadata { + tags: (0..100_000).map(|index| format!("tag:{index}")).collect(), + resources: Vec::new(), + }; + + let series = super::metrics::decode_v3_metric_data(&data, Some(&metadata)).unwrap(); + assert_eq!(series[0].tags.len(), 100_000); +} + +#[test] +fn decode_v3_rejects_excessive_metadata_string_entries_before_decode() { + let metadata_entry_count = 16 * 1024 * 1024 / std::mem::size_of::() + 1; + let mut metadata = Vec::with_capacity(metadata_entry_count * 2); + for _ in 0..metadata_entry_count { + metadata.extend([0x0a, 0x00]); + } + let mut payload = Vec::new(); + payload.push(0x12); + let mut length = metadata.len() as u64; + while length >= 0x80 { + payload.push((length as u8 & 0x7f) | 0x80); + length >>= 7; + } + payload.push(length as u8); + payload.extend(metadata); + + assert!(super::metrics::decode_ddseries_v3(Bytes::from(payload), &None, false).is_err()); +} + #[test] fn decode_v3_rejects_reference_accumulator_overflow() { let mut data = minimal_v3_metric_data(); From 6fa6c2d86ae08e7b9a0bae255253cd69cebed48b Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Sat, 5 Sep 2026 10:23:54 +0000 Subject: [PATCH 08/23] fix(datadog_agent): bound v3 predecode allocations --- src/sources/datadog_agent/metrics.rs | 159 ++++++++++++++++++++++----- src/sources/datadog_agent/tests.rs | 64 ++++++++--- 2 files changed, 182 insertions(+), 41 deletions(-) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 45a133f2ad30d..2f1540660fb9c 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -35,8 +35,7 @@ const MAX_V3_EXPANDED_TAGSET_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_RESOURCE_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_SERIES_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_EVENT_BYTES: usize = 16 * 1024 * 1024; -const MAX_V3_METADATA_STRING_ENTRIES: usize = - MAX_V3_EXPANDED_TAGSET_BYTES / std::mem::size_of::(); +const MAX_V3_DECODED_REPEATED_BYTES: usize = 16 * 1024 * 1024; #[derive(Deserialize, Serialize)] pub(crate) struct DatadogSeriesRequest { @@ -303,7 +302,7 @@ pub(crate) fn decode_ddseries_v3( api_key: &Option>, split_metric_namespace: bool, ) -> crate::Result> { - validate_v3_metadata_string_entries(&frame)?; + validate_v3_predecode_allocations(&frame)?; let payload = MetricPayloadV3::decode(frame)?; let series = payload.metric_data.map_or(Ok(Vec::new()), |metric_data| { decode_v3_metric_data(&metric_data, payload.metadata.as_ref()) @@ -443,7 +442,12 @@ pub(super) fn decode_v3_metric_data( }) .transpose()? .unwrap_or(resource_bytes); - let tag_resource_bytes = tag_bytes + let name = &names[name_ref as usize]; + let source_type = &source_types[source_type_ref as usize]; + let point_tag_bytes = tag_bytes + .checked_add(source_type.len()) + .ok_or("Datadog v3 expanded series size overflow")?; + let tag_resource_bytes = point_tag_bytes .checked_add(resource_bytes) .ok_or("Datadog v3 expanded series size overflow")?; let tag_resource_copies = point_count @@ -452,8 +456,6 @@ pub(super) fn decode_v3_metric_data( let expanded_tag_resource_bytes = tag_resource_bytes .checked_mul(tag_resource_copies) .ok_or("Datadog v3 expanded series size overflow")?; - let name = &names[name_ref as usize]; - let source_type = &source_types[source_type_ref as usize]; let dictionary_string_bytes = checked_v3_string_bytes(0, name)?; let dictionary_string_bytes = checked_v3_string_bytes(dictionary_string_bytes, &unit)?; let dictionary_string_bytes = @@ -624,7 +626,19 @@ fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { let bytes = &raw[offset..end]; let value = match std::str::from_utf8(bytes) { Ok(value) => value.to_owned(), - Err(_) if sanitize => String::from_utf8_lossy(bytes).into_owned(), + Err(_) if sanitize => { + let worst_case_length = length + .checked_mul(3) + .ok_or("Datadog v3 expanded string dictionary size overflow")?; + let worst_case_bytes = expanded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(worst_case_length)) + .ok_or("Datadog v3 expanded string dictionary size overflow")?; + if worst_case_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); + } + String::from_utf8_lossy(bytes).into_owned() + } Err(error) => return Err(error.into()), }; expanded_bytes = checked_v3_string_bytes(expanded_bytes, &value)?; @@ -654,24 +668,22 @@ fn decode_v3_varint(raw: &[u8]) -> crate::Result<(u64, usize)> { Err("truncated Datadog v3 varint".into()) } -fn validate_v3_metadata_string_entries(raw: &[u8]) -> crate::Result<()> { +fn validate_v3_predecode_allocations(raw: &[u8]) -> crate::Result<()> { let mut offset = 0; + let mut metadata_bytes = 0usize; + let mut repeated_numeric_bytes = 0usize; while offset < raw.len() { let (key, consumed) = decode_v3_varint(&raw[offset..])?; offset += consumed; + let field = key >> 3; let wire_type = key & 0x07; - if key >> 3 == 2 && wire_type == 2 { - let (length, consumed) = decode_v3_varint(&raw[offset..])?; - offset += consumed; - let length = - usize::try_from(length).map_err(|_| "Datadog v3 metadata length overflow")?; - let end = offset - .checked_add(length) - .ok_or("Datadog v3 metadata length overflow")?; - let metadata = raw - .get(offset..end) - .ok_or("truncated Datadog v3 metadata")?; - validate_v3_repeated_string_entries(metadata)?; + if matches!(field, 2 | 3) && wire_type == 2 { + let (message, end) = decode_v3_length_delimited(raw, offset)?; + if field == 2 { + validate_v3_metadata_allocations(message, &mut metadata_bytes)?; + } else { + validate_v3_metric_data_allocations(message, &mut repeated_numeric_bytes)?; + } offset = end; } else { offset = skip_v3_field(raw, offset, wire_type)?; @@ -680,27 +692,118 @@ fn validate_v3_metadata_string_entries(raw: &[u8]) -> crate::Result<()> { Ok(()) } -fn validate_v3_repeated_string_entries(raw: &[u8]) -> crate::Result<()> { +fn validate_v3_metadata_allocations(raw: &[u8], decoded_bytes: &mut usize) -> crate::Result<()> { let mut offset = 0; - let mut entries = 0usize; while offset < raw.len() { let (key, consumed) = decode_v3_varint(&raw[offset..])?; offset += consumed; let field = key >> 3; let wire_type = key & 0x07; if matches!(field, 1 | 2) && wire_type == 2 { - entries = entries - .checked_add(1) - .ok_or("Datadog v3 metadata string count overflow")?; - if entries > MAX_V3_METADATA_STRING_ENTRIES { - return Err("Datadog v3 metadata contains too many strings".into()); + let (value, end) = decode_v3_length_delimited(raw, offset)?; + *decoded_bytes = decoded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(value.len())) + .ok_or("Datadog v3 metadata string allocation overflow")?; + if *decoded_bytes > MAX_V3_DECODED_REPEATED_BYTES { + return Err("Datadog v3 metadata strings exceed size limit".into()); + } + offset = end; + } else { + offset = skip_v3_field(raw, offset, wire_type)?; + } + } + Ok(()) +} + +fn validate_v3_metric_data_allocations(raw: &[u8], decoded_bytes: &mut usize) -> crate::Result<()> { + let mut offset = 0; + while offset < raw.len() { + let (key, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let field = key >> 3; + let wire_type = key & 0x07; + let element_size = match field { + 3 | 5..=7 | 10..=17 | 19 | 20 | 23 | 24 | 26 => 8, + 9 | 18 | 21 | 22 => 4, + _ => { + offset = skip_v3_field(raw, offset, wire_type)?; + continue; + } + }; + let fixed_width = match field { + 18 => Some(4), + 19 => Some(8), + _ => None, + }; + let count = if wire_type == 2 { + let (packed, end) = decode_v3_length_delimited(raw, offset)?; + offset = end; + match fixed_width { + Some(width) => { + if packed.len() % width != 0 { + return Err("invalid packed Datadog v3 numeric column".into()); + } + packed.len() / width + } + None => count_v3_packed_varints(packed)?, } + } else if fixed_width.is_some_and(|width| wire_type == if width == 4 { 5 } else { 1 }) + || fixed_width.is_none() && wire_type == 0 + { + offset = skip_v3_field(raw, offset, wire_type)?; + 1 + } else { + offset = skip_v3_field(raw, offset, wire_type)?; + continue; + }; + let allocation = count + .checked_mul(element_size) + .ok_or("Datadog v3 numeric column allocation overflow")?; + *decoded_bytes = decoded_bytes + .checked_add(allocation) + .ok_or("Datadog v3 numeric column allocation overflow")?; + if *decoded_bytes > MAX_V3_DECODED_REPEATED_BYTES { + return Err("Datadog v3 numeric columns exceed size limit".into()); } - offset = skip_v3_field(raw, offset, wire_type)?; } Ok(()) } +fn count_v3_packed_varints(raw: &[u8]) -> crate::Result { + let mut offset = 0; + let mut count = 0usize; + while offset < raw.len() { + let (_, consumed) = decode_v3_varint(&raw[offset..])?; + offset = offset + .checked_add(consumed) + .ok_or("Datadog v3 packed numeric count overflow")?; + count = count + .checked_add(1) + .ok_or("Datadog v3 packed numeric count overflow")?; + } + Ok(count) +} + +fn decode_v3_length_delimited(raw: &[u8], offset: usize) -> crate::Result<(&[u8], usize)> { + let (length, consumed) = decode_v3_varint( + raw.get(offset..) + .ok_or("truncated Datadog v3 protobuf field")?, + )?; + let length = + usize::try_from(length).map_err(|_| "Datadog v3 protobuf field length overflow")?; + let start = offset + .checked_add(consumed) + .ok_or("Datadog v3 protobuf field length overflow")?; + let end = start + .checked_add(length) + .ok_or("Datadog v3 protobuf field length overflow")?; + let value = raw + .get(start..end) + .ok_or("truncated Datadog v3 protobuf field")?; + Ok((value, end)) +} + fn skip_v3_field(raw: &[u8], offset: usize, wire_type: u64) -> crate::Result { match wire_type { 0 => { diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 3447976c55118..ce9aa300f1afa 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2374,19 +2374,28 @@ fn v3_string_dictionary(values: &[&str]) -> Vec { values .iter() .flat_map(|value| { - let mut length = value.len() as u64; let mut encoded = Vec::new(); - while length >= 0x80 { - encoded.push((length as u8 & 0x7f) | 0x80); - length >>= 7; - } - encoded.push(length as u8); + encode_v3_varint(value.len() as u64, &mut encoded); encoded.extend_from_slice(value.as_bytes()); encoded }) .collect() } +fn encode_v3_varint(mut value: u64, encoded: &mut Vec) { + while value >= 0x80 { + encoded.push((value as u8 & 0x7f) | 0x80); + value >>= 7; + } + encoded.push(value as u8); +} + +fn encode_v3_length_delimited(field: u8, value: &[u8], encoded: &mut Vec) { + encode_v3_varint((u64::from(field) << 3) | 2, encoded); + encode_v3_varint(value.len() as u64, encoded); + encoded.extend_from_slice(value); +} + fn minimal_v3_metric_data() -> ddmetric_v3_proto::MetricData { ddmetric_v3_proto::MetricData { dict_name_str: v3_string_dictionary(&["metric"]), @@ -2602,18 +2611,47 @@ fn decode_v3_rejects_excessive_metadata_string_entries_before_decode() { metadata.extend([0x0a, 0x00]); } let mut payload = Vec::new(); - payload.push(0x12); - let mut length = metadata.len() as u64; - while length >= 0x80 { - payload.push((length as u8 & 0x7f) | 0x80); - length >>= 7; + encode_v3_length_delimited(2, &metadata, &mut payload); + + assert!(super::metrics::decode_ddseries_v3(Bytes::from(payload), &None, false).is_err()); +} + +#[test] +fn decode_v3_accumulates_metadata_allocations_across_fields() { + let metadata_entry_count = 16 * 1024 * 1024 / std::mem::size_of::() / 2 + 1; + let mut metadata = Vec::with_capacity(metadata_entry_count * 2); + for _ in 0..metadata_entry_count { + metadata.extend([0x0a, 0x00]); } - payload.push(length as u8); - payload.extend(metadata); + let mut payload = Vec::new(); + encode_v3_length_delimited(2, &metadata, &mut payload); + encode_v3_length_delimited(2, &metadata, &mut payload); + + assert!(super::metrics::decode_ddseries_v3(Bytes::from(payload), &None, false).is_err()); +} + +#[test] +fn decode_v3_rejects_excessive_numeric_columns_before_decode() { + let entry_count = 16 * 1024 * 1024 / std::mem::size_of::() + 1; + let packed_timestamps = vec![0; entry_count]; + let mut metric_data = Vec::new(); + encode_v3_length_delimited(16, &packed_timestamps, &mut metric_data); + let mut payload = Vec::new(); + encode_v3_length_delimited(3, &metric_data, &mut payload); assert!(super::metrics::decode_ddseries_v3(Bytes::from(payload), &None, false).is_err()); } +#[test] +fn decode_v3_rejects_lossy_tag_expansion_before_allocation() { + let mut data = minimal_v3_metric_data(); + let invalid_utf8 = vec![0xff; 6 * 1024 * 1024]; + encode_v3_varint(invalid_utf8.len() as u64, &mut data.dict_tag_str); + data.dict_tag_str.extend(invalid_utf8); + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + #[test] fn decode_v3_rejects_reference_accumulator_overflow() { let mut data = minimal_v3_metric_data(); From dddaf830f8b02fef428bbc9eaafcbffae9572e31 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:47:29 +0000 Subject: [PATCH 09/23] fix(datadog_agent): bound valid v3 strings before cloning --- src/sources/datadog_agent/metrics.rs | 7 +++++++ src/sources/datadog_agent/tests.rs | 10 ++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 2f1540660fb9c..c0fe2b2c1965d 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -624,6 +624,13 @@ fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { return Err("truncated Datadog v3 string dictionary".into()); } let bytes = &raw[offset..end]; + let valid_string_bytes = expanded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(length)) + .ok_or("Datadog v3 expanded string dictionary size overflow")?; + if valid_string_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); + } let value = match std::str::from_utf8(bytes) { Ok(value) => value.to_owned(), Err(_) if sanitize => { diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 1b440bdb2a22f..0ccf79e3bf70f 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2685,6 +2685,16 @@ fn decode_v3_rejects_lossy_tag_expansion_before_allocation() { assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); } +#[test] +fn decode_v3_rejects_valid_string_expansion_before_allocation() { + let mut data = minimal_v3_metric_data(); + let valid_utf8 = vec![b'x'; 16 * 1024 * 1024]; + encode_v3_varint(valid_utf8.len() as u64, &mut data.dict_tag_str); + data.dict_tag_str.extend(valid_utf8); + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + #[test] fn decode_v3_rejects_reference_accumulator_overflow() { let mut data = minimal_v3_metric_data(); From 502697bc072d5dbf3259462c5abbc0212eda2d66 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Sat, 12 Sep 2026 11:07:24 +0000 Subject: [PATCH 10/23] fix(datadog_agent): preserve protobuf compatibility --- proto/vector/dd_metric.proto | 3 +- src/sources/datadog_agent/metrics.rs | 86 ++++++++++++++++------------ src/sources/datadog_agent/tests.rs | 2 - 3 files changed, 51 insertions(+), 40 deletions(-) diff --git a/proto/vector/dd_metric.proto b/proto/vector/dd_metric.proto index f953e55ec03a2..0bf7856eb6956 100644 --- a/proto/vector/dd_metric.proto +++ b/proto/vector/dd_metric.proto @@ -14,8 +14,7 @@ message CommonMetadata { } message Origin { - reserved 1,2; - int32 metric_type = 3; + reserved 1,2,3; uint32 origin_product = 4; uint32 origin_category = 5; uint32 origin_service = 6; diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index c0fe2b2c1965d..c0a02e42f61a3 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -304,10 +304,24 @@ pub(crate) fn decode_ddseries_v3( ) -> crate::Result> { validate_v3_predecode_allocations(&frame)?; let payload = MetricPayloadV3::decode(frame)?; + // `metric_type` is reserved in the v2 Origin protobuf, so keep this v3-only flag out of + // the wire type and pass it alongside the translated series instead. + let metric_types = payload.metric_data.as_ref().map(|metric_data| { + metric_data + .types + .iter() + .map(|packed_type| (packed_type & 0x100 != 0).then_some(9)) + .collect::>() + }); let series = payload.metric_data.map_or(Ok(Vec::new()), |metric_data| { decode_v3_metric_data(&metric_data, payload.metadata.as_ref()) })?; - decode_ddseries(series, api_key, split_metric_namespace) + decode_ddseries( + series, + api_key, + split_metric_namespace, + metric_types.as_deref(), + ) } pub(super) fn decode_v3_metric_data( @@ -545,29 +559,18 @@ pub(super) fn decode_v3_metric_data( 3 => metric_payload::MetricType::Gauge, _ => metric_payload::MetricType::Unspecified, }; - let no_index = packed_type & 0x100 != 0; - let series_metadata = if origin_ref == 0 && !no_index { + let series_metadata = if origin_ref == 0 { None } else { - let (origin_product, origin_category, origin_service) = if origin_ref == 0 { - (0, 0, 0) - } else { - let offset = (origin_ref as usize - 1) * 3; - ( - u32::try_from(data.dict_origin_info[offset]) + let offset = (origin_ref as usize - 1) * 3; + Some(Metadata { + origin: Some(super::ddmetric_proto::Origin { + origin_product: u32::try_from(data.dict_origin_info[offset]) .map_err(|_| "invalid Datadog v3 origin product")?, - u32::try_from(data.dict_origin_info[offset + 1]) + origin_category: u32::try_from(data.dict_origin_info[offset + 1]) .map_err(|_| "invalid Datadog v3 origin category")?, - u32::try_from(data.dict_origin_info[offset + 2]) + origin_service: u32::try_from(data.dict_origin_info[offset + 2]) .map_err(|_| "invalid Datadog v3 origin service")?, - ) - }; - Some(Metadata { - origin: Some(super::ddmetric_proto::Origin { - metric_type: if no_index { 9 } else { 0 }, - origin_product, - origin_category, - origin_service, }), }) }; @@ -1003,23 +1006,28 @@ fn checked_v3_string_bytes(total: usize, value: &str) -> crate::Result { /// Builds Vector's `EventMetadata` from the received metadata. Currently this is only /// utilized for passing through origin metadata set by the Agent. -fn get_event_metadata(metadata: Option<&Metadata>) -> EventMetadata { - metadata - .and_then(|metadata| metadata.origin.as_ref()) - .map_or_else(EventMetadata::default, |origin| { - trace!( - "Deserialized origin_product: `{}` origin_category: `{}` origin_service: `{}`.", - origin.origin_product, origin.origin_category, origin.origin_service, - ); - EventMetadata::default().with_origin_metadata( - DatadogMetricOriginMetadata::new( +fn get_event_metadata(metadata: Option<&Metadata>, metric_type: Option) -> EventMetadata { + let origin = metadata.and_then(|metadata| metadata.origin.as_ref()); + if origin.is_none() && metric_type.is_none() { + EventMetadata::default() + } else { + let (origin_product, origin_category, origin_service) = + origin.map_or((None, None, None), |origin| { + trace!( + "Deserialized origin_product: `{}` origin_category: `{}` origin_service: `{}`.", + origin.origin_product, origin.origin_category, origin.origin_service, + ); + ( Some(origin.origin_product), Some(origin.origin_category), Some(origin.origin_service), ) - .with_metric_type((origin.metric_type != 0).then_some(origin.metric_type)), - ) - }) + }); + EventMetadata::default().with_origin_metadata( + DatadogMetricOriginMetadata::new(origin_product, origin_category, origin_service) + .with_metric_type(metric_type), + ) + } } pub(crate) fn decode_ddseries_v2( @@ -1028,17 +1036,19 @@ pub(crate) fn decode_ddseries_v2( split_metric_namespace: bool, ) -> crate::Result> { let payload = MetricPayload::decode(frame)?; - decode_ddseries(payload.series, api_key, split_metric_namespace) + decode_ddseries(payload.series, api_key, split_metric_namespace, None) } fn decode_ddseries( series: Vec, api_key: &Option>, split_metric_namespace: bool, + metric_types: Option<&[Option]>, ) -> crate::Result> { let decoded_metrics: Vec = series .into_iter() - .flat_map(|serie| { + .enumerate() + .flat_map(|(index, serie)| { let (namespace, name) = if split_metric_namespace { namespace_name_from_dd_metric(&serie.metric) } else { @@ -1046,7 +1056,11 @@ fn decode_ddseries( }; let mut tags = into_metric_tags(serie.tags); - let mut event_metadata = get_event_metadata(serie.metadata.as_ref()); + let metric_type = metric_types + .and_then(|metric_types| metric_types.get(index)) + .copied() + .flatten(); + let mut event_metadata = get_event_metadata(serie.metadata.as_ref(), metric_type); if !serie.unit.is_empty() { event_metadata.set_datadog_metric_unit(serie.unit.clone()); } @@ -1377,7 +1391,7 @@ pub(crate) fn decode_ddsketch( .host_key() .and_then(|key| tags.replace(key.to_string(), sketch_series.host.clone())); - let event_metadata = get_event_metadata(sketch_series.metadata.as_ref()); + let event_metadata = get_event_metadata(sketch_series.metadata.as_ref(), None); sketch_series.dogsketches.into_iter().map(move |sketch| { let k: Vec = sketch.k.iter().map(|k| *k as i16).collect(); diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 0ccf79e3bf70f..6df194eb689a8 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -1174,7 +1174,6 @@ async fn decode_sketches() { }], metadata: Some(ddmetric_proto::Metadata { origin: Some(ddmetric_proto::Origin { - metric_type: 0, origin_product: 10, origin_category: 11, origin_service: 9, @@ -2217,7 +2216,6 @@ async fn decode_series_endpoint_v2() { interval: 0, metadata: Some(ddmetric_proto::Metadata { origin: Some(ddmetric_proto::Origin { - metric_type: 0, origin_product: 10, origin_category: 10, origin_service: 42, From f042346916e71365eef5c812602d112a703d228a Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 22 Sep 2026 07:13:17 +0000 Subject: [PATCH 11/23] test(datadog): add v3 agent matrix entry --- tests/e2e/datadog-metrics/config/test.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/datadog-metrics/config/test.yaml b/tests/e2e/datadog-metrics/config/test.yaml index b46170124fed9..d7bdbd642b1ca 100644 --- a/tests/e2e/datadog-metrics/config/test.yaml +++ b/tests/e2e/datadog-metrics/config/test.yaml @@ -12,7 +12,9 @@ runner: FAKE_INTAKE_VECTOR_ENDPOINT: "http://fakeintake-vector:80" matrix: - agent_version: ["7", "6"] + # Keep the last Agent version before v3 series became the default, and also + # test the floating v7 tag to cover v3 series intake. + agent_version: ["7.80.4", "7", "6"] # validate both series API versions series_api_version: ["v1", "v2"] From bd4c29e483328fd361c3cd480a861416d7d24d20 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Wed, 23 Sep 2026 15:33:45 +0000 Subject: [PATCH 12/23] fix(datadog_agent): remove v3beta series route --- src/sources/datadog_agent/metrics.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index c0a02e42f61a3..2203f39820dc4 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -179,14 +179,7 @@ fn series_v3_service( source: DatadogAgentSource, ) -> BoxedFilter<(Response,)> { warp::post() - .and( - path!("api" / "intake" / "metrics" / "v3" / "series" / ..) - .or(path!( - "api" / "intake" / "metrics" / "v3beta" / "series" / .. - )) - .unify() - .and(warp::path::full()), - ) + .and(path!("api" / "intake" / "metrics" / "v3" / "series" / ..).and(warp::path::full())) .and(warp::header::optional::("content-encoding")) .and(warp::header::optional::("dd-api-key")) .and(warp::query::()) From 519a2355c18794feb38ff7c3fd695a6fb63ea73c Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 13 Oct 2026 07:03:29 +0000 Subject: [PATCH 13/23] fix(datadog): preserve v3 unit and no-index metadata --- src/common/datadog.rs | 2 + src/sinks/datadog/metrics/encoder_v3.rs | 55 ++++++++++++++++++++++--- src/sources/datadog_agent/metrics.rs | 9 ++-- 3 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/common/datadog.rs b/src/common/datadog.rs index 8117fb553c8b8..861b6d10e6efd 100644 --- a/src/common/datadog.rs +++ b/src/common/datadog.rs @@ -22,6 +22,8 @@ pub const DDTAGS: &str = "ddtags"; /// The datadog message event path. pub const MESSAGE: &str = "message"; pub(crate) const DATADOG_METRIC_RESOURCE_TAG_PREFIX: &str = "resource."; +/// Datadog origin metric-type value for metrics excluded from indexing. +pub(crate) const DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX: i32 = 9; /// Mapping of the semantic meaning of well known Datadog reserved attributes /// to the field name that Datadog intake expects. diff --git a/src/sinks/datadog/metrics/encoder_v3.rs b/src/sinks/datadog/metrics/encoder_v3.rs index 24d74c89b0b62..3cc2981490908 100644 --- a/src/sinks/datadog/metrics/encoder_v3.rs +++ b/src/sinks/datadog/metrics/encoder_v3.rs @@ -27,6 +27,7 @@ use super::{ generate_origin_metadata, split_series_tags, }, }; +use crate::common::datadog::DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX; use crate::sinks::util::{ Compression, Compressor, encode_namespace, request_builder::EncodeResult, }; @@ -221,8 +222,6 @@ fn encode_metric_to_v3( // `resource.` tags that the `datadog_agent` source produces from an upstream V2 // payload's resources back into structured resources. // - // `unit` is intentionally never set: V2 always sends it empty (`unit: "".to_string()` in - // `series_to_proto_message`), so a `dd.internal.unit` tag stays an ordinary tag here too. let SeriesTags { tags, resources, @@ -257,7 +256,8 @@ fn encode_metric_to_v3( let product = origin.product().unwrap_or(origin_product_value); let category = origin.category().unwrap_or(ORIGIN_CATEGORY_VALUE); let service = origin.service().unwrap_or(0); - builder.set_origin(product, category, service, false); + let no_index = origin.metric_type() == Some(DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX); + builder.set_origin(product, category, service, no_index); } // Interval — matches V2, which always stamps the interval field on the message @@ -266,8 +266,9 @@ fn encode_metric_to_v3( builder.set_interval(interval.into()); } - // Note: `unit` is intentionally never set — V2 always sends it empty (see - // `series_to_proto_message`'s `unit: "".to_string()`). + if let Some(unit) = event_metadata.datadog_metric_unit() { + builder.set_unit(unit); + } // ── Data points ───────────────────────────────────────────────────────── let timestamp = encode_timestamp(metric.timestamp()); @@ -312,8 +313,13 @@ mod tests { use super::super::config::SeriesApiVersion; use super::*; + use datadog_proto::metrics_v3::MetricData; + use prost::Message; use vector_lib::{ - event::{MetricKind, MetricValue, metric::MetricSketch}, + event::{ + DatadogMetricOriginMetadata, EventMetadata, MetricKind, MetricValue, + metric::MetricSketch, + }, metrics::AgentDDSketch, }; @@ -628,6 +634,43 @@ mod tests { assert_eq!(origin.service(), Some(211)); } + #[test] + fn v3_preserves_unit_and_no_index_metadata() { + const FLAG_NO_INDEX: u64 = 0x100; + const FLAG_HAS_UNIT: u64 = 0x200; + + let metadata = EventMetadata::default() + .with_origin_metadata( + DatadogMetricOriginMetadata::new(Some(10), Some(11), Some(42)) + .with_metric_type(Some(DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX)), + ) + .with_datadog_metric_unit("byte".to_string()); + let metric = Metric::new_with_metadata( + "preserved.metadata", + MetricKind::Absolute, + MetricValue::Gauge { value: 1.0 }, + metadata, + ); + + let mut writer = V3Writer::new(); + encode_metric_to_v3( + &mut writer, + &metric, + &None, + log_schema(), + *ORIGIN_PRODUCT_VALUE, + ) + .unwrap(); + let encoded = writer.finalize().unwrap(); + let data = MetricData::decode(encoded.payload.as_slice()).unwrap(); + + assert_eq!(data.types.len(), 1); + assert_eq!(data.types[0] & FLAG_NO_INDEX, FLAG_NO_INDEX); + assert_eq!(data.types[0] & FLAG_HAS_UNIT, FLAG_HAS_UNIT); + assert_eq!(data.dict_unit_str, b"\x04byte"); + assert_eq!(data.unit_refs, vec![1]); + } + #[test] fn v3_namespace_prepended() { let mut enc = DatadogMetricsV3Encoder::new( diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 2ca889a95b263..8d4dc4f370625 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -23,7 +23,10 @@ use super::ddmetric_v3_proto::{ use super::{ApiKeyQueryParams, DatadogAgentSource, RequestHandler}; use crate::{ common::{ - datadog::{DATADOG_METRIC_RESOURCE_TAG_PREFIX, DatadogMetricType, DatadogSeriesMetric}, + datadog::{ + DATADOG_METRIC_RESOURCE_TAG_PREFIX, DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX, + DatadogMetricType, DatadogSeriesMetric, + }, http::ErrorMessage, }, config::log_schema, @@ -54,10 +57,6 @@ const V3_VALUE_TYPE_SINT64: u64 = 0x10; const V3_VALUE_TYPE_FLOAT32: u64 = 0x20; const V3_VALUE_TYPE_FLOAT64: u64 = 0x30; -// Datadog's origin metric-type value for metrics excluded from indexing. This is event metadata, -// not the metric kind encoded in the low bits of `MetricData.types`. -const DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX: i32 = 9; - #[derive(Deserialize, Serialize)] pub(crate) struct DatadogSeriesRequest { pub(crate) series: Vec, From 9feff534c6fb6f61d15cd8e76b4f9f2418011cb2 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 13 Oct 2026 07:55:43 +0000 Subject: [PATCH 14/23] fix(datadog): address v3 review follow-ups --- src/sources/datadog_agent/metrics.rs | 26 +++++++++----------------- tests/e2e/datadog/metrics/series.rs | 8 +++++++- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 8d4dc4f370625..809da7a2f29b7 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -1219,6 +1219,9 @@ fn decode_ddseries( if !serie.unit.is_empty() { event_metadata.set_datadog_metric_unit(serie.unit.clone()); } + if let Some(api_key) = api_key { + event_metadata.set_datadog_api_key(Arc::clone(api_key)); + } // It is possible to receive non-rate metrics from the Agent with an interval set. // That interval can be applied with the `as_rate` function in the Datadog UI. @@ -1345,12 +1348,7 @@ fn decode_ddseries( } } }) - .map(|mut metric| { - if let Some(k) = &api_key { - metric.metadata_mut().set_datadog_api_key(Arc::clone(k)); - } - metric.into() - }) + .map(Event::from) .collect(); Ok(decoded_metrics) @@ -1418,6 +1416,10 @@ fn into_vector_metric( if let Some(unit) = dd_metric.unit.filter(|unit| !unit.is_empty()) { event_metadata.set_datadog_metric_unit(unit); } + if let Some(api_key) = api_key { + event_metadata.set_datadog_api_key(api_key); + } + event_metadata.set_schema_definition(schema_definition); if let Some(key) = log_schema().host_key() { dd_metric @@ -1505,17 +1507,7 @@ fn into_vector_metric( .collect::>(), } .into_iter() - .map(|mut metric| { - if let Some(k) = &api_key { - metric.metadata_mut().set_datadog_api_key(Arc::clone(k)); - } - - metric - .metadata_mut() - .set_schema_definition(schema_definition); - - metric.into() - }) + .map(Event::from) .collect() } diff --git a/tests/e2e/datadog/metrics/series.rs b/tests/e2e/datadog/metrics/series.rs index 04f5158115f23..f54e1c6490720 100644 --- a/tests/e2e/datadog/metrics/series.rs +++ b/tests/e2e/datadog/metrics/series.rs @@ -357,9 +357,15 @@ fn compare_intakes(agent_intake: &SeriesIntake, vector_intake: &SeriesIntake) { pub(super) async fn validate() { let api_version = std::env::var("CONFIG_SERIES_API_VERSION") .expect("CONFIG_SERIES_API_VERSION must be set ('v2' or 'v3')"); + let agent_version = std::env::var("CONFIG_AGENT_VERSION") + .expect("CONFIG_AGENT_VERSION must be set ('7.80.4', '7', or '6')"); info!("==== getting series data from agent-only pipeline ==== "); - let agent_intake = get_v2_series_from_pipeline(fake_intake_agent_address()).await; + let agent_intake = match agent_version.as_str() { + "7" => get_v3_series_from_pipeline(fake_intake_agent_address()).await, + "7.80.4" | "6" => get_v2_series_from_pipeline(fake_intake_agent_address()).await, + version => panic!("Unknown CONFIG_AGENT_VERSION: {version}"), + }; info!("==== getting series data from agent-vector pipeline ({api_version}) ===="); let vector_intake = match api_version.as_str() { From 3677136d5f25cd851128baebe270d764c9554b0f Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:35:02 +0000 Subject: [PATCH 15/23] fix(datadog): share metric units across events --- lib/vector-core/src/event/metadata.rs | 22 +++++++++++++++++----- lib/vector-core/src/event/proto.rs | 4 ++-- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index c7d35bd0de6cc..93694551f0297 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -118,7 +118,7 @@ pub(super) struct Inner { /// Datadog metric unit received from an upstream Agent payload. #[serde(default)] - pub(crate) datadog_metric_unit: Option, + pub(crate) datadog_metric_unit: Option>, /// Which legacy trace key layout produced this event. `None` means unmarked. /// Meaningful only on traces; always `None` for logs and metrics. @@ -329,8 +329,8 @@ impl EventMetadata { } /// Sets the Datadog metric unit. - pub fn set_datadog_metric_unit(&mut self, unit: String) { - self.get_mut().datadog_metric_unit = Some(unit); + pub fn set_datadog_metric_unit(&mut self, unit: impl Into>) { + self.get_mut().datadog_metric_unit = Some(unit.into()); } /// Returns a reference to the event id. @@ -447,8 +447,8 @@ impl EventMetadata { /// Sets the Datadog metric unit. #[must_use] - pub fn with_datadog_metric_unit(mut self, unit: String) -> Self { - self.get_mut().datadog_metric_unit = Some(unit); + pub fn with_datadog_metric_unit(mut self, unit: impl Into>) -> Self { + self.get_mut().datadog_metric_unit = Some(unit.into()); self } @@ -704,6 +704,18 @@ mod test { assert_eq!(metadata.splunk_hec_token().unwrap().as_ref(), SECRET2); } + #[test] + fn metadata_copy_on_write_keeps_datadog_metric_unit_shared() { + let metadata = EventMetadata::default().with_datadog_metric_unit("byte"); + let mut cloned = metadata.clone(); + + cloned.set_datadog_api_key(Arc::from(SECRET)); + + let original_unit = metadata.inner.datadog_metric_unit.as_ref().unwrap(); + let cloned_unit = cloned.inner.datadog_metric_unit.as_ref().unwrap(); + assert!(Arc::ptr_eq(original_unit, cloned_unit)); + } + #[test] fn secrets_merge() { let mut a = Secrets::new(); diff --git a/lib/vector-core/src/event/proto.rs b/lib/vector-core/src/event/proto.rs index 37dc01eba6d63..02b630e12844d 100644 --- a/lib/vector-core/src/event/proto.rs +++ b/lib/vector-core/src/event/proto.rs @@ -654,7 +654,7 @@ impl From for Metadata { upstream_id: upstream_id.map(|id| id.as_ref().clone()).map(Into::into), secrets, source_event_id: source_event_id.map_or(vec![], std::convert::Into::into), - datadog_metric_unit, + datadog_metric_unit: datadog_metric_unit.map(|unit| unit.to_string()), trace_layout: encode_trace_layout(trace_layout), } } @@ -712,7 +712,7 @@ impl TryFrom for EventMetadata { schema_definition: default_schema_definition(), dropped_fields: ObjectMap::new(), datadog_origin_metadata, - datadog_metric_unit, + datadog_metric_unit: datadog_metric_unit.map(Arc::from), trace_layout: decode_trace_layout(trace_layout), source_event_id, }), From f8f9919a6ac257c833badba422eb03781871786d Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:25:26 +0000 Subject: [PATCH 16/23] fix(datadog): address remaining v3 review threads --- src/sources/datadog_agent/metrics.rs | 924 ++---------------------- src/sources/datadog_agent/metrics/v3.rs | 885 +++++++++++++++++++++++ src/sources/datadog_agent/tests.rs | 52 ++ 3 files changed, 984 insertions(+), 877 deletions(-) create mode 100644 src/sources/datadog_agent/metrics/v3.rs diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 809da7a2f29b7..1d3745d228990 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, num::NonZeroU32, sync::Arc}; +use std::{num::NonZeroU32, sync::Arc}; use bytes::Bytes; use chrono::{TimeZone, Utc}; @@ -13,20 +13,11 @@ use vector_lib::{ }; use warp::{Filter, filters::BoxedFilter, path, path::FullPath, reply::Response}; -use super::ddmetric_proto::{ - Metadata, MetricPayload, SketchPayload, metric_payload, - metric_payload::{MetricPoint, MetricSeries, Resource}, -}; -use super::ddmetric_v3_proto::{ - Metadata as MetricMetadataV3, MetricData as MetricDataV3, Payload as MetricPayloadV3, -}; +use super::ddmetric_proto::{Metadata, MetricPayload, SketchPayload, metric_payload}; use super::{ApiKeyQueryParams, DatadogAgentSource, RequestHandler}; use crate::{ common::{ - datadog::{ - DATADOG_METRIC_RESOURCE_TAG_PREFIX, DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX, - DatadogMetricType, DatadogSeriesMetric, - }, + datadog::{DATADOG_METRIC_RESOURCE_TAG_PREFIX, DatadogMetricType, DatadogSeriesMetric}, http::ErrorMessage, }, config::log_schema, @@ -39,23 +30,13 @@ use crate::{ sources::util::{extract_tag_key_and_value, http::capped_body}, }; -const MAX_V3_EXPANDED_TAGSET_BYTES: usize = 16 * 1024 * 1024; -const MAX_V3_EXPANDED_RESOURCE_BYTES: usize = 16 * 1024 * 1024; -const MAX_V3_EXPANDED_SERIES_BYTES: usize = 16 * 1024 * 1024; -const MAX_V3_EXPANDED_EVENT_BYTES: usize = 16 * 1024 * 1024; -const MAX_V3_DECODED_REPEATED_BYTES: usize = 16 * 1024 * 1024; +mod v3; + +pub(crate) use v3::decode_ddseries_v3; +#[cfg(test)] +pub(crate) use v3::decode_v3_metric_data; -// `MetricData.types` packs three independent values into each integer. These masks mirror -// `metricType`, `valueType`, and `metricFlags` in Datadog's `intake_v3.proto`. -const V3_METRIC_TYPE_MASK: u64 = 0x0f; -const V3_VALUE_TYPE_MASK: u64 = 0xf0; -const V3_FLAG_NO_INDEX: u64 = 0x100; -const V3_FLAG_HAS_UNIT: u64 = 0x200; -const V3_METRIC_TYPE_SKETCH: u64 = 4; -const V3_VALUE_TYPE_ZERO: u64 = 0; -const V3_VALUE_TYPE_SINT64: u64 = 0x10; -const V3_VALUE_TYPE_FLOAT32: u64 = 0x20; -const V3_VALUE_TYPE_FLOAT64: u64 = 0x30; +const MAX_DATADOG_SERIES_UNIT_EVENT_BYTES: usize = 16 * 1024 * 1024; #[derive(Deserialize, Serialize)] pub(crate) struct DatadogSeriesRequest { @@ -310,855 +291,6 @@ fn decode_datadog_series_v3( Ok(metrics) } -pub(crate) fn decode_ddseries_v3( - frame: Bytes, - api_key: &Option>, - split_metric_namespace: bool, -) -> crate::Result> { - validate_v3_predecode_allocations(&frame)?; - let payload = MetricPayloadV3::decode(frame)?; - // `types` combines the metric kind, point value encoding, and flags. `flagNoIndex` is not - // represented by the v2 `MetricSeries` used during translation, so carry its Datadog origin - // value alongside each translated series until `decode_ddseries` builds the Vector events. - let metric_types = payload.metric_data.as_ref().map(|metric_data| { - metric_data - .types - .iter() - .map(|packed_type| { - (packed_type & V3_FLAG_NO_INDEX != 0).then_some(DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX) - }) - .collect::>() - }); - let series = payload.metric_data.map_or(Ok(Vec::new()), |metric_data| { - decode_v3_metric_data(&metric_data, payload.metadata.as_ref()) - })?; - decode_ddseries( - series, - api_key, - split_metric_namespace, - metric_types.as_deref(), - ) -} - -fn validate_v3_metric_columns( - data: &MetricDataV3, - metadata: Option<&MetricMetadataV3>, -) -> crate::Result<()> { - if metadata.is_some_and(|metadata| !metadata.resources.len().is_multiple_of(2)) { - return Err("Datadog v3 metadata resources must be pairs".into()); - } - - let series_count = data.types.len(); - if !data.dict_origin_info.len().is_multiple_of(3) - || data.name_refs.len() < series_count - || data.tagset_refs.len() < series_count - || data.resources_refs.len() < series_count - || data.intervals.len() < series_count - || data.num_points.len() < series_count - || data.source_type_name_refs.len() < series_count - || data.origin_info_refs.len() < series_count - { - return Err("invalid Datadog v3 metric columns".into()); - } - Ok(()) -} - -fn v3_string_list_bytes<'a>(values: impl IntoIterator) -> crate::Result { - values - .into_iter() - .try_fold(0, |total, value| checked_v3_string_bytes(total, value)) -} - -fn v3_resource_list_bytes(resources: &[(String, String)]) -> crate::Result { - resources - .iter() - .try_fold(0, |total, (resource_type, name)| { - let total = checked_v3_string_bytes(total, resource_type)?; - checked_v3_string_bytes(total, name) - }) -} - -#[derive(Default)] -struct V3ReferenceDecoder { - name: i64, - tagset: i64, - resources: i64, - source_type: i64, - origin: i64, - unit: i64, - unit_index: usize, -} - -struct V3References { - name: usize, - tagset: usize, - resources: usize, - source_type: usize, - origin: usize, -} - -impl V3ReferenceDecoder { - fn advance_reference( - current: &mut i64, - delta: i64, - dictionary_len: usize, - overflow_error: &'static str, - ) -> crate::Result { - *current = current.checked_add(delta).ok_or(overflow_error)?; - usize::try_from(*current) - .ok() - .filter(|index| *index < dictionary_len) - .ok_or_else(|| "invalid Datadog v3 dictionary reference".into()) - } - - fn advance( - &mut self, - data: &MetricDataV3, - index: usize, - dictionary_lengths: [usize; 5], - ) -> crate::Result { - Ok(V3References { - name: Self::advance_reference( - &mut self.name, - data.name_refs[index], - dictionary_lengths[0], - "Datadog v3 name reference overflow", - )?, - tagset: Self::advance_reference( - &mut self.tagset, - data.tagset_refs[index], - dictionary_lengths[1], - "Datadog v3 tagset reference overflow", - )?, - resources: Self::advance_reference( - &mut self.resources, - data.resources_refs[index], - dictionary_lengths[2], - "Datadog v3 resource reference overflow", - )?, - source_type: Self::advance_reference( - &mut self.source_type, - data.source_type_name_refs[index], - dictionary_lengths[3], - "Datadog v3 source type reference overflow", - )?, - origin: Self::advance_reference( - &mut self.origin, - data.origin_info_refs[index], - dictionary_lengths[4], - "Datadog v3 origin reference overflow", - )?, - }) - } - - fn decode_unit( - &mut self, - data: &MetricDataV3, - units: &[String], - packed_type: u64, - ) -> crate::Result { - if packed_type & V3_FLAG_HAS_UNIT == 0 { - return Ok(String::new()); - } - let delta = *data - .unit_refs - .get(self.unit_index) - .ok_or("invalid Datadog v3 unit column")?; - self.unit_index += 1; - let index = Self::advance_reference( - &mut self.unit, - delta, - units.len(), - "Datadog v3 unit reference overflow", - )?; - Ok(units[index].clone()) - } -} - -struct V3PointDecoder<'a> { - data: &'a MetricDataV3, - timestamp: i64, - timestamp_index: usize, - sint64_index: usize, - float32_index: usize, - float64_index: usize, -} - -impl<'a> V3PointDecoder<'a> { - const fn new(data: &'a MetricDataV3) -> Self { - Self { - data, - timestamp: 0, - timestamp_index: 0, - sint64_index: 0, - float32_index: 0, - float64_index: 0, - } - } - - fn validate_count(&self, value_type: u64, point_count: usize) -> crate::Result<()> { - let timestamp_count = self - .data - .timestamps - .len() - .saturating_sub(self.timestamp_index); - let value_count = match value_type { - V3_VALUE_TYPE_ZERO => timestamp_count, - V3_VALUE_TYPE_SINT64 => self - .data - .vals_sint64 - .len() - .saturating_sub(self.sint64_index), - V3_VALUE_TYPE_FLOAT32 => self - .data - .vals_float32 - .len() - .saturating_sub(self.float32_index), - V3_VALUE_TYPE_FLOAT64 => self - .data - .vals_float64 - .len() - .saturating_sub(self.float64_index), - _ => return Err("invalid Datadog v3 value type".into()), - }; - if point_count > timestamp_count || point_count > value_count { - return Err("invalid Datadog v3 point count".into()); - } - Ok(()) - } - - fn decode_points( - &mut self, - value_type: u64, - point_count: usize, - ) -> crate::Result> { - self.validate_count(value_type, point_count)?; - let mut points = Vec::with_capacity(point_count); - for _ in 0..point_count { - self.timestamp = self - .timestamp - .checked_add(self.data.timestamps[self.timestamp_index]) - .ok_or("Datadog v3 timestamp overflow")?; - if Utc.timestamp_opt(self.timestamp, 0).single().is_none() { - return Err("invalid Datadog v3 timestamp".into()); - } - self.timestamp_index += 1; - let value = match value_type { - V3_VALUE_TYPE_ZERO => 0.0, - V3_VALUE_TYPE_SINT64 => { - let value = self.data.vals_sint64[self.sint64_index] as f64; - self.sint64_index += 1; - value - } - V3_VALUE_TYPE_FLOAT32 => { - let value = f64::from(self.data.vals_float32[self.float32_index]); - self.float32_index += 1; - value - } - V3_VALUE_TYPE_FLOAT64 => { - let value = self.data.vals_float64[self.float64_index]; - self.float64_index += 1; - value - } - _ => return Err("invalid Datadog v3 value type".into()), - }; - points.push(MetricPoint { - value, - timestamp: self.timestamp, - }); - } - Ok(points) - } -} - -#[derive(Default)] -struct V3AllocationBudget { - series_bytes: usize, - event_bytes: usize, -} - -impl V3AllocationBudget { - /// Accounts for every clone and vector expansion performed for one series. - /// - /// Callers must invoke this before allocating points, tags, resources, or events for the - /// series. Successful return means both cumulative expansion limits still hold. - fn check_before_allocating( - &mut self, - point_count: usize, - name: &str, - unit: &str, - source_type: &str, - tag_bytes: usize, - resource_bytes: usize, - ) -> crate::Result<()> { - let tag_resource_bytes = tag_bytes - .checked_add(source_type.len()) - .and_then(|total| total.checked_add(resource_bytes)) - .ok_or("Datadog v3 expanded series size overflow")?; - let tag_resource_copies = point_count - .checked_add(1) - .ok_or("Datadog v3 expanded series size overflow")?; - let expanded_tag_resource_bytes = tag_resource_bytes - .checked_mul(tag_resource_copies) - .ok_or("Datadog v3 expanded series size overflow")?; - let dictionary_string_bytes = checked_v3_string_bytes(0, name)?; - let dictionary_string_bytes = checked_v3_string_bytes(dictionary_string_bytes, unit)?; - let dictionary_string_bytes = - checked_v3_string_bytes(dictionary_string_bytes, source_type)?; - let series_bytes = std::mem::size_of::() - .checked_add( - point_count - .checked_mul(std::mem::size_of::()) - .ok_or("Datadog v3 expanded series size overflow")?, - ) - .and_then(|total| total.checked_add(expanded_tag_resource_bytes)) - .and_then(|total| total.checked_add(dictionary_string_bytes)) - .ok_or("Datadog v3 expanded series size overflow")?; - self.series_bytes = self - .series_bytes - .checked_add(series_bytes) - .ok_or("Datadog v3 expanded series size overflow")?; - if self.series_bytes > MAX_V3_EXPANDED_SERIES_BYTES { - return Err("Datadog v3 expanded series exceed size limit".into()); - } - - let event_bytes = std::mem::size_of::() - .checked_add(name.len()) - .and_then(|total| total.checked_add(tag_bytes)) - .and_then(|total| total.checked_add(resource_bytes)) - .and_then(|total| total.checked_add(source_type.len())) - .and_then(|total| total.checked_add(unit.len())) - .ok_or("Datadog v3 expanded event size overflow")?; - self.event_bytes = self - .event_bytes - .checked_add( - event_bytes - .checked_mul(point_count) - .ok_or("Datadog v3 expanded event size overflow")?, - ) - .ok_or("Datadog v3 expanded event size overflow")?; - if self.event_bytes > MAX_V3_EXPANDED_EVENT_BYTES { - return Err("Datadog v3 expanded events exceed size limit".into()); - } - Ok(()) - } -} - -pub(super) fn decode_v3_metric_data( - data: &MetricDataV3, - metadata: Option<&MetricMetadataV3>, -) -> crate::Result> { - validate_v3_metric_columns(data, metadata)?; - let names = decode_v3_strings(&data.dict_name_str, false)?; - let tag_strings = decode_v3_strings(&data.dict_tag_str, true)?; - let units = decode_v3_strings(&data.dict_unit_str, false)?; - let tagsets = decode_v3_tagsets(&data.dict_tagsets, &tag_strings, metadata)?; - let resources = decode_v3_resources(data)?; - let source_types = decode_v3_strings(&data.dict_source_type_name, false)?; - let origin_count = data.dict_origin_info.len() / 3 + 1; - - // The size of a dictionary entry is stable, so calculate it once rather than rescanning the - // same tags and resources for every series that references it. - let tagset_bytes = tagsets - .iter() - .map(|tags| v3_string_list_bytes(tags)) - .collect::>>()?; - let metadata_resource_bytes = metadata - .map(|metadata| v3_string_list_bytes(&metadata.resources)) - .transpose()? - .unwrap_or(0); - let resource_bytes = resources - .iter() - .map(|entry| { - v3_resource_list_bytes(entry)? - .checked_add(metadata_resource_bytes) - .ok_or_else(|| "Datadog v3 expanded resource size overflow".into()) - }) - .collect::>>()?; - - let dictionary_lengths = [ - names.len(), - tagsets.len(), - resources.len(), - source_types.len(), - origin_count, - ]; - let mut references = V3ReferenceDecoder::default(); - let mut point_decoder = V3PointDecoder::new(data); - let mut allocation_budget = V3AllocationBudget::default(); - let mut series = Vec::new(); - - for index in 0..data.types.len() { - let refs = references.advance(data, index, dictionary_lengths)?; - let packed_type = data.types[index]; - let metric_type = packed_type & V3_METRIC_TYPE_MASK; - let value_type = packed_type & V3_VALUE_TYPE_MASK; - if metric_type == V3_METRIC_TYPE_SKETCH { - return Err("Datadog v3 series payload contains a sketch".into()); - } - let unit = references.decode_unit(data, &units, packed_type)?; - let point_count = usize::try_from(data.num_points[index]) - .map_err(|_| "invalid Datadog v3 point count")?; - let name = &names[refs.name]; - let source_type = &source_types[refs.source_type]; - allocation_budget.check_before_allocating( - point_count, - name, - &unit, - source_type, - tagset_bytes[refs.tagset], - resource_bytes[refs.resources], - )?; - let points = point_decoder.decode_points(value_type, point_count)?; - - let metric_type = match metric_type { - 1 => metric_payload::MetricType::Count, - 2 => metric_payload::MetricType::Rate, - 3 => metric_payload::MetricType::Gauge, - _ => metric_payload::MetricType::Unspecified, - }; - let series_metadata = if refs.origin == 0 { - None - } else { - let offset = (refs.origin - 1) * 3; - Some(Metadata { - origin: Some(super::ddmetric_proto::Origin { - origin_product: u32::try_from(data.dict_origin_info[offset]) - .map_err(|_| "invalid Datadog v3 origin product")?, - origin_category: u32::try_from(data.dict_origin_info[offset + 1]) - .map_err(|_| "invalid Datadog v3 origin category")?, - origin_service: u32::try_from(data.dict_origin_info[offset + 2]) - .map_err(|_| "invalid Datadog v3 origin service")?, - }), - }) - }; - let mut decoded_resources: Vec = resources[refs.resources] - .iter() - .map(|(resource_type, name)| Resource { - r#type: resource_type.clone(), - name: name.clone(), - }) - .collect(); - if let Some(payload_metadata) = metadata { - decoded_resources.extend(payload_metadata.resources.chunks_exact(2).map(|pair| { - Resource { - r#type: pair[0].clone(), - name: pair[1].clone(), - } - })); - } - let interval = - u32::try_from(data.intervals[index]).map_err(|_| "invalid Datadog v3 interval")?; - interval - .checked_mul(1000) - .ok_or("Datadog v3 interval milliseconds overflow")?; - series.push(MetricSeries { - resources: decoded_resources, - metric: name.clone(), - tags: tagsets[refs.tagset].clone(), - points, - r#type: metric_type as i32, - unit, - source_type_name: source_type.clone(), - interval: i64::from(interval), - metadata: series_metadata, - }); - } - Ok(series) -} - -fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { - let mut values = vec![String::new()]; - let mut expanded_bytes = std::mem::size_of::(); - let mut offset = 0; - while offset < raw.len() { - let (length, consumed) = decode_v3_varint(&raw[offset..])?; - offset += consumed; - let length = usize::try_from(length).map_err(|_| "Datadog v3 string length overflow")?; - let end = offset - .checked_add(length) - .ok_or("Datadog v3 string length overflow")?; - if end > raw.len() { - return Err("truncated Datadog v3 string dictionary".into()); - } - let bytes = &raw[offset..end]; - let valid_string_bytes = expanded_bytes - .checked_add(std::mem::size_of::()) - .and_then(|total| total.checked_add(length)) - .ok_or("Datadog v3 expanded string dictionary size overflow")?; - if valid_string_bytes > MAX_V3_EXPANDED_SERIES_BYTES { - return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); - } - let value = match std::str::from_utf8(bytes) { - Ok(value) => value.to_owned(), - Err(_) if sanitize => { - let worst_case_length = length - .checked_mul(3) - .ok_or("Datadog v3 expanded string dictionary size overflow")?; - let worst_case_bytes = expanded_bytes - .checked_add(std::mem::size_of::()) - .and_then(|total| total.checked_add(worst_case_length)) - .ok_or("Datadog v3 expanded string dictionary size overflow")?; - if worst_case_bytes > MAX_V3_EXPANDED_SERIES_BYTES { - return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); - } - String::from_utf8_lossy(bytes).into_owned() - } - Err(error) => return Err(error.into()), - }; - expanded_bytes = checked_v3_string_bytes(expanded_bytes, &value)?; - if expanded_bytes > MAX_V3_EXPANDED_SERIES_BYTES { - return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); - } - values.push(value); - offset = end; - } - Ok(values) -} - -fn decode_v3_varint(raw: &[u8]) -> crate::Result<(u64, usize)> { - let mut value = 0; - for (index, byte) in raw.iter().copied().enumerate() { - if index == 10 { - return Err("Datadog v3 varint overflow".into()); - } - if index == 9 && byte > 1 { - return Err("Datadog v3 varint overflow".into()); - } - value |= u64::from(byte & 0x7f) << (index * 7); - if byte & 0x80 == 0 { - return Ok((value, index + 1)); - } - } - Err("truncated Datadog v3 varint".into()) -} - -fn validate_v3_predecode_allocations(raw: &[u8]) -> crate::Result<()> { - let mut offset = 0; - let mut metadata_bytes = 0usize; - let mut repeated_numeric_bytes = 0usize; - while offset < raw.len() { - let (key, consumed) = decode_v3_varint(&raw[offset..])?; - offset += consumed; - let field = key >> 3; - let wire_type = key & 0x07; - if matches!(field, 2 | 3) && wire_type == 2 { - let (message, end) = decode_v3_length_delimited(raw, offset)?; - if field == 2 { - validate_v3_metadata_allocations(message, &mut metadata_bytes)?; - } else { - validate_v3_metric_data_allocations(message, &mut repeated_numeric_bytes)?; - } - offset = end; - } else { - offset = skip_v3_field(raw, offset, wire_type)?; - } - } - Ok(()) -} - -fn validate_v3_metadata_allocations(raw: &[u8], decoded_bytes: &mut usize) -> crate::Result<()> { - let mut offset = 0; - while offset < raw.len() { - let (key, consumed) = decode_v3_varint(&raw[offset..])?; - offset += consumed; - let field = key >> 3; - let wire_type = key & 0x07; - if matches!(field, 1 | 2) && wire_type == 2 { - let (value, end) = decode_v3_length_delimited(raw, offset)?; - *decoded_bytes = decoded_bytes - .checked_add(std::mem::size_of::()) - .and_then(|total| total.checked_add(value.len())) - .ok_or("Datadog v3 metadata string allocation overflow")?; - if *decoded_bytes > MAX_V3_DECODED_REPEATED_BYTES { - return Err("Datadog v3 metadata strings exceed size limit".into()); - } - offset = end; - } else { - offset = skip_v3_field(raw, offset, wire_type)?; - } - } - Ok(()) -} - -fn validate_v3_metric_data_allocations(raw: &[u8], decoded_bytes: &mut usize) -> crate::Result<()> { - let mut offset = 0; - while offset < raw.len() { - let (key, consumed) = decode_v3_varint(&raw[offset..])?; - offset += consumed; - let field = key >> 3; - let wire_type = key & 0x07; - let element_size = match field { - 3 | 5..=7 | 10..=17 | 19 | 20 | 23 | 24 | 26 => 8, - 9 | 18 | 21 | 22 => 4, - _ => { - offset = skip_v3_field(raw, offset, wire_type)?; - continue; - } - }; - let fixed_width = match field { - 18 => Some(4), - 19 => Some(8), - _ => None, - }; - let count = if wire_type == 2 { - let (packed, end) = decode_v3_length_delimited(raw, offset)?; - offset = end; - match fixed_width { - Some(width) => { - if packed.len() % width != 0 { - return Err("invalid packed Datadog v3 numeric column".into()); - } - packed.len() / width - } - None => count_v3_packed_varints(packed)?, - } - } else if fixed_width.is_some_and(|width| wire_type == if width == 4 { 5 } else { 1 }) - || fixed_width.is_none() && wire_type == 0 - { - offset = skip_v3_field(raw, offset, wire_type)?; - 1 - } else { - offset = skip_v3_field(raw, offset, wire_type)?; - continue; - }; - let allocation = count - .checked_mul(element_size) - .ok_or("Datadog v3 numeric column allocation overflow")?; - *decoded_bytes = decoded_bytes - .checked_add(allocation) - .ok_or("Datadog v3 numeric column allocation overflow")?; - if *decoded_bytes > MAX_V3_DECODED_REPEATED_BYTES { - return Err("Datadog v3 numeric columns exceed size limit".into()); - } - } - Ok(()) -} - -fn count_v3_packed_varints(raw: &[u8]) -> crate::Result { - let mut offset = 0; - let mut count = 0usize; - while offset < raw.len() { - let (_, consumed) = decode_v3_varint(&raw[offset..])?; - offset = offset - .checked_add(consumed) - .ok_or("Datadog v3 packed numeric count overflow")?; - count = count - .checked_add(1) - .ok_or("Datadog v3 packed numeric count overflow")?; - } - Ok(count) -} - -fn decode_v3_length_delimited(raw: &[u8], offset: usize) -> crate::Result<(&[u8], usize)> { - let (length, consumed) = decode_v3_varint( - raw.get(offset..) - .ok_or("truncated Datadog v3 protobuf field")?, - )?; - let length = - usize::try_from(length).map_err(|_| "Datadog v3 protobuf field length overflow")?; - let start = offset - .checked_add(consumed) - .ok_or("Datadog v3 protobuf field length overflow")?; - let end = start - .checked_add(length) - .ok_or("Datadog v3 protobuf field length overflow")?; - let value = raw - .get(start..end) - .ok_or("truncated Datadog v3 protobuf field")?; - Ok((value, end)) -} - -fn skip_v3_field(raw: &[u8], offset: usize, wire_type: u64) -> crate::Result { - match wire_type { - 0 => { - let (_, consumed) = decode_v3_varint( - raw.get(offset..) - .ok_or("truncated Datadog v3 protobuf field")?, - )?; - offset - .checked_add(consumed) - .ok_or_else(|| "Datadog v3 protobuf field length overflow".into()) - } - 1 => offset - .checked_add(8) - .filter(|&end| end <= raw.len()) - .ok_or_else(|| "truncated Datadog v3 protobuf field".into()), - 2 => { - let (length, consumed) = decode_v3_varint( - raw.get(offset..) - .ok_or("truncated Datadog v3 protobuf field")?, - )?; - let length = - usize::try_from(length).map_err(|_| "Datadog v3 protobuf field length overflow")?; - offset - .checked_add(consumed) - .and_then(|start| start.checked_add(length)) - .filter(|&end| end <= raw.len()) - .ok_or_else(|| "truncated Datadog v3 protobuf field".into()) - } - 5 => offset - .checked_add(4) - .filter(|&end| end <= raw.len()) - .ok_or_else(|| "truncated Datadog v3 protobuf field".into()), - _ => Err("invalid Datadog v3 protobuf wire type".into()), - } -} - -fn decode_v3_tagsets( - packed: &[i64], - dictionary: &[String], - metadata: Option<&super::ddmetric_v3_proto::Metadata>, -) -> crate::Result>> { - let mut tagsets: Vec> = vec![Vec::new()]; - let mut expanded_bytes = std::mem::size_of::>(); - let mut offset = 0; - while offset < packed.len() { - expanded_bytes = expanded_bytes - .checked_add(std::mem::size_of::>()) - .ok_or("Datadog v3 expanded tagset size overflow")?; - if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { - return Err("Datadog v3 expanded tagsets exceed size limit".into()); - } - let size = usize::try_from(packed[offset]).map_err(|_| "invalid Datadog v3 tagset size")?; - offset += 1; - let end = offset - .checked_add(size) - .ok_or("Datadog v3 tagset length overflow")?; - if end > packed.len() { - return Err("truncated Datadog v3 tagset dictionary".into()); - } - let mut tags = Vec::new(); - let mut reference: i64 = 0; - for delta in &packed[offset..end] { - reference = reference - .checked_add(*delta) - .ok_or("Datadog v3 tagset reference overflow")?; - if reference < 0 { - let index = usize::try_from( - reference - .checked_neg() - .ok_or("Datadog v3 tagset reference overflow")?, - ) - .map_err(|_| "invalid Datadog v3 tagset reference")?; - let referenced = tagsets - .get(index) - .ok_or("invalid Datadog v3 tagset reference")?; - let referenced_bytes = referenced.iter().try_fold(0usize, |total, tag| { - total - .checked_add(std::mem::size_of::()) - .and_then(|total| total.checked_add(tag.len())) - .ok_or("Datadog v3 expanded tagset size overflow") - })?; - expanded_bytes = expanded_bytes - .checked_add(referenced_bytes) - .ok_or("Datadog v3 expanded tagset size overflow")?; - if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { - return Err("Datadog v3 expanded tagsets exceed size limit".into()); - } - tags.extend(referenced.iter().cloned()); - } else { - let tag = dictionary - .get(reference as usize) - .ok_or("invalid Datadog v3 tag reference")?; - expanded_bytes = expanded_bytes - .checked_add(std::mem::size_of::()) - .and_then(|total| total.checked_add(tag.len())) - .ok_or("Datadog v3 expanded tagset size overflow")?; - if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { - return Err("Datadog v3 expanded tagsets exceed size limit".into()); - } - tags.push(tag.clone()); - } - } - tagsets.push(tags); - offset = end; - } - if let Some(metadata) = metadata { - let mut seen = HashSet::new(); - let metadata_tags = metadata - .tags - .iter() - .filter(|tag| seen.insert(tag.as_str())) - .collect::>(); - for tagset in &mut tagsets { - let mut tagset_members = tagset.iter().map(String::as_str).collect::>(); - let mut additional_tags = Vec::new(); - for &tag in &metadata_tags { - if tagset_members.insert(tag) { - expanded_bytes = checked_v3_string_bytes(expanded_bytes, tag)?; - if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { - return Err("Datadog v3 expanded tagsets exceed size limit".into()); - } - additional_tags.push(tag.clone()); - } - } - tagset.extend(additional_tags); - } - } - Ok(tagsets) -} - -fn decode_v3_resources( - data: &super::ddmetric_v3_proto::MetricData, -) -> crate::Result>> { - let dictionary = decode_v3_strings(&data.dict_resource_str, false)?; - let mut resources = vec![Vec::new()]; - let mut expanded_bytes = std::mem::size_of::>(); - let mut entry_offset: usize = 0; - for &resource_len in &data.dict_resource_len { - expanded_bytes = expanded_bytes - .checked_add(std::mem::size_of::>()) - .ok_or("Datadog v3 expanded resource size overflow")?; - if expanded_bytes > MAX_V3_EXPANDED_RESOURCE_BYTES { - return Err("Datadog v3 expanded resources exceed size limit".into()); - } - let size = usize::try_from(resource_len).map_err(|_| "invalid Datadog v3 resource size")?; - let end = entry_offset - .checked_add(size) - .ok_or("Datadog v3 resource length overflow")?; - if end > data.dict_resource_type.len() || end > data.dict_resource_name.len() { - return Err("truncated Datadog v3 resource dictionary".into()); - } - let mut set = Vec::new(); - let mut type_ref: i64 = 0; - let mut name_ref: i64 = 0; - for index in entry_offset..end { - type_ref = type_ref - .checked_add(data.dict_resource_type[index]) - .ok_or("Datadog v3 resource reference overflow")?; - name_ref = name_ref - .checked_add(data.dict_resource_name[index]) - .ok_or("Datadog v3 resource reference overflow")?; - if type_ref < 0 || name_ref < 0 { - return Err("invalid Datadog v3 resource reference".into()); - } - let resource_type = dictionary - .get(type_ref as usize) - .ok_or("invalid Datadog v3 resource reference")?; - let resource_name = dictionary - .get(name_ref as usize) - .ok_or("invalid Datadog v3 resource reference")?; - expanded_bytes = checked_v3_string_bytes(expanded_bytes, resource_type)?; - expanded_bytes = checked_v3_string_bytes(expanded_bytes, resource_name)?; - if expanded_bytes > MAX_V3_EXPANDED_RESOURCE_BYTES { - return Err("Datadog v3 expanded resources exceed size limit".into()); - } - set.push((resource_type.clone(), resource_name.clone())); - } - resources.push(set); - entry_offset = end; - } - Ok(resources) -} - -fn checked_v3_string_bytes(total: usize, value: &str) -> crate::Result { - total - .checked_add(std::mem::size_of::()) - .and_then(|total| total.checked_add(value.len())) - .ok_or_else(|| "Datadog v3 expanded string size overflow".into()) -} - /// Builds Vector's `EventMetadata` from the received metadata. Currently this is only /// utilized for passing through origin metadata set by the Agent. fn get_event_metadata(metadata: Option<&Metadata>, metric_type: Option) -> EventMetadata { @@ -1200,6 +332,12 @@ fn decode_ddseries( split_metric_namespace: bool, metric_types: Option<&[Option]>, ) -> crate::Result> { + validate_unit_event_expansion( + series + .iter() + .map(|series| (series.unit.as_str(), series.points.len())), + )?; + let decoded_metrics: Vec = series .into_iter() .enumerate() @@ -1354,6 +492,25 @@ fn decode_ddseries( Ok(decoded_metrics) } +fn validate_unit_event_expansion<'a>( + series: impl IntoIterator, +) -> crate::Result<()> { + let expanded_bytes = series + .into_iter() + .try_fold(0usize, |total, (unit, points)| { + unit.len() + .checked_mul(points) + .and_then(|bytes| total.checked_add(bytes)) + .ok_or("Datadog series unit expansion size overflow") + })?; + + if expanded_bytes > MAX_DATADOG_SERIES_UNIT_EVENT_BYTES { + return Err("Datadog series unit expansion exceeds size limit".into()); + } + + Ok(()) +} + fn decode_datadog_series_v1( body: Bytes, api_key: Option>, @@ -1374,6 +531,19 @@ fn decode_datadog_series_v1( ) })?; + validate_unit_event_expansion(metrics.series.iter().map(|series| { + ( + series.unit.as_deref().unwrap_or_default(), + series.points.len(), + ) + })) + .map_err(|error| { + ErrorMessage::new( + StatusCode::UNPROCESSABLE_ENTITY, + format!("Error decoding Datadog series: {error:?}"), + ) + })?; + let decoded_metrics: Vec = metrics .series .into_iter() diff --git a/src/sources/datadog_agent/metrics/v3.rs b/src/sources/datadog_agent/metrics/v3.rs new file mode 100644 index 0000000000000..5e2bea39fc7a3 --- /dev/null +++ b/src/sources/datadog_agent/metrics/v3.rs @@ -0,0 +1,885 @@ +use std::{collections::HashSet, sync::Arc}; + +use bytes::Bytes; +use chrono::{TimeZone, Utc}; +use prost::Message; + +use super::decode_ddseries; +use crate::{ + common::datadog::DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX, + event::Event, + sources::datadog_agent::{ + ddmetric_proto::{ + Metadata, Origin, + metric_payload::{self, MetricPoint, MetricSeries, Resource}, + }, + ddmetric_v3_proto::{ + Metadata as MetricMetadataV3, MetricData as MetricDataV3, Payload as MetricPayloadV3, + }, + }, +}; + +const MAX_V3_EXPANDED_TAGSET_BYTES: usize = 16 * 1024 * 1024; +const MAX_V3_EXPANDED_RESOURCE_BYTES: usize = 16 * 1024 * 1024; +const MAX_V3_EXPANDED_SERIES_BYTES: usize = 16 * 1024 * 1024; +const MAX_V3_EXPANDED_EVENT_BYTES: usize = 16 * 1024 * 1024; +const MAX_V3_DECODED_REPEATED_BYTES: usize = 16 * 1024 * 1024; + +// `MetricData.types` packs three independent values into each integer. These masks mirror +// `metricType`, `valueType`, and `metricFlags` in Datadog's `intake_v3.proto`. +const V3_METRIC_TYPE_MASK: u64 = 0x0f; +const V3_VALUE_TYPE_MASK: u64 = 0xf0; +const V3_FLAG_NO_INDEX: u64 = 0x100; +const V3_FLAG_HAS_UNIT: u64 = 0x200; +const V3_METRIC_TYPE_SKETCH: u64 = 4; +const V3_VALUE_TYPE_ZERO: u64 = 0; +const V3_VALUE_TYPE_SINT64: u64 = 0x10; +const V3_VALUE_TYPE_FLOAT32: u64 = 0x20; +const V3_VALUE_TYPE_FLOAT64: u64 = 0x30; + +pub(crate) fn decode_ddseries_v3( + frame: Bytes, + api_key: &Option>, + split_metric_namespace: bool, +) -> crate::Result> { + validate_v3_predecode_allocations(&frame)?; + let payload = MetricPayloadV3::decode(frame)?; + // `types` combines the metric kind, point value encoding, and flags. `flagNoIndex` is not + // represented by the v2 `MetricSeries` used during translation, so carry its Datadog origin + // value alongside each translated series until `decode_ddseries` builds the Vector events. + let metric_types = payload.metric_data.as_ref().map(|metric_data| { + metric_data + .types + .iter() + .map(|packed_type| { + (packed_type & V3_FLAG_NO_INDEX != 0).then_some(DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX) + }) + .collect::>() + }); + let series = payload.metric_data.map_or(Ok(Vec::new()), |metric_data| { + decode_v3_metric_data(&metric_data, payload.metadata.as_ref()) + })?; + decode_ddseries( + series, + api_key, + split_metric_namespace, + metric_types.as_deref(), + ) +} + +fn validate_v3_metric_columns( + data: &MetricDataV3, + metadata: Option<&MetricMetadataV3>, +) -> crate::Result<()> { + if metadata.is_some_and(|metadata| !metadata.resources.len().is_multiple_of(2)) { + return Err("Datadog v3 metadata resources must be pairs".into()); + } + + let series_count = data.types.len(); + if !data.dict_origin_info.len().is_multiple_of(3) + || data.name_refs.len() < series_count + || data.tagset_refs.len() < series_count + || data.resources_refs.len() < series_count + || data.intervals.len() < series_count + || data.num_points.len() < series_count + || data.source_type_name_refs.len() < series_count + || data.origin_info_refs.len() < series_count + { + return Err("invalid Datadog v3 metric columns".into()); + } + Ok(()) +} + +fn v3_string_list_bytes<'a>(values: impl IntoIterator) -> crate::Result { + values + .into_iter() + .try_fold(0, |total, value| checked_v3_string_bytes(total, value)) +} + +fn v3_resource_list_bytes(resources: &[(String, String)]) -> crate::Result { + resources + .iter() + .try_fold(0, |total, (resource_type, name)| { + let total = checked_v3_string_bytes(total, resource_type)?; + checked_v3_string_bytes(total, name) + }) +} + +#[derive(Default)] +struct V3ReferenceDecoder { + name: i64, + tagset: i64, + resources: i64, + source_type: i64, + origin: i64, + unit: i64, + unit_index: usize, +} + +struct V3References { + name: usize, + tagset: usize, + resources: usize, + source_type: usize, + origin: usize, +} + +impl V3ReferenceDecoder { + fn advance_reference( + current: &mut i64, + delta: i64, + dictionary_len: usize, + overflow_error: &'static str, + ) -> crate::Result { + *current = current.checked_add(delta).ok_or(overflow_error)?; + usize::try_from(*current) + .ok() + .filter(|index| *index < dictionary_len) + .ok_or_else(|| "invalid Datadog v3 dictionary reference".into()) + } + + fn advance( + &mut self, + data: &MetricDataV3, + index: usize, + dictionary_lengths: [usize; 5], + ) -> crate::Result { + Ok(V3References { + name: Self::advance_reference( + &mut self.name, + data.name_refs[index], + dictionary_lengths[0], + "Datadog v3 name reference overflow", + )?, + tagset: Self::advance_reference( + &mut self.tagset, + data.tagset_refs[index], + dictionary_lengths[1], + "Datadog v3 tagset reference overflow", + )?, + resources: Self::advance_reference( + &mut self.resources, + data.resources_refs[index], + dictionary_lengths[2], + "Datadog v3 resource reference overflow", + )?, + source_type: Self::advance_reference( + &mut self.source_type, + data.source_type_name_refs[index], + dictionary_lengths[3], + "Datadog v3 source type reference overflow", + )?, + origin: Self::advance_reference( + &mut self.origin, + data.origin_info_refs[index], + dictionary_lengths[4], + "Datadog v3 origin reference overflow", + )?, + }) + } + + fn decode_unit( + &mut self, + data: &MetricDataV3, + units: &[String], + packed_type: u64, + ) -> crate::Result { + if packed_type & V3_FLAG_HAS_UNIT == 0 { + return Ok(String::new()); + } + let delta = *data + .unit_refs + .get(self.unit_index) + .ok_or("invalid Datadog v3 unit column")?; + self.unit_index += 1; + let index = Self::advance_reference( + &mut self.unit, + delta, + units.len(), + "Datadog v3 unit reference overflow", + )?; + Ok(units[index].clone()) + } +} + +struct V3PointDecoder<'a> { + data: &'a MetricDataV3, + timestamp: i64, + timestamp_index: usize, + sint64_index: usize, + float32_index: usize, + float64_index: usize, +} + +impl<'a> V3PointDecoder<'a> { + const fn new(data: &'a MetricDataV3) -> Self { + Self { + data, + timestamp: 0, + timestamp_index: 0, + sint64_index: 0, + float32_index: 0, + float64_index: 0, + } + } + + fn validate_count(&self, value_type: u64, point_count: usize) -> crate::Result<()> { + let timestamp_count = self + .data + .timestamps + .len() + .saturating_sub(self.timestamp_index); + let value_count = match value_type { + V3_VALUE_TYPE_ZERO => timestamp_count, + V3_VALUE_TYPE_SINT64 => self + .data + .vals_sint64 + .len() + .saturating_sub(self.sint64_index), + V3_VALUE_TYPE_FLOAT32 => self + .data + .vals_float32 + .len() + .saturating_sub(self.float32_index), + V3_VALUE_TYPE_FLOAT64 => self + .data + .vals_float64 + .len() + .saturating_sub(self.float64_index), + _ => return Err("invalid Datadog v3 value type".into()), + }; + if point_count > timestamp_count || point_count > value_count { + return Err("invalid Datadog v3 point count".into()); + } + Ok(()) + } + + fn decode_points( + &mut self, + value_type: u64, + point_count: usize, + ) -> crate::Result> { + self.validate_count(value_type, point_count)?; + let mut points = Vec::with_capacity(point_count); + for _ in 0..point_count { + self.timestamp = self + .timestamp + .checked_add(self.data.timestamps[self.timestamp_index]) + .ok_or("Datadog v3 timestamp overflow")?; + if Utc.timestamp_opt(self.timestamp, 0).single().is_none() { + return Err("invalid Datadog v3 timestamp".into()); + } + self.timestamp_index += 1; + let value = match value_type { + V3_VALUE_TYPE_ZERO => 0.0, + V3_VALUE_TYPE_SINT64 => { + let value = self.data.vals_sint64[self.sint64_index] as f64; + self.sint64_index += 1; + value + } + V3_VALUE_TYPE_FLOAT32 => { + let value = f64::from(self.data.vals_float32[self.float32_index]); + self.float32_index += 1; + value + } + V3_VALUE_TYPE_FLOAT64 => { + let value = self.data.vals_float64[self.float64_index]; + self.float64_index += 1; + value + } + _ => return Err("invalid Datadog v3 value type".into()), + }; + points.push(MetricPoint { + value, + timestamp: self.timestamp, + }); + } + Ok(points) + } +} + +#[derive(Default)] +struct V3AllocationBudget { + series_bytes: usize, + event_bytes: usize, +} + +impl V3AllocationBudget { + /// Accounts for every clone and vector expansion performed for one series. + /// + /// Callers must invoke this before allocating points, tags, resources, or events for the + /// series. Successful return means both cumulative expansion limits still hold. + fn check_before_allocating( + &mut self, + point_count: usize, + name: &str, + unit: &str, + source_type: &str, + tag_bytes: usize, + resource_bytes: usize, + ) -> crate::Result<()> { + let tag_resource_bytes = tag_bytes + .checked_add(source_type.len()) + .and_then(|total| total.checked_add(resource_bytes)) + .ok_or("Datadog v3 expanded series size overflow")?; + let tag_resource_copies = point_count + .checked_add(1) + .ok_or("Datadog v3 expanded series size overflow")?; + let expanded_tag_resource_bytes = tag_resource_bytes + .checked_mul(tag_resource_copies) + .ok_or("Datadog v3 expanded series size overflow")?; + let dictionary_string_bytes = checked_v3_string_bytes(0, name)?; + let dictionary_string_bytes = checked_v3_string_bytes(dictionary_string_bytes, unit)?; + let dictionary_string_bytes = + checked_v3_string_bytes(dictionary_string_bytes, source_type)?; + let series_bytes = std::mem::size_of::() + .checked_add( + point_count + .checked_mul(std::mem::size_of::()) + .ok_or("Datadog v3 expanded series size overflow")?, + ) + .and_then(|total| total.checked_add(expanded_tag_resource_bytes)) + .and_then(|total| total.checked_add(dictionary_string_bytes)) + .ok_or("Datadog v3 expanded series size overflow")?; + self.series_bytes = self + .series_bytes + .checked_add(series_bytes) + .ok_or("Datadog v3 expanded series size overflow")?; + if self.series_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded series exceed size limit".into()); + } + + let event_bytes = std::mem::size_of::() + .checked_add(name.len()) + .and_then(|total| total.checked_add(tag_bytes)) + .and_then(|total| total.checked_add(resource_bytes)) + .and_then(|total| total.checked_add(source_type.len())) + .and_then(|total| total.checked_add(unit.len())) + .ok_or("Datadog v3 expanded event size overflow")?; + self.event_bytes = self + .event_bytes + .checked_add( + event_bytes + .checked_mul(point_count) + .ok_or("Datadog v3 expanded event size overflow")?, + ) + .ok_or("Datadog v3 expanded event size overflow")?; + if self.event_bytes > MAX_V3_EXPANDED_EVENT_BYTES { + return Err("Datadog v3 expanded events exceed size limit".into()); + } + Ok(()) + } +} + +pub(crate) fn decode_v3_metric_data( + data: &MetricDataV3, + metadata: Option<&MetricMetadataV3>, +) -> crate::Result> { + validate_v3_metric_columns(data, metadata)?; + let names = decode_v3_strings(&data.dict_name_str, false)?; + let tag_strings = decode_v3_strings(&data.dict_tag_str, true)?; + let units = decode_v3_strings(&data.dict_unit_str, false)?; + let tagsets = decode_v3_tagsets(&data.dict_tagsets, &tag_strings, metadata)?; + let resources = decode_v3_resources(data)?; + let source_types = decode_v3_strings(&data.dict_source_type_name, false)?; + let origin_count = data.dict_origin_info.len() / 3 + 1; + + // The size of a dictionary entry is stable, so calculate it once rather than rescanning the + // same tags and resources for every series that references it. + let tagset_bytes = tagsets + .iter() + .map(|tags| v3_string_list_bytes(tags)) + .collect::>>()?; + let metadata_resource_bytes = metadata + .map(|metadata| v3_string_list_bytes(&metadata.resources)) + .transpose()? + .unwrap_or(0); + let resource_bytes = resources + .iter() + .map(|entry| { + v3_resource_list_bytes(entry)? + .checked_add(metadata_resource_bytes) + .ok_or_else(|| "Datadog v3 expanded resource size overflow".into()) + }) + .collect::>>()?; + + let dictionary_lengths = [ + names.len(), + tagsets.len(), + resources.len(), + source_types.len(), + origin_count, + ]; + let mut references = V3ReferenceDecoder::default(); + let mut point_decoder = V3PointDecoder::new(data); + let mut allocation_budget = V3AllocationBudget::default(); + let mut series = Vec::new(); + + for index in 0..data.types.len() { + let refs = references.advance(data, index, dictionary_lengths)?; + let packed_type = data.types[index]; + let metric_type = packed_type & V3_METRIC_TYPE_MASK; + let value_type = packed_type & V3_VALUE_TYPE_MASK; + if metric_type == V3_METRIC_TYPE_SKETCH { + return Err("Datadog v3 series payload contains a sketch".into()); + } + let unit = references.decode_unit(data, &units, packed_type)?; + let point_count = usize::try_from(data.num_points[index]) + .map_err(|_| "invalid Datadog v3 point count")?; + let name = &names[refs.name]; + let source_type = &source_types[refs.source_type]; + allocation_budget.check_before_allocating( + point_count, + name, + &unit, + source_type, + tagset_bytes[refs.tagset], + resource_bytes[refs.resources], + )?; + let points = point_decoder.decode_points(value_type, point_count)?; + + let metric_type = match metric_type { + 1 => metric_payload::MetricType::Count, + 2 => metric_payload::MetricType::Rate, + 3 => metric_payload::MetricType::Gauge, + _ => metric_payload::MetricType::Unspecified, + }; + let series_metadata = if refs.origin == 0 { + None + } else { + let offset = (refs.origin - 1) * 3; + Some(Metadata { + origin: Some(Origin { + origin_product: u32::try_from(data.dict_origin_info[offset]) + .map_err(|_| "invalid Datadog v3 origin product")?, + origin_category: u32::try_from(data.dict_origin_info[offset + 1]) + .map_err(|_| "invalid Datadog v3 origin category")?, + origin_service: u32::try_from(data.dict_origin_info[offset + 2]) + .map_err(|_| "invalid Datadog v3 origin service")?, + }), + }) + }; + let mut decoded_resources: Vec = resources[refs.resources] + .iter() + .map(|(resource_type, name)| Resource { + r#type: resource_type.clone(), + name: name.clone(), + }) + .collect(); + if let Some(payload_metadata) = metadata { + decoded_resources.extend(payload_metadata.resources.chunks_exact(2).map(|pair| { + Resource { + r#type: pair[0].clone(), + name: pair[1].clone(), + } + })); + } + let interval = + u32::try_from(data.intervals[index]).map_err(|_| "invalid Datadog v3 interval")?; + interval + .checked_mul(1000) + .ok_or("Datadog v3 interval milliseconds overflow")?; + series.push(MetricSeries { + resources: decoded_resources, + metric: name.clone(), + tags: tagsets[refs.tagset].clone(), + points, + r#type: metric_type as i32, + unit, + source_type_name: source_type.clone(), + interval: i64::from(interval), + metadata: series_metadata, + }); + } + Ok(series) +} + +fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { + let mut values = vec![String::new()]; + let mut expanded_bytes = std::mem::size_of::(); + let mut offset = 0; + while offset < raw.len() { + let (length, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let length = usize::try_from(length).map_err(|_| "Datadog v3 string length overflow")?; + let end = offset + .checked_add(length) + .ok_or("Datadog v3 string length overflow")?; + if end > raw.len() { + return Err("truncated Datadog v3 string dictionary".into()); + } + let bytes = &raw[offset..end]; + let valid_string_bytes = expanded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(length)) + .ok_or("Datadog v3 expanded string dictionary size overflow")?; + if valid_string_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); + } + let value = match std::str::from_utf8(bytes) { + Ok(value) => value.to_owned(), + Err(_) if sanitize => { + let worst_case_length = length + .checked_mul(3) + .ok_or("Datadog v3 expanded string dictionary size overflow")?; + let worst_case_bytes = expanded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(worst_case_length)) + .ok_or("Datadog v3 expanded string dictionary size overflow")?; + if worst_case_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); + } + String::from_utf8_lossy(bytes).into_owned() + } + Err(error) => return Err(error.into()), + }; + expanded_bytes = checked_v3_string_bytes(expanded_bytes, &value)?; + if expanded_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); + } + values.push(value); + offset = end; + } + Ok(values) +} + +fn decode_v3_varint(raw: &[u8]) -> crate::Result<(u64, usize)> { + let mut value = 0; + for (index, byte) in raw.iter().copied().enumerate() { + if index == 10 { + return Err("Datadog v3 varint overflow".into()); + } + if index == 9 && byte > 1 { + return Err("Datadog v3 varint overflow".into()); + } + value |= u64::from(byte & 0x7f) << (index * 7); + if byte & 0x80 == 0 { + return Ok((value, index + 1)); + } + } + Err("truncated Datadog v3 varint".into()) +} + +fn validate_v3_predecode_allocations(raw: &[u8]) -> crate::Result<()> { + let mut offset = 0; + let mut metadata_bytes = 0usize; + let mut repeated_numeric_bytes = 0usize; + while offset < raw.len() { + let (key, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let field = key >> 3; + let wire_type = key & 0x07; + if matches!(field, 2 | 3) && wire_type == 2 { + let (message, end) = decode_v3_length_delimited(raw, offset)?; + if field == 2 { + validate_v3_metadata_allocations(message, &mut metadata_bytes)?; + } else { + validate_v3_metric_data_allocations(message, &mut repeated_numeric_bytes)?; + } + offset = end; + } else { + offset = skip_v3_field(raw, offset, wire_type)?; + } + } + Ok(()) +} + +fn validate_v3_metadata_allocations(raw: &[u8], decoded_bytes: &mut usize) -> crate::Result<()> { + let mut offset = 0; + while offset < raw.len() { + let (key, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let field = key >> 3; + let wire_type = key & 0x07; + if matches!(field, 1 | 2) && wire_type == 2 { + let (value, end) = decode_v3_length_delimited(raw, offset)?; + *decoded_bytes = decoded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(value.len())) + .ok_or("Datadog v3 metadata string allocation overflow")?; + if *decoded_bytes > MAX_V3_DECODED_REPEATED_BYTES { + return Err("Datadog v3 metadata strings exceed size limit".into()); + } + offset = end; + } else { + offset = skip_v3_field(raw, offset, wire_type)?; + } + } + Ok(()) +} + +fn validate_v3_metric_data_allocations(raw: &[u8], decoded_bytes: &mut usize) -> crate::Result<()> { + let mut offset = 0; + while offset < raw.len() { + let (key, consumed) = decode_v3_varint(&raw[offset..])?; + offset += consumed; + let field = key >> 3; + let wire_type = key & 0x07; + let element_size = match field { + 3 | 5..=7 | 10..=17 | 19 | 20 | 23 | 24 | 26 => 8, + 9 | 18 | 21 | 22 => 4, + _ => { + offset = skip_v3_field(raw, offset, wire_type)?; + continue; + } + }; + let fixed_width = match field { + 18 => Some(4), + 19 => Some(8), + _ => None, + }; + let count = if wire_type == 2 { + let (packed, end) = decode_v3_length_delimited(raw, offset)?; + offset = end; + match fixed_width { + Some(width) => { + if packed.len() % width != 0 { + return Err("invalid packed Datadog v3 numeric column".into()); + } + packed.len() / width + } + None => count_v3_packed_varints(packed)?, + } + } else if fixed_width.is_some_and(|width| wire_type == if width == 4 { 5 } else { 1 }) + || fixed_width.is_none() && wire_type == 0 + { + offset = skip_v3_field(raw, offset, wire_type)?; + 1 + } else { + offset = skip_v3_field(raw, offset, wire_type)?; + continue; + }; + let allocation = count + .checked_mul(element_size) + .ok_or("Datadog v3 numeric column allocation overflow")?; + *decoded_bytes = decoded_bytes + .checked_add(allocation) + .ok_or("Datadog v3 numeric column allocation overflow")?; + if *decoded_bytes > MAX_V3_DECODED_REPEATED_BYTES { + return Err("Datadog v3 numeric columns exceed size limit".into()); + } + } + Ok(()) +} + +fn count_v3_packed_varints(raw: &[u8]) -> crate::Result { + let mut offset = 0; + let mut count = 0usize; + while offset < raw.len() { + let (_, consumed) = decode_v3_varint(&raw[offset..])?; + offset = offset + .checked_add(consumed) + .ok_or("Datadog v3 packed numeric count overflow")?; + count = count + .checked_add(1) + .ok_or("Datadog v3 packed numeric count overflow")?; + } + Ok(count) +} + +fn decode_v3_length_delimited(raw: &[u8], offset: usize) -> crate::Result<(&[u8], usize)> { + let (length, consumed) = decode_v3_varint( + raw.get(offset..) + .ok_or("truncated Datadog v3 protobuf field")?, + )?; + let length = + usize::try_from(length).map_err(|_| "Datadog v3 protobuf field length overflow")?; + let start = offset + .checked_add(consumed) + .ok_or("Datadog v3 protobuf field length overflow")?; + let end = start + .checked_add(length) + .ok_or("Datadog v3 protobuf field length overflow")?; + let value = raw + .get(start..end) + .ok_or("truncated Datadog v3 protobuf field")?; + Ok((value, end)) +} + +fn skip_v3_field(raw: &[u8], offset: usize, wire_type: u64) -> crate::Result { + match wire_type { + 0 => { + let (_, consumed) = decode_v3_varint( + raw.get(offset..) + .ok_or("truncated Datadog v3 protobuf field")?, + )?; + offset + .checked_add(consumed) + .ok_or_else(|| "Datadog v3 protobuf field length overflow".into()) + } + 1 => offset + .checked_add(8) + .filter(|&end| end <= raw.len()) + .ok_or_else(|| "truncated Datadog v3 protobuf field".into()), + 2 => { + let (length, consumed) = decode_v3_varint( + raw.get(offset..) + .ok_or("truncated Datadog v3 protobuf field")?, + )?; + let length = + usize::try_from(length).map_err(|_| "Datadog v3 protobuf field length overflow")?; + offset + .checked_add(consumed) + .and_then(|start| start.checked_add(length)) + .filter(|&end| end <= raw.len()) + .ok_or_else(|| "truncated Datadog v3 protobuf field".into()) + } + 5 => offset + .checked_add(4) + .filter(|&end| end <= raw.len()) + .ok_or_else(|| "truncated Datadog v3 protobuf field".into()), + _ => Err("invalid Datadog v3 protobuf wire type".into()), + } +} + +fn decode_v3_tagsets( + packed: &[i64], + dictionary: &[String], + metadata: Option<&MetricMetadataV3>, +) -> crate::Result>> { + let mut tagsets: Vec> = vec![Vec::new()]; + let mut expanded_bytes = std::mem::size_of::>(); + let mut offset = 0; + while offset < packed.len() { + expanded_bytes = expanded_bytes + .checked_add(std::mem::size_of::>()) + .ok_or("Datadog v3 expanded tagset size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { + return Err("Datadog v3 expanded tagsets exceed size limit".into()); + } + let size = usize::try_from(packed[offset]).map_err(|_| "invalid Datadog v3 tagset size")?; + offset += 1; + let end = offset + .checked_add(size) + .ok_or("Datadog v3 tagset length overflow")?; + if end > packed.len() { + return Err("truncated Datadog v3 tagset dictionary".into()); + } + let mut tags = Vec::new(); + let mut reference: i64 = 0; + for delta in &packed[offset..end] { + reference = reference + .checked_add(*delta) + .ok_or("Datadog v3 tagset reference overflow")?; + if reference < 0 { + let index = usize::try_from( + reference + .checked_neg() + .ok_or("Datadog v3 tagset reference overflow")?, + ) + .map_err(|_| "invalid Datadog v3 tagset reference")?; + let referenced = tagsets + .get(index) + .ok_or("invalid Datadog v3 tagset reference")?; + let referenced_bytes = referenced.iter().try_fold(0usize, |total, tag| { + total + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(tag.len())) + .ok_or("Datadog v3 expanded tagset size overflow") + })?; + expanded_bytes = expanded_bytes + .checked_add(referenced_bytes) + .ok_or("Datadog v3 expanded tagset size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { + return Err("Datadog v3 expanded tagsets exceed size limit".into()); + } + tags.extend(referenced.iter().cloned()); + } else { + let tag = dictionary + .get(reference as usize) + .ok_or("invalid Datadog v3 tag reference")?; + expanded_bytes = expanded_bytes + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(tag.len())) + .ok_or("Datadog v3 expanded tagset size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { + return Err("Datadog v3 expanded tagsets exceed size limit".into()); + } + tags.push(tag.clone()); + } + } + tagsets.push(tags); + offset = end; + } + if let Some(metadata) = metadata { + let mut seen = HashSet::new(); + let metadata_tags = metadata + .tags + .iter() + .filter(|tag| seen.insert(tag.as_str())) + .collect::>(); + for tagset in &mut tagsets { + let mut tagset_members = tagset.iter().map(String::as_str).collect::>(); + let mut additional_tags = Vec::new(); + for &tag in &metadata_tags { + if tagset_members.insert(tag) { + expanded_bytes = checked_v3_string_bytes(expanded_bytes, tag)?; + if expanded_bytes > MAX_V3_EXPANDED_TAGSET_BYTES { + return Err("Datadog v3 expanded tagsets exceed size limit".into()); + } + additional_tags.push(tag.clone()); + } + } + tagset.extend(additional_tags); + } + } + Ok(tagsets) +} + +fn decode_v3_resources(data: &MetricDataV3) -> crate::Result>> { + let dictionary = decode_v3_strings(&data.dict_resource_str, false)?; + let mut resources = vec![Vec::new()]; + let mut expanded_bytes = std::mem::size_of::>(); + let mut entry_offset: usize = 0; + for &resource_len in &data.dict_resource_len { + expanded_bytes = expanded_bytes + .checked_add(std::mem::size_of::>()) + .ok_or("Datadog v3 expanded resource size overflow")?; + if expanded_bytes > MAX_V3_EXPANDED_RESOURCE_BYTES { + return Err("Datadog v3 expanded resources exceed size limit".into()); + } + let size = usize::try_from(resource_len).map_err(|_| "invalid Datadog v3 resource size")?; + let end = entry_offset + .checked_add(size) + .ok_or("Datadog v3 resource length overflow")?; + if end > data.dict_resource_type.len() || end > data.dict_resource_name.len() { + return Err("truncated Datadog v3 resource dictionary".into()); + } + let mut set = Vec::new(); + let mut type_ref: i64 = 0; + let mut name_ref: i64 = 0; + for index in entry_offset..end { + type_ref = type_ref + .checked_add(data.dict_resource_type[index]) + .ok_or("Datadog v3 resource reference overflow")?; + name_ref = name_ref + .checked_add(data.dict_resource_name[index]) + .ok_or("Datadog v3 resource reference overflow")?; + if type_ref < 0 || name_ref < 0 { + return Err("invalid Datadog v3 resource reference".into()); + } + let resource_type = dictionary + .get(type_ref as usize) + .ok_or("invalid Datadog v3 resource reference")?; + let resource_name = dictionary + .get(name_ref as usize) + .ok_or("invalid Datadog v3 resource reference")?; + expanded_bytes = checked_v3_string_bytes(expanded_bytes, resource_type)?; + expanded_bytes = checked_v3_string_bytes(expanded_bytes, resource_name)?; + if expanded_bytes > MAX_V3_EXPANDED_RESOURCE_BYTES { + return Err("Datadog v3 expanded resources exceed size limit".into()); + } + set.push((resource_type.clone(), resource_name.clone())); + } + resources.push(set); + entry_offset = end; + } + Ok(resources) +} + +fn checked_v3_string_bytes(total: usize, value: &str) -> crate::Result { + total + .checked_add(std::mem::size_of::()) + .and_then(|total| total.checked_add(value.len())) + .ok_or_else(|| "Datadog v3 expanded string size overflow".into()) +} diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 53594230f68fe..09b0a0a663e88 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2571,6 +2571,58 @@ async fn decode_series_endpoint_v2() { .await; } +#[test] +fn decode_series_v2_rejects_excessive_unit_event_expansion() { + let point = ddmetric_proto::metric_payload::MetricPoint { + value: 1.0, + timestamp: 1_542_182_950, + }; + let payload = ddmetric_proto::MetricPayload { + series: vec![ddmetric_proto::metric_payload::MetricSeries { + metric: "unit.expansion".to_string(), + points: vec![point; 16_385], + r#type: ddmetric_proto::metric_payload::MetricType::Gauge as i32, + unit: "u".repeat(1024), + ..Default::default() + }], + }; + let mut body = Vec::new(); + payload.encode(&mut body).unwrap(); + + assert!(super::metrics::decode_ddseries_v2(Bytes::from(body), &None, false).is_err()); +} + +#[tokio::test] +async fn decode_series_v1_rejects_excessive_unit_event_expansion() { + let (rx, _, _, addr, _guard) = source(EventStatus::Delivered, true, true, false, true).await; + let request = DatadogSeriesRequest { + series: vec![DatadogSeriesMetric { + metric: "unit.expansion".to_string(), + r#type: DatadogMetricType::Gauge, + interval: None, + points: vec![DatadogPoint(1_542_182_950, 1.0); 16_385], + tags: None, + host: None, + source_type_name: None, + device: None, + unit: Some("u".repeat(1024)), + metadata: None, + }], + }; + + assert_eq!( + 422, + send_with_path( + addr, + serde_json::to_vec(&request).unwrap(), + dd_api_key_headers(), + DD_API_SERIES_V1_PATH, + ) + .await + ); + assert!(crate::test_util::collect_ready(rx).is_empty()); +} + fn v3_string_dictionary(values: &[&str]) -> Vec { values .iter() From 0bbe585d8c98521871ea9673ece58e3ecaeeaa4e Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Wed, 14 Oct 2026 06:16:51 +0000 Subject: [PATCH 17/23] fix(datadog): preserve no-index in v2 series --- lib/datadog-proto/proto/datadog/agentpayload.proto | 6 +++++- src/sinks/datadog/metrics/encoder.rs | 2 ++ src/sources/datadog_agent/metrics.rs | 5 +++++ src/sources/datadog_agent/metrics/v3.rs | 1 + src/sources/datadog_agent/tests.rs | 10 ++++++++++ 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/datadog-proto/proto/datadog/agentpayload.proto b/lib/datadog-proto/proto/datadog/agentpayload.proto index 471447d11f801..93e1276bf0892 100644 --- a/lib/datadog-proto/proto/datadog/agentpayload.proto +++ b/lib/datadog-proto/proto/datadog/agentpayload.proto @@ -14,7 +14,11 @@ message CommonMetadata { } message Origin { - reserved 1, 2, 3; + reserved 1, 2; + // The Agent's v2 serializer writes field 3 for no-index series even though the public + // agent-payload schema reserves it. Keep the wire field so Vector can preserve no-index + // metadata when receiving or forwarding v2 series. + int32 metric_type = 3; uint32 origin_product = 4; uint32 origin_category = 5; uint32 origin_service = 6; diff --git a/src/sinks/datadog/metrics/encoder.rs b/src/sinks/datadog/metrics/encoder.rs index fa577428b7300..f164d4d8dd949 100644 --- a/src/sinks/datadog/metrics/encoder.rs +++ b/src/sinks/datadog/metrics/encoder.rs @@ -444,6 +444,7 @@ fn generate_proto_metadata( } ddmetric_proto::Metadata { origin: Some(ddmetric_proto::Origin { + metric_type: origin.metric_type().unwrap_or_default(), origin_product: origin.product().unwrap_or_default(), origin_category: origin.category().unwrap_or_default(), origin_service: origin.service().unwrap_or_default(), @@ -1273,6 +1274,7 @@ mod tests { .unwrap(); let generated_origin = series_proto.metadata.unwrap().origin.unwrap(); + assert_eq!(generated_origin.metric_type, 9); assert_eq!(generated_origin.origin_product, product); assert_eq!(generated_origin.origin_category, category); assert_eq!(generated_origin.origin_service, service); diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 1d3745d228990..d005d7d707a69 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -295,6 +295,11 @@ fn decode_datadog_series_v3( /// utilized for passing through origin metadata set by the Agent. fn get_event_metadata(metadata: Option<&Metadata>, metric_type: Option) -> EventMetadata { let origin = metadata.and_then(|metadata| metadata.origin.as_ref()); + let metric_type = metric_type.or_else(|| { + origin + .map(|origin| origin.metric_type) + .filter(|metric_type| *metric_type != 0) + }); if origin.is_none() && metric_type.is_none() { EventMetadata::default() } else { diff --git a/src/sources/datadog_agent/metrics/v3.rs b/src/sources/datadog_agent/metrics/v3.rs index 5e2bea39fc7a3..9710a33a990c2 100644 --- a/src/sources/datadog_agent/metrics/v3.rs +++ b/src/sources/datadog_agent/metrics/v3.rs @@ -450,6 +450,7 @@ pub(crate) fn decode_v3_metric_data( let offset = (refs.origin - 1) * 3; Some(Metadata { origin: Some(Origin { + metric_type: 0, origin_product: u32::try_from(data.dict_origin_info[offset]) .map_err(|_| "invalid Datadog v3 origin product")?, origin_category: u32::try_from(data.dict_origin_info[offset + 1]) diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 09b0a0a663e88..9868fe28d32f1 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -1204,6 +1204,7 @@ async fn decode_sketches() { }], metadata: Some(ddmetric_proto::Metadata { origin: Some(ddmetric_proto::Origin { + metric_type: 0, origin_product: 10, origin_category: 11, origin_service: 9, @@ -2387,6 +2388,7 @@ async fn decode_series_endpoint_v2() { interval: 0, metadata: Some(ddmetric_proto::Metadata { origin: Some(ddmetric_proto::Origin { + metric_type: 9, origin_product: 10, origin_category: 10, origin_service: 42, @@ -2566,6 +2568,14 @@ async fn decode_series_endpoint_v2() { .unwrap(), 42 ); + assert_eq!( + events[3] + .metadata() + .datadog_origin_metadata() + .unwrap() + .metric_type(), + Some(9) + ); } }) .await; From 22ba240f717f44a2081bdc55bf938b92e111944e Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:10:59 +0000 Subject: [PATCH 18/23] fix(datadog_agent): share v3 dictionary budget --- src/sources/datadog_agent/metrics/v3.rs | 54 ++++++++++++++++--------- src/sources/datadog_agent/tests.rs | 12 ++++++ 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/src/sources/datadog_agent/metrics/v3.rs b/src/sources/datadog_agent/metrics/v3.rs index 9710a33a990c2..883a662f53982 100644 --- a/src/sources/datadog_agent/metrics/v3.rs +++ b/src/sources/datadog_agent/metrics/v3.rs @@ -376,12 +376,17 @@ pub(crate) fn decode_v3_metric_data( metadata: Option<&MetricMetadataV3>, ) -> crate::Result> { validate_v3_metric_columns(data, metadata)?; - let names = decode_v3_strings(&data.dict_name_str, false)?; - let tag_strings = decode_v3_strings(&data.dict_tag_str, true)?; - let units = decode_v3_strings(&data.dict_unit_str, false)?; + let mut string_dictionary_bytes = 0; + let names = decode_v3_strings(&data.dict_name_str, false, &mut string_dictionary_bytes)?; + let tag_strings = decode_v3_strings(&data.dict_tag_str, true, &mut string_dictionary_bytes)?; + let units = decode_v3_strings(&data.dict_unit_str, false, &mut string_dictionary_bytes)?; let tagsets = decode_v3_tagsets(&data.dict_tagsets, &tag_strings, metadata)?; - let resources = decode_v3_resources(data)?; - let source_types = decode_v3_strings(&data.dict_source_type_name, false)?; + let resources = decode_v3_resources(data, &mut string_dictionary_bytes)?; + let source_types = decode_v3_strings( + &data.dict_source_type_name, + false, + &mut string_dictionary_bytes, + )?; let origin_count = data.dict_origin_info.len() / 3 + 1; // The size of a dictionary entry is stable, so calculate it once rather than rescanning the @@ -495,9 +500,16 @@ pub(crate) fn decode_v3_metric_data( Ok(series) } -fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { +fn decode_v3_strings( + raw: &[u8], + sanitize: bool, + expanded_bytes: &mut usize, +) -> crate::Result> { + *expanded_bytes = checked_v3_string_bytes(*expanded_bytes, "")?; + if *expanded_bytes > MAX_V3_EXPANDED_SERIES_BYTES { + return Err("Datadog v3 expanded string dictionaries exceed size limit".into()); + } let mut values = vec![String::new()]; - let mut expanded_bytes = std::mem::size_of::(); let mut offset = 0; while offset < raw.len() { let (length, consumed) = decode_v3_varint(&raw[offset..])?; @@ -510,34 +522,35 @@ fn decode_v3_strings(raw: &[u8], sanitize: bool) -> crate::Result> { return Err("truncated Datadog v3 string dictionary".into()); } let bytes = &raw[offset..end]; - let valid_string_bytes = expanded_bytes + let valid_string_bytes = (*expanded_bytes) .checked_add(std::mem::size_of::()) .and_then(|total| total.checked_add(length)) .ok_or("Datadog v3 expanded string dictionary size overflow")?; if valid_string_bytes > MAX_V3_EXPANDED_SERIES_BYTES { - return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); + return Err("Datadog v3 expanded string dictionaries exceed size limit".into()); } let value = match std::str::from_utf8(bytes) { - Ok(value) => value.to_owned(), + Ok(value) => { + *expanded_bytes = valid_string_bytes; + value.to_owned() + } Err(_) if sanitize => { let worst_case_length = length .checked_mul(3) .ok_or("Datadog v3 expanded string dictionary size overflow")?; - let worst_case_bytes = expanded_bytes + let worst_case_bytes = (*expanded_bytes) .checked_add(std::mem::size_of::()) .and_then(|total| total.checked_add(worst_case_length)) .ok_or("Datadog v3 expanded string dictionary size overflow")?; if worst_case_bytes > MAX_V3_EXPANDED_SERIES_BYTES { - return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); + return Err("Datadog v3 expanded string dictionaries exceed size limit".into()); } - String::from_utf8_lossy(bytes).into_owned() + let value = String::from_utf8_lossy(bytes).into_owned(); + *expanded_bytes = checked_v3_string_bytes(*expanded_bytes, &value)?; + value } Err(error) => return Err(error.into()), }; - expanded_bytes = checked_v3_string_bytes(expanded_bytes, &value)?; - if expanded_bytes > MAX_V3_EXPANDED_SERIES_BYTES { - return Err("Datadog v3 expanded string dictionary exceeds size limit".into()); - } values.push(value); offset = end; } @@ -827,8 +840,11 @@ fn decode_v3_tagsets( Ok(tagsets) } -fn decode_v3_resources(data: &MetricDataV3) -> crate::Result>> { - let dictionary = decode_v3_strings(&data.dict_resource_str, false)?; +fn decode_v3_resources( + data: &MetricDataV3, + string_dictionary_bytes: &mut usize, +) -> crate::Result>> { + let dictionary = decode_v3_strings(&data.dict_resource_str, false, string_dictionary_bytes)?; let mut resources = vec![Vec::new()]; let mut expanded_bytes = std::mem::size_of::>(); let mut entry_offset: usize = 0; diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 9868fe28d32f1..ff248f873b5b5 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2823,6 +2823,18 @@ fn decode_v3_rejects_excessive_empty_string_entries() { assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); } +#[test] +fn decode_v3_accumulates_string_dictionary_expansion_across_fields() { + let mut data = minimal_v3_metric_data(); + data.dict_name_str = vec![0; 300_000]; + data.dict_tag_str = vec![0; 300_000]; + data.dict_unit_str = vec![0; 300_000]; + data.dict_resource_str = vec![0; 300_000]; + data.dict_source_type_name = vec![0; 300_000]; + + assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); +} + #[test] fn decode_v3_rejects_excessive_empty_tagsets() { let mut data = minimal_v3_metric_data(); From 734abdb47edee5bc2f644d716d4d70d5b28218f3 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:19:28 +0000 Subject: [PATCH 19/23] fix(events): account for shared metric units --- lib/vector-core/src/event/metadata.rs | 36 ++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index 93694551f0297..903c0f19082b8 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -387,10 +387,20 @@ pub(super) fn default_schema_definition() -> Arc { impl ByteSizeOf for EventMetadata { fn allocated_bytes(&self) -> usize { - // NOTE we don't count the `str` here because it's allocated somewhere - // else. We're just moving around the pointer, which is already captured - // by `ByteSizeOf::size_of`. - self.inner.finalizers.allocated_bytes() + // Most Arc-backed metadata points to allocations owned or accounted for elsewhere. The + // Datadog metric unit is retained solely by event metadata, so include its allocation. + let datadog_metric_unit_bytes = self.inner.datadog_metric_unit.as_ref().map_or(0, |unit| { + // Amortize the shared string allocation across both copies of this metadata wrapper + // and copy-on-write `Inner` values that retain the same unit. This keeps a collection + // of cloned events accountable for the allocation without charging its full size to + // every event. + let owners = Arc::strong_count(&self.inner) + .saturating_mul(Arc::strong_count(unit)) + .max(1); + unit.len().div_ceil(owners) + }); + + self.inner.finalizers.allocated_bytes() + datadog_metric_unit_bytes } } @@ -716,6 +726,24 @@ mod test { assert!(Arc::ptr_eq(original_unit, cloned_unit)); } + #[test] + fn metadata_byte_size_accounts_for_shared_datadog_metric_unit_once() { + let unit = "x".repeat(1024); + let metadata = EventMetadata::default().with_datadog_metric_unit(unit); + let mut shared_inner = metadata.clone(); + + assert_eq!( + metadata.allocated_bytes() + shared_inner.allocated_bytes(), + 1024 + ); + + shared_inner.set_datadog_api_key(Arc::from(SECRET)); + assert_eq!( + metadata.allocated_bytes() + shared_inner.allocated_bytes(), + 1024 + ); + } + #[test] fn secrets_merge() { let mut a = Secrets::new(); From 733cd3e5905b678a9c20fbc8968a3c4f51b93052 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:31:33 +0000 Subject: [PATCH 20/23] fix(metrics): stabilize unit metadata handling --- lib/vector-core/src/event/metadata.rs | 37 +++++++++++++-------------- src/transforms/log_to_metric.rs | 29 ++++++++++++++++++++- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index 903c0f19082b8..1ddf451f0c8d8 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -333,6 +333,11 @@ impl EventMetadata { self.get_mut().datadog_metric_unit = Some(unit.into()); } + /// Clears the Datadog metric unit. + pub fn clear_datadog_metric_unit(&mut self) { + self.get_mut().datadog_metric_unit = None; + } + /// Returns a reference to the event id. pub fn source_event_id(&self) -> Option { self.inner.source_event_id @@ -389,16 +394,14 @@ impl ByteSizeOf for EventMetadata { fn allocated_bytes(&self) -> usize { // Most Arc-backed metadata points to allocations owned or accounted for elsewhere. The // Datadog metric unit is retained solely by event metadata, so include its allocation. - let datadog_metric_unit_bytes = self.inner.datadog_metric_unit.as_ref().map_or(0, |unit| { - // Amortize the shared string allocation across both copies of this metadata wrapper - // and copy-on-write `Inner` values that retain the same unit. This keeps a collection - // of cloned events accountable for the allocation without charging its full size to - // every event. - let owners = Arc::strong_count(&self.inner) - .saturating_mul(Arc::strong_count(unit)) - .max(1); - unit.len().div_ceil(owners) - }); + // Charge each event conservatively. Arc owner counts are transient: fanout branches can + // disappear after a memory queue has snapshotted this value, leaving that queue as the sole + // owner without updating its reservation. + let datadog_metric_unit_bytes = self + .inner + .datadog_metric_unit + .as_ref() + .map_or(0, |unit| unit.len()); self.inner.finalizers.allocated_bytes() + datadog_metric_unit_bytes } @@ -727,21 +730,17 @@ mod test { } #[test] - fn metadata_byte_size_accounts_for_shared_datadog_metric_unit_once() { + fn metadata_byte_size_conservatively_accounts_for_shared_datadog_metric_unit() { let unit = "x".repeat(1024); let metadata = EventMetadata::default().with_datadog_metric_unit(unit); let mut shared_inner = metadata.clone(); - assert_eq!( - metadata.allocated_bytes() + shared_inner.allocated_bytes(), - 1024 - ); + assert_eq!(metadata.allocated_bytes(), 1024); + assert_eq!(shared_inner.allocated_bytes(), 1024); shared_inner.set_datadog_api_key(Arc::from(SECRET)); - assert_eq!( - metadata.allocated_bytes() + shared_inner.allocated_bytes(), - 1024 - ); + assert_eq!(metadata.allocated_bytes(), 1024); + assert_eq!(shared_inner.allocated_bytes(), 1024); } #[test] diff --git a/src/transforms/log_to_metric.rs b/src/transforms/log_to_metric.rs index 26bed316c92f8..aaeb7b6d22ca8 100644 --- a/src/transforms/log_to_metric.rs +++ b/src/transforms/log_to_metric.rs @@ -376,7 +376,7 @@ fn to_metric_with_config(config: &MetricConfig, event: &Event) -> Result Result Date: Wed, 14 Oct 2026 13:20:06 +0000 Subject: [PATCH 21/23] fix(datadog_agent): budget metric tag entries --- src/sources/datadog_agent/metrics/v3.rs | 48 ++++++++++++++++++++----- src/sources/datadog_agent/tests.rs | 11 ++++++ 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/sources/datadog_agent/metrics/v3.rs b/src/sources/datadog_agent/metrics/v3.rs index 883a662f53982..b1f4dba0fa899 100644 --- a/src/sources/datadog_agent/metrics/v3.rs +++ b/src/sources/datadog_agent/metrics/v3.rs @@ -7,7 +7,7 @@ use prost::Message; use super::decode_ddseries; use crate::{ common::datadog::DATADOG_ORIGIN_METRIC_TYPE_NO_INDEX, - event::Event, + event::{Event, metric::TagValueSet}, sources::datadog_agent::{ ddmetric_proto::{ Metadata, Origin, @@ -24,6 +24,10 @@ const MAX_V3_EXPANDED_RESOURCE_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_SERIES_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_EXPANDED_EVENT_BYTES: usize = 16 * 1024 * 1024; const MAX_V3_DECODED_REPEATED_BYTES: usize = 16 * 1024 * 1024; +// `decode_ddseries` converts every wire tag/resource into a BTreeMap entry. Account for the +// key/value containers plus a conservative three-pointer share of the tree node allocation. +const V3_METRIC_TAG_ENTRY_BYTES: usize = + std::mem::size_of::<(String, TagValueSet)>() + 3 * std::mem::size_of::(); // `MetricData.types` packs three independent values into each integer. These masks mirror // `metricType`, `valueType`, and `metricFlags` in Datadog's `intake_v3.proto`. @@ -304,6 +308,13 @@ struct V3AllocationBudget { event_bytes: usize, } +struct V3TagResourceSize { + tag_bytes: usize, + tag_count: usize, + resource_bytes: usize, + resource_count: usize, +} + impl V3AllocationBudget { /// Accounts for every clone and vector expansion performed for one series. /// @@ -315,12 +326,12 @@ impl V3AllocationBudget { name: &str, unit: &str, source_type: &str, - tag_bytes: usize, - resource_bytes: usize, + tag_resources: V3TagResourceSize, ) -> crate::Result<()> { - let tag_resource_bytes = tag_bytes + let tag_resource_bytes = tag_resources + .tag_bytes .checked_add(source_type.len()) - .and_then(|total| total.checked_add(resource_bytes)) + .and_then(|total| total.checked_add(tag_resources.resource_bytes)) .ok_or("Datadog v3 expanded series size overflow")?; let tag_resource_copies = point_count .checked_add(1) @@ -349,12 +360,21 @@ impl V3AllocationBudget { return Err("Datadog v3 expanded series exceed size limit".into()); } + let event_tag_count = tag_resources + .tag_count + .checked_add(tag_resources.resource_count) + .and_then(|total| total.checked_add(usize::from(!source_type.is_empty()))) + .ok_or("Datadog v3 expanded event size overflow")?; + let event_tag_entry_bytes = event_tag_count + .checked_mul(V3_METRIC_TAG_ENTRY_BYTES) + .ok_or("Datadog v3 expanded event size overflow")?; let event_bytes = std::mem::size_of::() .checked_add(name.len()) - .and_then(|total| total.checked_add(tag_bytes)) - .and_then(|total| total.checked_add(resource_bytes)) + .and_then(|total| total.checked_add(tag_resources.tag_bytes)) + .and_then(|total| total.checked_add(tag_resources.resource_bytes)) .and_then(|total| total.checked_add(source_type.len())) .and_then(|total| total.checked_add(unit.len())) + .and_then(|total| total.checked_add(event_tag_entry_bytes)) .ok_or("Datadog v3 expanded event size overflow")?; self.event_bytes = self .event_bytes @@ -395,6 +415,7 @@ pub(crate) fn decode_v3_metric_data( .iter() .map(|tags| v3_string_list_bytes(tags)) .collect::>>()?; + let tagset_counts = tagsets.iter().map(Vec::len).collect::>(); let metadata_resource_bytes = metadata .map(|metadata| v3_string_list_bytes(&metadata.resources)) .transpose()? @@ -407,6 +428,11 @@ pub(crate) fn decode_v3_metric_data( .ok_or_else(|| "Datadog v3 expanded resource size overflow".into()) }) .collect::>>()?; + let metadata_resource_count = metadata.map_or(0, |metadata| metadata.resources.len() / 2); + let resource_counts = resources + .iter() + .map(|entry| entry.len() + metadata_resource_count) + .collect::>(); let dictionary_lengths = [ names.len(), @@ -438,8 +464,12 @@ pub(crate) fn decode_v3_metric_data( name, &unit, source_type, - tagset_bytes[refs.tagset], - resource_bytes[refs.resources], + V3TagResourceSize { + tag_bytes: tagset_bytes[refs.tagset], + tag_count: tagset_counts[refs.tagset], + resource_bytes: resource_bytes[refs.resources], + resource_count: resource_counts[refs.resources], + }, )?; let points = point_decoder.decode_points(value_type, point_count)?; diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index ff248f873b5b5..d8b5e3e47780a 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2773,6 +2773,17 @@ fn decode_v3_rejects_excessive_point_tag_cloning() { assert!(super::metrics::decode_v3_metric_data(&data, None).is_err()); } +#[test] +fn decode_v3_accounts_for_metric_tag_entry_storage() { + let data = minimal_v3_metric_data(); + let metadata = ddmetric_v3_proto::Metadata { + tags: (0..200_000).map(|index| format!("t{index}")).collect(), + resources: Vec::new(), + }; + + assert!(super::metrics::decode_v3_metric_data(&data, Some(&metadata)).is_err()); +} + #[test] fn decode_v3_rejects_excessive_event_expansion() { let mut data = minimal_v3_metric_data(); From 45ccfea8561ef278cab88075c1812555551a7b01 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Sat, 17 Oct 2026 10:18:49 +0000 Subject: [PATCH 22/23] fix(datadog_agent): address remaining review feedback --- lib/vector-core/src/event/metadata.rs | 6 ----- src/sources/datadog_agent/metrics.rs | 7 ++++++ src/sources/datadog_agent/tests.rs | 21 +++++++++++++++++ src/transforms/aggregate/tests.rs | 34 ++++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index 1ddf451f0c8d8..6cd86c95e1380 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -481,12 +481,6 @@ impl EventMetadata { let other = other.into_owned(); inner.finalizers.merge(other.finalizers); inner.secrets.merge(other.secrets); - if inner.datadog_origin_metadata.is_none() { - inner.datadog_origin_metadata = other.datadog_origin_metadata; - } - if inner.datadog_metric_unit.is_none() { - inner.datadog_metric_unit = other.datadog_metric_unit; - } // Update `source_event_id` if necessary. if inner.source_event_id.is_none() { diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index d005d7d707a69..123255be436c4 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -347,6 +347,13 @@ fn decode_ddseries( .into_iter() .enumerate() .flat_map(|(index, serie)| { + // A series without points cannot emit an event. Skip it before expanding its tags + // into a MetricTags map, since a compressed v3 payload can otherwise make this an + // arbitrarily large allocation that is immediately discarded. + if serie.points.is_empty() { + return Vec::new(); + } + let (namespace, name) = if split_metric_namespace { namespace_name_from_dd_metric(&serie.metric) } else { diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index d8b5e3e47780a..032bcc986f8dd 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2889,6 +2889,27 @@ fn decode_v3_accepts_many_distinct_metadata_tags() { assert_eq!(series[0].tags.len(), 100_000); } +#[test] +fn decode_v3_skips_tag_expansion_for_zero_point_series() { + let mut data = minimal_v3_metric_data(); + data.num_points[0] = 0; + data.timestamps.clear(); + data.vals_float64.clear(); + let payload = ddmetric_v3_proto::Payload { + metadata: Some(ddmetric_v3_proto::Metadata { + tags: (0..100_000).map(|index| format!("tag:{index}")).collect(), + resources: Vec::new(), + }), + metric_data: Some(data), + }; + let mut body = Vec::new(); + payload.encode(&mut body).unwrap(); + + let events = super::metrics::decode_ddseries_v3(Bytes::from(body), &None, false).unwrap(); + + assert!(events.is_empty()); +} + #[test] fn decode_v3_rejects_excessive_metadata_string_entries_before_decode() { let metadata_entry_count = 16 * 1024 * 1024 / std::mem::size_of::() + 1; diff --git a/src/transforms/aggregate/tests.rs b/src/transforms/aggregate/tests.rs index d5f934320aa32..35d336c6b61cc 100644 --- a/src/transforms/aggregate/tests.rs +++ b/src/transforms/aggregate/tests.rs @@ -15,7 +15,8 @@ use super::{ use crate::{ config::{OutputId, TransformConfig, TransformContext}, event::{ - BatchNotifier, BatchStatus, BatchStatusReceiver, Event, EventStatus, Metric, + BatchNotifier, BatchStatus, BatchStatusReceiver, DatadogMetricOriginMetadata, Event, + EventMetadata, EventStatus, Metric, metric::{MetricKind, MetricValue}, }, schema::Definition, @@ -1528,6 +1529,37 @@ fn event_time_metadata_is_merged_and_diff_finalizers_are_released() { assert_gauge(&out[0], 2.0); } + // The selected sample's absent Datadog attribution must not be filled from a discarded one. + { + let mut agg = + Aggregate::new(&event_time_config(interval_ms, AggregationMode::Auto)).unwrap(); + let attributed_metadata = EventMetadata::default() + .with_origin_metadata(DatadogMetricOriginMetadata::new(Some(1), Some(2), Some(3))) + .with_datadog_metric_unit("byte"); + let attributed = Event::Metric( + Metric::new_with_metadata( + "latest_attribution", + MetricKind::Absolute, + MetricValue::Gauge { value: 1.0 }, + attributed_metadata, + ) + .with_timestamp(Some(later)), + ); + let selected = make_metric_with_timestamp( + "latest_attribution", + MetricKind::Absolute, + MetricValue::Gauge { value: 2.0 }, + latest_ts, + ); + + agg.record(attributed); + agg.record(selected); + let out = flush_final(&mut agg); + assert!(out[0].metadata().datadog_origin_metadata().is_none()); + assert_eq!(out[0].metadata().datadog_metric_unit(), None); + assert_gauge(&out[0], 2.0); + } + // Auto kind switch: retained sample's data and source_type must stay paired. // Inc→Abs: Absolute wins even if older. Abs→Inc: newer Incremental replaces. for (first_kind, first_value, first_src, second_kind, second_value, second_src, expect_kind) in [ From 7b555a323324a01ab2739d7629cb1c4a9a867ef9 Mon Sep 17 00:00:00 2001 From: arfa79 <80360132+arfa79@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:15:01 +0000 Subject: [PATCH 23/23] fix(datadog_agent): skip empty v1 series expansion --- src/sources/datadog_agent/metrics.rs | 5 +++++ src/sources/datadog_agent/tests.rs | 31 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/sources/datadog_agent/metrics.rs b/src/sources/datadog_agent/metrics.rs index 123255be436c4..b2e0f556d6179 100644 --- a/src/sources/datadog_agent/metrics.rs +++ b/src/sources/datadog_agent/metrics.rs @@ -587,6 +587,11 @@ fn into_vector_metric( schema_definition: &Arc, split_metric_namespace: bool, ) -> Vec { + // Avoid copying large units or expanding tags for a series that cannot emit an event. + if dd_metric.points.is_empty() { + return Vec::new(); + } + let mut tags = into_metric_tags(dd_metric.tags.unwrap_or_default()); let mut event_metadata = dd_metric .metadata diff --git a/src/sources/datadog_agent/tests.rs b/src/sources/datadog_agent/tests.rs index 032bcc986f8dd..8d6616f2b86e0 100644 --- a/src/sources/datadog_agent/tests.rs +++ b/src/sources/datadog_agent/tests.rs @@ -2633,6 +2633,37 @@ async fn decode_series_v1_rejects_excessive_unit_event_expansion() { assert!(crate::test_util::collect_ready(rx).is_empty()); } +#[tokio::test] +async fn decode_series_v1_skips_unit_materialization_for_zero_point_series() { + let (rx, _, _, addr, _guard) = source(EventStatus::Delivered, true, true, false, true).await; + let request = DatadogSeriesRequest { + series: vec![DatadogSeriesMetric { + metric: "empty".to_string(), + r#type: DatadogMetricType::Gauge, + interval: None, + points: Vec::new(), + tags: None, + host: None, + source_type_name: None, + device: None, + unit: Some("u".repeat(1024 * 1024)), + metadata: None, + }], + }; + + assert_eq!( + 200, + send_with_path( + addr, + serde_json::to_vec(&request).unwrap(), + dd_api_key_headers(), + DD_API_SERIES_V1_PATH, + ) + .await + ); + assert!(crate::test_util::collect_ready(rx).is_empty()); +} + fn v3_string_dictionary(values: &[&str]) -> Vec { values .iter()