Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions bin/agent-data-plane/src/cli/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,8 @@ fn add_mrf_metrics_pipeline_to_blueprint(

let mrf_gateway_config = MrfMetricsGatewayConfiguration::new(mrf_config.clone(), config.clone());
let mrf_metrics_config = DatadogMetricsConfiguration::from_configuration(config)
.error_context("Failed to configure Multi-Region Failover Datadog Metrics encoder.")?;
.error_context("Failed to configure Multi-Region Failover Datadog Metrics encoder.")?
.with_metrics_endpoint_override(mrf_dd_url.clone());

let mrf_forwarder_config = DatadogForwarderConfiguration::from_configuration(config)
.map(|config| {
Expand Down Expand Up @@ -571,7 +572,8 @@ fn add_autoscaling_failover_metrics_pipeline_to_blueprint(

let af_gateway_config = AutoscalingFailoverGatewayConfiguration::new(af_config);
let af_metrics_config = DatadogMetricsConfiguration::from_configuration(config)
.error_context("Failed to configure autoscaling failover metrics encoder.")?;
.error_context("Failed to configure autoscaling failover metrics encoder.")?
.with_v2_series_only();
let cluster_agent_forwarder_config =
ClusterAgentForwarderConfiguration::from_configuration(config, ca_url, ca_token)
.error_context("Failed to configure Cluster Agent forwarder.")?;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ crate::declare_annotations! {
support_level: SupportLevel::Full,
additional_yaml_paths: &[],
env_var_override: None,
used_by: &[structs::FORWARDER_CONFIGURATION],
used_by: &[structs::DATADOG_METRICS_CONFIGURATION, structs::FORWARDER_CONFIGURATION],
value_type_override: None,
test_json: None,
pipeline_affinity: PipelineAffinity::CrossCutting,
Expand All @@ -33,7 +33,7 @@ crate::declare_annotations! {
support_level: SupportLevel::Full,
additional_yaml_paths: &[],
env_var_override: None,
used_by: &[structs::FORWARDER_CONFIGURATION],
used_by: &[structs::DATADOG_METRICS_CONFIGURATION, structs::FORWARDER_CONFIGURATION],
value_type_override: None,
test_json: None,
pipeline_affinity: PipelineAffinity::CrossCutting,
Expand Down
70 changes: 51 additions & 19 deletions lib/datadog-agent/config/build/datadog_config_gen.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@
//! cannot render it faithfully. Parsing the duration once, at the deserialization boundary, keeps a
//! bare number from reaching the translator as an ambiguous unit.
//!
//! Every `Vec<String>` leaf also gets a shape-tolerant deserializer: a config file or the remote
//! Agent stream supplies a real sequence, but an environment variable supplies a single
//! space-separated string (`DD_DOGSTATSD_TAGS="env:prod team:core"`), and the field must accept
//! both. Handling that in the deserializer (rather than pre-splitting env values elsewhere) keeps
//! the concern in one place; see `stringlistize` and `crate::list_de`.
//! String-list fields also get shape-tolerant decoders. A standalone `Vec<String>` may arrive
//! as a real sequence or a space-separated environment string, while a `HashMap<String,
//! Vec<String>>` may carry each map value as either one scalar string or a sequence. Handling those
//! shapes at the deserialization boundary keeps downstream types consistent; see `stringlistize`
//! and `crate::list_de`.
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::Path;

Expand Down Expand Up @@ -400,32 +400,36 @@ fn duration_defaults_module(durations: &BTreeMap<String, u64>) -> String {
module
}

/// Give every `Vec<String>` leaf a shape-tolerant deserializer.
/// Give string-list leaves shape-tolerant decoders.
///
/// String lists arrive as a real sequence from a file or the remote Agent stream, but as a single
/// space-separated string from an environment variable (`DD_DOGSTATSD_TAGS="a b"`), so each field
/// must accept both. We push an extra `#[serde(deserialize_with = ...)]` attribute rather than
/// replacing the field's existing serde attributes: serde merges multiple `#[serde(...)]`, so the
/// field keeps its `default`/`skip_serializing_if`/`alias` and only gains the tolerant reader (the
/// same additive technique as `inject_serde_aliases`).
/// must accept both. Map values containing string lists may likewise arrive as either one scalar or
/// a sequence and are normalized into vectors.
///
/// Fields are matched by their Rust type, not by name: `Vec<String>` is unambiguous, needs no
/// schema lookup, and correctly skips `HashMap<String, Vec<String>>` (a map, not a list) and the
/// duration leaves (already retyped to `Duration`). Runs after `PathShortener`, so the type reads
/// as `Vec<String>` rather than `::std::vec::Vec<::std::string::String>`.
/// We push an extra `#[serde(deserialize_with = ...)]` attribute rather than replacing the field's
/// existing serde attributes: serde merges multiple `#[serde(...)]`, so the field keeps its
/// `default`/`skip_serializing_if`/`alias` and only gains the tolerant reader (the same additive
/// technique as `inject_serde_aliases`).
///
/// Fields are matched by their Rust type, not by name. Runs after `PathShortener`, so the types read
/// as `Vec<String>` and `HashMap<String, Vec<String>>` rather than fully qualified prelude paths.
fn stringlistize(file: &mut syn::File) {
for item in &mut file.items {
let Item::Struct(s) = item else { continue };
let syn::Fields::Named(fields) = &mut s.fields else {
continue;
};
for field in &mut fields.named {
if !is_vec_string(&field.ty) {
continue;
if is_vec_string(&field.ty) {
field.attrs.push(parse_quote!(
#[serde(deserialize_with = "crate::list_de::deserialize_space_separated_or_seq")]
));
} else if is_string_map_vec_string(&field.ty) {
field.attrs.push(parse_quote!(
#[serde(deserialize_with = "crate::list_de::deserialize_string_map_scalar_or_seq")]
));
}
field.attrs.push(parse_quote!(
#[serde(deserialize_with = "crate::list_de::deserialize_space_separated_or_seq")]
));
}
}
}
Expand All @@ -449,6 +453,34 @@ fn is_vec_string(ty: &syn::Type) -> bool {
inner.path.segments.last().is_some_and(|seg| seg.ident == "String")
}

/// Returns whether `ty` is exactly `HashMap<String, Vec<String>>`.
fn is_string_map_vec_string(ty: &syn::Type) -> bool {
let syn::Type::Path(tp) = ty else { return false };
let Some(last) = tp.path.segments.last() else {
return false;
};
if last.ident != "HashMap" {
return false;
}
let syn::PathArguments::AngleBracketed(args) = &last.arguments else {
return false;
};
let mut args = args.args.iter();
let Some(syn::GenericArgument::Type(key)) = args.next() else {
return false;
};
let Some(syn::GenericArgument::Type(value)) = args.next() else {
return false;
};
is_string(key) && is_vec_string(value)
}

/// Returns whether `ty` is exactly `String`.
fn is_string(ty: &syn::Type) -> bool {
let syn::Type::Path(tp) = ty else { return false };
tp.path.segments.last().is_some_and(|seg| seg.ident == "String")
}

/// Make nested-section fields non-optional with `#[serde(default)]`.
///
/// typify renders each non-required object property as `Option<SectionStruct>`. Every section
Expand Down
4 changes: 2 additions & 2 deletions lib/datadog-agent/config/schema/schema_overlay.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -713,7 +713,7 @@ inventory:
pipelines: [cross_cutting]
description: "Override intake endpoint URL"
test_support:
used_by: [ForwarderConfiguration]
used_by: [DatadogMetricsConfiguration, ForwarderConfiguration]
additional_attributes:
config_registry_filename: forwarder.rs

Expand Down Expand Up @@ -2388,7 +2388,7 @@ inventory:
pipelines: [cross_cutting]
description: "Datadog site domain"
test_support:
used_by: [ForwarderConfiguration]
used_by: [DatadogMetricsConfiguration, ForwarderConfiguration]
additional_attributes:
config_registry_filename: forwarder.rs

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ pub mod error {
#[derive(::serde::Deserialize, ::serde::Serialize, Clone, Debug)]
pub struct DatadogConfiguration {
#[serde(default, skip_serializing_if = ":: std :: collections :: HashMap::is_empty")]
#[serde(deserialize_with = "crate::list_de::deserialize_string_map_scalar_or_seq")]
pub additional_endpoints: HashMap<String, Vec<String>>,

#[serde(default)]
Expand Down
35 changes: 35 additions & 0 deletions lib/datadog-agent/config/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,41 @@ mod string_list_shape_tests {
}
}

#[cfg(test)]
mod string_map_list_shape_tests {
use super::DatadogConfiguration;

#[test]
fn additional_endpoints_accept_scalar_values() {
let config: DatadogConfiguration = serde_json::from_value(serde_json::json!({
"additional_endpoints": {
"https://agent.datadoghq.com.": "ENC[vault://api-key]"
}
}))
.expect("scalar additional endpoint API key deserializes");

assert_eq!(
config.additional_endpoints["https://agent.datadoghq.com."],
["ENC[vault://api-key]"]
);
}

#[test]
fn additional_endpoints_accept_sequence_values() {
let config: DatadogConfiguration = serde_json::from_value(serde_json::json!({
"additional_endpoints": {
"https://agent.datadoghq.com.": ["first", "second"]
}
}))
.expect("additional endpoint API key sequence deserializes");

assert_eq!(
config.additional_endpoints["https://agent.datadoghq.com."],
["first", "second"]
);
}
}

#[cfg(test)]
mod byte_size_shape_tests {
use super::DatadogConfiguration;
Expand Down
60 changes: 58 additions & 2 deletions lib/datadog-agent/config/src/list_de.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@
//! space-separated string (for example, `DD_DOGSTATSD_TAGS="env:prod team:core"`). A single field
//! must accept both forms.
//!
//! Deserializing here keeps that difference at the boundary: every string-list field accepts either
//! shape, while downstream consumers always receive a `Vec<String>`.
//! Map values containing string lists have a similar compatibility shape: a single value can arrive
//! as a scalar string, while multiple values arrive as a sequence. Deserializing here keeps those
//! differences at the boundary, while downstream consumers always receive a `Vec<String>`.
Comment on lines +8 to +10

use std::collections::HashMap;
use std::fmt;

use serde::de::{self, Deserializer, SeqAccess, Visitor};
use serde::Deserialize;

/// Deserialize a `Vec<String>` from either a sequence or a space-separated string.
///
Expand Down Expand Up @@ -45,6 +48,36 @@ where
deserializer.deserialize_any(SpaceSeparatedOrSeq)
}

/// Deserialize string-list map values from either scalar strings or sequences.
///
/// Scalar values are normalized into one-element vectors. Unlike standalone string-list fields,
/// scalar map values are not split on whitespace because each scalar represents one complete value.
pub(crate) fn deserialize_string_map_scalar_or_seq<'de, D>(
deserializer: D,
) -> Result<HashMap<String, Vec<String>>, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum ScalarOrSeq {
Scalar(String),
Seq(Vec<String>),
}

let values = HashMap::<String, ScalarOrSeq>::deserialize(deserializer)?;
Ok(values
.into_iter()
.map(|(key, value)| {
let value = match value {
ScalarOrSeq::Scalar(value) => vec![value],
ScalarOrSeq::Seq(values) => values,
};
(key, value)
})
.collect())
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down Expand Up @@ -84,4 +117,27 @@ mod tests {
fn wrong_shape_is_rejected() {
assert!(serde_json::from_str::<Holder>(r#"{"list": 5}"#).is_err());
}

#[derive(serde::Deserialize)]
struct MapHolder {
#[serde(deserialize_with = "deserialize_string_map_scalar_or_seq")]
map: HashMap<String, Vec<String>>,
}

fn parse_map(json: &str) -> HashMap<String, Vec<String>> {
serde_json::from_str::<MapHolder>(json).unwrap().map
}

#[test]
fn string_map_accepts_scalar_and_sequence_values() {
let parsed = parse_map(r#"{"map":{"one":"api-key","many":["first","second"],"none":[]}}"#);
assert_eq!(parsed["one"], ["api-key"]);
assert_eq!(parsed["many"], ["first", "second"]);
assert!(parsed["none"].is_empty());
}

#[test]
fn string_map_rejects_non_string_values() {
assert!(serde_json::from_str::<MapHolder>(r#"{"map":{"endpoint":5}}"#).is_err());
}
}
6 changes: 6 additions & 0 deletions lib/saluki-components/src/common/datadog/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -395,6 +395,12 @@ impl ForwarderConfiguration {
self.opw_metrics = OpwMetricsConfiguration::default();
}

/// Forces series metrics routing to accept only V2 payloads.
pub(crate) fn force_v2_series(&mut self) {
self.data_plane_metrics_v3_series_enabled = false;
self.v3_api.series.shadow_sites.clear();
}

/// Builds resolved endpoints with routing metadata.
///
/// The normal primary and OPW metrics primary endpoints share the same dynamic API key source.
Expand Down
6 changes: 3 additions & 3 deletions lib/saluki-components/src/common/datadog/endpoints.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ pub const DEFAULT_SITE: &str = "datadoghq.com";
/// A `dd_url` equal to this constant carries no override intent and must not shadow `site`.
const DEFAULT_PRIMARY_ENDPOINT: &str = "https://app.datadoghq.com";

fn default_site() -> String {
pub(crate) fn default_site() -> String {
DEFAULT_SITE.to_owned()
}

Expand All @@ -43,7 +43,7 @@ fn default_site() -> String {
/// value equal to the default is treated as `None`, allowing `site` to determine the endpoint. This
/// only affects the serde path; programmatic callers such as `set_dd_url` bypass serde and are
/// unaffected.
fn deserialize_dd_url<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
pub(crate) fn deserialize_dd_url<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: serde::Deserializer<'de>,
{
Expand Down Expand Up @@ -895,7 +895,7 @@ fn add_data_plane_version_prefix(mut endpoint: Url) -> Result<Url, EndpointError
///
/// If an override URL is provided and can't be parsed, or if a valid endpoint can't be constructed from the given
/// site, an error will be returned.
fn calculate_resolved_endpoint(
pub(crate) fn calculate_resolved_endpoint(
override_url: Option<&str>, site: &str, api_key: &str,
) -> Result<ResolvedEndpoint, EndpointError> {
let raw_endpoint = match override_url {
Expand Down
Loading