From 3fd70d5dd5a2934f9a6d6abc148afdcc13c383db Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 12:57:08 -0400 Subject: [PATCH 01/44] feat(components): expose aggregate context snapshots Coordinate retained-context snapshots through the aggregate owner task. Clone only shared context handles and small metric metadata so serialization can happen without holding up ingestion. --- lib/saluki-components/Cargo.toml | 1 + .../src/transforms/aggregate/mod.rs | 475 +++++++++++++++++- lib/saluki-components/src/transforms/mod.rs | 9 +- 3 files changed, 483 insertions(+), 2 deletions(-) diff --git a/lib/saluki-components/Cargo.toml b/lib/saluki-components/Cargo.toml index 435166fb74e..353e888a89e 100644 --- a/lib/saluki-components/Cargo.toml +++ b/lib/saluki-components/Cargo.toml @@ -11,6 +11,7 @@ workspace = true [features] default = [] fips = ["saluki-io/fips"] +test-util = [] [dependencies] agent-data-plane-config = { workspace = true } diff --git a/lib/saluki-components/src/transforms/aggregate/mod.rs b/lib/saluki-components/src/transforms/aggregate/mod.rs index 0c56d500ec9..866f2797539 100644 --- a/lib/saluki-components/src/transforms/aggregate/mod.rs +++ b/lib/saluki-components/src/transforms/aggregate/mod.rs @@ -1,5 +1,7 @@ use std::{ + future::pending, num::NonZeroU64, + sync::Mutex, time::{Duration, Instant}, }; @@ -17,13 +19,14 @@ use saluki_core::{ topology::{interconnect::BufferedDispatcher, OutputDefinition}, topology::{EventsBuffer, EventsDispatcher}, }; -use saluki_error::GenericError; +use saluki_error::{generic_error, GenericError}; use saluki_metrics::MetricsBuilder; use serde::Deserialize; use smallvec::SmallVec; use stringtheory::MetaString; use tokio::{ pin, select, + sync::{mpsc, oneshot}, time::{interval, interval_at}, }; use tracing::{debug, error, info, trace, warn}; @@ -35,6 +38,194 @@ mod config; use self::config::{HistogramConfiguration, HistogramStatistic}; const PASSTHROUGH_IDLE_FLUSH_CHECK_INTERVAL: Duration = Duration::from_secs(2); +const CONTEXT_SNAPSHOT_REQUEST_CHANNEL_CAPACITY: usize = 1; + +/// The shape of metric values retained by the aggregate transform. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AggregateMetricType { + /// Counter values. + Counter, + + /// Rate values. + Rate, + + /// Gauge values. + Gauge, + + /// Set values. + Set, + + /// Histogram values. + Histogram, + + /// Distribution values. + Distribution, +} + +impl From<&MetricValues> for AggregateMetricType { + fn from(values: &MetricValues) -> Self { + match values { + MetricValues::Counter(_) => Self::Counter, + MetricValues::Rate(_, _) => Self::Rate, + MetricValues::Gauge(_) => Self::Gauge, + MetricValues::Set(_) => Self::Set, + MetricValues::Histogram(_) => Self::Histogram, + MetricValues::Distribution(_) => Self::Distribution, + } + } +} + +/// A retained metric context and its small aggregation metadata. +/// +/// Cloning an entry shares the underlying context name and tags rather than copying their contents. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct AggregateContextSnapshotEntry { + context: Context, + metric_type: AggregateMetricType, + unit: MetaString, +} + +impl AggregateContextSnapshotEntry { + /// Returns the retained metric context. + pub fn context(&self) -> &Context { + &self.context + } + + /// Returns the shape of the retained metric values. + pub fn metric_type(&self) -> AggregateMetricType { + self.metric_type + } + + /// Returns the unit attached to the retained metric values. + /// + /// An empty value indicates that no unit is attached. + pub fn unit(&self) -> &MetaString { + &self.unit + } + + /// Creates a snapshot entry for downstream test and benchmark fixtures. + #[cfg(any(test, feature = "test-util"))] + pub fn for_test(context: Context, metric_type: AggregateMetricType, unit: MetaString) -> Self { + Self { + context, + metric_type, + unit, + } + } +} + +type AggregateContextSnapshot = Vec; +type AggregateContextSnapshotRequest = oneshot::Sender; +type AggregateContextSnapshotRequestReceiver = mpsc::Receiver; + +/// A handle for requesting retained-context snapshots from an aggregate transform. +/// +/// Snapshot construction runs on the aggregate owner task. The returned entries contain shared context handles and +/// small metric metadata, allowing callers to perform heavier processing after the owner resumes ingestion. +#[derive(Clone, Debug)] +pub struct AggregateContextSnapshotHandle { + requests: mpsc::Sender, +} + +impl AggregateContextSnapshotHandle { + /// Requests the aggregate transform's current retained contexts. + /// + /// # Errors + /// + /// Returns an error if the aggregate owner is unavailable or stops before responding. + pub async fn snapshot(&self) -> Result, GenericError> { + let (response_tx, response_rx) = oneshot::channel(); + self.requests + .send(response_tx) + .await + .map_err(|_| generic_error!("aggregate context snapshot owner is unavailable"))?; + + response_rx + .await + .map_err(|_| generic_error!("aggregate context snapshot owner stopped before responding")) + } +} + +fn aggregate_context_snapshot_channel() -> (AggregateContextSnapshotHandle, AggregateContextSnapshotRequestReceiver) { + let (requests, receiver) = mpsc::channel(CONTEXT_SNAPSHOT_REQUEST_CHANNEL_CAPACITY); + (AggregateContextSnapshotHandle { requests }, receiver) +} + +/// A responder for retained-context snapshot test fixtures. +#[cfg(any(test, feature = "test-util"))] +pub struct AggregateContextSnapshotResponder { + receiver: AggregateContextSnapshotRequestReceiver, +} + +#[cfg(any(test, feature = "test-util"))] +impl AggregateContextSnapshotResponder { + /// Waits for one snapshot request and responds with the supplied entries. + /// + /// A canceled requester is treated as a successful no-op delivery. + /// + /// # Errors + /// + /// Returns an error if the request channel closes before a request arrives. + pub async fn respond(&mut self, snapshot: Vec) -> Result<(), GenericError> { + let response = self + .receiver + .recv() + .await + .ok_or_else(|| generic_error!("aggregate context snapshot request channel is closed"))?; + let _ = response.send(snapshot); + Ok(()) + } +} + +/// Creates a retained-context snapshot handle and owner-side responder for tests. +#[cfg(any(test, feature = "test-util"))] +pub fn aggregate_context_snapshot_channel_for_test( +) -> (AggregateContextSnapshotHandle, AggregateContextSnapshotResponder) { + let (handle, receiver) = aggregate_context_snapshot_channel(); + (handle, AggregateContextSnapshotResponder { receiver }) +} + +struct AggregateContextSnapshotChannel { + handle: AggregateContextSnapshotHandle, + receiver: Mutex>, +} + +impl AggregateContextSnapshotChannel { + fn take_receiver(&self) -> Result { + let mut receiver = self + .receiver + .lock() + .map_err(|_| generic_error!("aggregate context snapshot receiver lock is poisoned"))?; + receiver + .take() + .ok_or_else(|| generic_error!("aggregate context snapshot receiver has already been taken")) + } +} + +impl Default for AggregateContextSnapshotChannel { + fn default() -> Self { + let (handle, receiver) = aggregate_context_snapshot_channel(); + Self { + handle, + receiver: Mutex::new(Some(receiver)), + } + } +} + +#[cfg(test)] +impl std::fmt::Debug for AggregateContextSnapshotChannel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AggregateContextSnapshotChannel") + .finish_non_exhaustive() + } +} + +#[cfg(test)] +impl PartialEq for AggregateContextSnapshotChannel { + fn eq(&self, _other: &Self) -> bool { + true + } +} const fn default_window_duration_seconds() -> NonZeroU64 { NonZeroU64::new(10).expect("not zero") @@ -185,6 +376,12 @@ pub struct AggregateConfiguration { /// distribution aggregation). #[serde(flatten)] hist_config: HistogramConfiguration, + + /// Owner-side channel used to coordinate retained-context snapshots. + /// + /// This runtime-only channel is never read from or written to configuration data. + #[serde(skip, default)] + context_snapshot_channel: AggregateContextSnapshotChannel, } impl AggregateConfiguration { @@ -204,13 +401,20 @@ impl AggregateConfiguration { passthrough_timestamped_metrics: default_passthrough_timestamped_metrics(), passthrough_idle_flush_timeout: default_passthrough_idle_flush_timeout(), hist_config: HistogramConfiguration::default(), + context_snapshot_channel: AggregateContextSnapshotChannel::default(), } } + + /// Returns a handle for requesting retained-context snapshots from the built aggregate transform. + pub fn context_snapshot_handle(&self) -> AggregateContextSnapshotHandle { + self.context_snapshot_channel.handle.clone() + } } #[async_trait] impl TransformBuilder for AggregateConfiguration { async fn build(&self, context: ComponentContext) -> Result, GenericError> { + let context_snapshot_requests = self.context_snapshot_channel.take_receiver()?; let metrics_builder = MetricsBuilder::from_component_context(&context); let telemetry = Telemetry::new(&metrics_builder); @@ -236,6 +440,7 @@ impl TransformBuilder for AggregateConfiguration { flush_open_windows: self.flush_open_windows, passthrough_batcher, passthrough_timestamped_metrics: self.passthrough_timestamped_metrics, + context_snapshot_requests: Some(context_snapshot_requests), })) } @@ -285,6 +490,7 @@ pub struct Aggregate { flush_open_windows: bool, passthrough_batcher: PassthroughBatcher, passthrough_timestamped_metrics: bool, + context_snapshot_requests: Option, } #[async_trait] @@ -345,6 +551,15 @@ impl Transform for Aggregate { } }, _ = passthrough_flush.tick() => self.passthrough_batcher.try_flush(context.dispatcher()).await, + snapshot_request = receive_context_snapshot_request(&mut self.context_snapshot_requests) => { + match snapshot_request { + Some(response) => { + let snapshot = self.state.snapshot_contexts(); + let _ = response.send(snapshot); + } + None => self.context_snapshot_requests = None, + } + }, maybe_events = context.events().next(), if !final_primary_flush => match maybe_events { Some(events) => { trace!(events_len = events.len(), "Received events."); @@ -415,6 +630,15 @@ impl Transform for Aggregate { } } +async fn receive_context_snapshot_request( + receiver: &mut Option, +) -> Option { + match receiver { + Some(receiver) => receiver.recv().await, + None => pending().await, + } +} + fn try_split_timestamped_values(mut metric: Metric) -> (Option, Option) { if metric.values().all_timestamped() { (Some(metric), None) @@ -568,6 +792,18 @@ impl AggregationState { self.contexts.is_empty() } + fn snapshot_contexts(&self) -> Vec { + let mut snapshot = Vec::with_capacity(self.contexts.len()); + for (context, aggregated) in &self.contexts { + snapshot.push(AggregateContextSnapshotEntry { + context: context.clone(), + metric_type: AggregateMetricType::from(&aggregated.values), + unit: aggregated.metadata.unit.clone(), + }); + } + snapshot + } + fn insert(&mut self, timestamp: u64, metric: Metric) -> bool { // If we haven't seen this context yet, and it would put us over the limit to insert it, then return early. if !self.contexts.contains_key(metric.context()) && self.contexts.len() >= self.context_limit { @@ -773,6 +1009,49 @@ impl AggregationState { } } +/// A benchmark fixture backed by a real aggregate state populated before snapshot timing begins. +#[cfg(any(test, feature = "test-util"))] +pub struct AggregateContextSnapshotBenchmarkHarness { + state: AggregationState, +} + +#[cfg(any(test, feature = "test-util"))] +impl AggregateContextSnapshotBenchmarkHarness { + /// Creates a harness retaining exactly `context_count` distinct gauge contexts. + pub fn with_contexts(context_count: usize) -> Self { + let mut state = AggregationState::new( + default_window_duration_seconds(), + context_count, + None, + HistogramConfiguration::default(), + Telemetry::new(&MetricsBuilder::default()), + ); + + for index in 0..context_count { + let context = Context::from_parts( + format!("aggregate.snapshot.benchmark.{index}"), + saluki_context::tags::TagSet::default(), + ); + assert!( + state.insert(0, Metric::gauge(context, 0.0)), + "benchmark fixture must retain every requested context" + ); + } + + Self { state } + } + + /// Takes a retained-context snapshot from the populated aggregate state. + pub fn snapshot(&self) -> Vec { + self.state.snapshot_contexts() + } + + /// Returns the exact number of contexts retained by the aggregate state. + pub fn retained_len(&self) -> usize { + self.state.contexts.len() + } +} + async fn transform_and_push_metric( context: Context, mut values: MetricValues, metadata: MetricMetadata, bucket_width_secs: NonZeroU64, hist_config: &HistogramConfiguration, dispatcher: &mut BufferedDispatcher<'_, EventsBuffer>, @@ -896,6 +1175,7 @@ const fn is_bucket_closed( #[cfg(test)] mod tests { use float_cmp::ApproxEqRatio as _; + use saluki_context::tags::{Tag, TagSet}; use saluki_core::{ components::ComponentContext, topology::{interconnect::Dispatcher, OutputName}, @@ -1063,6 +1343,199 @@ mod tests { }; } + #[test] + fn aggregate_metric_type_matches_every_metric_shape() { + let cases = [ + (MetricValues::counter(1.0), AggregateMetricType::Counter), + ( + MetricValues::rate(1.0, Duration::from_secs(10)), + AggregateMetricType::Rate, + ), + (MetricValues::gauge(1.0), AggregateMetricType::Gauge), + (MetricValues::set("value"), AggregateMetricType::Set), + (MetricValues::histogram([1.0]), AggregateMetricType::Histogram), + ( + MetricValues::distribution(&[1.0][..]), + AggregateMetricType::Distribution, + ), + ]; + + for (values, expected) in cases { + assert_eq!(AggregateMetricType::from(&values), expected); + } + } + + #[test] + fn snapshot_contexts_preserves_full_context_shape_and_unit() { + let mut state = AggregationState::new( + BUCKET_WIDTH_SECS, + 10, + COUNTER_EXPIRE, + HistogramConfiguration::default(), + Telemetry::noop(), + ); + + let histogram_context = Context::from_static_parts("request.duration", &["env:prod"]) + .with_host(Some(MetaString::from_static("host-a"))) + .with_origin_tags(TagSet::from(Tag::from_static("container:one"))); + let gauge_context = Context::from_static_parts("request.duration", &["env:prod"]) + .with_host(Some(MetaString::from_static("host-b"))) + .with_origin_tags(TagSet::from(Tag::from_static("container:two"))); + let histogram = Metric::from_parts( + histogram_context.clone(), + MetricValues::histogram([12.0]), + MetricMetadata::default().with_unit(MetaString::from_static("millisecond")), + ); + let gauge = Metric::gauge(gauge_context.clone(), 2.0); + + assert!(state.insert(insert_ts(1), histogram)); + assert!(state.insert(insert_ts(1), gauge)); + + let mut snapshot = state.snapshot_contexts(); + assert_eq!(snapshot.len(), 2); + assert_eq!(snapshot.capacity(), state.contexts.len()); + snapshot.sort_by(|a, b| a.context().host().cmp(&b.context().host())); + + assert_eq!(snapshot[0].context(), &histogram_context); + assert_eq!(snapshot[0].context().tags().len(), 1); + assert_eq!(snapshot[0].context().origin_tags().len(), 1); + assert_eq!(snapshot[0].metric_type(), AggregateMetricType::Histogram); + assert_eq!(snapshot[0].unit(), "millisecond"); + + assert_eq!(snapshot[1].context(), &gauge_context); + assert_eq!(snapshot[1].metric_type(), AggregateMetricType::Gauge); + assert!(snapshot[1].unit().is_empty()); + } + + #[tokio::test] + async fn snapshot_contexts_follows_ordinary_context_lifecycle() { + let mut state = AggregationState::new( + BUCKET_WIDTH_SECS, + 10, + COUNTER_EXPIRE, + HistogramConfiguration::default(), + Telemetry::noop(), + ); + let context = Context::from_static_name("active.gauge"); + + assert!(state.insert(insert_ts(1), Metric::gauge(context.clone(), 1.0))); + assert_eq!(state.snapshot_contexts().len(), 1); + + let _ = get_flushed_metrics(flush_ts(1), &mut state).await; + assert!(state.snapshot_contexts().is_empty()); + } + + #[tokio::test] + async fn snapshot_contexts_retains_idle_counter_until_expiry() { + let mut state = AggregationState::new( + BUCKET_WIDTH_SECS, + 10, + COUNTER_EXPIRE, + HistogramConfiguration::default(), + Telemetry::noop(), + ); + let context = Context::from_static_name("sparse.counter"); + + assert!(state.insert(insert_ts(1), Metric::counter(context.clone(), 1.0))); + let _ = get_flushed_metrics(flush_ts(1), &mut state).await; + let first_snapshot = state.snapshot_contexts(); + assert_eq!(first_snapshot.len(), 1); + assert_eq!(first_snapshot[0].context(), &context); + assert_eq!(first_snapshot[0].metric_type(), AggregateMetricType::Counter); + + let _ = get_flushed_metrics(flush_ts(2), &mut state).await; + assert_eq!(state.snapshot_contexts().len(), 1); + + let _ = get_flushed_metrics(flush_ts(3), &mut state).await; + assert!(state.snapshot_contexts().is_empty()); + } + + #[tokio::test] + async fn snapshot_handle_round_trips_through_responder() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let expected = vec![AggregateContextSnapshotEntry::for_test( + Context::from_static_name("round.trip"), + AggregateMetricType::Gauge, + MetaString::from_static("widget"), + )]; + let snapshot_task = tokio::spawn(async move { handle.snapshot().await }); + + responder + .respond(expected.clone()) + .await + .expect("responder should receive and fulfill a snapshot request"); + + let actual = snapshot_task + .await + .expect("snapshot task should complete") + .expect("snapshot request should succeed"); + assert_eq!(actual, expected); + } + + #[tokio::test] + async fn snapshot_handle_reports_dropped_receiver() { + let (handle, responder) = aggregate_context_snapshot_channel_for_test(); + drop(responder); + + let error = handle + .snapshot() + .await + .expect_err("snapshot should fail after its owner is dropped"); + assert!(error.to_string().contains("unavailable")); + } + + #[tokio::test] + async fn aggregate_configuration_receiver_can_only_be_taken_once() { + let config = AggregateConfiguration::with_defaults(); + let _handle = config.context_snapshot_handle(); + let first = config.build(ComponentContext::test_transform("aggregate_one")).await; + assert!(first.is_ok()); + + let second = config.build(ComponentContext::test_transform("aggregate_two")).await; + let error = match second { + Ok(_) => panic!("second build should not take the snapshot receiver again"), + Err(error) => error, + }; + assert!(error.to_string().contains("already been taken")); + } + + #[tokio::test] + async fn snapshot_responder_reports_closed_request_channel() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + drop(handle); + + let error = responder + .respond(Vec::new()) + .await + .expect_err("responder should fail when every request handle is dropped"); + assert!(error.to_string().contains("closed")); + } + + #[tokio::test] + async fn snapshot_responder_ignores_canceled_requester() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let snapshot_task = tokio::spawn(async move { handle.snapshot().await }); + tokio::task::yield_now().await; + snapshot_task.abort(); + let _ = snapshot_task.await; + + responder + .respond(Vec::new()) + .await + .expect("a canceled requester should be a successful no-op delivery"); + } + + #[test] + fn snapshot_benchmark_harness_populates_exact_context_count() { + let empty_harness = AggregateContextSnapshotBenchmarkHarness::with_contexts(0); + assert_eq!(empty_harness.retained_len(), 0); + assert!(empty_harness.snapshot().is_empty()); + + let harness = AggregateContextSnapshotBenchmarkHarness::with_contexts(32); + assert_eq!(harness.retained_len(), 32); + assert_eq!(harness.snapshot().len(), 32); + } + #[test] fn bucket_is_closed() { // Cases are defined as: diff --git a/lib/saluki-components/src/transforms/mod.rs b/lib/saluki-components/src/transforms/mod.rs index 10c2c615bf1..c467be42a0c 100644 --- a/lib/saluki-components/src/transforms/mod.rs +++ b/lib/saluki-components/src/transforms/mod.rs @@ -4,7 +4,14 @@ mod autoscaling_failover_gateway; pub use self::autoscaling_failover_gateway::AutoscalingFailoverGatewayConfiguration; mod aggregate; -pub use self::aggregate::AggregateConfiguration; +#[cfg(feature = "test-util")] +pub use self::aggregate::{ + aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotBenchmarkHarness, + AggregateContextSnapshotResponder, +}; +pub use self::aggregate::{ + AggregateConfiguration, AggregateContextSnapshotEntry, AggregateContextSnapshotHandle, AggregateMetricType, +}; mod chained; pub use self::chained::ChainedConfiguration; From 20edec24e32676749c762c71e67fc1e2738a44c0 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 13:04:34 -0400 Subject: [PATCH 02/44] fix(components): expose optional snapshot units Represent absent snapshot units as None so downstream serializers can distinguish missing metadata without inspecting an empty string. --- .../src/transforms/aggregate/mod.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/saluki-components/src/transforms/aggregate/mod.rs b/lib/saluki-components/src/transforms/aggregate/mod.rs index 866f2797539..ff0a8da4608 100644 --- a/lib/saluki-components/src/transforms/aggregate/mod.rs +++ b/lib/saluki-components/src/transforms/aggregate/mod.rs @@ -96,11 +96,13 @@ impl AggregateContextSnapshotEntry { self.metric_type } - /// Returns the unit attached to the retained metric values. - /// - /// An empty value indicates that no unit is attached. - pub fn unit(&self) -> &MetaString { - &self.unit + /// Returns the unit attached to the retained metric values, if one is set. + pub fn unit(&self) -> Option<&str> { + if self.unit.is_empty() { + None + } else { + Some(&self.unit) + } } /// Creates a snapshot entry for downstream test and benchmark fixtures. @@ -1400,11 +1402,11 @@ mod tests { assert_eq!(snapshot[0].context().tags().len(), 1); assert_eq!(snapshot[0].context().origin_tags().len(), 1); assert_eq!(snapshot[0].metric_type(), AggregateMetricType::Histogram); - assert_eq!(snapshot[0].unit(), "millisecond"); + assert_eq!(snapshot[0].unit(), Some("millisecond")); assert_eq!(snapshot[1].context(), &gauge_context); assert_eq!(snapshot[1].metric_type(), AggregateMetricType::Gauge); - assert!(snapshot[1].unit().is_empty()); + assert_eq!(snapshot[1].unit(), None); } #[tokio::test] From 686911b342a9f3c168bc7a3c4562b9ae99345508 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 13:19:30 -0400 Subject: [PATCH 03/44] fix(components): account for snapshot lifecycle Account for the peak owner-side snapshot allocation alongside retained aggregation state. Exercise snapshot delivery, canceled request handling, and clean shutdown through the production aggregate run loop. --- .../src/transforms/aggregate/mod.rs | 208 +++++++++++++++++- 1 file changed, 204 insertions(+), 4 deletions(-) diff --git a/lib/saluki-components/src/transforms/aggregate/mod.rs b/lib/saluki-components/src/transforms/aggregate/mod.rs index ff0a8da4608..b7edf3f9c5c 100644 --- a/lib/saluki-components/src/transforms/aggregate/mod.rs +++ b/lib/saluki-components/src/transforms/aggregate/mod.rs @@ -132,6 +132,9 @@ pub struct AggregateContextSnapshotHandle { impl AggregateContextSnapshotHandle { /// Requests the aggregate transform's current retained contexts. /// + /// The returned snapshot uses O(context count) memory. After delivery, the caller owns this memory, so callers that + /// retain snapshots must include their retained size in their own memory accounting. + /// /// # Errors /// /// Returns an error if the aggregate owner is unavailable or stops before responding. @@ -481,6 +484,12 @@ impl MemoryBounds for AggregateConfiguration { UsageExpr::struct_size::("aggregated metric"), ), UsageExpr::config("aggregate_context_limit", self.context_limit), + )) + // A snapshot is constructed while the aggregation state remains live, so its peak allocation is additive. + .with_expr(UsageExpr::product( + "retained context snapshot", + UsageExpr::struct_size::("snapshot entry"), + UsageExpr::config("aggregate_context_limit", self.context_limit), )); } } @@ -1176,15 +1185,25 @@ const fn is_bucket_closed( // then we run it to make sure that we are always generating sequential timestamps for data points, etc. #[cfg(test)] mod tests { + use std::mem::size_of; + use float_cmp::ApproxEqRatio as _; use saluki_context::tags::{Tag, TagSet}; use saluki_core::{ - components::ComponentContext, - topology::{interconnect::Dispatcher, OutputName}, + accounting::{ComponentRegistry, MemoryLimiter}, + components::{ + destinations::{Destination, DestinationBuilder, DestinationContext}, + sources::{Source, SourceBuilder, SourceContext}, + ComponentContext, + }, + health::HealthRegistry, + runtime::Supervisor, + support::SubsystemIdentifier, + topology::{interconnect::Dispatcher, OutputDefinition, OutputName, TopologyBlueprint}, }; use saluki_metrics::test::TestRecorder; use stringtheory::MetaString; - use tokio::sync::mpsc; + use tokio::sync::{mpsc, oneshot}; use super::config::HistogramStatistic; use super::*; @@ -1209,6 +1228,83 @@ mod tests { BUCKET_WIDTH_SECS.get() * (step + 1) } + struct ControlledMetricSource { + events: mpsc::Receiver, + } + + #[async_trait] + impl Source for ControlledMetricSource { + async fn run(mut self: Box, mut context: SourceContext) -> Result<(), GenericError> { + let shutdown = context.take_shutdown_handle(); + tokio::pin!(shutdown); + let mut events_open = true; + + loop { + select! { + _ = &mut shutdown => break, + maybe_event = self.events.recv(), if events_open => match maybe_event { + Some(event) => context.dispatcher().dispatch_one(event).await?, + None => events_open = false, + } + } + } + Ok(()) + } + } + + struct ControlledMetricSourceBuilder { + events: Mutex>>, + outputs: Vec>, + } + + #[async_trait] + impl SourceBuilder for ControlledMetricSourceBuilder { + fn outputs(&self) -> &[OutputDefinition] { + &self.outputs + } + + async fn build(&self, _context: ComponentContext) -> Result, GenericError> { + let events = self + .events + .lock() + .map_err(|_| generic_error!("controlled metric source receiver lock is poisoned"))? + .take() + .ok_or_else(|| generic_error!("controlled metric source receiver has already been taken"))?; + Ok(Box::new(ControlledMetricSource { events })) + } + } + + impl MemoryBounds for ControlledMetricSourceBuilder { + fn specify_bounds(&self, _builder: &mut MemoryBoundsBuilder) {} + } + + struct DrainingMetricDestination; + + #[async_trait] + impl Destination for DrainingMetricDestination { + async fn run(self: Box, mut context: DestinationContext) -> Result<(), GenericError> { + while context.events().next().await.is_some() {} + Ok(()) + } + } + + struct DrainingMetricDestinationBuilder; + + #[async_trait] + impl DestinationBuilder for DrainingMetricDestinationBuilder { + fn input_event_type(&self) -> EventType { + EventType::Metric + } + + async fn build(&self, _context: ComponentContext) -> Result, GenericError> { + Ok(Box::new(DrainingMetricDestination)) + } + } + + impl MemoryBounds for DrainingMetricDestinationBuilder { + fn specify_bounds(&self, _builder: &mut MemoryBoundsBuilder) {} + } + struct DispatcherReceiver { receiver: mpsc::Receiver, } @@ -1395,7 +1491,7 @@ mod tests { let mut snapshot = state.snapshot_contexts(); assert_eq!(snapshot.len(), 2); - assert_eq!(snapshot.capacity(), state.contexts.len()); + assert!(snapshot.capacity() >= state.contexts.len()); snapshot.sort_by(|a, b| a.context().host().cmp(&b.context().host())); assert_eq!(snapshot[0].context(), &histogram_context); @@ -1452,6 +1548,110 @@ mod tests { assert!(state.snapshot_contexts().is_empty()); } + #[test] + fn aggregate_memory_bounds_include_peak_context_snapshot() { + let mut config = AggregateConfiguration::with_defaults(); + config.context_limit = 17; + let registry = ComponentRegistry::default(); + config.specify_bounds(&mut registry.bounds_builder(&SubsystemIdentifier::from_dotted("test"))); + let bounds = registry.as_bounds(); + + let expected_minimum = size_of::(); + let aggregation_state_bytes = config.context_limit * (size_of::() + size_of::()); + let context_snapshot_bytes = config.context_limit * size_of::(); + + assert_eq!(bounds.total_minimum_required_bytes(), expected_minimum); + assert_eq!( + bounds.total_firm_limit_bytes(), + expected_minimum + aggregation_state_bytes + context_snapshot_bytes + ); + } + + #[tokio::test] + async fn production_owner_loop_serves_snapshots_and_stops_after_cancellation() { + tokio::time::timeout(Duration::from_secs(5), async { + let mut config = AggregateConfiguration::with_defaults(); + config.primary_flush_interval = Duration::from_secs(60); + let snapshot_handle = config.context_snapshot_handle(); + + let available_request_capacity = snapshot_handle.requests.capacity(); + let canceled_request = tokio::spawn({ + let snapshot_handle = snapshot_handle.clone(); + async move { snapshot_handle.snapshot().await } + }); + while snapshot_handle.requests.capacity() == available_request_capacity { + tokio::task::yield_now().await; + } + canceled_request.abort(); + assert!(canceled_request + .await + .expect_err("snapshot requester should be canceled") + .is_cancelled()); + + let (events_tx, events_rx) = mpsc::channel(1); + let source = ControlledMetricSourceBuilder { + events: Mutex::new(Some(events_rx)), + outputs: vec![OutputDefinition::default_output(EventType::Metric)], + }; + let component_registry = ComponentRegistry::default(); + let mut blueprint = TopologyBlueprint::new("aggregate_snapshot_owner", &component_registry); + blueprint + .add_source("source", source) + .expect("controlled source should be accepted") + .add_transform("aggregate", config) + .expect("aggregate transform should be accepted") + .add_destination("destination", DrainingMetricDestinationBuilder) + .expect("draining destination should be accepted"); + blueprint + .connect_components_in_order(["source", "aggregate", "destination"]) + .expect("test topology should connect"); + blueprint + .with_health_registry(HealthRegistry::new()) + .with_memory_limiter(MemoryLimiter::noop()) + .with_ambient_worker_pool(); + + let mut supervisor = + Supervisor::new("aggregate-snapshot-owner").expect("test supervisor should be created"); + supervisor.add_worker(blueprint); + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let topology_task = tokio::spawn(async move { supervisor.run_with_shutdown(shutdown_rx).await }); + + let expected_context = Context::from_static_name("owner.loop.gauge"); + events_tx + .send(Event::Metric(Metric::gauge(expected_context.clone(), 1.0))) + .await + .expect("controlled source should accept an event"); + + let snapshot = loop { + let snapshot = snapshot_handle + .snapshot() + .await + .expect("running aggregate should fulfill snapshots"); + if snapshot.iter().any(|entry| entry.context() == &expected_context) { + break snapshot; + } + tokio::task::yield_now().await; + }; + let entry = snapshot + .iter() + .find(|entry| entry.context() == &expected_context) + .expect("snapshot should retain the inserted context"); + assert_eq!(entry.metric_type(), AggregateMetricType::Gauge); + assert_eq!(entry.unit(), None); + + drop(events_tx); + drop(snapshot_handle); + shutdown_tx.send(()).expect("test topology should still be running"); + let topology_result = topology_task.await.expect("topology task should not panic"); + assert!( + topology_result.is_ok(), + "topology should stop cleanly: {topology_result:?}" + ); + }) + .await + .expect("production aggregate owner loop should complete without spinning or hanging"); + } + #[tokio::test] async fn snapshot_handle_round_trips_through_responder() { let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); From 49d42fca577b0d01da2279b7d2b3cb0034719472 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 13:32:52 -0400 Subject: [PATCH 04/44] feat(adp): analyze DogStatsD context artifacts Read plain or zstd Agent NDJSON by magic bytes and render retained-context summaries with Agent-compatible grouping, ordering, cardinality, and limit behavior. --- Cargo.lock | 2 + bin/agent-data-plane/Cargo.toml | 2 + .../src/dogstatsd_contexts/artifact.rs | 299 +++++++++++++++ .../src/dogstatsd_contexts/mod.rs | 48 +++ .../src/dogstatsd_contexts/report.rs | 340 ++++++++++++++++++ bin/agent-data-plane/src/main.rs | 2 + .../fixtures/dogstatsd_contexts_agent.ndjson | 4 + .../dogstatsd_contexts_agent.ndjson.zstd | Bin 0 -> 192 bytes 8 files changed, 697 insertions(+) create mode 100644 bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs create mode 100644 bin/agent-data-plane/src/dogstatsd_contexts/mod.rs create mode 100644 bin/agent-data-plane/src/dogstatsd_contexts/report.rs create mode 100644 bin/agent-data-plane/tests/fixtures/dogstatsd_contexts_agent.ndjson create mode 100644 bin/agent-data-plane/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd diff --git a/Cargo.lock b/Cargo.lock index d06491af906..ce859e4099e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,12 +64,14 @@ dependencies = [ "serde_json", "serde_with", "stringtheory", + "tempfile", "tikv-jemallocator", "tokio", "tokio-util", "tonic", "tracing", "uuid", + "zstd", ] [[package]] diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index a9cc5f27a6e..3237e688def 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -67,6 +67,7 @@ tokio-util = { workspace = true } tonic = { workspace = true } tracing = { workspace = true } uuid = { workspace = true, features = ["v4"] } +zstd = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] tokio = { workspace = true, features = ["taskdump"] } @@ -86,3 +87,4 @@ saluki-components = { workspace = true } saluki-config = { workspace = true, features = ["test-util"] } saluki-core = { workspace = true, features = ["test-util"] } saluki-metrics = { workspace = true, features = ["test"] } +tempfile = { workspace = true } diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs new file mode 100644 index 00000000000..76cc468920f --- /dev/null +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -0,0 +1,299 @@ +use std::error::Error; +use std::fmt; +use std::fs::File; +use std::io::{BufRead, BufReader, Read}; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +const ZSTD_MAGIC: &[u8; 4] = b"\x28\xb5\x2f\xfd"; + +#[derive(Debug, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "PascalCase")] +pub(super) struct AgentContextRecord { + pub(super) name: String, + pub(super) host: String, + #[serde(rename = "Type")] + pub(super) metric_type: String, + pub(super) tagger_tags: Vec, + pub(super) metric_tags: Vec, + pub(super) no_index: bool, + pub(super) source: u32, +} + +#[derive(Debug)] +pub(super) struct ArtifactError { + path: PathBuf, + operation: &'static str, + record_index: usize, + source: Box, +} + +impl ArtifactError { + fn new( + path: &Path, operation: &'static str, record_index: usize, source: impl Error + Send + Sync + 'static, + ) -> Self { + Self { + path: path.to_owned(), + operation, + record_index, + source: Box::new(source), + } + } + + #[cfg(test)] + pub(super) fn path(&self) -> &Path { + &self.path + } + + #[cfg(test)] + pub(super) fn operation(&self) -> &'static str { + self.operation + } + + #[cfg(test)] + pub(super) fn record_index(&self) -> usize { + self.record_index + } +} + +impl fmt::Display for ArtifactError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{}: failed to {} record {}: {}", + self.path.display(), + self.operation, + self.record_index, + self.source + ) + } +} + +impl Error for ArtifactError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + Some(self.source.as_ref()) + } +} + +pub(super) fn for_each_record(path: &Path, mut consume: impl FnMut(AgentContextRecord)) -> Result<(), ArtifactError> { + let file = File::open(path).map_err(|error| ArtifactError::new(path, "open", 0, error))?; + let mut reader = BufReader::new(file); + let is_compressed = reader + .fill_buf() + .map_err(|error| ArtifactError::new(path, "inspect compression magic for", 0, error))? + .starts_with(ZSTD_MAGIC); + + if is_compressed { + let decoder = zstd::stream::read::Decoder::with_buffer(reader) + .map_err(|error| ArtifactError::new(path, "initialize zstd decoder for", 0, error))?; + decode_records(path, decoder, &mut consume) + } else { + decode_records(path, reader, &mut consume) + } +} + +fn decode_records( + path: &Path, reader: impl Read, consume: &mut impl FnMut(AgentContextRecord), +) -> Result<(), ArtifactError> { + let records = serde_json::Deserializer::from_reader(reader).into_iter::(); + for (index, record) in records.enumerate() { + let record_index = index + 1; + let record = record.map_err(|error| ArtifactError::new(path, "decode", record_index, error))?; + consume(record); + } + Ok(()) +} + +#[cfg(test)] +fn collect_records(path: &Path) -> Result, ArtifactError> { + let mut records = Vec::new(); + for_each_record(path, |record| records.push(record))?; + Ok(records) +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::PathBuf; + + use super::{collect_records, AgentContextRecord}; + + const PLAIN_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson" + )); + const COMPRESSED_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd" + )); + const PLAIN_FIXTURE_PATH: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson" + ); + const COMPRESSED_FIXTURE_PATH: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd" + ); + + // These fixtures and expectations mirror ContextDebugRepr and the `dogstatsd top` command at Datadog Agent + // commit 9de2a8371cf8d95794f238939709491907893288. + fn expected_fixture_records() -> Vec { + vec![ + AgentContextRecord { + name: "z.metric".to_owned(), + host: "node-a".to_owned(), + metric_type: "Gauge".to_owned(), + tagger_tags: vec!["pod_name:web-a".to_owned()], + metric_tags: vec![ + "env:prod".to_owned(), + "image:registry/repo:v1".to_owned(), + "bare".to_owned(), + ], + no_index: false, + source: 1, + }, + AgentContextRecord { + name: "a.metric".to_owned(), + host: "node-a".to_owned(), + metric_type: "Counter".to_owned(), + tagger_tags: vec!["pod_name:web-a".to_owned()], + metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + AgentContextRecord { + name: "a.metric".to_owned(), + host: "node-b".to_owned(), + metric_type: "Counter".to_owned(), + tagger_tags: vec!["pod_name:web-b".to_owned()], + metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + AgentContextRecord { + name: "a.metric".to_owned(), + host: "node-c".to_owned(), + metric_type: "Counter".to_owned(), + tagger_tags: vec!["pod_name:web-c".to_owned()], + metric_tags: vec!["env:staging".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + ] + } + + #[test] + fn detects_plain_and_compressed_artifacts_by_magic_bytes() { + let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); + let compressed_without_suffix = temporary_directory.path().join("compressed-contexts"); + let plain_with_compressed_suffix = temporary_directory.path().join("plain-contexts.zstd"); + fs::write(&compressed_without_suffix, COMPRESSED_FIXTURE_BYTES).expect("compressed fixture should be copied"); + fs::write(&plain_with_compressed_suffix, PLAIN_FIXTURE_BYTES).expect("plain fixture should be copied"); + + let cases = [ + PathBuf::from(PLAIN_FIXTURE_PATH), + PathBuf::from(COMPRESSED_FIXTURE_PATH), + compressed_without_suffix, + plain_with_compressed_suffix, + ]; + let expected = expected_fixture_records(); + + for path in cases { + assert_eq!( + collect_records(&path).expect("artifact should decode"), + expected, + "{path:?}" + ); + } + } + + #[test] + fn checked_in_compressed_fixture_expands_to_the_plain_fixture_exactly() { + let decompressed = zstd::stream::decode_all(COMPRESSED_FIXTURE_BYTES) + .expect("checked-in compressed fixture should decompress"); + + assert_eq!(decompressed, PLAIN_FIXTURE_BYTES); + assert_eq!(PLAIN_FIXTURE_BYTES.last(), Some(&b'\n')); + } + + #[test] + fn rejects_invalid_artifacts_with_path_operation_and_record_index() { + let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); + let valid_record = concat!( + "{\"Name\":\"metric\",\"Host\":\"host\",\"Type\":\"Gauge\",", + "\"TaggerTags\":[],\"MetricTags\":[],\"NoIndex\":false,\"Source\":1}" + ); + let truncated_length = COMPRESSED_FIXTURE_BYTES.len() - 4; + let cases = [ + InvalidArtifactCase { + filename: "array.ndjson", + bytes: format!("[{valid_record}]").into_bytes(), + expected_record_index: 1, + }, + InvalidArtifactCase { + filename: "missing-field.ndjson", + bytes: br#"{"Name":"metric"}"#.to_vec(), + expected_record_index: 1, + }, + InvalidArtifactCase { + filename: "malformed-trailing.ndjson", + bytes: format!("{valid_record}\nnot-json").into_bytes(), + expected_record_index: 2, + }, + InvalidArtifactCase { + filename: "truncated.zstd", + bytes: COMPRESSED_FIXTURE_BYTES[..truncated_length].to_vec(), + expected_record_index: 5, + }, + ]; + + for case in cases { + let path = temporary_directory.path().join(case.filename); + fs::write(&path, case.bytes).expect("invalid artifact should be written"); + + let error = collect_records(&path).expect_err("invalid artifact should be rejected"); + + assert_eq!(error.path(), path.as_path(), "{}", case.filename); + assert_eq!(error.operation(), "decode", "{}", case.filename); + assert_eq!(error.record_index(), case.expected_record_index, "{}", case.filename); + assert!(error.to_string().contains(&path.display().to_string()), "{error}"); + assert!( + error + .to_string() + .contains(&format!("decode record {}", case.expected_record_index)), + "{error}" + ); + } + } + + struct InvalidArtifactCase { + filename: &'static str, + bytes: Vec, + expected_record_index: usize, + } + + #[test] + fn ignores_unknown_fields_but_requires_every_agent_field() { + let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); + let record = concat!( + "{\"Name\":\"metric\",\"Host\":\"host\",\"Type\":\"Gauge\",", + "\"TaggerTags\":[],\"MetricTags\":[\"env:prod\"],\"NoIndex\":false,\"Source\":7,", + "\"FutureField\":\"ignored\"}\n" + ); + fs::write(artifact.path(), record).expect("artifact should be written"); + + assert_eq!( + collect_records(artifact.path()).expect("record with unknown field should decode"), + vec![AgentContextRecord { + name: "metric".to_owned(), + host: "host".to_owned(), + metric_type: "Gauge".to_owned(), + tagger_tags: Vec::new(), + metric_tags: vec!["env:prod".to_owned()], + no_index: false, + source: 7, + }] + ); + } +} diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs new file mode 100644 index 00000000000..e3d5b42e3c5 --- /dev/null +++ b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs @@ -0,0 +1,48 @@ +use std::path::Path; + +use self::artifact::ArtifactError; +use self::report::ContextReport; + +mod artifact; +mod report; + +fn read_report(path: &Path) -> Result { + let mut report = ContextReport::new(); + artifact::for_each_record(path, |record| report.ingest(record))?; + Ok(report) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::read_report; + + const HEADING: &str = " Contexts\tMetric name\t(number of unique values for each tag)\n"; + const PLAIN_FIXTURE: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson" + ); + + #[test] + fn renders_agent_fixture_as_a_golden_report() { + let report = read_report(Path::new(PLAIN_FIXTURE)).expect("fixture should decode"); + + assert_eq!( + report.render(usize::MAX, usize::MAX), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 3\ta.metric\t(2 env, 1 service)\n", + " 1\tz.metric\t(1 bare, 1 env, 1 image)\n", + ) + ); + } + + #[test] + fn empty_artifact_renders_only_the_heading() { + let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); + let report = read_report(artifact.path()).expect("empty artifact should decode"); + + assert_eq!(report.render(10, 10), HEADING); + } +} diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/report.rs b/bin/agent-data-plane/src/dogstatsd_contexts/report.rs new file mode 100644 index 00000000000..58ccb225852 --- /dev/null +++ b/bin/agent-data-plane/src/dogstatsd_contexts/report.rs @@ -0,0 +1,340 @@ +use std::collections::{HashMap, HashSet}; +use std::fmt::Write as _; + +use super::artifact::AgentContextRecord; + +const HEADING: &str = " Contexts\tMetric name\t(number of unique values for each tag)\n"; + +#[derive(Default)] +pub(super) struct ContextReport { + metrics: HashMap, +} + +impl ContextReport { + pub(super) fn new() -> Self { + Self::default() + } + + pub(super) fn ingest(&mut self, record: AgentContextRecord) { + let AgentContextRecord { + name, + host: _, + metric_type: _, + tagger_tags: _, + metric_tags, + no_index: _, + source: _, + } = record; + let summary = self.metrics.entry(name).or_default(); + summary.context_count += 1; + summary.metric_tags.extend(metric_tags); + } + + pub(super) fn render(&self, metric_limit: usize, tag_limit: usize) -> String { + let mut output = String::from(HEADING); + let mut metrics: Vec<_> = self.metrics.iter().collect(); + metrics.sort_unstable_by(|(left_name, left), (right_name, right)| { + right + .context_count + .cmp(&left.context_count) + .then_with(|| left_name.cmp(right_name)) + }); + + let visible_count = visible_item_count(metrics.len(), metric_limit); + for (name, summary) in &metrics[..visible_count] { + write!(output, " {:>10}\t{}\t(", summary.context_count, name).expect("writing to a String should not fail"); + summary.write_tags(&mut output, tag_limit); + output.push_str(")\n"); + } + + let hidden_metrics = &metrics[visible_count..]; + if !hidden_metrics.is_empty() { + let hidden_context_count: usize = hidden_metrics.iter().map(|(_, summary)| summary.context_count).sum(); + writeln!( + output, + " {hidden_context_count:>10}\t(other {} metrics)", + hidden_metrics.len() + ) + .expect("writing to a String should not fail"); + } + + output + } +} + +#[derive(Default)] +struct MetricSummary { + context_count: usize, + metric_tags: HashSet, +} + +impl MetricSummary { + fn write_tags(&self, output: &mut String, tag_limit: usize) { + let mut cardinalities: HashMap<&str, usize> = HashMap::new(); + for tag in &self.metric_tags { + let key = tag.split_once(':').map_or(tag.as_str(), |(key, _)| key); + *cardinalities.entry(key).or_default() += 1; + } + + let mut tags: Vec<_> = cardinalities.into_iter().collect(); + tags.sort_unstable_by(|(left_key, left_count), (right_key, right_count)| { + right_count.cmp(left_count).then_with(|| left_key.cmp(right_key)) + }); + + let visible_count = visible_item_count(tags.len(), tag_limit); + for (index, (key, cardinality)) in tags[..visible_count].iter().enumerate() { + if index > 0 { + output.push_str(", "); + } + write!(output, "{cardinality} {key}").expect("writing to a String should not fail"); + } + + let hidden_tags = &tags[visible_count..]; + if !hidden_tags.is_empty() { + let hidden_value_count: usize = hidden_tags.iter().map(|(_, cardinality)| cardinality).sum(); + if visible_count > 0 { + output.push_str(", "); + } + write!( + output, + "{hidden_value_count} values in {} other tags", + hidden_tags.len() + ) + .expect("writing to a String should not fail"); + } + } +} + +fn visible_item_count(item_count: usize, limit: usize) -> usize { + if item_count.saturating_sub(limit) > 1 { + limit + } else { + item_count + } +} + +#[cfg(test)] +mod tests { + use super::ContextReport; + use crate::dogstatsd_contexts::artifact::AgentContextRecord; + + const HEADING: &str = " Contexts\tMetric name\t(number of unique values for each tag)\n"; + + #[test] + fn sorts_metric_and_tag_ties_lexically() { + let mut report = ContextReport::new(); + report.ingest(record("beta", "host", &[], &["z:value", "a:value"])); + report.ingest(record("alpha", "host", &[], &["z:value", "a:value"])); + + assert_eq!( + report.render(10, 10), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 1\talpha\t(1 a, 1 z)\n", + " 1\tbeta\t(1 a, 1 z)\n", + ) + ); + } + + #[test] + fn deduplicates_full_metric_tags_before_calculating_cardinality() { + let mut report = ContextReport::new(); + report.ingest(record( + "metric", + "host-a", + &[], + &["env:prod", "env:prod", "env:staging"], + )); + report.ingest(record("metric", "host-b", &[], &["env:prod"])); + + assert_eq!( + report.render(10, 10), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 2\tmetric\t(2 env)\n", + ) + ); + } + + #[test] + fn counts_host_and_origin_variants_but_excludes_tagger_tags() { + let mut report = ContextReport::new(); + report.ingest(record("metric", "host-a", &["pod_name:a"], &["env:prod"])); + + let mut second = record("metric", "host-b", &["pod_name:b"], &["env:prod"]); + second.metric_type = "Counter".to_owned(); + second.no_index = true; + second.source = 99; + report.ingest(second); + + report.ingest(record("metric", "host-c", &["pod_name:c"], &["env:prod"])); + + assert_eq!( + report.render(10, 10), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 3\tmetric\t(1 env)\n", + ) + ); + } + + #[test] + fn uses_the_whole_colonless_tag_or_the_prefix_before_only_the_first_colon() { + let mut report = ContextReport::new(); + report.ingest(record( + "metric", + "host", + &[], + &["bare", "image:registry/repo:v1", "image:registry/repo:v2"], + )); + + assert_eq!( + report.render(10, 10), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 1\tmetric\t(2 image, 1 bare)\n", + ) + ); + } + + #[test] + fn applies_agent_metric_limit_plus_one_rule_without_overflow() { + let cases = [ + MetricLimitCase { + name: "zero limit shows its one allowed extra item", + counts: &[("a", 1)], + limit: 0, + expected_body: " 1\ta\t()\n", + }, + MetricLimitCase { + name: "zero limit summarizes two items", + counts: &[("a", 2), ("b", 1)], + limit: 0, + expected_body: " 3\t(other 2 metrics)\n", + }, + MetricLimitCase { + name: "item count equal to limit is shown", + counts: &[("a", 2), ("b", 1)], + limit: 2, + expected_body: " 2\ta\t()\n 1\tb\t()\n", + }, + MetricLimitCase { + name: "limit plus one items are all shown", + counts: &[("a", 2), ("b", 1)], + limit: 1, + expected_body: " 2\ta\t()\n 1\tb\t()\n", + }, + MetricLimitCase { + name: "limit plus two items hide the remainder and sum their contexts", + counts: &[("a", 3), ("b", 2), ("c", 1)], + limit: 1, + expected_body: " 3\ta\t()\n 3\t(other 2 metrics)\n", + }, + MetricLimitCase { + name: "maximum limit does not overflow", + counts: &[("a", 3), ("b", 2), ("c", 1)], + limit: usize::MAX, + expected_body: " 3\ta\t()\n 2\tb\t()\n 1\tc\t()\n", + }, + ]; + + for case in cases { + let report = report_with_counts(case.counts); + assert_eq!( + report.render(case.limit, usize::MAX), + format!("{HEADING}{}", case.expected_body), + "{}", + case.name + ); + } + } + + #[test] + fn applies_agent_tag_limit_plus_one_rule_without_overflow() { + let cases = [ + TagLimitCase { + name: "zero limit shows its one allowed extra tag", + tags: &["a:1"], + limit: 0, + expected_tags: "1 a", + }, + TagLimitCase { + name: "zero limit summarizes two tags and their values", + tags: &["a:1", "a:2", "b:1"], + limit: 0, + expected_tags: "3 values in 2 other tags", + }, + TagLimitCase { + name: "tag count equal to limit is shown", + tags: &["a:1", "b:1"], + limit: 2, + expected_tags: "1 a, 1 b", + }, + TagLimitCase { + name: "limit plus one tags are all shown", + tags: &["a:1", "b:1"], + limit: 1, + expected_tags: "1 a, 1 b", + }, + TagLimitCase { + name: "limit plus two tags hide the remainder and sum their values", + tags: &["a:1", "a:2", "a:3", "b:1", "b:2", "c:1"], + limit: 1, + expected_tags: "3 a, 3 values in 2 other tags", + }, + TagLimitCase { + name: "maximum limit does not overflow", + tags: &["a:1", "a:2", "a:3", "b:1", "b:2", "c:1"], + limit: usize::MAX, + expected_tags: "3 a, 2 b, 1 c", + }, + ]; + + for case in cases { + let mut report = ContextReport::new(); + report.ingest(record("metric", "host", &[], case.tags)); + assert_eq!( + report.render(usize::MAX, case.limit), + format!("{HEADING} 1\tmetric\t({})\n", case.expected_tags), + "{}", + case.name + ); + } + } + + fn report_with_counts(counts: &[(&str, usize)]) -> ContextReport { + let mut report = ContextReport::new(); + for (name, count) in counts { + for index in 0..*count { + report.ingest(record(name, &format!("host-{index}"), &[], &[])); + } + } + report + } + + fn record(name: &str, host: &str, tagger_tags: &[&str], metric_tags: &[&str]) -> AgentContextRecord { + AgentContextRecord { + name: name.to_owned(), + host: host.to_owned(), + metric_type: "Gauge".to_owned(), + tagger_tags: tagger_tags.iter().map(|tag| (*tag).to_owned()).collect(), + metric_tags: metric_tags.iter().map(|tag| (*tag).to_owned()).collect(), + no_index: false, + source: 1, + } + } + + struct MetricLimitCase { + name: &'static str, + counts: &'static [(&'static str, usize)], + limit: usize, + expected_body: &'static str, + } + + struct TagLimitCase { + name: &'static str, + tags: &'static [&'static str], + limit: usize, + expected_tags: &'static str, + } +} diff --git a/bin/agent-data-plane/src/main.rs b/bin/agent-data-plane/src/main.rs index 8cec1ad27e6..1c93766845b 100644 --- a/bin/agent-data-plane/src/main.rs +++ b/bin/agent-data-plane/src/main.rs @@ -29,6 +29,8 @@ use crate::internal::logging::LoggingConfigurationTranslator; mod components; mod config; +#[allow(dead_code)] +mod dogstatsd_contexts; mod internal; pub(crate) mod state; diff --git a/bin/agent-data-plane/tests/fixtures/dogstatsd_contexts_agent.ndjson b/bin/agent-data-plane/tests/fixtures/dogstatsd_contexts_agent.ndjson new file mode 100644 index 00000000000..0d91ad8024a --- /dev/null +++ b/bin/agent-data-plane/tests/fixtures/dogstatsd_contexts_agent.ndjson @@ -0,0 +1,4 @@ +{"Name":"z.metric","Host":"node-a","Type":"Gauge","TaggerTags":["pod_name:web-a"],"MetricTags":["env:prod","image:registry/repo:v1","bare"],"NoIndex":false,"Source":1} +{"Name":"a.metric","Host":"node-a","Type":"Counter","TaggerTags":["pod_name:web-a"],"MetricTags":["env:prod","service:web"],"NoIndex":false,"Source":1} +{"Name":"a.metric","Host":"node-b","Type":"Counter","TaggerTags":["pod_name:web-b"],"MetricTags":["env:prod","service:web"],"NoIndex":false,"Source":1} +{"Name":"a.metric","Host":"node-c","Type":"Counter","TaggerTags":["pod_name:web-c"],"MetricTags":["env:staging","service:web"],"NoIndex":false,"Source":1} diff --git a/bin/agent-data-plane/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd b/bin/agent-data-plane/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd new file mode 100644 index 0000000000000000000000000000000000000000..0dc488722bbd31fd902fd9ea2543d5622147f3c0 GIT binary patch literal 192 zcmV;x06+gIwJ-f-a{-kF0D_7kAaF~X13+-p-9*xs49O)X5z)8U3-q7?5cV|YQu@I^ z6S-7kV+fst)Us#$wiFe>M1y+lP;edQvw?!we1MNgU?5BVTca^ u+APz6a)5{tkO~kTHzwg~ZAr^W2IIamI?$EXp|r4sBQov1Sc(jlwYfMRnOckh literal 0 HcmV?d00001 From db0ffb661c0d53b5f8e311b6b5ed4d106b642742 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 13:46:27 -0400 Subject: [PATCH 05/44] perf(adp): buffer context dump decoding Buffer decompressed zstd output before serde decoding while retaining buffered compressed input, preserving bounded-memory streaming and avoiding small-read throughput penalties. Document the temporary module-level dead-code allowance until subsequent DogStatsD CLI and API tasks wire it. --- .../src/dogstatsd_contexts/artifact.rs | 38 +++++++++++++++---- bin/agent-data-plane/src/main.rs | 5 ++- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index 76cc468920f..a540c3e85ed 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -1,7 +1,7 @@ use std::error::Error; use std::fmt; use std::fs::File; -use std::io::{BufRead, BufReader, Read}; +use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use serde::Deserialize; @@ -77,24 +77,29 @@ impl Error for ArtifactError { } pub(super) fn for_each_record(path: &Path, mut consume: impl FnMut(AgentContextRecord)) -> Result<(), ArtifactError> { + let reader = open_buffered_reader(path)?; + decode_records(path, reader, &mut consume) +} + +fn open_buffered_reader(path: &Path) -> Result, ArtifactError> { let file = File::open(path).map_err(|error| ArtifactError::new(path, "open", 0, error))?; - let mut reader = BufReader::new(file); - let is_compressed = reader + let mut compressed_input = BufReader::new(file); + let is_compressed = compressed_input .fill_buf() .map_err(|error| ArtifactError::new(path, "inspect compression magic for", 0, error))? .starts_with(ZSTD_MAGIC); if is_compressed { - let decoder = zstd::stream::read::Decoder::with_buffer(reader) + let decoder = zstd::stream::read::Decoder::with_buffer(compressed_input) .map_err(|error| ArtifactError::new(path, "initialize zstd decoder for", 0, error))?; - decode_records(path, decoder, &mut consume) + Ok(Box::new(BufReader::new(decoder))) } else { - decode_records(path, reader, &mut consume) + Ok(Box::new(compressed_input)) } } fn decode_records( - path: &Path, reader: impl Read, consume: &mut impl FnMut(AgentContextRecord), + path: &Path, reader: impl BufRead, consume: &mut impl FnMut(AgentContextRecord), ) -> Result<(), ArtifactError> { let records = serde_json::Deserializer::from_reader(reader).into_iter::(); for (index, record) in records.enumerate() { @@ -115,9 +120,10 @@ fn collect_records(path: &Path) -> Result, ArtifactError #[cfg(test)] mod tests { use std::fs; + use std::io::BufRead; use std::path::PathBuf; - use super::{collect_records, AgentContextRecord}; + use super::{collect_records, open_buffered_reader, AgentContextRecord}; const PLAIN_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( env!("CARGO_MANIFEST_DIR"), @@ -208,6 +214,22 @@ mod tests { } } + #[test] + fn opens_plain_and_compressed_artifacts_as_buffered_readers() { + for path in [ + PathBuf::from(PLAIN_FIXTURE_PATH), + PathBuf::from(COMPRESSED_FIXTURE_PATH), + ] { + let mut reader: Box = open_buffered_reader(&path).expect("artifact reader should open"); + let buffered = reader.fill_buf().expect("artifact reader should fill its buffer"); + + assert!( + buffered.starts_with(br#"{"Name":"z.metric","Host":"node-a""#), + "{path:?}" + ); + } + } + #[test] fn checked_in_compressed_fixture_expands_to_the_plain_fixture_exactly() { let decompressed = zstd::stream::decode_all(COMPRESSED_FIXTURE_BYTES) diff --git a/bin/agent-data-plane/src/main.rs b/bin/agent-data-plane/src/main.rs index 1c93766845b..1bccea0a22c 100644 --- a/bin/agent-data-plane/src/main.rs +++ b/bin/agent-data-plane/src/main.rs @@ -29,7 +29,10 @@ use crate::internal::logging::LoggingConfigurationTranslator; mod components; mod config; -#[allow(dead_code)] +#[allow( + dead_code, + reason = "wired into the DogStatsD CLI and API by subsequent feature tasks" +)] mod dogstatsd_contexts; mod internal; From bc312455325bf6bcd58af4cd10c6d2c58a8f151b Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 13:51:10 -0400 Subject: [PATCH 06/44] perf(adp): keep context readers monomorphic Branch on zstd magic while readers are concrete and monomorphize serde decoding for plain and compressed streams. This removes heap allocation and per-byte vtable dispatch when processing huge context artifacts. --- .../src/dogstatsd_contexts/artifact.rs | 34 ++++--------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index a540c3e85ed..2655ad6a5a2 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -1,7 +1,7 @@ use std::error::Error; use std::fmt; use std::fs::File; -use std::io::{BufRead, BufReader}; +use std::io::{BufRead, BufReader, Read}; use std::path::{Path, PathBuf}; use serde::Deserialize; @@ -77,11 +77,6 @@ impl Error for ArtifactError { } pub(super) fn for_each_record(path: &Path, mut consume: impl FnMut(AgentContextRecord)) -> Result<(), ArtifactError> { - let reader = open_buffered_reader(path)?; - decode_records(path, reader, &mut consume) -} - -fn open_buffered_reader(path: &Path) -> Result, ArtifactError> { let file = File::open(path).map_err(|error| ArtifactError::new(path, "open", 0, error))?; let mut compressed_input = BufReader::new(file); let is_compressed = compressed_input @@ -92,14 +87,14 @@ fn open_buffered_reader(path: &Path) -> Result, ArtifactError> if is_compressed { let decoder = zstd::stream::read::Decoder::with_buffer(compressed_input) .map_err(|error| ArtifactError::new(path, "initialize zstd decoder for", 0, error))?; - Ok(Box::new(BufReader::new(decoder))) + decode_records(path, BufReader::new(decoder), &mut consume) } else { - Ok(Box::new(compressed_input)) + decode_records(path, compressed_input, &mut consume) } } -fn decode_records( - path: &Path, reader: impl BufRead, consume: &mut impl FnMut(AgentContextRecord), +fn decode_records( + path: &Path, reader: R, consume: &mut impl FnMut(AgentContextRecord), ) -> Result<(), ArtifactError> { let records = serde_json::Deserializer::from_reader(reader).into_iter::(); for (index, record) in records.enumerate() { @@ -120,10 +115,9 @@ fn collect_records(path: &Path) -> Result, ArtifactError #[cfg(test)] mod tests { use std::fs; - use std::io::BufRead; use std::path::PathBuf; - use super::{collect_records, open_buffered_reader, AgentContextRecord}; + use super::{collect_records, AgentContextRecord}; const PLAIN_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( env!("CARGO_MANIFEST_DIR"), @@ -214,22 +208,6 @@ mod tests { } } - #[test] - fn opens_plain_and_compressed_artifacts_as_buffered_readers() { - for path in [ - PathBuf::from(PLAIN_FIXTURE_PATH), - PathBuf::from(COMPRESSED_FIXTURE_PATH), - ] { - let mut reader: Box = open_buffered_reader(&path).expect("artifact reader should open"); - let buffered = reader.fill_buf().expect("artifact reader should fill its buffer"); - - assert!( - buffered.starts_with(br#"{"Name":"z.metric","Host":"node-a""#), - "{path:?}" - ); - } - } - #[test] fn checked_in_compressed_fixture_expands_to_the_plain_fixture_exactly() { let decompressed = zstd::stream::decode_all(COMPRESSED_FIXTURE_BYTES) From a82c8868a1f57a6d7046a91d80ac566437d481b0 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 14:05:36 -0400 Subject: [PATCH 07/44] feat(adp): publish secure DogStatsD context dumps Stream Agent-compatible records through zstd into a mode-0600 temporary file, finalize and sync it, then atomically replace the fixed run-path artifact without exposing partial output. --- Cargo.lock | 1 + bin/agent-data-plane/Cargo.toml | 8 +- .../src/dogstatsd_contexts/artifact.rs | 822 +++++++++++++++++- .../src/dogstatsd_contexts/mod.rs | 2 + 4 files changed, 828 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce859e4099e..3f380aa194e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,7 @@ dependencies = [ "tonic", "tracing", "uuid", + "windows-sys 0.61.2", "zstd", ] diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index 3237e688def..1d1b47414ec 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -77,13 +77,19 @@ tikv-jemallocator = { workspace = true, features = [ "stats", ] } +[target.'cfg(target_os = "windows")'.dependencies] +windows-sys = { workspace = true, features = [ + "Win32_Foundation", + "Win32_Storage_FileSystem", +] } + [build-dependencies] chrono = { workspace = true } [dev-dependencies] datadog-agent-config-testing = { workspace = true } derive-where = { workspace = true } -saluki-components = { workspace = true } +saluki-components = { workspace = true, features = ["test-util"] } saluki-config = { workspace = true, features = ["test-util"] } saluki-core = { workspace = true, features = ["test-util"] } saluki-metrics = { workspace = true, features = ["test"] } diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index 2655ad6a5a2..8a80ecd4694 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -1,12 +1,291 @@ use std::error::Error; use std::fmt; -use std::fs::File; -use std::io::{BufRead, BufReader, Read}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; -use serde::Deserialize; +use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; +use saluki_context::tags::TagSet; +use saluki_error::{ErrorContext as _, GenericError}; +use serde::{ser::SerializeStruct as _, Deserialize, Serialize, Serializer}; +use uuid::Uuid; const ZSTD_MAGIC: &[u8; 4] = b"\x28\xb5\x2f\xfd"; +pub(crate) const CONTEXT_DUMP_FILENAME: &str = "dogstatsd_contexts.json.zstd"; + +struct AgentContextSnapshotRecord<'a>(&'a AggregateContextSnapshotEntry); + +impl Serialize for AgentContextSnapshotRecord<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let entry = self.0; + let context = entry.context(); + let name: &str = context.name().as_ref(); + let mut record = serializer.serialize_struct("AgentContextRecord", 7)?; + record.serialize_field("Name", name)?; + record.serialize_field("Host", context.host().unwrap_or_default())?; + record.serialize_field("Type", agent_metric_type(entry))?; + record.serialize_field("TaggerTags", &BorrowedTags(context.origin_tags()))?; + record.serialize_field("MetricTags", &BorrowedTags(context.tags()))?; + record.serialize_field("NoIndex", &false)?; + record.serialize_field("Source", &1u32)?; + record.end() + } +} + +struct BorrowedTags<'a>(&'a TagSet); + +impl Serialize for BorrowedTags<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.collect_seq(self.0.into_iter().map(|tag| tag.as_str())) + } +} + +fn agent_metric_type(entry: &AggregateContextSnapshotEntry) -> &'static str { + match entry.metric_type() { + AggregateMetricType::Counter => "Counter", + AggregateMetricType::Rate => "Rate", + AggregateMetricType::Gauge => "Gauge", + AggregateMetricType::Set => "Set", + AggregateMetricType::Distribution => "Distribution", + AggregateMetricType::Histogram if entry.unit() == Some("millisecond") => "Historate", + AggregateMetricType::Histogram => "Histogram", + } +} + +fn write_snapshot_records( + writer: &mut W, snapshot: &[AggregateContextSnapshotEntry], path: &Path, +) -> Result<(), GenericError> { + for (index, entry) in snapshot.iter().enumerate() { + serde_json::to_writer(&mut *writer, &AgentContextSnapshotRecord(entry)).with_error_context(|| { + format!( + "Failed to serialize DogStatsD context dump record {} for '{}'.", + index + 1, + path.display() + ) + })?; + writer.write_all(b"\n").with_error_context(|| { + format!( + "Failed to write DogStatsD context dump record {} terminator for '{}'.", + index + 1, + path.display() + ) + })?; + } + Ok(()) +} + +pub(crate) fn publish_context_dump( + run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], +) -> Result { + publish_context_dump_with(run_path, snapshot, &FileSystemAtomicReplace) +} + +fn publish_context_dump_with( + run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, +) -> Result { + let target = run_path.join(CONTEXT_DUMP_FILENAME); + if run_path.as_os_str().is_empty() { + return Err(saluki_error::generic_error!( + "Failed to validate run path for DogStatsD context dump target '{}': the run path is empty.", + target.display() + )); + } + + let temporary_path = run_path.join(format!( + ".{CONTEXT_DUMP_FILENAME}.{}.tmp", + Uuid::new_v4().as_hyphenated() + )); + let file = open_temporary_file(&temporary_path).with_error_context(|| { + format!( + "Failed to create temporary DogStatsD context dump for target '{}'.", + target.display() + ) + })?; + + publish_open_temporary(file, &temporary_path, &target, snapshot, replacer)?; + Ok(target) +} + +fn open_temporary_file(path: &Path) -> io::Result { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + options.open(path) +} + +fn publish_open_temporary( + writer: W, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, +) -> Result<(), GenericError> +where + W: Write + SyncAll, + R: AtomicReplace, +{ + publish_buffered_temporary(BufWriter::new(writer), temporary_path, target, snapshot, replacer) +} + +fn publish_buffered_temporary( + buffer: BufWriter, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], + replacer: &R, +) -> Result<(), GenericError> +where + W: Write + SyncAll, + R: AtomicReplace, +{ + let mut cleanup = TemporaryPathCleanup::new(temporary_path); + let writer = write_compressed_snapshot(buffer, snapshot, target)?; + writer.sync_all().with_error_context(|| { + format!( + "Failed to sync temporary DogStatsD context dump for target '{}'.", + target.display() + ) + })?; + drop(writer); + + replacer.replace(temporary_path, target).with_error_context(|| { + format!( + "Failed to atomically replace DogStatsD context dump target '{}'.", + target.display() + ) + })?; + cleanup.disarm(); + Ok(()) +} + +fn write_compressed_snapshot( + buffer: BufWriter, snapshot: &[AggregateContextSnapshotEntry], target: &Path, +) -> Result { + let mut encoder = zstd::stream::write::Encoder::new(buffer, 0).with_error_context(|| { + format!( + "Failed to initialize zstd stream for DogStatsD context dump target '{}'.", + target.display() + ) + })?; + write_snapshot_records(&mut encoder, snapshot, target)?; + let buffer = finish_zstd_encoder(encoder, target)?; + finish_buffered_writer(buffer, target) +} + +fn finish_zstd_encoder<'a, W: Write>( + encoder: zstd::stream::write::Encoder<'a, W>, target: &Path, +) -> Result { + encoder.finish().with_error_context(|| { + format!( + "Failed to finalize zstd stream for DogStatsD context dump target '{}'.", + target.display() + ) + }) +} + +fn finish_buffered_writer(buffer: BufWriter, target: &Path) -> Result { + buffer + .into_inner() + .map_err(|error| error.into_error()) + .with_error_context(|| { + format!( + "Failed to finalize buffered DogStatsD context dump output for target '{}'.", + target.display() + ) + }) +} + +trait SyncAll { + fn sync_all(&self) -> io::Result<()>; +} + +impl SyncAll for File { + fn sync_all(&self) -> io::Result<()> { + File::sync_all(self) + } +} + +trait AtomicReplace { + fn replace(&self, source: &Path, target: &Path) -> io::Result<()>; +} + +struct FileSystemAtomicReplace; + +impl AtomicReplace for FileSystemAtomicReplace { + fn replace(&self, source: &Path, target: &Path) -> io::Result<()> { + #[cfg(unix)] + { + fs::rename(source, target) + } + + #[cfg(windows)] + { + move_file_replace(source, target) + } + } +} + +#[cfg(windows)] +fn move_file_replace(source: &Path, target: &Path) -> io::Result<()> { + use std::os::windows::ffi::OsStrExt as _; + + use windows_sys::Win32::Storage::FileSystem::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; + + fn nul_terminated(path: &Path) -> io::Result> { + let mut encoded: Vec<_> = path.as_os_str().encode_wide().collect(); + if encoded.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "path contains an interior NUL", + )); + } + encoded.push(0); + Ok(encoded) + } + + let source = nul_terminated(source)?; + let target = nul_terminated(target)?; + // SAFETY: Both path buffers are NUL-terminated, contain no interior NULs, and remain alive for the duration of the + // call. The flags request same-filesystem replacement with write-through semantics. + let result = unsafe { + MoveFileExW( + source.as_ptr(), + target.as_ptr(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, + ) + }; + if result == 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +struct TemporaryPathCleanup<'a> { + path: &'a Path, + armed: bool, +} + +impl<'a> TemporaryPathCleanup<'a> { + fn new(path: &'a Path) -> Self { + Self { path, armed: true } + } + + fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for TemporaryPathCleanup<'_> { + fn drop(&mut self) { + if self.armed { + let _ = fs::remove_file(self.path); + } + } +} #[derive(Debug, Deserialize, Eq, PartialEq)] #[serde(rename_all = "PascalCase")] @@ -115,9 +394,27 @@ fn collect_records(path: &Path) -> Result, ArtifactError #[cfg(test)] mod tests { use std::fs; + use std::io::{self, BufWriter, Write as _}; use std::path::PathBuf; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; - use super::{collect_records, AgentContextRecord}; + use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; + use saluki_context::{ + tags::{Tag, TagSet}, + Context, + }; + use serde_json::json; + use stringtheory::MetaString; + + use super::{ + collect_records, finish_buffered_writer, finish_zstd_encoder, publish_buffered_temporary, publish_context_dump, + publish_context_dump_with, publish_open_temporary, write_snapshot_records, AgentContextRecord, AtomicReplace, + SyncAll, CONTEXT_DUMP_FILENAME, + }; + use crate::dogstatsd_contexts::read_report; const PLAIN_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( env!("CARGO_MANIFEST_DIR"), @@ -183,6 +480,523 @@ mod tests { ] } + #[test] + fn streams_exact_agent_schema_for_every_metric_type() { + let cases = [ + (AggregateMetricType::Counter, None, "Counter"), + (AggregateMetricType::Rate, None, "Rate"), + (AggregateMetricType::Gauge, None, "Gauge"), + (AggregateMetricType::Set, None, "Set"), + (AggregateMetricType::Distribution, None, "Distribution"), + (AggregateMetricType::Histogram, None, "Histogram"), + (AggregateMetricType::Histogram, Some("second"), "Histogram"), + (AggregateMetricType::Histogram, Some("millisecond"), "Historate"), + ]; + let snapshot: Vec<_> = cases + .iter() + .enumerate() + .map(|(index, (metric_type, unit, _))| { + snapshot_entry( + Box::leak(format!("metric.{index}").into_boxed_str()), + (index == 0).then_some("node-a"), + *metric_type, + unit.unwrap_or_default(), + &["client:value", "bare"], + &["origin:value"], + ) + }) + .collect(); + let mut output = Vec::new(); + + write_snapshot_records(&mut output, &snapshot, PathBuf::from("plain-stream").as_path()) + .expect("snapshot should serialize"); + + let output = String::from_utf8(output).expect("serialized output should be UTF-8"); + let lines: Vec<_> = output.lines().collect(); + assert_eq!(lines.len(), cases.len()); + assert_eq!(output.bytes().filter(|byte| *byte == b'\n').count(), cases.len()); + assert!(output.ends_with('\n')); + for (index, ((_, _, expected_type), line)) in cases.iter().zip(&lines).enumerate() { + let expected_host = if index == 0 { "node-a" } else { "" }; + let expected = json!({ + "Name": format!("metric.{index}"), + "Host": expected_host, + "Type": expected_type, + "TaggerTags": ["origin:value"], + "MetricTags": ["client:value", "bare"], + "NoIndex": false, + "Source": 1, + }); + assert_eq!(serde_json::from_str::(line).unwrap(), expected); + assert_eq!( + *line, + format!( + "{{\"Name\":\"metric.{index}\",\"Host\":\"{expected_host}\",\"Type\":\"{expected_type}\",\"TaggerTags\":[\"origin:value\"],\"MetricTags\":[\"client:value\",\"bare\"],\"NoIndex\":false,\"Source\":1}}" + ) + ); + } + } + + #[test] + fn plain_stream_round_trips_through_reader_and_report() { + let snapshot = vec![ + snapshot_entry( + "shared.metric", + Some("node-a"), + AggregateMetricType::Gauge, + "", + &["env:prod", "service:web"], + &["pod_name:web-a"], + ), + snapshot_entry( + "shared.metric", + None, + AggregateMetricType::Counter, + "", + &["env:staging", "service:web"], + &["pod_name:web-b"], + ), + ]; + let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); + let mut file = artifact.reopen().expect("temporary artifact should reopen"); + + write_snapshot_records(&mut file, &snapshot, artifact.path()).expect("snapshot should serialize"); + file.flush().expect("plain stream should flush"); + + assert_eq!( + collect_records(artifact.path()).expect("plain stream should decode"), + vec![ + AgentContextRecord { + name: "shared.metric".to_owned(), + host: "node-a".to_owned(), + metric_type: "Gauge".to_owned(), + tagger_tags: vec!["pod_name:web-a".to_owned()], + metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + AgentContextRecord { + name: "shared.metric".to_owned(), + host: String::new(), + metric_type: "Counter".to_owned(), + tagger_tags: vec!["pod_name:web-b".to_owned()], + metric_tags: vec!["env:staging".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + ] + ); + assert_eq!( + read_report(artifact.path()) + .expect("plain stream report should build") + .render(10, 10), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 2\tshared.metric\t(2 env, 1 service)\n", + ) + ); + } + + #[test] + fn surfaces_a_partial_writer_failure_with_record_and_path_context() { + let snapshot = [snapshot_entry( + "write.failure", + None, + AggregateMetricType::Gauge, + "", + &["env:test"], + &[], + )]; + let path = PathBuf::from("/run/test/dogstatsd_contexts.json.zstd"); + let mut writer = FailAfterBytes::new(12); + + let error = write_snapshot_records(&mut writer, &snapshot, &path).expect_err("write failure should surface"); + + assert_eq!(writer.written.len(), 12); + assert!(error.to_string().contains("serialize DogStatsD context dump record 1")); + assert!(error.to_string().contains(&path.display().to_string())); + assert!(format!("{error:#}").contains("injected write failure")); + } + + #[test] + fn publishes_a_decodable_mode_0600_dump_and_replaces_the_canonical_artifact() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + fs::write(&target, b"old canonical artifact").expect("old canonical artifact should be written"); + let snapshot = vec![ + snapshot_entry( + "published.metric", + Some("node-a"), + AggregateMetricType::Histogram, + "millisecond", + &["env:prod"], + &["pod_name:web-a"], + ), + snapshot_entry( + "published.metric", + None, + AggregateMetricType::Distribution, + "", + &["env:staging"], + &["pod_name:web-b"], + ), + ]; + + let published = publish_context_dump(run_directory.path(), &snapshot).expect("snapshot should publish"); + + assert_eq!(published, target); + assert_eq!( + collect_records(&target) + .expect("published artifact should decode") + .len(), + 2 + ); + assert_eq!( + read_report(&target) + .expect("published report should build") + .render(10, 10), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 2\tpublished.metric\t(2 env)\n", + ) + ); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + + assert_eq!(fs::metadata(&target).unwrap().permissions().mode() & 0o777, 0o600); + } + } + + #[test] + fn rejects_empty_missing_and_non_directory_run_paths_with_target_and_operation_context() { + let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); + let missing = temporary_directory.path().join("missing"); + let not_directory = temporary_directory.path().join("not-a-directory"); + fs::write(¬_directory, b"file").expect("non-directory fixture should be written"); + let cases = [ + (PathBuf::new(), "validate run path"), + (missing, "create temporary"), + (not_directory, "create temporary"), + ]; + + for (run_path, expected_operation) in cases { + let error = publish_context_dump(&run_path, &[]).expect_err("invalid run path should fail"); + let message = format!("{error:#}"); + assert!(message.contains(CONTEXT_DUMP_FILENAME), "{message}"); + assert!(message.contains(expected_operation), "{message}"); + } + } + + #[cfg(unix)] + #[test] + fn rejects_an_unwritable_run_directory_with_target_and_operation_context() { + use std::os::unix::fs::PermissionsExt as _; + + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let original_permissions = fs::metadata(run_directory.path()).unwrap().permissions(); + fs::set_permissions(run_directory.path(), fs::Permissions::from_mode(0o500)).unwrap(); + + let result = publish_context_dump(run_directory.path(), &[]); + fs::set_permissions(run_directory.path(), original_permissions).unwrap(); + + let error = result.expect_err("unwritable run path should fail"); + let message = format!("{error:#}"); + assert!(message.contains(CONTEXT_DUMP_FILENAME), "{message}"); + assert!(message.contains("create temporary"), "{message}"); + assert_eq!(directory_entries(run_directory.path()), Vec::::new()); + } + + #[test] + fn injected_replace_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let snapshot = [snapshot_entry( + "replacement.failure", + None, + AggregateMetricType::Gauge, + "", + &[], + &[], + )]; + + let error = publish_context_dump_with(run_directory.path(), &snapshot, &FailingReplace) + .expect_err("replacement should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("atomically replace"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + } + + #[test] + fn zstd_finalization_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let temporary = run_directory.path().join(".injected-zstd-failure.tmp"); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let file = fs::File::create(&temporary).expect("temporary fixture should be created"); + let writer = InjectableFile { + file, + fail_writes: true, + fail_sync: false, + }; + let buffer = BufWriter::with_capacity(0, writer); + let snapshot = [snapshot_entry( + "zstd.failure", + None, + AggregateMetricType::Gauge, + "", + &[], + &[], + )]; + + let error = publish_buffered_temporary(buffer, &temporary, &target, &snapshot, &FailingReplace) + .expect_err("zstd finalization should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("finalize zstd stream"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected write failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + } + + #[test] + fn buffered_finalization_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let temporary = run_directory.path().join(".injected-buffer-failure.tmp"); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let file = fs::File::create(&temporary).expect("temporary fixture should be created"); + let writer = InjectableFile { + file, + fail_writes: true, + fail_sync: false, + }; + let snapshot = [snapshot_entry( + "buffer.failure", + None, + AggregateMetricType::Gauge, + "", + &[], + &[], + )]; + + let error = publish_open_temporary(writer, &temporary, &target, &snapshot, &FailingReplace) + .expect_err("buffer finalization should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("finalize buffered"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected write failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + } + + #[test] + fn sync_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let temporary = run_directory.path().join(".injected-sync-failure.tmp"); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let file = fs::File::create(&temporary).expect("temporary fixture should be created"); + let writer = InjectableFile { + file, + fail_writes: false, + fail_sync: true, + }; + + let error = + publish_open_temporary(writer, &temporary, &target, &[], &FailingReplace).expect_err("sync should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("sync temporary"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected sync failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + } + + #[test] + fn zstd_and_buffer_finalizers_surface_their_own_error_context() { + let target = PathBuf::from("/run/test/dogstatsd_contexts.json.zstd"); + let snapshot = [snapshot_entry( + "finalize.failure", + None, + AggregateMetricType::Gauge, + "", + &[], + &[], + )]; + + let zstd_failure = Arc::new(AtomicBool::new(false)); + let mut encoder = zstd::stream::write::Encoder::new(ToggleFailureWriter::new(zstd_failure.clone()), 0) + .expect("encoder should initialize"); + write_snapshot_records(&mut encoder, &snapshot, &target).expect("records should stream before failure"); + zstd_failure.store(true, Ordering::Relaxed); + let error = finish_zstd_encoder(encoder, &target).expect_err("zstd finalization should fail"); + let message = format!("{error:#}"); + assert!(message.contains("finalize zstd stream"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected finalization failure"), "{message}"); + + let buffer_failure = Arc::new(AtomicBool::new(false)); + let buffer = BufWriter::new(ToggleFailureWriter::new(buffer_failure.clone())); + let mut encoder = zstd::stream::write::Encoder::new(buffer, 0).expect("encoder should initialize"); + write_snapshot_records(&mut encoder, &snapshot, &target).expect("records should stream before failure"); + let buffer = finish_zstd_encoder(encoder, &target).expect("zstd stream should finalize"); + buffer_failure.store(true, Ordering::Relaxed); + let error = finish_buffered_writer(buffer, &target).expect_err("buffer finalization should fail"); + let message = format!("{error:#}"); + assert!(message.contains("finalize buffered"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected finalization failure"), "{message}"); + } + + struct InjectableFile { + file: fs::File, + fail_writes: bool, + fail_sync: bool, + } + + impl io::Write for InjectableFile { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.fail_writes { + Err(io::Error::other("injected write failure")) + } else { + self.file.write(bytes) + } + } + + fn flush(&mut self) -> io::Result<()> { + if self.fail_writes { + Err(io::Error::other("injected write failure")) + } else { + self.file.flush() + } + } + } + + impl SyncAll for InjectableFile { + fn sync_all(&self) -> io::Result<()> { + if self.fail_sync { + Err(io::Error::other("injected sync failure")) + } else { + self.file.sync_all() + } + } + } + + #[derive(Debug)] + struct ToggleFailureWriter { + failure_enabled: Arc, + } + + impl ToggleFailureWriter { + fn new(failure_enabled: Arc) -> Self { + Self { failure_enabled } + } + } + + impl io::Write for ToggleFailureWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.failure_enabled.load(Ordering::Relaxed) { + Err(io::Error::other("injected finalization failure")) + } else { + Ok(bytes.len()) + } + } + + fn flush(&mut self) -> io::Result<()> { + if self.failure_enabled.load(Ordering::Relaxed) { + Err(io::Error::other("injected finalization failure")) + } else { + Ok(()) + } + } + } + + struct FailingReplace; + + impl AtomicReplace for FailingReplace { + fn replace(&self, _source: &std::path::Path, _target: &std::path::Path) -> io::Result<()> { + Err(io::Error::other("injected replacement failure")) + } + } + + fn directory_entries(path: &std::path::Path) -> Vec { + let mut entries: Vec<_> = fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + entries.sort_unstable(); + entries + } + + struct FailAfterBytes { + remaining: usize, + written: Vec, + } + + impl FailAfterBytes { + fn new(remaining: usize) -> Self { + Self { + remaining, + written: Vec::new(), + } + } + } + + impl io::Write for FailAfterBytes { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.remaining == 0 { + return Err(io::Error::other("injected write failure")); + } + let written = self.remaining.min(bytes.len()); + self.written.extend_from_slice(&bytes[..written]); + self.remaining -= written; + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + fn snapshot_entry( + name: &'static str, host: Option<&'static str>, metric_type: AggregateMetricType, unit: &'static str, + metric_tags: &[&'static str], origin_tags: &[&'static str], + ) -> AggregateContextSnapshotEntry { + let context = Context::from_static_parts(name, metric_tags) + .with_host(host.map(MetaString::from_static)) + .with_origin_tags(origin_tags.iter().map(|tag| Tag::from_static(tag)).collect::()); + AggregateContextSnapshotEntry::for_test(context, metric_type, MetaString::from_static(unit)) + } + #[test] fn detects_plain_and_compressed_artifacts_by_magic_bytes() { let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs index e3d5b42e3c5..659f6dcf8ba 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs @@ -1,6 +1,8 @@ use std::path::Path; use self::artifact::ArtifactError; +#[allow(unused_imports, reason = "wired into the DogStatsD API by a subsequent feature task")] +pub(crate) use self::artifact::{publish_context_dump, CONTEXT_DUMP_FILENAME}; use self::report::ContextReport; mod artifact; From fda80c8df64e8ca42bb685d1e4c393b89e3d4a48 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 14:24:14 -0400 Subject: [PATCH 08/44] fix(adp): harden context dump staging Normalize newly created Unix staging files to exact owner-only permissions before writing, generate UUID v4 names from fallible entropy, and report explicit cleanup failures alongside the primary publication error while retaining best-effort fallback cleanup. --- Cargo.lock | 1 + Cargo.toml | 1 + bin/agent-data-plane/Cargo.toml | 1 + .../src/dogstatsd_contexts/artifact.rs | 371 +++++++++++++++++- 4 files changed, 355 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f380aa194e..75c0aad0dbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,7 @@ dependencies = [ "derive-where", "foldhash 0.2.0", "futures", + "getrandom 0.4.2", "hashbrown 0.17.1", "http", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index df9440893b4..23726193137 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -223,6 +223,7 @@ iana-time-zone = { version = "0.1", default-features = false } backon = { version = "1", default-features = false } http-serde-ext = { version = "1", default-features = false } uuid = { version = "1.23", default-features = false } +getrandom = { version = "0.4", default-features = false } rcgen = { version = "0.14", default-features = false } tikv-jemallocator = { version = "0.7", default-features = false } axum-extra = { version = "0.12", default-features = false } diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index 1d1b47414ec..9ede6d4f2db 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -29,6 +29,7 @@ datadog-agent-config = { workspace = true } datadog-protos = { workspace = true } foldhash = { workspace = true } futures = { workspace = true } +getrandom = { workspace = true } hashbrown = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index 8a80ecd4694..a511c6a986e 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -89,6 +89,26 @@ pub(crate) fn publish_context_dump( fn publish_context_dump_with( run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, ) -> Result { + publish_context_dump_with_services( + run_path, + snapshot, + replacer, + &SystemEntropy, + &OwnerOnlyPermissions, + &FileSystemCleanup, + ) +} + +fn publish_context_dump_with_services( + run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, entropy: &E, permissions: &P, + cleanup: &C, +) -> Result +where + R: AtomicReplace, + E: EntropySource, + P: PermissionNormalizer, + C: TemporaryFileCleanup, +{ let target = run_path.join(CONTEXT_DUMP_FILENAME); if run_path.as_os_str().is_empty() { return Err(saluki_error::generic_error!( @@ -97,10 +117,7 @@ fn publish_context_dump_with( )); } - let temporary_path = run_path.join(format!( - ".{CONTEXT_DUMP_FILENAME}.{}.tmp", - Uuid::new_v4().as_hyphenated() - )); + let temporary_path = temporary_context_dump_path(run_path, &target, entropy)?; let file = open_temporary_file(&temporary_path).with_error_context(|| { format!( "Failed to create temporary DogStatsD context dump for target '{}'.", @@ -108,10 +125,71 @@ fn publish_context_dump_with( ) })?; - publish_open_temporary(file, &temporary_path, &target, snapshot, replacer)?; + run_with_temporary_cleanup(&temporary_path, &target, cleanup, || { + permissions.normalize(&file).with_error_context(|| { + format!( + "Failed to set owner-only permissions on temporary DogStatsD context dump '{}' for target '{}'.", + temporary_path.display(), + target.display() + ) + })?; + publish_buffered_temporary_unmanaged(BufWriter::new(file), &temporary_path, &target, snapshot, replacer) + })?; Ok(target) } +fn temporary_context_dump_path( + run_path: &Path, target: &Path, entropy: &E, +) -> Result { + let mut random_bytes = [0u8; 16]; + entropy.fill(&mut random_bytes).with_error_context(|| { + format!( + "Failed to generate random temporary filename for DogStatsD context dump target '{}'.", + target.display() + ) + })?; + random_bytes[6] = (random_bytes[6] & 0x0f) | 0x40; + random_bytes[8] = (random_bytes[8] & 0x3f) | 0x80; + let identifier = Uuid::from_bytes(random_bytes); + Ok(run_path.join(format!(".{CONTEXT_DUMP_FILENAME}.{}.tmp", identifier.as_hyphenated()))) +} + +trait EntropySource { + fn fill(&self, bytes: &mut [u8]) -> io::Result<()>; +} + +struct SystemEntropy; + +impl EntropySource for SystemEntropy { + fn fill(&self, bytes: &mut [u8]) -> io::Result<()> { + getrandom::fill(bytes).map_err(|error| io::Error::other(error.to_string())) + } +} + +trait PermissionNormalizer { + fn normalize(&self, file: &File) -> io::Result<()>; +} + +struct OwnerOnlyPermissions; + +impl PermissionNormalizer for OwnerOnlyPermissions { + fn normalize(&self, file: &File) -> io::Result<()> { + set_owner_only_permissions(file) + } +} + +#[cfg(unix)] +fn set_owner_only_permissions(file: &File) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt as _; + + file.set_permissions(fs::Permissions::from_mode(0o600)) +} + +#[cfg(not(unix))] +fn set_owner_only_permissions(_file: &File) -> io::Result<()> { + Ok(()) +} + fn open_temporary_file(path: &Path) -> io::Result { let mut options = OpenOptions::new(); options.write(true).create_new(true); @@ -130,7 +208,9 @@ where W: Write + SyncAll, R: AtomicReplace, { - publish_buffered_temporary(BufWriter::new(writer), temporary_path, target, snapshot, replacer) + run_with_temporary_cleanup(temporary_path, target, &FileSystemCleanup, || { + publish_buffered_temporary_unmanaged(BufWriter::new(writer), temporary_path, target, snapshot, replacer) + }) } fn publish_buffered_temporary( @@ -141,7 +221,19 @@ where W: Write + SyncAll, R: AtomicReplace, { - let mut cleanup = TemporaryPathCleanup::new(temporary_path); + run_with_temporary_cleanup(temporary_path, target, &FileSystemCleanup, || { + publish_buffered_temporary_unmanaged(buffer, temporary_path, target, snapshot, replacer) + }) +} + +fn publish_buffered_temporary_unmanaged( + buffer: BufWriter, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], + replacer: &R, +) -> Result<(), GenericError> +where + W: Write + SyncAll, + R: AtomicReplace, +{ let writer = write_compressed_snapshot(buffer, snapshot, target)?; writer.sync_all().with_error_context(|| { format!( @@ -156,9 +248,33 @@ where "Failed to atomically replace DogStatsD context dump target '{}'.", target.display() ) - })?; - cleanup.disarm(); - Ok(()) + }) +} + +fn run_with_temporary_cleanup( + temporary_path: &Path, target: &Path, cleanup: &C, operation: impl FnOnce() -> Result, +) -> Result +where + C: TemporaryFileCleanup + ?Sized, +{ + let mut guard = TemporaryPathCleanup::new(temporary_path, cleanup); + match operation() { + Ok(value) => { + guard.disarm(); + Ok(value) + } + Err(primary_error) => match guard.cleanup() { + Ok(()) => Err(primary_error), + Err(cleanup_error) => Err(saluki_error::generic_error!( + "Failed to publish DogStatsD context dump target '{}': {:#}. Additionally, failed to remove temporary \ + DogStatsD context dump '{}': {}", + target.display(), + primary_error, + temporary_path.display(), + cleanup_error + )), + }, + } } fn write_compressed_snapshot( @@ -264,14 +380,37 @@ fn move_file_replace(source: &Path, target: &Path) -> io::Result<()> { } } -struct TemporaryPathCleanup<'a> { +trait TemporaryFileCleanup { + fn remove(&self, path: &Path) -> io::Result<()>; +} + +struct FileSystemCleanup; + +impl TemporaryFileCleanup for FileSystemCleanup { + fn remove(&self, path: &Path) -> io::Result<()> { + fs::remove_file(path) + } +} + +struct TemporaryPathCleanup<'a, C: TemporaryFileCleanup + ?Sized> { path: &'a Path, + cleanup: &'a C, armed: bool, } -impl<'a> TemporaryPathCleanup<'a> { - fn new(path: &'a Path) -> Self { - Self { path, armed: true } +impl<'a, C: TemporaryFileCleanup + ?Sized> TemporaryPathCleanup<'a, C> { + fn new(path: &'a Path, cleanup: &'a C) -> Self { + Self { + path, + cleanup, + armed: true, + } + } + + fn cleanup(&mut self) -> io::Result<()> { + self.cleanup.remove(self.path)?; + self.disarm(); + Ok(()) } fn disarm(&mut self) { @@ -279,10 +418,10 @@ impl<'a> TemporaryPathCleanup<'a> { } } -impl Drop for TemporaryPathCleanup<'_> { +impl Drop for TemporaryPathCleanup<'_, C> { fn drop(&mut self) { if self.armed { - let _ = fs::remove_file(self.path); + let _ = self.cleanup.remove(self.path); } } } @@ -397,7 +536,7 @@ mod tests { use std::io::{self, BufWriter, Write as _}; use std::path::PathBuf; use std::sync::{ - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, }; @@ -411,8 +550,10 @@ mod tests { use super::{ collect_records, finish_buffered_writer, finish_zstd_encoder, publish_buffered_temporary, publish_context_dump, - publish_context_dump_with, publish_open_temporary, write_snapshot_records, AgentContextRecord, AtomicReplace, - SyncAll, CONTEXT_DUMP_FILENAME, + publish_context_dump_with, publish_context_dump_with_services, publish_open_temporary, + set_owner_only_permissions, temporary_context_dump_path, write_snapshot_records, AgentContextRecord, + AtomicReplace, EntropySource, FileSystemAtomicReplace, FileSystemCleanup, OwnerOnlyPermissions, + PermissionNormalizer, SyncAll, TemporaryFileCleanup, TemporaryPathCleanup, CONTEXT_DUMP_FILENAME, }; use crate::dogstatsd_contexts::read_report; @@ -673,6 +814,143 @@ mod tests { } } + #[cfg(unix)] + #[test] + fn publication_normalizes_restrictive_temporary_permissions_to_exactly_0600() { + use std::os::unix::fs::PermissionsExt as _; + + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let normalizer = RestrictiveThenOwnerOnly::default(); + + let target = publish_context_dump_with_services( + run_directory.path(), + &[], + &FileSystemAtomicReplace, + &FixedEntropy, + &normalizer, + &FileSystemCleanup, + ) + .expect("snapshot should publish"); + + assert!(normalizer.called.load(Ordering::Relaxed)); + assert_eq!(fs::metadata(target).unwrap().permissions().mode() & 0o777, 0o600); + } + + #[test] + fn entropy_failure_is_contextual_and_creates_no_temporary_artifact() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + + let error = publish_context_dump_with_services( + run_directory.path(), + &[], + &FileSystemAtomicReplace, + &FailingEntropy, + &OwnerOnlyPermissions, + &FileSystemCleanup, + ) + .expect_err("entropy failure should surface"); + + let message = format!("{error:#}"); + assert!(message.contains("generate random temporary filename"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected entropy failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + } + + #[test] + fn temporary_name_sets_rfc_4122_version_and_variant_bits() { + let run_path = std::path::Path::new("/run/test"); + let target = run_path.join(CONTEXT_DUMP_FILENAME); + + let temporary_path = temporary_context_dump_path(run_path, &target, &FixedEntropy).unwrap(); + + assert_eq!( + temporary_path.file_name().unwrap(), + ".dogstatsd_contexts.json.zstd.11111111-1111-4111-9111-111111111111.tmp" + ); + } + + #[test] + fn permission_failure_closes_and_removes_the_new_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + + let error = publish_context_dump_with_services( + run_directory.path(), + &[], + &FileSystemAtomicReplace, + &FixedEntropy, + &FailingPermissions, + &FileSystemCleanup, + ) + .expect_err("permission failure should surface"); + + let message = format!("{error:#}"); + assert!(message.contains("set owner-only permissions"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected permission failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + } + + #[test] + fn cleanup_method_surfaces_removal_failure_and_remains_armed() { + let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); + let temporary_path = temporary_directory.path().join("staged-contexts.tmp"); + fs::write(&temporary_path, b"staged").unwrap(); + let cleanup = FailingCleanup::default(); + let mut guard = TemporaryPathCleanup::new(&temporary_path, &cleanup); + + let error = guard.cleanup().expect_err("cleanup failure should surface"); + + assert!(error.to_string().contains("injected cleanup failure")); + assert!(temporary_path.exists()); + assert_eq!(cleanup.attempts.load(Ordering::Relaxed), 1); + } + + #[test] + fn reports_primary_and_cleanup_failures_without_replacing_the_canonical_artifact() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let cleanup = FailingCleanup::default(); + + let error = publish_context_dump_with_services( + run_directory.path(), + &[], + &FailingReplace, + &FixedEntropy, + &OwnerOnlyPermissions, + &cleanup, + ) + .expect_err("replacement and cleanup should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("atomically replace"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("remove temporary DogStatsD context dump"), "{message}"); + assert!(message.contains("injected cleanup failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!(directory_entries(run_directory.path()).len(), 2); + assert!(directory_entries(run_directory.path()) + .iter() + .any(|entry| entry.starts_with(&format!(".{CONTEXT_DUMP_FILENAME}.")) && entry.ends_with(".tmp"))); + assert!(cleanup.attempts.load(Ordering::Relaxed) >= 1); + } + #[test] fn rejects_empty_missing_and_non_directory_run_paths_with_target_and_operation_context() { let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); @@ -940,6 +1218,61 @@ mod tests { } } + struct FixedEntropy; + + impl EntropySource for FixedEntropy { + fn fill(&self, bytes: &mut [u8]) -> io::Result<()> { + bytes.fill(0x11); + Ok(()) + } + } + + struct FailingEntropy; + + impl EntropySource for FailingEntropy { + fn fill(&self, _bytes: &mut [u8]) -> io::Result<()> { + Err(io::Error::other("injected entropy failure")) + } + } + + struct FailingPermissions; + + impl PermissionNormalizer for FailingPermissions { + fn normalize(&self, _file: &fs::File) -> io::Result<()> { + Err(io::Error::other("injected permission failure")) + } + } + + #[cfg(unix)] + #[derive(Default)] + struct RestrictiveThenOwnerOnly { + called: AtomicBool, + } + + #[cfg(unix)] + impl PermissionNormalizer for RestrictiveThenOwnerOnly { + fn normalize(&self, file: &fs::File) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt as _; + + file.set_permissions(fs::Permissions::from_mode(0o000))?; + set_owner_only_permissions(file)?; + self.called.store(true, Ordering::Relaxed); + Ok(()) + } + } + + #[derive(Default)] + struct FailingCleanup { + attempts: AtomicUsize, + } + + impl TemporaryFileCleanup for FailingCleanup { + fn remove(&self, _path: &std::path::Path) -> io::Result<()> { + self.attempts.fetch_add(1, Ordering::Relaxed); + Err(io::Error::other("injected cleanup failure")) + } + } + struct FailingReplace; impl AtomicReplace for FailingReplace { From a7ff331c787c1f367f1173833c2a8dc3e40604b8 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 14:43:46 -0400 Subject: [PATCH 09/44] feat(adp): expose authenticated context dump API Protect dump creation with the Agent bearer token, coordinate bounded owner snapshots, serialize fixed-path requests, and return only the completed artifact path. --- Cargo.lock | 2 + Cargo.toml | 1 + bin/agent-data-plane/Cargo.toml | 2 + .../src/dogstatsd_contexts/api.rs | 863 ++++++++++++++++++ .../src/dogstatsd_contexts/mod.rs | 6 + 5 files changed, 874 insertions(+) create mode 100644 bin/agent-data-plane/src/dogstatsd_contexts/api.rs diff --git a/Cargo.lock b/Cargo.lock index 75c0aad0dbe..163db22e38c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,11 +65,13 @@ dependencies = [ "serde_json", "serde_with", "stringtheory", + "subtle", "tempfile", "tikv-jemallocator", "tokio", "tokio-util", "tonic", + "tower", "tracing", "uuid", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index 23726193137..4075577a109 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,6 +158,7 @@ rustls-webpki = { version = "0.103", default-features = false } schemars = { version = "0.8", default-features = false } similar-asserts = { version = "2.0", default-features = false } slab = { version = "0.4.12", default-features = false } +subtle = { version = "2.6", default-features = false } syn = { version = "2", default-features = false, features = ["full", "parsing", "visit-mut"] } tokio-util = { version = "0.7.18", default-features = false } tower = { version = "0.5", default-features = false } diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index 9ede6d4f2db..0e766a5c800 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -56,6 +56,7 @@ serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true } stringtheory = { workspace = true } +subtle = { workspace = true } tokio = { workspace = true, features = [ "macros", "net", @@ -95,3 +96,4 @@ saluki-config = { workspace = true, features = ["test-util"] } saluki-core = { workspace = true, features = ["test-util"] } saluki-metrics = { workspace = true, features = ["test"] } tempfile = { workspace = true } +tower = { workspace = true, features = ["util"] } diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs new file mode 100644 index 00000000000..15ac017e303 --- /dev/null +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs @@ -0,0 +1,863 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::try_join_all; +use http::{header::AUTHORIZATION, HeaderMap, StatusCode}; +use saluki_api::{ + extract::State, + response::{IntoResponse as _, Response}, + routing::{post, Router}, + APIHandler, Json, +}; +use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateContextSnapshotHandle}; +use saluki_error::{generic_error, GenericError}; +use subtle::ConstantTimeEq as _; +use tokio::sync::Mutex; + +use super::publish_context_dump; + +const PRODUCTION_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(30); +const AUTHENTICATION_REQUIRED: &str = "Authentication required."; +const SNAPSHOT_UNAVAILABLE: &str = + "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running."; +const SNAPSHOT_TIMED_OUT: &str = "Timed out waiting for the DogStatsD context snapshot; retry the request."; +const PUBLICATION_FAILED: &str = + "Failed to publish DogStatsD context dump; check the configured run path and permissions."; +const PUBLICATION_TASK_FAILED: &str = "DogStatsD context dump publication did not complete; retry the request."; +const NON_UTF8_RESULT_PATH: &str = + "The completed DogStatsD context dump path is not valid UTF-8; configure a UTF-8 run path."; + +#[derive(Clone)] +struct AuthenticationSecret(Arc<[u8]>); + +impl AuthenticationSecret { + fn new(token: &[u8]) -> Result { + if token.is_empty() { + return Err(generic_error!("Agent authentication token must not be empty.")); + } + if !is_valid_bearer_token(token) { + return Err(generic_error!( + "Agent authentication token cannot be represented as an HTTP bearer credential." + )); + } + Ok(Self(Arc::from(token))) + } + + fn authenticates(&self, headers: &HeaderMap) -> bool { + let mut authorization_values = headers.get_all(AUTHORIZATION).iter(); + let Some(authorization) = authorization_values.next() else { + return false; + }; + if authorization_values.next().is_some() { + return false; + } + + let value = authorization.as_bytes(); + let Some(separator) = value.iter().position(|byte| *byte == b' ') else { + return false; + }; + if !value[..separator].eq_ignore_ascii_case(b"bearer") { + return false; + } + let supplied = &value[separator..]; + let Some(first_credential_byte) = supplied.iter().position(|byte| *byte != b' ') else { + return false; + }; + let supplied = &supplied[first_credential_byte..]; + if supplied.len() != self.0.len() { + return false; + } + + bool::from(supplied.ct_eq(self.0.as_ref())) + } +} + +fn is_valid_bearer_token(token: &[u8]) -> bool { + let core_length = token + .iter() + .rposition(|byte| *byte != b'=') + .map_or(0, |index| index + 1); + core_length > 0 + && token[..core_length] + .iter() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~' | b'+' | b'/')) + && token[core_length..].iter().all(|byte| *byte == b'=') +} + +#[derive(Clone)] +struct ContextSnapshotCoordinator { + handles: Arc<[AggregateContextSnapshotHandle]>, + timeout: Duration, +} + +impl ContextSnapshotCoordinator { + fn new(handles: Vec, timeout: Duration) -> Self { + Self { + handles: handles.into(), + timeout, + } + } + + async fn snapshot(&self) -> Result, SnapshotError> { + if self.handles.is_empty() { + return Err(SnapshotError::SnapshotUnavailable); + } + + let requests = self.handles.iter().map(AggregateContextSnapshotHandle::snapshot); + let owner_snapshots = tokio::time::timeout(self.timeout, try_join_all(requests)) + .await + .map_err(|_| SnapshotError::SnapshotTimedOut)? + .map_err(|_| SnapshotError::SnapshotUnavailable)?; + Ok(owner_snapshots.into_iter().flatten().collect()) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SnapshotError { + SnapshotUnavailable, + SnapshotTimedOut, +} + +trait ContextDumpPublisher: Send + Sync { + fn publish(&self, run_path: PathBuf, snapshot: Vec) + -> Result; +} + +struct FileSystemContextDumpPublisher; + +impl ContextDumpPublisher for FileSystemContextDumpPublisher { + fn publish( + &self, run_path: PathBuf, snapshot: Vec, + ) -> Result { + publish_context_dump(&run_path, &snapshot) + } +} + +#[derive(Clone)] +pub(crate) struct DogStatsDContextDumpAPIState { + auth_secret: AuthenticationSecret, + coordinator: ContextSnapshotCoordinator, + run_path: PathBuf, + publication_lock: Arc>, + publisher: Arc, +} + +#[derive(Clone)] +pub(crate) struct DogStatsDContextDumpAPIHandler { + state: DogStatsDContextDumpAPIState, +} + +impl DogStatsDContextDumpAPIHandler { + pub(crate) fn new( + auth_token: impl AsRef<[u8]>, handles: Vec, run_path: impl Into, + ) -> Result { + Self::new_with_services( + auth_token.as_ref(), + handles, + run_path.into(), + PRODUCTION_SNAPSHOT_TIMEOUT, + Arc::new(FileSystemContextDumpPublisher), + ) + } + + #[cfg(test)] + fn new_for_test( + auth_token: impl AsRef<[u8]>, handles: Vec, run_path: PathBuf, + snapshot_timeout: Duration, publisher: Arc, + ) -> Result { + Self::new_with_services(auth_token.as_ref(), handles, run_path, snapshot_timeout, publisher) + } + + fn new_with_services( + auth_token: &[u8], handles: Vec, run_path: PathBuf, snapshot_timeout: Duration, + publisher: Arc, + ) -> Result { + let auth_secret = AuthenticationSecret::new(auth_token)?; + Ok(Self { + state: DogStatsDContextDumpAPIState { + auth_secret, + coordinator: ContextSnapshotCoordinator::new(handles, snapshot_timeout), + run_path, + publication_lock: Arc::new(Mutex::new(())), + publisher, + }, + }) + } + + async fn dump_handler(State(state): State, headers: HeaderMap) -> Response { + if !state.auth_secret.authenticates(&headers) { + return (StatusCode::UNAUTHORIZED, AUTHENTICATION_REQUIRED).into_response(); + } + + let publication_guard = state.publication_lock.clone().lock_owned().await; + let snapshot = match state.coordinator.snapshot().await { + Ok(snapshot) => snapshot, + Err(SnapshotError::SnapshotUnavailable) => { + return (StatusCode::SERVICE_UNAVAILABLE, SNAPSHOT_UNAVAILABLE).into_response(); + } + Err(SnapshotError::SnapshotTimedOut) => { + return (StatusCode::GATEWAY_TIMEOUT, SNAPSHOT_TIMED_OUT).into_response(); + } + }; + + let run_path = state.run_path.clone(); + let publisher = state.publisher.clone(); + let publication = tokio::task::spawn_blocking(move || { + let _publication_guard = publication_guard; + publisher.publish(run_path, snapshot) + }) + .await; + + match publication { + Ok(Ok(path)) => match path.into_os_string().into_string() { + Ok(path) => Json(path).into_response(), + Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, NON_UTF8_RESULT_PATH).into_response(), + }, + Ok(Err(_)) => (StatusCode::INTERNAL_SERVER_ERROR, PUBLICATION_FAILED).into_response(), + Err(_) => (StatusCode::INTERNAL_SERVER_ERROR, PUBLICATION_TASK_FAILED).into_response(), + } + } +} + +impl APIHandler for DogStatsDContextDumpAPIHandler { + type State = DogStatsDContextDumpAPIState; + + fn generate_initial_state(&self) -> Self::State { + self.state.clone() + } + + fn generate_routes(&self) -> Router { + Router::new().route("/dogstatsd/contexts/dump", post(Self::dump_handler)) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::{Path, PathBuf}; + use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + mpsc, Arc, Condvar, Mutex as StdMutex, + }; + use std::time::Duration; + + use http::{ + header::{AUTHORIZATION, CONTENT_TYPE}, + HeaderValue, Request, StatusCode, + }; + use http_body_util::{BodyExt as _, Empty}; + use hyper::body::Bytes; + use saluki_api::{response::Response, APIHandler}; + use saluki_components::transforms::{ + aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotEntry, AggregateContextSnapshotHandle, + AggregateMetricType, + }; + use saluki_context::Context; + use saluki_error::{generic_error, GenericError}; + use stringtheory::MetaString; + use tower::ServiceExt as _; + + use super::{ + ContextDumpPublisher, ContextSnapshotCoordinator, DogStatsDContextDumpAPIHandler, + FileSystemContextDumpPublisher, SnapshotError, + }; + use crate::dogstatsd_contexts::{artifact::for_each_record, publish_context_dump, CONTEXT_DUMP_FILENAME}; + + const AUTH_TOKEN: &str = "expected-agent-token"; + const ROUTE: &str = "/dogstatsd/contexts/dump"; + + #[test] + fn constructor_rejects_empty_and_non_visible_http_tokens_without_exposing_them() { + let invalid_tokens: &[&[u8]] = &[ + b"", + b"contains space", + b"line\nbreak", + b"tab\tvalue", + b"nul\0value", + b"\x7f", + b"\xff", + ]; + + for token in invalid_tokens { + let error = match DogStatsDContextDumpAPIHandler::new(*token, Vec::new(), PathBuf::from("run")) { + Ok(_) => panic!("invalid authentication token should be rejected"), + Err(error) => error, + }; + let message = format!("{error:#}"); + if !token.is_empty() { + assert!(!message.as_bytes().windows(token.len()).any(|window| window == *token)); + } + } + } + + #[tokio::test] + async fn rejects_missing_duplicate_malformed_and_wrong_authorization_with_an_exact_safe_body() { + let publisher = Arc::new(NoOpPublisher); + let handler = test_handler(Vec::new(), PathBuf::from("unused"), publisher); + let mut duplicate = post_request(None); + duplicate + .headers_mut() + .append(AUTHORIZATION, HeaderValue::from_static("Bearer expected-agent-token")); + duplicate + .headers_mut() + .append(AUTHORIZATION, HeaderValue::from_static("Bearer expected-agent-token")); + let mut non_utf8 = post_request(None); + non_utf8.headers_mut().insert( + AUTHORIZATION, + HeaderValue::from_bytes(b"Bearer \xff").expect("HTTP headers allow opaque non-ASCII bytes"), + ); + let cases = [ + post_request(None), + duplicate, + non_utf8, + post_request(Some("Basic expected-agent-token")), + post_request(Some("Bearer")), + post_request(Some("Bearersecret")), + post_request(Some("Bearer\texpected-agent-token")), + post_request(Some("Bearer expected-agent-token trailing")), + post_request(Some("Bearer supplied-secret")), + post_request(Some("Bearer unexpect-agent-token")), + ]; + + for request in cases { + let response = send(&handler, request).await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let body = response_body(response).await; + assert_eq!(body, "Authentication required."); + assert!(!body.contains(AUTH_TOKEN)); + assert!(!body.contains("supplied-secret")); + } + } + + #[tokio::test] + async fn accepts_bearer_scheme_case_insensitively() { + for scheme in ["bearer", "BEARER", "BeArEr"] { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, post_request(Some(&format!("{scheme} {AUTH_TOKEN}")))).await; + + assert_eq!(response.status(), StatusCode::OK); + owner.await.unwrap().unwrap(); + } + } + + #[tokio::test] + async fn accepts_only_the_exact_bearer_credential_bytes() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, post_request(Some("Bearer expected-agent-token"))).await; + + assert_eq!(response.status(), StatusCode::OK); + owner.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn coordinator_returns_one_owner_snapshot_unchanged() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let expected = vec![snapshot_entry("owner.one.first"), snapshot_entry("owner.one.second")]; + let owner_snapshot = expected.clone(); + let owner = tokio::spawn(async move { responder.respond(owner_snapshot).await }); + let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); + + let actual = coordinator.snapshot().await.expect("snapshot should succeed"); + + assert_eq!(actual, expected); + owner.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn coordinator_requests_all_owners_concurrently_and_flattens_each_snapshot_once_in_owner_order() { + let (first_handle, mut first_responder) = aggregate_context_snapshot_channel_for_test(); + let (second_handle, mut second_responder) = aggregate_context_snapshot_channel_for_test(); + let first_snapshot = vec![snapshot_entry("owner.one.first"), snapshot_entry("owner.one.second")]; + let second_snapshot = vec![snapshot_entry("owner.two.only")]; + let expected = [first_snapshot.clone(), second_snapshot.clone()].concat(); + let (second_responded_tx, second_responded_rx) = tokio::sync::oneshot::channel(); + let first_owner = tokio::spawn(async move { + second_responded_rx + .await + .expect("second owner should receive its request"); + first_responder.respond(first_snapshot).await + }); + let second_owner = tokio::spawn(async move { + let result = second_responder.respond(second_snapshot).await; + let _ = second_responded_tx.send(()); + result + }); + let coordinator = + ContextSnapshotCoordinator::new(vec![first_handle, second_handle], Duration::from_millis(250)); + + let actual = coordinator.snapshot().await.expect("both snapshots should succeed"); + + assert_eq!(actual, expected); + first_owner.await.unwrap().unwrap(); + second_owner.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn coordinator_reports_no_handles_as_typed_unavailable() { + let coordinator = ContextSnapshotCoordinator::new(Vec::new(), Duration::from_secs(1)); + + let error = coordinator.snapshot().await.expect_err("empty owner set should fail"); + + assert_eq!(error, SnapshotError::SnapshotUnavailable); + } + + #[tokio::test] + async fn coordinator_reports_a_closed_owner_as_typed_unavailable() { + let (handle, responder) = aggregate_context_snapshot_channel_for_test(); + drop(responder); + let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); + + let error = coordinator.snapshot().await.expect_err("closed owner should fail"); + + assert_eq!(error, SnapshotError::SnapshotUnavailable); + } + + #[tokio::test] + async fn coordinator_reports_elapsed_deadline_as_typed_timeout() { + let (handle, _responder) = aggregate_context_snapshot_channel_for_test(); + let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_millis(10)); + + let error = coordinator + .snapshot() + .await + .expect_err("unresponsive owner should time out"); + + assert_eq!(error, SnapshotError::SnapshotTimedOut); + } + + #[tokio::test] + async fn canceled_coordinator_request_does_not_make_a_late_owner_response_panic_or_fail() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); + let request = tokio::spawn(async move { coordinator.snapshot().await }); + tokio::task::yield_now().await; + + request.abort(); + let _ = request.await; + + responder + .respond(vec![snapshot_entry("late.owner.response")]) + .await + .expect("a canceled response receiver should be ignored"); + } + + #[tokio::test] + async fn generated_router_rejects_get_with_method_not_allowed() { + let handler = test_handler(Vec::new(), PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let request = Request::builder() + .method("GET") + .uri(ROUTE) + .body(Empty::::new()) + .unwrap(); + + let response = send(&handler, request).await; + + assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); + } + + #[tokio::test] + async fn unauthorized_post_does_not_request_a_snapshot_or_create_an_artifact() { + let run_directory = tempfile::tempdir().unwrap(); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_directory.path().to_owned()) + .expect("handler should be valid"); + + let response = send(&handler, post_request(None)).await; + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert!(!run_directory.path().join(CONTEXT_DUMP_FILENAME).exists()); + drop(handler); + assert!(responder.respond(Vec::new()).await.is_err()); + } + + #[tokio::test] + async fn authorized_router_post_returns_only_the_json_path_to_a_complete_decodable_artifact() { + let run_directory = tempfile::tempdir().unwrap(); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let snapshot = vec![snapshot_entry("published.first"), snapshot_entry("published.second")]; + let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN.as_bytes(), vec![handle], run_directory.path()) + .expect("handler should be valid"); + let owner = tokio::spawn(async move { responder.respond(snapshot).await }); + + let response = send(&handler, post_request(Some("Bearer expected-agent-token"))).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers().get(CONTENT_TYPE).unwrap(), "application/json"); + assert_eq!( + response_body(response).await, + serde_json::to_string(target.to_str().unwrap()).unwrap() + ); + owner.await.unwrap().unwrap(); + let mut names = Vec::new(); + for_each_record(&target, |record| names.push(record.name)).expect("published artifact should decode"); + assert_eq!(names, ["published.first", "published.second"]); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + } + + #[tokio::test] + async fn no_handles_and_closed_owner_return_service_unavailable() { + let no_handles = test_handler(Vec::new(), PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let response = send(&no_handles, authorized_post()).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response_body(response).await, + "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." + ); + + let (handle, responder) = aggregate_context_snapshot_channel_for_test(); + drop(responder); + let closed_owner = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let response = send(&closed_owner, authorized_post()).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response_body(response).await, + "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." + ); + } + + #[tokio::test] + async fn snapshot_deadline_returns_gateway_timeout() { + let (handle, _responder) = aggregate_context_snapshot_channel_for_test(); + let handler = DogStatsDContextDumpAPIHandler::new_for_test( + AUTH_TOKEN, + vec![handle], + PathBuf::from("unused"), + Duration::from_millis(10), + Arc::new(NoOpPublisher), + ) + .unwrap(); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::GATEWAY_TIMEOUT); + assert_eq!( + response_body(response).await, + "Timed out waiting for the DogStatsD context snapshot; retry the request." + ); + } + + #[tokio::test] + async fn empty_and_non_directory_run_paths_reach_publication_and_return_safe_internal_errors() { + let temporary_directory = tempfile::tempdir().unwrap(); + let non_directory = temporary_directory.path().join("not-a-directory"); + fs::write(&non_directory, b"fixture").unwrap(); + + for run_path in [PathBuf::new(), non_directory] { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_path).unwrap(); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = response_body(response).await; + assert_eq!( + body, + "Failed to publish DogStatsD context dump; check the configured run path and permissions." + ); + assert!(!body.contains(AUTH_TOKEN)); + owner.await.unwrap().unwrap(); + } + assert_eq!( + directory_entries(temporary_directory.path()), + vec!["not-a-directory".to_owned()] + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn unwritable_run_path_returns_internal_error_without_an_artifact() { + use std::os::unix::fs::PermissionsExt as _; + + let run_directory = tempfile::tempdir().unwrap(); + let original_permissions = fs::metadata(run_directory.path()).unwrap().permissions(); + fs::set_permissions(run_directory.path(), fs::Permissions::from_mode(0o500)).unwrap(); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_directory.path()).unwrap(); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, authorized_post()).await; + fs::set_permissions(run_directory.path(), original_permissions).unwrap(); + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + response_body(response).await, + "Failed to publish DogStatsD context dump; check the configured run path and permissions." + ); + assert!(!run_directory.path().join(CONTEXT_DUMP_FILENAME).exists()); + owner.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn publication_failure_and_blocking_task_panic_return_safe_internal_errors() { + for publisher in [ + Arc::new(FailingPublisher) as Arc, + Arc::new(PanickingPublisher) as Arc, + ] { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), publisher); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = response_body(response).await; + assert!( + body == "Failed to publish DogStatsD context dump; check the configured run path and permissions." + || body == "DogStatsD context dump publication did not complete; retry the request." + ); + assert!(!body.contains("injected")); + assert!(!body.contains(AUTH_TOKEN)); + owner.await.unwrap().unwrap(); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn authorized_requests_serialize_fixed_path_publisher_executions() { + let (started_tx, started_rx) = mpsc::sync_channel(1); + let release = Arc::new((StdMutex::new(false), Condvar::new())); + let publisher = Arc::new(BlockingFirstPublisher { + active: AtomicUsize::new(0), + max_active: AtomicUsize::new(0), + calls: AtomicUsize::new(0), + first_started: started_tx, + release: release.clone(), + }); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), publisher.clone()); + let owner = tokio::spawn(async move { + responder.respond(Vec::new()).await.unwrap(); + responder.respond(Vec::new()).await.unwrap(); + }); + + let first_handler = handler.clone(); + let first = tokio::spawn(async move { send(&first_handler, authorized_post()).await }); + tokio::task::spawn_blocking(move || started_rx.recv_timeout(Duration::from_secs(1))) + .await + .unwrap() + .expect("first publisher should start"); + let second_handler = handler.clone(); + let second = tokio::spawn(async move { send(&second_handler, authorized_post()).await }); + tokio::time::sleep(Duration::from_millis(25)).await; + assert_eq!(publisher.calls.load(Ordering::SeqCst), 1); + { + let (released, wake) = &*release; + *released.lock().unwrap() = true; + wake.notify_all(); + } + + assert_eq!(first.await.unwrap().status(), StatusCode::OK); + assert_eq!(second.await.unwrap().status(), StatusCode::OK); + owner.await.unwrap(); + assert_eq!(publisher.calls.load(Ordering::SeqCst), 2); + assert_eq!(publisher.max_active.load(Ordering::SeqCst), 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn canceling_after_spawn_blocking_starts_still_completes_atomic_publication() { + let run_directory = tempfile::tempdir().unwrap(); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + fs::write(&target, b"original canonical artifact").unwrap(); + let (started_tx, started_rx) = mpsc::sync_channel(1); + let release = Arc::new((StdMutex::new(false), Condvar::new())); + let completed = Arc::new(AtomicBool::new(false)); + let publisher = Arc::new(BlockingRealPublisher { + started: started_tx, + release: release.clone(), + completed: completed.clone(), + }); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], run_directory.path().to_owned(), publisher); + let owner = tokio::spawn(async move { responder.respond(vec![snapshot_entry("after.cancel")]).await }); + let request_handler = handler.clone(); + let request = tokio::spawn(async move { send(&request_handler, authorized_post()).await }); + tokio::task::spawn_blocking(move || started_rx.recv_timeout(Duration::from_secs(1))) + .await + .unwrap() + .expect("publisher should start"); + + request.abort(); + let _ = request.await; + assert_eq!(fs::read(&target).unwrap(), b"original canonical artifact"); + { + let (released, wake) = &*release; + *released.lock().unwrap() = true; + wake.notify_all(); + } + tokio::time::timeout(Duration::from_secs(2), async { + while !completed.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("detached publication should complete"); + + owner.await.unwrap().unwrap(); + let mut names = Vec::new(); + for_each_record(&target, |record| names.push(record.name)).expect("canonical artifact should be complete"); + assert_eq!(names, ["after.cancel"]); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + } + + fn test_handler( + handles: Vec, run_path: PathBuf, publisher: Arc, + ) -> DogStatsDContextDumpAPIHandler { + DogStatsDContextDumpAPIHandler::new_for_test( + AUTH_TOKEN, + handles, + run_path, + Duration::from_millis(100), + publisher, + ) + .expect("test handler should be valid") + } + + fn post_request(authorization: Option<&str>) -> Request> { + let mut builder = Request::builder().method("POST").uri(ROUTE); + if let Some(authorization) = authorization { + builder = builder.header(AUTHORIZATION, authorization); + } + builder.body(Empty::new()).unwrap() + } + + fn authorized_post() -> Request> { + post_request(Some("Bearer expected-agent-token")) + } + + async fn send(handler: &DogStatsDContextDumpAPIHandler, request: Request>) -> Response { + handler + .generate_routes() + .with_state(handler.generate_initial_state()) + .oneshot(request) + .await + .unwrap() + } + + async fn response_body(response: Response) -> String { + String::from_utf8(response.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap() + } + + fn snapshot_entry(name: &'static str) -> AggregateContextSnapshotEntry { + AggregateContextSnapshotEntry::for_test( + Context::from_static_name(name), + AggregateMetricType::Gauge, + MetaString::empty(), + ) + } + + fn directory_entries(path: &Path) -> Vec { + let mut entries: Vec<_> = fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + entries.sort_unstable(); + entries + } + + struct NoOpPublisher; + + impl ContextDumpPublisher for NoOpPublisher { + fn publish( + &self, run_path: PathBuf, _snapshot: Vec, + ) -> Result { + Ok(run_path.join(CONTEXT_DUMP_FILENAME)) + } + } + + struct FailingPublisher; + + impl ContextDumpPublisher for FailingPublisher { + fn publish( + &self, _run_path: PathBuf, _snapshot: Vec, + ) -> Result { + Err(generic_error!("injected publisher failure with {AUTH_TOKEN}")) + } + } + + struct PanickingPublisher; + + impl ContextDumpPublisher for PanickingPublisher { + fn publish( + &self, _run_path: PathBuf, _snapshot: Vec, + ) -> Result { + panic!("injected publisher panic") + } + } + + struct BlockingFirstPublisher { + active: AtomicUsize, + max_active: AtomicUsize, + calls: AtomicUsize, + first_started: mpsc::SyncSender<()>, + release: Arc<(StdMutex, Condvar)>, + } + + impl ContextDumpPublisher for BlockingFirstPublisher { + fn publish( + &self, run_path: PathBuf, _snapshot: Vec, + ) -> Result { + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.max_active.fetch_max(active, Ordering::SeqCst); + let call = self.calls.fetch_add(1, Ordering::SeqCst); + if call == 0 { + self.first_started.send(()).unwrap(); + let (released, wake) = &*self.release; + let mut released = released.lock().unwrap(); + while !*released { + released = wake.wait(released).unwrap(); + } + } + self.active.fetch_sub(1, Ordering::SeqCst); + Ok(run_path.join(CONTEXT_DUMP_FILENAME)) + } + } + + struct BlockingRealPublisher { + started: mpsc::SyncSender<()>, + release: Arc<(StdMutex, Condvar)>, + completed: Arc, + } + + impl ContextDumpPublisher for BlockingRealPublisher { + fn publish( + &self, run_path: PathBuf, snapshot: Vec, + ) -> Result { + self.started.send(()).unwrap(); + let (released, wake) = &*self.release; + let mut released = released.lock().unwrap(); + while !*released { + released = wake.wait(released).unwrap(); + } + drop(released); + let result = publish_context_dump(&run_path, &snapshot); + self.completed.store(true, Ordering::SeqCst); + result + } + } + + #[test] + fn production_publisher_type_uses_the_real_artifact_publisher() { + let run_directory = tempfile::tempdir().unwrap(); + let path = FileSystemContextDumpPublisher + .publish(run_directory.path().to_owned(), vec![snapshot_entry("real.publisher")]) + .unwrap(); + + let mut count = 0; + for_each_record(&path, |_| count += 1).unwrap(); + assert_eq!(count, 1); + } +} diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs index 659f6dcf8ba..31fe3fe6d1b 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs @@ -1,10 +1,16 @@ use std::path::Path; +#[allow( + unused_imports, + reason = "registered with the privileged API by a subsequent feature task" +)] +pub(crate) use self::api::DogStatsDContextDumpAPIHandler; use self::artifact::ArtifactError; #[allow(unused_imports, reason = "wired into the DogStatsD API by a subsequent feature task")] pub(crate) use self::artifact::{publish_context_dump, CONTEXT_DUMP_FILENAME}; use self::report::ContextReport; +mod api; mod artifact; mod report; From f482dabef98c33f49d932df1682badf134013e05 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 14:53:19 -0400 Subject: [PATCH 10/44] test(adp): cover stopped snapshot owners Add a test-only responder that accepts a snapshot request and drops its response sender. Verify the resulting stopped-owner error maps to the same safe 503 response as other unavailable snapshots. --- .../src/dogstatsd_contexts/api.rs | 16 +++++++++ .../src/transforms/aggregate/mod.rs | 35 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs index 15ac017e303..581c99100c4 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs @@ -526,6 +526,22 @@ mod tests { ); } + #[tokio::test] + async fn owner_stopped_after_accepting_request_returns_service_unavailable() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let owner = tokio::spawn(async move { responder.stop_after_receiving().await }); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response_body(response).await, + "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." + ); + owner.await.unwrap().unwrap(); + } + #[tokio::test] async fn snapshot_deadline_returns_gateway_timeout() { let (handle, _responder) = aggregate_context_snapshot_channel_for_test(); diff --git a/lib/saluki-components/src/transforms/aggregate/mod.rs b/lib/saluki-components/src/transforms/aggregate/mod.rs index b7edf3f9c5c..b2adbcc3ac0 100644 --- a/lib/saluki-components/src/transforms/aggregate/mod.rs +++ b/lib/saluki-components/src/transforms/aggregate/mod.rs @@ -180,6 +180,24 @@ impl AggregateContextSnapshotResponder { let _ = response.send(snapshot); Ok(()) } + + /// Waits for one snapshot request and stops without responding. + /// + /// This accepts the request from the owner channel before dropping its one-shot response sender, allowing tests to + /// distinguish an owner that stops mid-request from an owner whose request channel is unavailable. + /// + /// # Errors + /// + /// Returns an error if the request channel closes before a request arrives. + pub async fn stop_after_receiving(&mut self) -> Result<(), GenericError> { + let response = self + .receiver + .recv() + .await + .ok_or_else(|| generic_error!("aggregate context snapshot request channel is closed"))?; + drop(response); + Ok(()) + } } /// Creates a retained-context snapshot handle and owner-side responder for tests. @@ -1686,6 +1704,23 @@ mod tests { assert!(error.to_string().contains("unavailable")); } + #[tokio::test] + async fn snapshot_responder_stops_after_accepting_request_and_cancels_response() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let snapshot_task = tokio::spawn(async move { handle.snapshot().await }); + + responder + .stop_after_receiving() + .await + .expect("responder should accept the snapshot request before stopping"); + + let error = snapshot_task + .await + .expect("snapshot task should complete") + .expect_err("snapshot should fail when the accepted response is dropped"); + assert!(error.to_string().contains("stopped before responding")); + } + #[tokio::test] async fn aggregate_configuration_receiver_can_only_be_taken_once() { let config = AggregateConfiguration::with_defaults(); From c3f11ef30c60f399ca54d81cde4328f5276b2337 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 15:11:46 -0400 Subject: [PATCH 11/44] perf(adp): bound snapshot coordination memory Reuse the first owner snapshot as the combined buffer to bound peak coordination memory. Make cancellation fixtures deterministic and ensure blocking test publishers are always released during unwinding. --- .../src/dogstatsd_contexts/api.rs | 89 +++++++++++++++---- .../src/dogstatsd_contexts/mod.rs | 8 +- .../src/transforms/aggregate/mod.rs | 50 ++++++++--- 3 files changed, 113 insertions(+), 34 deletions(-) diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs index 581c99100c4..f697c59224e 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs @@ -109,7 +109,16 @@ impl ContextSnapshotCoordinator { .await .map_err(|_| SnapshotError::SnapshotTimedOut)? .map_err(|_| SnapshotError::SnapshotUnavailable)?; - Ok(owner_snapshots.into_iter().flatten().collect()) + let total_len = owner_snapshots.iter().map(Vec::len).sum::(); + let mut owner_snapshots = owner_snapshots.into_iter(); + let mut combined = owner_snapshots + .next() + .expect("a non-empty snapshot handle set always returns at least one owner snapshot"); + combined.reserve(total_len - combined.len()); + for snapshot in owner_snapshots { + combined.extend(snapshot); + } + Ok(combined) } } @@ -357,16 +366,20 @@ mod tests { } #[tokio::test] - async fn coordinator_returns_one_owner_snapshot_unchanged() { + async fn coordinator_reuses_the_one_owner_snapshot_allocation() { let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let expected = vec![snapshot_entry("owner.one.first"), snapshot_entry("owner.one.second")]; - let owner_snapshot = expected.clone(); + let owner_snapshot = vec![snapshot_entry("owner.one.first"), snapshot_entry("owner.one.second")]; + let expected = owner_snapshot.clone(); + let original_pointer = owner_snapshot.as_ptr() as usize; + let original_capacity = owner_snapshot.capacity(); let owner = tokio::spawn(async move { responder.respond(owner_snapshot).await }); let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); let actual = coordinator.snapshot().await.expect("snapshot should succeed"); assert_eq!(actual, expected); + assert_eq!(actual.as_ptr() as usize, original_pointer); + assert_eq!(actual.capacity(), original_capacity); owner.await.unwrap().unwrap(); } @@ -437,15 +450,15 @@ mod tests { let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); let request = tokio::spawn(async move { coordinator.snapshot().await }); - tokio::task::yield_now().await; + let pending_response = responder + .receive() + .await + .expect("the owner should accept the snapshot request"); request.abort(); let _ = request.await; - responder - .respond(vec![snapshot_entry("late.owner.response")]) - .await - .expect("a canceled response receiver should be ignored"); + pending_response.respond(vec![snapshot_entry("late.owner.response")]); } #[tokio::test] @@ -639,6 +652,31 @@ mod tests { } } + #[test] + fn blocking_publisher_release_guard_releases_and_notifies_on_drop() { + let release = Arc::new((StdMutex::new(false), Condvar::new())); + let (waiting_tx, waiting_rx) = mpsc::sync_channel(1); + + std::thread::scope(|scope| { + let waiter_release = release.clone(); + let waiter = scope.spawn(move || { + let (released, wake) = &*waiter_release; + let released = released.lock().unwrap(); + waiting_tx.send(()).unwrap(); + let (released, timeout) = wake + .wait_timeout_while(released, Duration::from_secs(1), |released| !*released) + .unwrap(); + assert!(!timeout.timed_out(), "release guard should notify the waiter"); + assert!(*released); + }); + waiting_rx.recv().unwrap(); + + let release_guard = BlockingPublisherReleaseGuard::new(release); + drop(release_guard); + waiter.join().unwrap(); + }); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn authorized_requests_serialize_fixed_path_publisher_executions() { let (started_tx, started_rx) = mpsc::sync_channel(1); @@ -663,15 +701,12 @@ mod tests { .await .unwrap() .expect("first publisher should start"); + let release_guard = BlockingPublisherReleaseGuard::new(release); let second_handler = handler.clone(); let second = tokio::spawn(async move { send(&second_handler, authorized_post()).await }); tokio::time::sleep(Duration::from_millis(25)).await; assert_eq!(publisher.calls.load(Ordering::SeqCst), 1); - { - let (released, wake) = &*release; - *released.lock().unwrap() = true; - wake.notify_all(); - } + drop(release_guard); assert_eq!(first.await.unwrap().status(), StatusCode::OK); assert_eq!(second.await.unwrap().status(), StatusCode::OK); @@ -702,15 +737,12 @@ mod tests { .await .unwrap() .expect("publisher should start"); + let release_guard = BlockingPublisherReleaseGuard::new(release); request.abort(); let _ = request.await; assert_eq!(fs::read(&target).unwrap(), b"original canonical artifact"); - { - let (released, wake) = &*release; - *released.lock().unwrap() = true; - wake.notify_all(); - } + drop(release_guard); tokio::time::timeout(Duration::from_secs(2), async { while !completed.load(Ordering::SeqCst) { tokio::time::sleep(Duration::from_millis(5)).await; @@ -814,6 +846,25 @@ mod tests { } } + struct BlockingPublisherReleaseGuard { + release: Arc<(StdMutex, Condvar)>, + } + + impl BlockingPublisherReleaseGuard { + fn new(release: Arc<(StdMutex, Condvar)>) -> Self { + Self { release } + } + } + + impl Drop for BlockingPublisherReleaseGuard { + fn drop(&mut self) { + let (released, wake) = &*self.release; + let mut released = released.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + *released = true; + wake.notify_all(); + } + } + struct BlockingFirstPublisher { active: AtomicUsize, max_active: AtomicUsize, diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs index 31fe3fe6d1b..45d51b94cfb 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs @@ -5,9 +5,13 @@ use std::path::Path; reason = "registered with the privileged API by a subsequent feature task" )] pub(crate) use self::api::DogStatsDContextDumpAPIHandler; +pub(crate) use self::artifact::publish_context_dump; use self::artifact::ArtifactError; -#[allow(unused_imports, reason = "wired into the DogStatsD API by a subsequent feature task")] -pub(crate) use self::artifact::{publish_context_dump, CONTEXT_DUMP_FILENAME}; +#[allow( + unused_imports, + reason = "used by the DogStatsD CLI and topology wiring in subsequent feature tasks" +)] +pub(crate) use self::artifact::CONTEXT_DUMP_FILENAME; use self::report::ContextReport; mod api; diff --git a/lib/saluki-components/src/transforms/aggregate/mod.rs b/lib/saluki-components/src/transforms/aggregate/mod.rs index b2adbcc3ac0..426f2f90ac9 100644 --- a/lib/saluki-components/src/transforms/aggregate/mod.rs +++ b/lib/saluki-components/src/transforms/aggregate/mod.rs @@ -156,6 +156,22 @@ fn aggregate_context_snapshot_channel() -> (AggregateContextSnapshotHandle, Aggr (AggregateContextSnapshotHandle { requests }, receiver) } +/// An accepted retained-context snapshot request for test fixtures. +#[cfg(any(test, feature = "test-util"))] +pub struct AggregateContextSnapshotPendingResponse { + response: AggregateContextSnapshotRequest, +} + +#[cfg(any(test, feature = "test-util"))] +impl AggregateContextSnapshotPendingResponse { + /// Responds to the accepted snapshot request with the supplied entries. + /// + /// If the requester was canceled after the request was accepted, the response is discarded. + pub fn respond(self, snapshot: Vec) { + let _ = self.response.send(snapshot); + } +} + /// A responder for retained-context snapshot test fixtures. #[cfg(any(test, feature = "test-util"))] pub struct AggregateContextSnapshotResponder { @@ -172,13 +188,25 @@ impl AggregateContextSnapshotResponder { /// /// Returns an error if the request channel closes before a request arrives. pub async fn respond(&mut self, snapshot: Vec) -> Result<(), GenericError> { + self.receive().await?.respond(snapshot); + Ok(()) + } + + /// Waits for one snapshot request and returns its pending response. + /// + /// The returned response lets tests deterministically control whether the owner responds, stops, or outlives a + /// canceled requester after accepting the request. + /// + /// # Errors + /// + /// Returns an error if the request channel closes before a request arrives. + pub async fn receive(&mut self) -> Result { let response = self .receiver .recv() .await .ok_or_else(|| generic_error!("aggregate context snapshot request channel is closed"))?; - let _ = response.send(snapshot); - Ok(()) + Ok(AggregateContextSnapshotPendingResponse { response }) } /// Waits for one snapshot request and stops without responding. @@ -190,12 +218,7 @@ impl AggregateContextSnapshotResponder { /// /// Returns an error if the request channel closes before a request arrives. pub async fn stop_after_receiving(&mut self) -> Result<(), GenericError> { - let response = self - .receiver - .recv() - .await - .ok_or_else(|| generic_error!("aggregate context snapshot request channel is closed"))?; - drop(response); + drop(self.receive().await?); Ok(()) } } @@ -1752,14 +1775,15 @@ mod tests { async fn snapshot_responder_ignores_canceled_requester() { let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); let snapshot_task = tokio::spawn(async move { handle.snapshot().await }); - tokio::task::yield_now().await; + let pending_response = responder + .receive() + .await + .expect("responder should accept the snapshot request"); + snapshot_task.abort(); let _ = snapshot_task.await; - responder - .respond(Vec::new()) - .await - .expect("a canceled requester should be a successful no-op delivery"); + pending_response.respond(Vec::new()); } #[test] From 4a835be5cbac37f486077f3b2cd1025e4c52e650 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 15:21:04 -0400 Subject: [PATCH 12/44] fix(components): export pending snapshot responses Re-export the accepted snapshot response wrapper with the test utilities so downstream fixtures can name the type returned by AggregateContextSnapshotResponder::receive. --- bin/agent-data-plane/src/dogstatsd_contexts/api.rs | 4 ++-- lib/saluki-components/src/transforms/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs index f697c59224e..23a219643ad 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs @@ -260,7 +260,7 @@ mod tests { use saluki_api::{response::Response, APIHandler}; use saluki_components::transforms::{ aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotEntry, AggregateContextSnapshotHandle, - AggregateMetricType, + AggregateContextSnapshotPendingResponse, AggregateMetricType, }; use saluki_context::Context; use saluki_error::{generic_error, GenericError}; @@ -450,7 +450,7 @@ mod tests { let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); let request = tokio::spawn(async move { coordinator.snapshot().await }); - let pending_response = responder + let pending_response: AggregateContextSnapshotPendingResponse = responder .receive() .await .expect("the owner should accept the snapshot request"); diff --git a/lib/saluki-components/src/transforms/mod.rs b/lib/saluki-components/src/transforms/mod.rs index c467be42a0c..1899fc7f912 100644 --- a/lib/saluki-components/src/transforms/mod.rs +++ b/lib/saluki-components/src/transforms/mod.rs @@ -7,7 +7,7 @@ mod aggregate; #[cfg(feature = "test-util")] pub use self::aggregate::{ aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotBenchmarkHarness, - AggregateContextSnapshotResponder, + AggregateContextSnapshotPendingResponse, AggregateContextSnapshotResponder, }; pub use self::aggregate::{ AggregateConfiguration, AggregateContextSnapshotEntry, AggregateContextSnapshotHandle, AggregateMetricType, From 165e9d7579b4239fe6751666a8592df9c6df3b96 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 15:37:18 -0400 Subject: [PATCH 13/44] feat(adp): authenticate context dump requests Allow the HTTP client to consume the Agent IPC TLS configuration, pin the server certificate, and attach the Agent bearer token to DogStatsD context dump requests. --- Cargo.lock | 4 + bin/agent-data-plane/Cargo.toml | 11 + bin/agent-data-plane/src/cli/utils.rs | 418 +++++++++++++++++++- lib/saluki-io/src/net/client/http/client.rs | 60 ++- 4 files changed, 480 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 163db22e38c..1a3b8ef3e0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,6 +48,8 @@ dependencies = [ "process-memory", "prost", "prost-types", + "rcgen", + "rustls", "saluki-antithesis", "saluki-api", "saluki-app", @@ -61,6 +63,7 @@ dependencies = [ "saluki-io", "saluki-metadata", "saluki-metrics", + "saluki-tls", "serde", "serde_json", "serde_with", @@ -69,6 +72,7 @@ dependencies = [ "tempfile", "tikv-jemallocator", "tokio", + "tokio-rustls", "tokio-util", "tonic", "tower", diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index 0e766a5c800..ea6d6252b64 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -58,6 +58,7 @@ serde_with = { workspace = true } stringtheory = { workspace = true } subtle = { workspace = true } tokio = { workspace = true, features = [ + "fs", "macros", "net", "rt", @@ -91,9 +92,19 @@ chrono = { workspace = true } [dev-dependencies] datadog-agent-config-testing = { workspace = true } derive-where = { workspace = true } +rcgen = { workspace = true, features = ["crypto", "pem"] } +rustls = { workspace = true } saluki-components = { workspace = true, features = ["test-util"] } saluki-config = { workspace = true, features = ["test-util"] } saluki-core = { workspace = true, features = ["test-util"] } saluki-metrics = { workspace = true, features = ["test"] } +saluki-tls = { workspace = true } tempfile = { workspace = true } +tokio-rustls = { workspace = true } tower = { workspace = true, features = ["util"] } + +[target.'cfg(not(windows))'.dev-dependencies] +rcgen = { workspace = true, features = ["aws_lc_rs"] } + +[target.'cfg(windows)'.dev-dependencies] +rcgen = { workspace = true, features = ["ring"] } diff --git a/bin/agent-data-plane/src/cli/utils.rs b/bin/agent-data-plane/src/cli/utils.rs index e8255951e28..1eb2b5aa873 100644 --- a/bin/agent-data-plane/src/cli/utils.rs +++ b/bin/agent-data-plane/src/cli/utils.rs @@ -1,5 +1,15 @@ +use std::{ + path::{Path, PathBuf}, + time::Duration, +}; + +use datadog_agent_commons::ipc::{config::IpcAuthConfiguration, tls::build_ipc_client_ipc_tls_config}; use futures::TryFutureExt as _; -use http::{header::CONTENT_TYPE, uri::PathAndQuery, Request, Response, StatusCode, Uri}; +use http::{ + header::{HeaderValue, AUTHORIZATION, CONTENT_TYPE}, + uri::PathAndQuery, + Request, Response, StatusCode, Uri, +}; use http_body_util::BodyExt as _; #[cfg(target_os = "linux")] use http_body_util::Full; @@ -10,7 +20,10 @@ use hyper::body::Incoming; use prost::Message as _; use saluki_config::GenericConfiguration; use saluki_error::{generic_error, ErrorContext as _, GenericError}; -use saluki_io::net::{client::http::HttpClient, ListenAddress}; +use saluki_io::net::{ + client::http::{HttpClient, HttpClientBuilder}, + ListenAddress, +}; use serde::{Deserialize, Serialize}; use crate::config::DataPlaneConfiguration; @@ -19,6 +32,8 @@ use crate::config::DataPlaneConfiguration; pub struct DataPlaneAPIClient { client: HttpClient, authority: String, + #[allow(dead_code, reason = "stored for authenticated privileged API commands")] + authorization: Option, } #[derive(Serialize)] @@ -50,10 +65,50 @@ impl DataPlaneAPIClient { pub fn from_config(config: &GenericConfiguration) -> Result { let dp_config = DataPlaneConfiguration::from_configuration(config)?; - let listen_address = dp_config.secure_api_listen_address(); - let builder = HttpClient::builder().with_tls_config(|b| b.danger_accept_invalid_certs()); + Self::from_builder(builder, dp_config.secure_api_listen_address(), None) + } + + /// Creates an authenticated `DataPlaneAPIClient` for commands that use the Agent IPC credentials. + /// + /// This client pins the privileged API server certificate to the configured IPC certificate and authenticates + /// requests with the raw contents of the configured Agent authentication token file. Unlike the general-purpose + /// client, it does not impose a whole-request timeout because context dump generation can legitimately take longer + /// than the default timeout. + /// + /// # Errors + /// + /// If the data plane or IPC authentication configuration is invalid, the IPC certificate or authentication token + /// cannot be read, the token cannot be represented as an HTTP header, or the HTTP client cannot be built, an error + /// is returned. + #[allow( + dead_code, + reason = "public constructor reserved for authenticated privileged API commands" + )] + pub async fn from_config_with_ipc_auth(config: &GenericConfiguration) -> Result { + let dp_config = DataPlaneConfiguration::from_configuration(config)?; + let ipc_config = IpcAuthConfiguration::from_configuration(config)?; + let tls_config = build_ipc_client_ipc_tls_config(ipc_config.ipc_cert_file_path()).await?; + let auth_token_path = ipc_config.auth_token_file_path(); + let raw_token = tokio::fs::read(&auth_token_path).await.map_err(|error| { + generic_error!( + "Failed to read Agent authentication token from file '{}' ({}).", + auth_token_path.display(), + error.kind() + ) + })?; + let authorization = authorization_from_token_file_contents(&raw_token, &auth_token_path)?; + + let builder = HttpClient::builder() + .with_client_tls_config(tls_config) + .with_connect_timeout(Duration::from_secs(20)) + .without_request_timeout(); + Self::from_builder(builder, dp_config.secure_api_listen_address(), Some(authorization)) + } + fn from_builder( + builder: HttpClientBuilder, listen_address: &ListenAddress, authorization: Option, + ) -> Result { let (builder, authority) = match listen_address { ListenAddress::Tcp(_) => { let local_address = listen_address @@ -77,7 +132,11 @@ impl DataPlaneAPIClient { .build() .error_context("Failed to construct API client for privileged API endpoint.")?; - Ok(Self { client, authority }) + Ok(Self { + client, + authority, + authorization, + }) } fn build_uri(&self, path: &str, query: Option<&str>) -> Uri { @@ -193,6 +252,35 @@ impl DataPlaneAPIClient { .and_then(body_when_success) } + /// Requests an authenticated DogStatsD context dump and returns its path on the server. + /// + /// The response contains only the server-local artifact path; this method does not download the dump contents. + /// + /// # Errors + /// + /// If this client was not created with [`Self::from_config_with_ipc_auth`], the request fails, the server rejects + /// the request, or the successful response does not contain a JSON string path, an error is returned. + #[allow( + dead_code, + reason = "public operation reserved for authenticated context dump commands" + )] + pub async fn dogstatsd_contexts_dump(&mut self) -> Result { + let authorization = self.authorization.as_ref().ok_or_else(|| { + generic_error!( + "DogStatsD context dumps require an authenticated API client; construct it with `from_config_with_ipc_auth`." + ) + })?; + let uri = self.build_uri("/dogstatsd/contexts/dump", None); + let request = build_dogstatsd_contexts_dump_request(uri, authorization); + let response = self + .client + .send(request) + .await + .error_context("Failed to request a DogStatsD context dump from the privileged API endpoint.")?; + let response = process_response_body(response).await?; + path_when_context_dump_success(response) + } + /// Starts a DogStatsD traffic capture. /// /// # Errors @@ -331,6 +419,52 @@ impl DataPlaneAPIClient { } } +fn authorization_from_token_file_contents(raw_token: &[u8], token_path: &Path) -> Result { + if raw_token.is_empty() { + return Err(generic_error!( + "Agent authentication token file '{}' is empty.", + token_path.display() + )); + } + + let mut raw_authorization = Vec::with_capacity("Bearer ".len() + raw_token.len()); + raw_authorization.extend_from_slice(b"Bearer "); + raw_authorization.extend_from_slice(raw_token); + let mut authorization = HeaderValue::from_bytes(&raw_authorization).map_err(|_| { + generic_error!( + "Agent authentication token file '{}' contains characters that are invalid in an HTTP Authorization header.", + token_path.display() + ) + })?; + authorization.set_sensitive(true); + Ok(authorization) +} + +#[allow(dead_code, reason = "request helper for the authenticated context dump operation")] +fn build_dogstatsd_contexts_dump_request(uri: Uri, authorization: &HeaderValue) -> Request { + Request::post(uri) + .header(AUTHORIZATION, authorization.clone()) + .body(String::new()) + .expect("valid DogStatsD context dump request") +} + +#[allow(dead_code, reason = "response helper for the authenticated context dump operation")] +fn path_when_context_dump_success(resp: Response) -> Result { + match resp.status() { + status if status.is_success() => serde_json::from_str::(resp.body()) + .error_context("Failed to decode DogStatsD context dump path from response JSON."), + StatusCode::UNAUTHORIZED => Err(generic_error!( + "DogStatsD context dump authentication failed ({}); verify the configured Agent authentication token file.", + resp.status() + )), + status => Err(generic_error!( + "Failed to create DogStatsD context dump ({}): {}.", + status, + resp.into_body() + )), + } +} + async fn collect_body(body: Incoming) -> Option { // `Collected::to_bytes()` merges all frames. Do not use `Buf::chunk().to_vec()` on an aggregated body: `chunk()` // is only the first contiguous slice (often ~16 KiB), which truncates large JSON such as `/config` responses. @@ -416,9 +550,22 @@ fn empty_when_replay_session_success(resp: Response) -> Result<(), Gener #[cfg(test)] mod tests { - use http::{Response, StatusCode}; - - use super::body_when_capture_success; + use std::{ + path::{Path, PathBuf}, + sync::Arc, + time::Duration, + }; + + use datadog_agent_commons::ipc::tls::{build_ipc_client_ipc_tls_config, build_ipc_server_tls_config}; + use http::{header::AUTHORIZATION, HeaderValue, Method, Response, StatusCode, Uri}; + use rcgen::{generate_simple_self_signed, CertifiedKey}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + use tokio_rustls::TlsAcceptor; + + use super::{ + authorization_from_token_file_contents, body_when_capture_success, build_dogstatsd_contexts_dump_request, + path_when_context_dump_success, DataPlaneAPIClient, + }; #[cfg(target_os = "linux")] use super::{body_when_replay_session_success, empty_when_replay_session_success}; @@ -480,4 +627,259 @@ mod tests { assert_eq!(error.to_string(), "session does not own active replay"); } + + #[test] + fn dogstatsd_contexts_authorization_preserves_raw_token_and_is_sensitive() { + let authorization = authorization_from_token_file_contents(b" token with spaces ", Path::new("token-file")) + .expect("visible token bytes should be accepted"); + + assert_eq!(authorization, "Bearer token with spaces "); + assert!(authorization.is_sensitive()); + } + + #[test] + fn dogstatsd_contexts_authorization_rejects_empty_or_invalid_token_without_exposing_it() { + let path = Path::new("/private/auth_token"); + let empty_error = authorization_from_token_file_contents(b"", path).expect_err("empty token must fail"); + assert!(empty_error.to_string().contains("/private/auth_token")); + + let invalid_error = + authorization_from_token_file_contents(b"secret\ntoken", path).expect_err("newline in token must fail"); + let message = invalid_error.to_string(); + assert!(message.contains("/private/auth_token")); + assert!(!message.contains("secret")); + } + + #[test] + fn dogstatsd_contexts_request_is_authenticated_empty_post() { + let mut auth = HeaderValue::from_static("Bearer exact-token"); + auth.set_sensitive(true); + let uri = Uri::from_static("https://127.0.0.1:5101/dogstatsd/contexts/dump"); + + let request = build_dogstatsd_contexts_dump_request(uri, &auth); + + assert_eq!(request.method(), Method::POST); + assert_eq!(request.uri().path(), "/dogstatsd/contexts/dump"); + assert!(request.body().is_empty()); + let request_auth = request.headers().get(AUTHORIZATION).expect("authorization header"); + assert_eq!(request_auth, "Bearer exact-token"); + assert!(request_auth.is_sensitive()); + } + + #[tokio::test] + async fn dogstatsd_contexts_unauthenticated_client_errors_before_network() { + let _ = saluki_tls::initialize_default_crypto_provider(); + let client = saluki_io::net::client::http::HttpClient::builder() + .with_tls_config(|builder| builder.danger_accept_invalid_certs()) + .build() + .expect("test client should build"); + let mut client = DataPlaneAPIClient { + client, + authority: "unresolvable.invalid:1".to_string(), + authorization: None, + }; + + let error = client + .dogstatsd_contexts_dump() + .await + .expect_err("an unauthenticated client must be rejected"); + + assert!(error.to_string().contains("authenticated")); + } + + #[test] + fn dogstatsd_contexts_success_decodes_json_path() { + let response = Response::builder() + .status(StatusCode::OK) + .body(r#""/var/run/datadog/dogstatsd_contexts.json.zstd""#.to_string()) + .expect("valid response"); + + let path = path_when_context_dump_success(response).expect("successful response should decode"); + + assert_eq!(path, PathBuf::from("/var/run/datadog/dogstatsd_contexts.json.zstd")); + } + + #[test] + fn dogstatsd_contexts_malformed_success_is_actionable() { + let response = Response::builder() + .status(StatusCode::OK) + .body("not-json".to_string()) + .expect("valid response"); + + let error = path_when_context_dump_success(response).expect_err("malformed JSON should fail"); + + assert!(error.to_string().contains("DogStatsD context dump path")); + } + + #[test] + fn dogstatsd_contexts_unauthorized_error_is_safe() { + let response = Response::builder() + .status(StatusCode::UNAUTHORIZED) + .body("rejected Bearer exact-token".to_string()) + .expect("valid response"); + + let error = path_when_context_dump_success(response).expect_err("unauthorized response should fail"); + let message = error.to_string(); + + assert!(message.contains("authentication")); + assert!(message.contains("401")); + assert!(!message.contains("rejected")); + assert!(!message.contains("exact-token")); + } + + #[test] + fn dogstatsd_contexts_server_errors_include_status_and_body() { + for status in [ + StatusCode::INTERNAL_SERVER_ERROR, + StatusCode::SERVICE_UNAVAILABLE, + StatusCode::GATEWAY_TIMEOUT, + ] { + let response = Response::builder() + .status(status) + .body("dump unavailable".to_string()) + .expect("valid response"); + + let error = path_when_context_dump_success(response).expect_err("server error should fail"); + let message = error.to_string(); + + assert!(message.contains(status.as_str()), "missing status in: {message}"); + assert!( + message.contains("dump unavailable"), + "missing response body in: {message}" + ); + } + } + + fn write_generated_ipc_certificate(path: &std::path::Path) { + let CertifiedKey { cert, signing_key } = + generate_simple_self_signed(["localhost".to_string()]).expect("certificate generation should succeed"); + std::fs::write(path, format!("{}{}", cert.pem(), signing_key.serialize_pem())) + .expect("certificate file should be written"); + } + + async fn start_context_dump_tls_server( + certificate_path: &std::path::Path, + ) -> ( + std::net::SocketAddr, + tokio::task::JoinHandle, Box>>, + ) { + let server_config = build_ipc_server_tls_config(certificate_path) + .await + .expect("server TLS configuration should build"); + let acceptor = TlsAcceptor::from(Arc::new(server_config)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("test listener should bind"); + let address = listener.local_addr().expect("test listener should have an address"); + let task = tokio::spawn(async move { + let (stream, _) = listener.accept().await?; + let mut stream = match acceptor.accept(stream).await { + Ok(stream) => stream, + Err(error) => return Ok(Some(error.to_string())), + }; + let mut request = Vec::new(); + loop { + let mut buffer = [0_u8; 1024]; + let read = stream.read(&mut buffer).await?; + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let request = String::from_utf8(request)?; + assert!(request.starts_with("POST /dogstatsd/contexts/dump HTTP/1.1\r\n")); + assert!( + request + .lines() + .any(|line| line.eq_ignore_ascii_case("authorization: Bearer exact-token")), + "request must contain the exact bearer header" + ); + + let body = r#""/tmp/dogstatsd_contexts.json.zstd""#; + let response = format!( + "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ); + stream.write_all(response.as_bytes()).await?; + stream.shutdown().await?; + Ok(None) + }); + + (address, task) + } + + fn authenticated_test_client( + address: std::net::SocketAddr, tls_config: rustls::ClientConfig, + ) -> DataPlaneAPIClient { + let mut authorization = HeaderValue::from_static("Bearer exact-token"); + authorization.set_sensitive(true); + let client = saluki_io::net::client::http::HttpClient::builder() + .with_client_tls_config(tls_config) + .with_connect_timeout(Duration::from_secs(2)) + .without_request_timeout() + .build() + .expect("test HTTP client should build"); + DataPlaneAPIClient { + client, + authority: format!("localhost:{}", address.port()), + authorization: Some(authorization), + } + } + + #[tokio::test] + async fn dogstatsd_contexts_certificate_pinning_accepts_match_and_rejects_mismatch() { + tokio::time::timeout(Duration::from_secs(10), async { + let _ = saluki_tls::initialize_default_crypto_provider(); + let directory = tempfile::tempdir().expect("temporary directory should be created"); + let certificate_a = directory.path().join("ipc-a.pem"); + let certificate_b = directory.path().join("ipc-b.pem"); + write_generated_ipc_certificate(&certificate_a); + write_generated_ipc_certificate(&certificate_b); + + let (matching_address, matching_server) = start_context_dump_tls_server(&certificate_a).await; + let matching_tls = build_ipc_client_ipc_tls_config(&certificate_a) + .await + .expect("matching client TLS configuration should build"); + let mut matching_client = authenticated_test_client(matching_address, matching_tls); + let path = matching_client + .dogstatsd_contexts_dump() + .await + .expect("matching pinned certificate should succeed"); + assert_eq!(path, PathBuf::from("/tmp/dogstatsd_contexts.json.zstd")); + assert!(matching_server + .await + .expect("matching server task should join") + .expect("matching server should complete") + .is_none()); + + let (mismatched_address, mismatched_server) = start_context_dump_tls_server(&certificate_a).await; + let mismatched_tls = build_ipc_client_ipc_tls_config(&certificate_b) + .await + .expect("mismatched client TLS configuration should build"); + let mut mismatched_client = authenticated_test_client(mismatched_address, mismatched_tls); + let error = mismatched_client + .dogstatsd_contexts_dump() + .await + .expect_err("mismatched pinned certificate must fail"); + let chain = format!("{error:#}").to_ascii_lowercase(); + assert!( + chain.contains("certificate") || chain.contains("unknownissuer") || chain.contains("unknown issuer"), + "expected a certificate verification failure, got: {error:#}" + ); + assert!( + mismatched_server + .await + .expect("mismatched server task should join") + .expect("mismatched server should complete") + .is_some(), + "the HTTP server must not receive a request after TLS verification fails" + ); + }) + .await + .expect("certificate pinning test should finish within its timeout"); + } } diff --git a/lib/saluki-io/src/net/client/http/client.rs b/lib/saluki-io/src/net/client/http/client.rs index 31ad4067239..9f0f81c53c0 100644 --- a/lib/saluki-io/src/net/client/http/client.rs +++ b/lib/saluki-io/src/net/client/http/client.rs @@ -19,9 +19,10 @@ use hyper_util::{ }; use metrics::Counter; use pin_project::pin_project; +use rustls::ClientConfig; use saluki_error::GenericError; use saluki_metrics::MetricsBuilder; -use saluki_tls::{ClientTLSConfigBuilder, TlsMinimumVersion}; +use saluki_tls::{ensure_client_config_fips_compliant, ClientTLSConfigBuilder, TlsMinimumVersion}; use stringtheory::MetaString; use tower::{timeout::TimeoutLayer, util::BoxCloneService, BoxError, Service, ServiceBuilder, ServiceExt as _}; @@ -165,6 +166,7 @@ pub struct HttpClientBuilder { connector_builder: HttpsCapableConnectorBuilder, hyper_builder: Builder, tls_builder: ClientTLSConfigBuilder, + client_tls_config: Option, request_timeout: Option, endpoint_telemetry: Option, proxies: Option>, @@ -283,9 +285,19 @@ impl HttpClientBuilder { self } + /// Sets a complete client TLS configuration. + /// + /// The supplied configuration takes precedence over all options applied through [`Self::with_tls_config`] and + /// [`Self::with_min_tls_version`], regardless of call order. + pub fn with_client_tls_config(mut self, config: ClientConfig) -> Self { + self.client_tls_config = Some(config); + self + } + /// Sets the TLS configuration. /// - /// A TLS configuration builder is provided to allow for more advanced configuration of the TLS connection. + /// A TLS configuration builder is provided to allow for more advanced configuration of the TLS connection. These + /// options are ignored when a complete configuration is supplied through [`Self::with_client_tls_config`]. pub fn with_tls_config(mut self, f: F) -> Self where F: FnOnce(ClientTLSConfigBuilder) -> ClientTLSConfigBuilder, @@ -299,7 +311,8 @@ impl HttpClientBuilder { /// Defaults to TLS 1.2. /// /// This updates the same TLS builder configured by [`Self::with_tls_config`], so call order matters when both - /// methods change the minimum TLS version. + /// methods change the minimum TLS version. This option is ignored when a complete configuration is supplied + /// through [`Self::with_client_tls_config`]. pub fn with_min_tls_version(mut self, version: TlsMinimumVersion) -> Self { self.tls_builder = self.tls_builder.with_min_tls_version(version); self @@ -332,9 +345,15 @@ impl HttpClientBuilder { /// /// # Errors /// - /// If there was an error building the TLS configuration for the client, an error will be returned. + /// If there was an error building or validating the TLS configuration for the client, an error will be returned. pub fn build(self) -> Result { - let tls_config = self.tls_builder.build()?; + let tls_config = match self.client_tls_config { + Some(config) => { + ensure_client_config_fips_compliant(&config)?; + config + } + None => self.tls_builder.build()?, + }; let connector = self.connector_builder.build(tls_config)?; // TODO(fips): Look into updating `hyper-http-proxy` to use the provided connector for establishing the // connection to the proxy itself, even when the proxy is at an HTTPS URL, to ensure our desired TLS stack is @@ -369,6 +388,7 @@ impl Default for HttpClientBuilder { connector_builder: HttpsCapableConnectorBuilder::default(), hyper_builder, tls_builder: ClientTLSConfigBuilder::new(), + client_tls_config: None, request_timeout: Some(Duration::from_secs(20)), endpoint_telemetry: None, proxies: None, @@ -390,4 +410,34 @@ mod tests { assert_eq!(Some(5), converted.size_hint().exact()); } + + #[cfg(not(windows))] + fn test_crypto_provider() -> rustls::crypto::CryptoProvider { + rustls::crypto::aws_lc_rs::default_provider() + } + + #[cfg(windows)] + fn test_crypto_provider() -> rustls::crypto::CryptoProvider { + rustls_cng_crypto::default_provider() + } + + #[test] + fn accepts_complete_client_tls_configuration() { + let tls_config = rustls::ClientConfig::builder_with_provider(test_crypto_provider().into()) + .with_safe_default_protocol_versions() + .expect("default protocol versions should be valid") + .with_root_certificates(rustls::RootCertStore::empty()) + .with_no_client_auth(); + + HttpClient::builder() + .with_client_tls_config(tls_config.clone()) + .with_min_tls_version(TlsMinimumVersion::Tls13) + .build() + .expect("a complete TLS configuration should take precedence over later builder options"); + HttpClient::builder() + .with_min_tls_version(TlsMinimumVersion::Tls13) + .with_client_tls_config(tls_config) + .build() + .expect("a complete TLS configuration should take precedence over earlier builder options"); + } } From 80d58316e02b21e1d6fa86c8c888df1564a653f3 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 15:50:55 -0400 Subject: [PATCH 14/44] fix(io): normalize supplied HTTP ALPN Clear preconfigured ALPN values from complete client TLS configurations before constructing the hyper-rustls connector. This prevents its non-empty ALPN panic and keeps HTTP protocol advertisement owned by HttpClientBuilder. --- lib/saluki-io/src/net/client/http/client.rs | 30 ++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/lib/saluki-io/src/net/client/http/client.rs b/lib/saluki-io/src/net/client/http/client.rs index 9f0f81c53c0..04ea191dc8d 100644 --- a/lib/saluki-io/src/net/client/http/client.rs +++ b/lib/saluki-io/src/net/client/http/client.rs @@ -288,7 +288,9 @@ impl HttpClientBuilder { /// Sets a complete client TLS configuration. /// /// The supplied configuration takes precedence over all options applied through [`Self::with_tls_config`] and - /// [`Self::with_min_tls_version`], regardless of call order. + /// [`Self::with_min_tls_version`], regardless of call order. All supplied settings are preserved except ALPN: + /// [`ClientConfig::alpn_protocols`] is cleared before connector construction so [`Self::with_http_protocol`] + /// selects the advertised protocols. By default, [`HttpProtocol::Auto`] advertises HTTP/2 with HTTP/1.1 fallback. pub fn with_client_tls_config(mut self, config: ClientConfig) -> Self { self.client_tls_config = Some(config); self @@ -348,8 +350,9 @@ impl HttpClientBuilder { /// If there was an error building or validating the TLS configuration for the client, an error will be returned. pub fn build(self) -> Result { let tls_config = match self.client_tls_config { - Some(config) => { + Some(mut config) => { ensure_client_config_fips_compliant(&config)?; + config.alpn_protocols.clear(); config } None => self.tls_builder.build()?, @@ -421,13 +424,17 @@ mod tests { rustls_cng_crypto::default_provider() } - #[test] - fn accepts_complete_client_tls_configuration() { - let tls_config = rustls::ClientConfig::builder_with_provider(test_crypto_provider().into()) + fn empty_client_tls_config() -> rustls::ClientConfig { + rustls::ClientConfig::builder_with_provider(test_crypto_provider().into()) .with_safe_default_protocol_versions() .expect("default protocol versions should be valid") .with_root_certificates(rustls::RootCertStore::empty()) - .with_no_client_auth(); + .with_no_client_auth() + } + + #[test] + fn accepts_complete_client_tls_configuration() { + let tls_config = empty_client_tls_config(); HttpClient::builder() .with_client_tls_config(tls_config.clone()) @@ -440,4 +447,15 @@ mod tests { .build() .expect("a complete TLS configuration should take precedence over earlier builder options"); } + + #[test] + fn accepts_complete_client_tls_configuration_with_alpn() { + let mut tls_config = empty_client_tls_config(); + tls_config.alpn_protocols = vec![b"h2".to_vec()]; + + HttpClient::builder() + .with_client_tls_config(tls_config) + .build() + .expect("the HTTP client should normalize ALPN from a supplied TLS configuration"); + } } From d0cd029af7e9412daa223279a3f9fc37897058ac Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 16:08:27 -0400 Subject: [PATCH 15/44] feat(adp): add DogStatsD top commands Support authenticated online dump-and-analyze, offline Agent artifact analysis, and dump-only workflows with compatible flags, ordering, limits, and output text. --- bin/agent-data-plane/src/cli/dogstatsd.rs | 70 ++- bin/agent-data-plane/src/cli/dogstatsd/top.rs | 432 ++++++++++++++++++ bin/agent-data-plane/src/cli/utils.rs | 11 - .../src/dogstatsd_contexts/mod.rs | 12 +- .../src/dogstatsd_contexts/report.rs | 4 +- 5 files changed, 488 insertions(+), 41 deletions(-) create mode 100644 bin/agent-data-plane/src/cli/dogstatsd/top.rs diff --git a/bin/agent-data-plane/src/cli/dogstatsd.rs b/bin/agent-data-plane/src/cli/dogstatsd.rs index 2896b4d917c..8e814d468a1 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd.rs @@ -34,6 +34,9 @@ use tracing::{error, info}; use crate::cli::utils::DataPlaneAPIClient; +mod top; +use self::top::{handle_dogstatsd_dump_contexts, handle_dogstatsd_top, DumpContextsCommand, TopCommand}; + /// DogStatsD-specific debugging commands. #[derive(FromArgs, Debug)] #[argh(subcommand, name = "dogstatsd")] @@ -48,6 +51,8 @@ enum DogstatsdSubcommand { Stats(StatsCommand), Capture(CaptureCommand), Replay(ReplayCommand), + Top(TopCommand), + DumpContexts(DumpContextsCommand), } /// Prints basic statistics about the metrics received by the data plane. @@ -168,34 +173,38 @@ struct StatsResponse<'a> { /// Entrypoint for the `dogstatsd` commands. pub async fn handle_dogstatsd_command(bootstrap_config: &GenericConfiguration, cmd: DogstatsdCommand) { - let mut api_client = match DataPlaneAPIClient::from_config(bootstrap_config) { - Ok(client) => client, - Err(e) => { - error!("Failed to create data plane API client: {:#}", e); - std::process::exit(1); - } - }; + if let Err(error) = run_dogstatsd_command(bootstrap_config, cmd).await { + error!("{:#}", error); + std::process::exit(1); + } +} +async fn run_dogstatsd_command( + bootstrap_config: &GenericConfiguration, cmd: DogstatsdCommand, +) -> Result<(), GenericError> { match cmd.subcommand { DogstatsdSubcommand::Stats(config) => { - if let Err(e) = handle_dogstatsd_stats(&mut api_client, config).await { - error!("Failed to run stats subcommand: {:#}", e); - std::process::exit(1); - } + let mut api_client = DataPlaneAPIClient::from_config(bootstrap_config) + .error_context("Failed to create data plane API client")?; + handle_dogstatsd_stats(&mut api_client, config) + .await + .error_context("Failed to run stats subcommand") } DogstatsdSubcommand::Capture(config) => { - if let Err(e) = handle_dogstatsd_capture(&mut api_client, config).await { - error!("Failed to start DogStatsD capture: {:#}", e); - std::process::exit(1); - } + let mut api_client = DataPlaneAPIClient::from_config(bootstrap_config) + .error_context("Failed to create data plane API client")?; + handle_dogstatsd_capture(&mut api_client, config) + .await + .error_context("Failed to start DogStatsD capture") } DogstatsdSubcommand::Replay(config) => { #[cfg(target_os = "linux")] { - if let Err(e) = handle_dogstatsd_replay(&mut api_client, bootstrap_config, config).await { - error!("Failed to replay DogStatsD traffic: {:#}", e); - std::process::exit(1); - } + let mut api_client = DataPlaneAPIClient::from_config(bootstrap_config) + .error_context("Failed to create data plane API client")?; + handle_dogstatsd_replay(&mut api_client, bootstrap_config, config) + .await + .error_context("Failed to replay DogStatsD traffic") } #[cfg(not(target_os = "linux"))] @@ -205,10 +214,29 @@ pub async fn handle_dogstatsd_command(bootstrap_config: &GenericConfiguration, c loops, } = config; let _ = (replay_file_path, loops); - error!("DogStatsD replay is only supported on Linux."); - std::process::exit(1); + Err(saluki_error::generic_error!( + "DogStatsD replay is only supported on Linux." + )) + } + } + DogstatsdSubcommand::Top(config) => { + let mut output = std::io::stdout(); + if config.is_offline() { + handle_dogstatsd_top(None, config, &mut output).await + } else { + let mut api_client = DataPlaneAPIClient::from_config_with_ipc_auth(bootstrap_config) + .await + .error_context("Failed to create authenticated data plane API client for DogStatsD top.")?; + handle_dogstatsd_top(Some(&mut api_client), config, &mut output).await } } + DogstatsdSubcommand::DumpContexts(_) => { + let mut api_client = DataPlaneAPIClient::from_config_with_ipc_auth(bootstrap_config) + .await + .error_context("Failed to create authenticated data plane API client for DogStatsD context dump.")?; + let mut output = std::io::stdout(); + handle_dogstatsd_dump_contexts(&mut api_client, &mut output).await + } } } diff --git a/bin/agent-data-plane/src/cli/dogstatsd/top.rs b/bin/agent-data-plane/src/cli/dogstatsd/top.rs new file mode 100644 index 00000000000..1629bbd0b50 --- /dev/null +++ b/bin/agent-data-plane/src/cli/dogstatsd/top.rs @@ -0,0 +1,432 @@ +use std::io::Write; +use std::path::{Path, PathBuf}; + +use argh::FromArgs; +use async_trait::async_trait; +use saluki_error::{ErrorContext as _, GenericError}; + +use crate::cli::utils::DataPlaneAPIClient; +use crate::dogstatsd_contexts::read_report; + +/// Displays the DogStatsD contexts with the highest cardinality. +#[derive(FromArgs, Debug)] +#[argh(subcommand, name = "top")] +pub(super) struct TopCommand { + /// read a context dump artifact instead of requesting a new dump + #[argh(option, short = 'p', long = "path")] + path: Option, + + /// set the maximum number of metrics to display + #[argh(option, short = 'm', long = "num-metrics", default = "10")] + num_metrics: usize, + + /// set the maximum number of tags to display for each metric + #[argh(option, short = 't', long = "num-tags")] + num_tags: Option, + + /// use the legacy `--mum-tags` typo for `--num-tags` + #[argh(option, long = "mum-tags")] + legacy_num_tags: Option, +} + +impl TopCommand { + pub(super) fn is_offline(&self) -> bool { + self.path.is_some() + } + + fn effective_num_tags(&self) -> Result { + match (self.num_tags, self.legacy_num_tags) { + (Some(_), Some(_)) => Err(saluki_error::generic_error!( + "Cannot use `--num-tags` and legacy `--mum-tags` together; use `--num-tags`." + )), + (Some(limit), None) | (None, Some(limit)) => Ok(limit), + (None, None) => Ok(5), + } + } +} + +/// Writes a DogStatsD context dump artifact. +#[derive(FromArgs, Debug)] +#[argh(subcommand, name = "dump-contexts")] +pub(super) struct DumpContextsCommand {} + +#[async_trait(?Send)] +pub(super) trait DogStatsDContextDumpRequester { + async fn request_context_dump(&mut self) -> Result; +} + +#[async_trait(?Send)] +impl DogStatsDContextDumpRequester for DataPlaneAPIClient { + async fn request_context_dump(&mut self) -> Result { + self.dogstatsd_contexts_dump().await + } +} + +pub(super) async fn handle_dogstatsd_top( + requester: Option<&mut dyn DogStatsDContextDumpRequester>, cmd: TopCommand, output: &mut dyn Write, +) -> Result<(), GenericError> { + let num_tags = cmd.effective_num_tags()?; + let path = match cmd.path { + Some(path) => path, + None => { + let requester = requester.ok_or_else(|| { + saluki_error::generic_error!("Online DogStatsD top requires a context dump requester.") + })?; + let path = requester + .request_context_dump() + .await + .error_context("Failed to request a DogStatsD context dump.")?; + write_dump_path(output, &path)?; + path + } + }; + + let report = read_report(&path) + .with_error_context(|| format!("Failed to read DogStatsD context report from '{}'.", path.display()))?; + let rendered = report.render(cmd.num_metrics, num_tags); + output + .write_all(rendered.as_bytes()) + .with_error_context(|| format!("Failed to write DogStatsD context report for '{}'.", path.display()))?; + output + .flush() + .with_error_context(|| format!("Failed to flush DogStatsD context report for '{}'.", path.display()))?; + Ok(()) +} + +pub(super) async fn handle_dogstatsd_dump_contexts( + requester: &mut dyn DogStatsDContextDumpRequester, output: &mut dyn Write, +) -> Result<(), GenericError> { + let path = requester + .request_context_dump() + .await + .error_context("Failed to request a DogStatsD context dump.")?; + write_dump_path(output, &path) +} + +fn write_dump_path(output: &mut dyn Write, path: &Path) -> Result<(), GenericError> { + writeln!(output, "Wrote {}", path.display()) + .with_error_context(|| format!("Failed to write the DogStatsD context dump path '{}'.", path.display()))?; + output + .flush() + .with_error_context(|| format!("Failed to flush the DogStatsD context dump path '{}'.", path.display()))?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::io; + use std::path::{Path, PathBuf}; + + use argh::FromArgs as _; + use async_trait::async_trait; + use saluki_error::{generic_error, GenericError}; + + use super::{handle_dogstatsd_dump_contexts, handle_dogstatsd_top, DogStatsDContextDumpRequester, TopCommand}; + use crate::cli::dogstatsd::{DogstatsdCommand, DogstatsdSubcommand}; + + const PLAIN_FIXTURE: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson" + ); + const GOLDEN_REPORT: &str = concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 3\ta.metric\t(2 env, 1 service)\n", + " 1\tz.metric\t(1 bare, 1 env, 1 image)\n", + ); + + #[test] + fn dogstatsd_top_parses_defaults() { + let command = parse_dogstatsd(&["top"]).expect("top defaults should parse"); + let DogstatsdSubcommand::Top(top) = command.subcommand else { + panic!("expected top subcommand"); + }; + + assert_eq!(top.path, None); + assert_eq!(top.num_metrics, 10); + assert_eq!(top.num_tags, None); + assert_eq!(top.legacy_num_tags, None); + assert_eq!(top.effective_num_tags().unwrap(), 5); + } + + #[test] + fn dogstatsd_top_parses_all_short_flags() { + let command = + parse_dogstatsd(&["top", "-p", PLAIN_FIXTURE, "-m", "7", "-t", "3"]).expect("top short flags should parse"); + let DogstatsdSubcommand::Top(top) = command.subcommand else { + panic!("expected top subcommand"); + }; + + assert_eq!(top.path, Some(PathBuf::from(PLAIN_FIXTURE))); + assert_eq!(top.num_metrics, 7); + assert_eq!(top.num_tags, Some(3)); + assert_eq!(top.legacy_num_tags, None); + assert_eq!(top.effective_num_tags().unwrap(), 3); + } + + #[test] + fn dogstatsd_top_parses_corrected_long_num_tags() { + let command = parse_dogstatsd(&["top", "--num-tags", "4"]).expect("corrected option should parse"); + let DogstatsdSubcommand::Top(top) = command.subcommand else { + panic!("expected top subcommand"); + }; + + assert_eq!(top.num_tags, Some(4)); + assert_eq!(top.legacy_num_tags, None); + assert_eq!(top.effective_num_tags().unwrap(), 4); + } + + #[test] + fn dogstatsd_top_parses_legacy_mum_tags() { + let command = parse_dogstatsd(&["top", "--mum-tags", "6"]).expect("legacy option should parse"); + let DogstatsdSubcommand::Top(top) = command.subcommand else { + panic!("expected top subcommand"); + }; + + assert_eq!(top.num_tags, None); + assert_eq!(top.legacy_num_tags, Some(6)); + assert_eq!(top.effective_num_tags().unwrap(), 6); + } + + #[test] + fn dogstatsd_top_rejects_both_num_tags_spellings() { + let command = parse_dogstatsd(&["top", "--num-tags", "4", "--mum-tags", "6"]) + .expect("each spelling should parse before conflict validation"); + let DogstatsdSubcommand::Top(top) = command.subcommand else { + panic!("expected top subcommand"); + }; + + assert_eq!( + top.effective_num_tags().unwrap_err().to_string(), + "Cannot use `--num-tags` and legacy `--mum-tags` together; use `--num-tags`." + ); + } + + #[test] + fn dogstatsd_top_rejects_negative_limits_and_extra_arguments() { + for args in [ + &["top", "--num-metrics", "-1"][..], + &["top", "--num-tags", "-1"][..], + &["top", "--mum-tags", "-1"][..], + &["top", "artifact.ndjson"][..], + ] { + assert!(parse_dogstatsd(args).is_err(), "arguments should be rejected: {args:?}"); + } + } + + #[test] + fn dogstatsd_dump_contexts_parses_without_arguments_and_rejects_extras() { + let command = parse_dogstatsd(&["dump-contexts"]).expect("dump-contexts should parse"); + assert!(matches!(command.subcommand, DogstatsdSubcommand::DumpContexts(_))); + + let error = parse_dogstatsd(&["dump-contexts", "extra"]).expect_err("extra argument should fail"); + assert!( + error.output.contains("Unrecognized argument: extra"), + "{}", + error.output + ); + } + + #[tokio::test] + async fn dogstatsd_top_reads_an_offline_fixture_without_triggering_a_dump() { + let command = top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 10, None, None); + let mut requester = FakeRequester::returning_path("unused"); + let mut output = RecordingWriter::default(); + + handle_dogstatsd_top(Some(&mut requester), command, &mut output) + .await + .expect("offline top should succeed"); + + assert_eq!(requester.calls, 0); + assert_eq!(output.text(), GOLDEN_REPORT); + } + + #[tokio::test] + async fn dogstatsd_top_triggers_one_online_dump_before_rendering_the_report() { + let mut requester = FakeRequester::returning_path(PLAIN_FIXTURE); + let mut output = RecordingWriter::default(); + + handle_dogstatsd_top(Some(&mut requester), top_command(None, 10, None, None), &mut output) + .await + .expect("online top should succeed"); + + assert_eq!(requester.calls, 1); + assert_eq!(output.text(), format!("Wrote {PLAIN_FIXTURE}\n{GOLDEN_REPORT}")); + assert_eq!( + output.flushes.first().unwrap(), + format!("Wrote {PLAIN_FIXTURE}\n").as_bytes() + ); + } + + #[tokio::test] + async fn dogstatsd_dump_contexts_triggers_once_and_prints_only_the_path() { + let mut requester = FakeRequester::returning_path(PLAIN_FIXTURE); + let mut output = RecordingWriter::default(); + + handle_dogstatsd_dump_contexts(&mut requester, &mut output) + .await + .expect("dump-contexts should succeed"); + + assert_eq!(requester.calls, 1); + assert_eq!(output.text(), format!("Wrote {PLAIN_FIXTURE}\n")); + assert_eq!(output.flushes, vec![format!("Wrote {PLAIN_FIXTURE}\n").into_bytes()]); + } + + #[tokio::test] + async fn dogstatsd_dump_contexts_prints_nothing_when_transport_fails() { + let mut requester = FakeRequester::returning_error("injected dump transport failure"); + let mut output = RecordingWriter::default(); + + let error = handle_dogstatsd_dump_contexts(&mut requester, &mut output) + .await + .expect_err("transport failure should propagate"); + + assert_eq!(requester.calls, 1); + assert!(format!("{error:#}").contains("injected dump transport failure")); + assert_eq!(output.text(), ""); + assert!(output.flushes.is_empty()); + } + + #[tokio::test] + async fn dogstatsd_top_keeps_the_flushed_path_when_online_artifact_parsing_fails() { + let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); + std::fs::write(artifact.path(), b"not-json").expect("corrupt artifact should be written"); + let mut requester = FakeRequester::returning_path(artifact.path()); + let mut output = RecordingWriter::default(); + + let error = handle_dogstatsd_top(Some(&mut requester), top_command(None, 10, None, None), &mut output) + .await + .expect_err("corrupt artifact should fail"); + + let wrote_line = format!("Wrote {}\n", artifact.path().display()); + assert_eq!(requester.calls, 1); + assert_eq!(output.text(), wrote_line); + assert_eq!(output.flushes, vec![wrote_line.into_bytes()]); + let error_chain = format!("{error:#}"); + assert!(error_chain.contains("DogStatsD context report"), "{error_chain}"); + assert!(error_chain.contains("decode record 1"), "{error_chain}"); + } + + #[tokio::test] + async fn dogstatsd_top_renders_only_the_heading_for_an_empty_artifact() { + let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); + let mut output = RecordingWriter::default(); + + handle_dogstatsd_top( + None, + top_command(Some(artifact.path().to_owned()), 10, None, None), + &mut output, + ) + .await + .expect("empty artifact should render"); + + assert_eq!( + output.text(), + " Contexts\tMetric name\t(number of unique values for each tag)\n" + ); + } + + #[tokio::test] + async fn dogstatsd_top_applies_custom_and_zero_report_limits() { + let cases = [ + ( + top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 0, Some(5), None), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 4\t(other 2 metrics)\n", + ), + ), + ( + top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 10, Some(0), None), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 3\ta.metric\t(3 values in 2 other tags)\n", + " 1\tz.metric\t(3 values in 3 other tags)\n", + ), + ), + ( + top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 1, Some(1), None), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 3\ta.metric\t(2 env, 1 service)\n", + " 1\tz.metric\t(1 bare, 2 values in 2 other tags)\n", + ), + ), + ]; + + for (command, expected) in cases { + let mut output = RecordingWriter::default(); + handle_dogstatsd_top(None, command, &mut output) + .await + .expect("offline report should render"); + assert_eq!(output.text(), expected); + } + } + + fn parse_dogstatsd(args: &[&str]) -> Result { + DogstatsdCommand::from_args(&["agent-data-plane", "dogstatsd"], args) + } + + fn top_command( + path: Option, num_metrics: usize, num_tags: Option, legacy_num_tags: Option, + ) -> TopCommand { + TopCommand { + path, + num_metrics, + num_tags, + legacy_num_tags, + } + } + + struct FakeRequester { + calls: usize, + response: Option>, + } + + impl FakeRequester { + fn returning_path(path: impl AsRef) -> Self { + Self { + calls: 0, + response: Some(Ok(path.as_ref().to_owned())), + } + } + + fn returning_error(message: &'static str) -> Self { + Self { + calls: 0, + response: Some(Err(generic_error!(message))), + } + } + } + + #[async_trait(?Send)] + impl DogStatsDContextDumpRequester for FakeRequester { + async fn request_context_dump(&mut self) -> Result { + self.calls += 1; + self.response.take().expect("fake requester called more than once") + } + } + + #[derive(Default)] + struct RecordingWriter { + bytes: Vec, + flushes: Vec>, + } + + impl RecordingWriter { + fn text(&self) -> String { + String::from_utf8(self.bytes.clone()).expect("test output should be UTF-8") + } + } + + impl io::Write for RecordingWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + self.bytes.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.flushes.push(self.bytes.clone()); + Ok(()) + } + } +} diff --git a/bin/agent-data-plane/src/cli/utils.rs b/bin/agent-data-plane/src/cli/utils.rs index 1eb2b5aa873..7e8a1569de0 100644 --- a/bin/agent-data-plane/src/cli/utils.rs +++ b/bin/agent-data-plane/src/cli/utils.rs @@ -32,7 +32,6 @@ use crate::config::DataPlaneConfiguration; pub struct DataPlaneAPIClient { client: HttpClient, authority: String, - #[allow(dead_code, reason = "stored for authenticated privileged API commands")] authorization: Option, } @@ -81,10 +80,6 @@ impl DataPlaneAPIClient { /// If the data plane or IPC authentication configuration is invalid, the IPC certificate or authentication token /// cannot be read, the token cannot be represented as an HTTP header, or the HTTP client cannot be built, an error /// is returned. - #[allow( - dead_code, - reason = "public constructor reserved for authenticated privileged API commands" - )] pub async fn from_config_with_ipc_auth(config: &GenericConfiguration) -> Result { let dp_config = DataPlaneConfiguration::from_configuration(config)?; let ipc_config = IpcAuthConfiguration::from_configuration(config)?; @@ -260,10 +255,6 @@ impl DataPlaneAPIClient { /// /// If this client was not created with [`Self::from_config_with_ipc_auth`], the request fails, the server rejects /// the request, or the successful response does not contain a JSON string path, an error is returned. - #[allow( - dead_code, - reason = "public operation reserved for authenticated context dump commands" - )] pub async fn dogstatsd_contexts_dump(&mut self) -> Result { let authorization = self.authorization.as_ref().ok_or_else(|| { generic_error!( @@ -440,7 +431,6 @@ fn authorization_from_token_file_contents(raw_token: &[u8], token_path: &Path) - Ok(authorization) } -#[allow(dead_code, reason = "request helper for the authenticated context dump operation")] fn build_dogstatsd_contexts_dump_request(uri: Uri, authorization: &HeaderValue) -> Request { Request::post(uri) .header(AUTHORIZATION, authorization.clone()) @@ -448,7 +438,6 @@ fn build_dogstatsd_contexts_dump_request(uri: Uri, authorization: &HeaderValue) .expect("valid DogStatsD context dump request") } -#[allow(dead_code, reason = "response helper for the authenticated context dump operation")] fn path_when_context_dump_success(resp: Response) -> Result { match resp.status() { status if status.is_success() => serde_json::from_str::(resp.body()) diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs index 45d51b94cfb..fc591b61ace 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs @@ -1,24 +1,22 @@ use std::path::Path; +use saluki_error::GenericError; + #[allow( unused_imports, reason = "registered with the privileged API by a subsequent feature task" )] pub(crate) use self::api::DogStatsDContextDumpAPIHandler; pub(crate) use self::artifact::publish_context_dump; -use self::artifact::ArtifactError; -#[allow( - unused_imports, - reason = "used by the DogStatsD CLI and topology wiring in subsequent feature tasks" -)] +#[allow(unused_imports, reason = "used by topology wiring in a subsequent feature task")] pub(crate) use self::artifact::CONTEXT_DUMP_FILENAME; -use self::report::ContextReport; +pub(crate) use self::report::ContextReport; mod api; mod artifact; mod report; -fn read_report(path: &Path) -> Result { +pub(crate) fn read_report(path: &Path) -> Result { let mut report = ContextReport::new(); artifact::for_each_record(path, |record| report.ingest(record))?; Ok(report) diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/report.rs b/bin/agent-data-plane/src/dogstatsd_contexts/report.rs index 58ccb225852..b2b96eed6b0 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/report.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/report.rs @@ -6,7 +6,7 @@ use super::artifact::AgentContextRecord; const HEADING: &str = " Contexts\tMetric name\t(number of unique values for each tag)\n"; #[derive(Default)] -pub(super) struct ContextReport { +pub(crate) struct ContextReport { metrics: HashMap, } @@ -30,7 +30,7 @@ impl ContextReport { summary.metric_tags.extend(metric_tags); } - pub(super) fn render(&self, metric_limit: usize, tag_limit: usize) -> String { + pub(crate) fn render(&self, metric_limit: usize, tag_limit: usize) -> String { let mut output = String::from(HEADING); let mut metrics: Vec<_> = self.metrics.iter().collect(); metrics.sort_unstable_by(|(left_name, left), (right_name, right)| { From dff7a9f06353650e325c41fda76b904532ec4618 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 16:58:39 -0400 Subject: [PATCH 16/44] fix(adp): redact context artifact errors Sanitize JSON decode failures to retain only safe path, record, category, and location diagnostics so malformed artifact values cannot enter CLI error chains. Consume and validate DogStatsD top options before constructing the authenticated client so conflicting tag flags fail locally. --- bin/agent-data-plane/src/cli/dogstatsd.rs | 1 + bin/agent-data-plane/src/cli/dogstatsd/top.rs | 100 ++++++++++++++---- .../src/dogstatsd_contexts/artifact.rs | 68 +++++++++++- 3 files changed, 147 insertions(+), 22 deletions(-) diff --git a/bin/agent-data-plane/src/cli/dogstatsd.rs b/bin/agent-data-plane/src/cli/dogstatsd.rs index 8e814d468a1..a6261945aac 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd.rs @@ -220,6 +220,7 @@ async fn run_dogstatsd_command( } } DogstatsdSubcommand::Top(config) => { + let config = config.validate()?; let mut output = std::io::stdout(); if config.is_offline() { handle_dogstatsd_top(None, config, &mut output).await diff --git a/bin/agent-data-plane/src/cli/dogstatsd/top.rs b/bin/agent-data-plane/src/cli/dogstatsd/top.rs index 1629bbd0b50..c7472dca891 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd/top.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd/top.rs @@ -30,18 +30,35 @@ pub(super) struct TopCommand { } impl TopCommand { - pub(super) fn is_offline(&self) -> bool { - self.path.is_some() + pub(super) fn validate(self) -> Result { + let num_tags = match (self.num_tags, self.legacy_num_tags) { + (Some(_), Some(_)) => { + return Err(saluki_error::generic_error!( + "Cannot use `--num-tags` and legacy `--mum-tags` together; use `--num-tags`." + )); + } + (Some(limit), None) | (None, Some(limit)) => limit, + (None, None) => 5, + }; + + Ok(ValidatedTopCommand { + path: self.path, + num_metrics: self.num_metrics, + num_tags, + }) } +} - fn effective_num_tags(&self) -> Result { - match (self.num_tags, self.legacy_num_tags) { - (Some(_), Some(_)) => Err(saluki_error::generic_error!( - "Cannot use `--num-tags` and legacy `--mum-tags` together; use `--num-tags`." - )), - (Some(limit), None) | (None, Some(limit)) => Ok(limit), - (None, None) => Ok(5), - } +#[derive(Debug)] +pub(super) struct ValidatedTopCommand { + path: Option, + num_metrics: usize, + num_tags: usize, +} + +impl ValidatedTopCommand { + pub(super) fn is_offline(&self) -> bool { + self.path.is_some() } } @@ -63,9 +80,8 @@ impl DogStatsDContextDumpRequester for DataPlaneAPIClient { } pub(super) async fn handle_dogstatsd_top( - requester: Option<&mut dyn DogStatsDContextDumpRequester>, cmd: TopCommand, output: &mut dyn Write, + requester: Option<&mut dyn DogStatsDContextDumpRequester>, cmd: ValidatedTopCommand, output: &mut dyn Write, ) -> Result<(), GenericError> { - let num_tags = cmd.effective_num_tags()?; let path = match cmd.path { Some(path) => path, None => { @@ -83,7 +99,7 @@ pub(super) async fn handle_dogstatsd_top( let report = read_report(&path) .with_error_context(|| format!("Failed to read DogStatsD context report from '{}'.", path.display()))?; - let rendered = report.render(cmd.num_metrics, num_tags); + let rendered = report.render(cmd.num_metrics, cmd.num_tags); output .write_all(rendered.as_bytes()) .with_error_context(|| format!("Failed to write DogStatsD context report for '{}'.", path.display()))?; @@ -121,7 +137,10 @@ mod tests { use async_trait::async_trait; use saluki_error::{generic_error, GenericError}; - use super::{handle_dogstatsd_dump_contexts, handle_dogstatsd_top, DogStatsDContextDumpRequester, TopCommand}; + use super::{ + handle_dogstatsd_dump_contexts, handle_dogstatsd_top, DogStatsDContextDumpRequester, TopCommand, + ValidatedTopCommand, + }; use crate::cli::dogstatsd::{DogstatsdCommand, DogstatsdSubcommand}; const PLAIN_FIXTURE: &str = concat!( @@ -145,7 +164,10 @@ mod tests { assert_eq!(top.num_metrics, 10); assert_eq!(top.num_tags, None); assert_eq!(top.legacy_num_tags, None); - assert_eq!(top.effective_num_tags().unwrap(), 5); + let validated = top.validate().expect("defaults should pass preflight"); + assert_eq!(validated.path, None); + assert_eq!(validated.num_metrics, 10); + assert_eq!(validated.num_tags, 5); } #[test] @@ -160,7 +182,10 @@ mod tests { assert_eq!(top.num_metrics, 7); assert_eq!(top.num_tags, Some(3)); assert_eq!(top.legacy_num_tags, None); - assert_eq!(top.effective_num_tags().unwrap(), 3); + let validated = top.validate().expect("short options should pass preflight"); + assert_eq!(validated.path, Some(PathBuf::from(PLAIN_FIXTURE))); + assert_eq!(validated.num_metrics, 7); + assert_eq!(validated.num_tags, 3); } #[test] @@ -172,7 +197,7 @@ mod tests { assert_eq!(top.num_tags, Some(4)); assert_eq!(top.legacy_num_tags, None); - assert_eq!(top.effective_num_tags().unwrap(), 4); + assert_eq!(top.validate().unwrap().num_tags, 4); } #[test] @@ -184,19 +209,20 @@ mod tests { assert_eq!(top.num_tags, None); assert_eq!(top.legacy_num_tags, Some(6)); - assert_eq!(top.effective_num_tags().unwrap(), 6); + assert_eq!(top.validate().unwrap().num_tags, 6); } #[test] - fn dogstatsd_top_rejects_both_num_tags_spellings() { + fn dogstatsd_top_preflight_rejects_both_num_tags_spellings() { let command = parse_dogstatsd(&["top", "--num-tags", "4", "--mum-tags", "6"]) .expect("each spelling should parse before conflict validation"); let DogstatsdSubcommand::Top(top) = command.subcommand else { panic!("expected top subcommand"); }; + let error = top.validate().expect_err("preflight should reject conflicting options"); assert_eq!( - top.effective_num_tags().unwrap_err().to_string(), + error.to_string(), "Cannot use `--num-tags` and legacy `--mum-tags` together; use `--num-tags`." ); } @@ -306,6 +332,36 @@ mod tests { assert!(error_chain.contains("decode record 1"), "{error_chain}"); } + #[tokio::test] + async fn dogstatsd_top_redacts_malformed_artifact_values_from_errors() { + const SENTINEL: &str = "SECRET_TENANT_TAG"; + + let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); + let record = format!( + "{{\"Name\":\"metric\",\"Host\":\"host\",\"Type\":\"Gauge\",\"TaggerTags\":[],\"MetricTags\":[],\"NoIndex\":\"{SENTINEL}\",\"Source\":1}}\n" + ); + std::fs::write(artifact.path(), record).expect("malformed artifact should be written"); + let mut output = RecordingWriter::default(); + + let error = handle_dogstatsd_top( + None, + top_command(Some(artifact.path().to_owned()), 10, None, None), + &mut output, + ) + .await + .expect_err("wrong-typed artifact field should fail"); + + let error_chain = format!("{error:#}"); + assert!(!error_chain.contains(SENTINEL), "{error_chain}"); + assert!( + error_chain.contains(&artifact.path().display().to_string()), + "{error_chain}" + ); + assert!(error_chain.contains("decode record 1"), "{error_chain}"); + assert!(error_chain.contains("line 1"), "{error_chain}"); + assert!(error_chain.contains("column"), "{error_chain}"); + } + #[tokio::test] async fn dogstatsd_top_renders_only_the_heading_for_an_empty_artifact() { let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); @@ -368,13 +424,15 @@ mod tests { fn top_command( path: Option, num_metrics: usize, num_tags: Option, legacy_num_tags: Option, - ) -> TopCommand { + ) -> ValidatedTopCommand { TopCommand { path, num_metrics, num_tags, legacy_num_tags, } + .validate() + .expect("test command should pass preflight") } struct FakeRequester { diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index a511c6a986e..bc8db687179 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -439,6 +439,65 @@ pub(super) struct AgentContextRecord { pub(super) source: u32, } +#[derive(Clone, Copy, Debug)] +enum JsonErrorCategory { + Io, + Syntax, + Data, + Eof, +} + +impl JsonErrorCategory { + fn from_serde(category: serde_json::error::Category) -> Self { + match category { + serde_json::error::Category::Io => Self::Io, + serde_json::error::Category::Syntax => Self::Syntax, + serde_json::error::Category::Data => Self::Data, + serde_json::error::Category::Eof => Self::Eof, + } + } + + fn label(self) -> &'static str { + match self { + Self::Io => "I/O", + Self::Syntax => "syntax", + Self::Data => "data", + Self::Eof => "end-of-file", + } + } +} + +#[derive(Debug)] +struct ArtifactDecodeError { + category: JsonErrorCategory, + line: usize, + column: usize, +} + +impl ArtifactDecodeError { + fn from_serde(error: serde_json::Error) -> Self { + Self { + category: JsonErrorCategory::from_serde(error.classify()), + line: error.line(), + column: error.column(), + } + } +} + +impl fmt::Display for ArtifactDecodeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "JSON {} error at line {}, column {}", + self.category.label(), + self.line, + self.column + ) + } +} + +impl Error for ArtifactDecodeError {} + #[derive(Debug)] pub(super) struct ArtifactError { path: PathBuf, @@ -459,6 +518,10 @@ impl ArtifactError { } } + fn decode(path: &Path, record_index: usize, source: serde_json::Error) -> Self { + Self::new(path, "decode", record_index, ArtifactDecodeError::from_serde(source)) + } + #[cfg(test)] pub(super) fn path(&self) -> &Path { &self.path @@ -517,7 +580,7 @@ fn decode_records( let records = serde_json::Deserializer::from_reader(reader).into_iter::(); for (index, record) in records.enumerate() { let record_index = index + 1; - let record = record.map_err(|error| ArtifactError::new(path, "decode", record_index, error))?; + let record = record.map_err(|error| ArtifactError::decode(path, record_index, error))?; consume(record); } Ok(()) @@ -1411,6 +1474,9 @@ mod tests { .contains(&format!("decode record {}", case.expected_record_index)), "{error}" ); + assert!(error.to_string().contains("JSON"), "{error}"); + assert!(error.to_string().contains("line"), "{error}"); + assert!(error.to_string().contains("column"), "{error}"); } } From 6c098ef5a9088ea867ef893d927d6c29b62ad959 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 21 Jul 2026 17:14:08 -0400 Subject: [PATCH 17/44] feat(adp): wire retained contexts into DogStatsD control Connect the post-processing aggregate owner to the authenticated privileged dump route using the configured run path and Agent IPC credentials. --- bin/agent-data-plane/src/cli/run.rs | 195 +++++++++++++++++- .../src/dogstatsd_contexts/api.rs | 5 +- .../src/dogstatsd_contexts/artifact.rs | 2 + .../src/dogstatsd_contexts/mod.rs | 6 - .../src/internal/control_surfaces.rs | 93 +++++++++ bin/agent-data-plane/src/main.rs | 4 - 6 files changed, 290 insertions(+), 15 deletions(-) diff --git a/bin/agent-data-plane/src/cli/run.rs b/bin/agent-data-plane/src/cli/run.rs index 5afb6bb1a92..884d7210623 100644 --- a/bin/agent-data-plane/src/cli/run.rs +++ b/bin/agent-data-plane/src/cli/run.rs @@ -7,7 +7,7 @@ use std::{ use agent_data_plane_config_system::{ConfigurationSystem, EnvPrecedence, LoadedConfiguration}; use argh::FromArgs; -use datadog_agent_commons::platform::PlatformSettings; +use datadog_agent_commons::{ipc::config::IpcAuthConfiguration, platform::PlatformSettings}; use datadog_agent_config::classifier::{ConfigClassifier, Pipeline, PipelineAffinity, Severity, SupportLevel}; use saluki_app::{ accounting::{initialize_memory_bounds, MemoryBoundsConfiguration}, @@ -27,9 +27,10 @@ use saluki_components::{ relays::otlp::OtlpRelayConfiguration, sources::{ChecksIPCConfiguration, DogStatsDConfiguration, OtlpConfiguration}, transforms::{ - AggregateConfiguration, ApmStatsTransformConfiguration, AutoscalingFailoverGatewayConfiguration, - ChainedConfiguration, DogStatsDMapperConfiguration, HostEnrichmentConfiguration, - MrfMetricsGatewayConfiguration, TraceObfuscationConfiguration, TraceSamplerConfiguration, + AggregateConfiguration, AggregateContextSnapshotHandle, ApmStatsTransformConfiguration, + AutoscalingFailoverGatewayConfiguration, ChainedConfiguration, DogStatsDMapperConfiguration, + HostEnrichmentConfiguration, MrfMetricsGatewayConfiguration, TraceObfuscationConfiguration, + TraceSamplerConfiguration, }, }; use saluki_config::GenericConfiguration; @@ -49,6 +50,7 @@ use crate::{ ottl_filter_processor::OttlFilterConfiguration, ottl_transform_processor::OttlTransformConfiguration, tag_filterlist::TagFilterlistConfiguration, }, + dogstatsd_contexts::DogStatsDContextDumpAPIHandler, internal::{ create_internal_supervisor, logging::LoggingConfigurationTranslator, remote_agent::RemoteAgentBootstrap, DogStatsDControlSurface, TopologyControlSurfaces, @@ -677,6 +679,27 @@ async fn add_baseline_traces_pipeline_to_blueprint( Ok(()) } +async fn build_dogstatsd_context_dump_api_handler( + config: &GenericConfiguration, snapshot_handle: AggregateContextSnapshotHandle, +) -> Result { + let ipc_config = IpcAuthConfiguration::from_configuration(config)?; + let auth_token_path = ipc_config.auth_token_file_path(); + let auth_token = tokio::fs::read(&auth_token_path).await.map_err(|error| { + generic_error!( + "Failed to read Agent authentication token from file '{}' ({}).", + auth_token_path.display(), + error.kind() + ) + })?; + let run_path = config + .try_get_typed::("run_path") + .error_context("Failed to read configured `run_path` for DogStatsD context dumps.")? + .unwrap_or_default(); + + DogStatsDContextDumpAPIHandler::new(auth_token, vec![snapshot_handle], run_path) + .error_context("Failed to configure DogStatsD context dump API handler.") +} + async fn add_dsd_pipeline_to_blueprint( blueprint: &mut TopologyBlueprint, config: &GenericConfiguration, @@ -738,6 +761,7 @@ async fn add_dsd_pipeline_to_blueprint( .error_context("Failed to configure metric tag filterlist transform.")?; let dsd_agg_config = AggregateConfiguration::from_configuration(config).error_context("Failed to configure aggregate transform.")?; + let dsd_context_snapshot_handle = dsd_agg_config.context_snapshot_handle(); let dsd_post_agg_filter_config = DogStatsDPostAggregateFilterConfiguration::from_configuration(config) .error_context("Failed to configure DogStatsD post-aggregate filter transform.")?; let events_enrich_config = ChainedConfiguration::default().with_transform_builder( @@ -758,6 +782,8 @@ async fn add_dsd_pipeline_to_blueprint( let stats_api_handler = dsd_stats_config.api_handler(); let capture_api_handler = dsd_config.capture_api_handler(); let replay_api_handler = dsd_config.replay_api_handler(); + let context_dump_api_handler = + build_dogstatsd_context_dump_api_handler(config, dsd_context_snapshot_handle).await?; blueprint // Components. @@ -801,6 +827,7 @@ async fn add_dsd_pipeline_to_blueprint( stats_api_handler, capture_api_handler, replay_api_handler, + context_dump_api_handler, }) } @@ -896,3 +923,163 @@ fn write_sizing_guide(bounds: ComponentBounds) -> Result<(), GenericError> { Ok(()) } + +#[cfg(test)] +mod tests { + use std::{fs, path::Path}; + + use http::{header::AUTHORIZATION, Request, StatusCode}; + use http_body_util::{BodyExt as _, Empty}; + use hyper::body::Bytes; + use saluki_api::{response::Response, APIHandler as _}; + use saluki_components::transforms::{ + aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotEntry, AggregateMetricType, + }; + use saluki_config::{config_from, GenericConfiguration}; + use saluki_context::Context; + use serde_json::json; + use stringtheory::MetaString; + use tower::ServiceExt as _; + + use super::build_dogstatsd_context_dump_api_handler; + use crate::dogstatsd_contexts::DogStatsDContextDumpAPIHandler; + + const AUTH_TOKEN: &[u8] = b"configured-agent-token"; + const CONTEXT_DUMP_FILENAME: &str = "dogstatsd_contexts.json.zstd"; + const CONTEXT_DUMP_ROUTE: &str = "/dogstatsd/contexts/dump"; + + #[tokio::test] + async fn context_dump_handler_reads_configured_credentials_and_run_path_and_uses_supplied_owner() { + let run_directory = tempfile::tempdir().expect("run directory should be created"); + let token_file = run_directory.path().join("auth_token"); + fs::write(&token_file, AUTH_TOKEN).expect("token should be written"); + let config = context_dump_config(Some(run_directory.path()), &token_file).await; + let (snapshot_handle, mut snapshot_responder) = aggregate_context_snapshot_channel_for_test(); + + let handler = build_dogstatsd_context_dump_api_handler(&config, snapshot_handle) + .await + .expect("configured handler should build"); + let owner = tokio::spawn(async move { + snapshot_responder + .respond(vec![snapshot_entry("from.supplied.aggregate")]) + .await + }); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::OK); + let artifact_path = run_directory.path().join(CONTEXT_DUMP_FILENAME); + assert_eq!( + response_body(response).await, + serde_json::to_string(&artifact_path).unwrap() + ); + assert!(artifact_path.is_file()); + owner.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn context_dump_handler_reports_missing_and_unreadable_token_paths_with_io_kind() { + let directory = tempfile::tempdir().expect("temporary directory should be created"); + let cases = [directory.path().join("missing-token"), directory.path().to_owned()]; + + for token_path in cases { + let expected_kind = tokio::fs::read(&token_path) + .await + .expect_err("token fixture should be unreadable") + .kind(); + let config = context_dump_config(Some(directory.path()), &token_path).await; + let (snapshot_handle, _snapshot_responder) = aggregate_context_snapshot_channel_for_test(); + + let error = match build_dogstatsd_context_dump_api_handler(&config, snapshot_handle).await { + Ok(_) => panic!("unreadable token should fail handler construction"), + Err(error) => error, + }; + let message = format!("{error:#}"); + assert!(message.contains(&token_path.display().to_string()), "{message}"); + assert!(message.contains(&expected_kind.to_string()), "{message}"); + } + } + + #[tokio::test] + async fn context_dump_handler_rejects_empty_and_invalid_raw_token_contents() { + let directory = tempfile::tempdir().expect("temporary directory should be created"); + let token_file = directory.path().join("auth_token"); + + for token in [b"".as_slice(), b"token-with-newline\n".as_slice()] { + fs::write(&token_file, token).expect("token fixture should be written"); + let config = context_dump_config(Some(directory.path()), &token_file).await; + let (snapshot_handle, _snapshot_responder) = aggregate_context_snapshot_channel_for_test(); + + if build_dogstatsd_context_dump_api_handler(&config, snapshot_handle) + .await + .is_ok() + { + panic!("invalid raw token should fail closed"); + } + } + } + + #[tokio::test] + async fn context_dump_handler_keeps_missing_and_empty_run_path_empty_until_authorized_publication() { + let directory = tempfile::tempdir().expect("temporary directory should be created"); + let token_file = directory.path().join("auth_token"); + fs::write(&token_file, AUTH_TOKEN).expect("token should be written"); + let cwd_artifact = std::env::current_dir().unwrap().join(CONTEXT_DUMP_FILENAME); + assert!(!cwd_artifact.exists(), "test requires no pre-existing cwd artifact"); + + for run_path in [None, Some(Path::new(""))] { + let config = context_dump_config(run_path, &token_file).await; + let (snapshot_handle, mut snapshot_responder) = aggregate_context_snapshot_channel_for_test(); + let handler = build_dogstatsd_context_dump_api_handler(&config, snapshot_handle) + .await + .expect("empty run path should not weaken authentication setup"); + let owner = tokio::spawn(async move { snapshot_responder.respond(Vec::new()).await }); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(!cwd_artifact.exists()); + owner.await.unwrap().unwrap(); + } + } + + async fn context_dump_config(run_path: Option<&Path>, token_path: &Path) -> GenericConfiguration { + let mut values = json!({ + "auth_token_file_path": token_path, + }); + if let Some(run_path) = run_path { + values["run_path"] = json!(run_path); + } + config_from(values).await + } + + fn authorized_post() -> Request> { + Request::builder() + .method("POST") + .uri(CONTEXT_DUMP_ROUTE) + .header(AUTHORIZATION, "Bearer configured-agent-token") + .body(Empty::new()) + .unwrap() + } + + async fn send(handler: &DogStatsDContextDumpAPIHandler, request: Request>) -> Response { + handler + .generate_routes() + .with_state(handler.generate_initial_state()) + .oneshot(request) + .await + .unwrap() + } + + async fn response_body(response: Response) -> String { + String::from_utf8(response.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap() + } + + fn snapshot_entry(name: &'static str) -> AggregateContextSnapshotEntry { + AggregateContextSnapshotEntry::for_test( + Context::from_static_name(name), + AggregateMetricType::Gauge, + MetaString::empty(), + ) + } +} diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs index 23a219643ad..ee2b53a43da 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs @@ -271,7 +271,10 @@ mod tests { ContextDumpPublisher, ContextSnapshotCoordinator, DogStatsDContextDumpAPIHandler, FileSystemContextDumpPublisher, SnapshotError, }; - use crate::dogstatsd_contexts::{artifact::for_each_record, publish_context_dump, CONTEXT_DUMP_FILENAME}; + use crate::dogstatsd_contexts::{ + artifact::{for_each_record, CONTEXT_DUMP_FILENAME}, + publish_context_dump, + }; const AUTH_TOKEN: &str = "expected-agent-token"; const ROUTE: &str = "/dogstatsd/contexts/dump"; diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index bc8db687179..329b8f29664 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -201,6 +201,7 @@ fn open_temporary_file(path: &Path) -> io::Result { options.open(path) } +#[cfg(test)] fn publish_open_temporary( writer: W, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, ) -> Result<(), GenericError> @@ -213,6 +214,7 @@ where }) } +#[cfg(test)] fn publish_buffered_temporary( buffer: BufWriter, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs index fc591b61ace..4cbfcb0baa9 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs @@ -2,14 +2,8 @@ use std::path::Path; use saluki_error::GenericError; -#[allow( - unused_imports, - reason = "registered with the privileged API by a subsequent feature task" -)] pub(crate) use self::api::DogStatsDContextDumpAPIHandler; pub(crate) use self::artifact::publish_context_dump; -#[allow(unused_imports, reason = "used by topology wiring in a subsequent feature task")] -pub(crate) use self::artifact::CONTEXT_DUMP_FILENAME; pub(crate) use self::report::ContextReport; mod api; diff --git a/bin/agent-data-plane/src/internal/control_surfaces.rs b/bin/agent-data-plane/src/internal/control_surfaces.rs index 6bc45d43196..ea6236a35b3 100644 --- a/bin/agent-data-plane/src/internal/control_surfaces.rs +++ b/bin/agent-data-plane/src/internal/control_surfaces.rs @@ -4,6 +4,8 @@ use saluki_components::{ sources::{DogStatsDCaptureAPIHandler, DogStatsDReplayAPIHandler}, }; +use crate::dogstatsd_contexts::DogStatsDContextDumpAPIHandler; + /// Combined set of control surfaces to expose from the privileged API endpoint. #[derive(Default)] pub struct TopologyControlSurfaces { @@ -34,6 +36,8 @@ pub struct DogStatsDControlSurface { pub(crate) capture_api_handler: DogStatsDCaptureAPIHandler, /// API handler for the `/dogstatsd/replay/session` endpoints. pub(crate) replay_api_handler: DogStatsDReplayAPIHandler, + /// API handler for the `/dogstatsd/contexts/dump` endpoint. + pub(crate) context_dump_api_handler: DogStatsDContextDumpAPIHandler, } impl DogStatsDControlSurface { @@ -42,5 +46,94 @@ impl DogStatsDControlSurface { .with_handler(self.stats_api_handler) .with_handler(self.capture_api_handler) .with_handler(self.replay_api_handler) + .with_handler(self.context_dump_api_handler) + } +} + +#[cfg(test)] +mod tests { + use std::{net::TcpListener, time::Duration}; + + use http::StatusCode; + use saluki_api::EndpointType; + use saluki_app::dynamic_api::DynamicAPIBuilder; + use saluki_components::{destinations::DogStatsDStatisticsConfiguration, sources::DogStatsDConfiguration}; + use saluki_config::config_from; + use saluki_core::runtime::Supervisor; + use saluki_io::net::ListenAddress; + use tokio::{ + io::{AsyncReadExt as _, AsyncWriteExt as _}, + net::TcpStream, + sync::oneshot, + time::Instant, + }; + + use super::DogStatsDControlSurface; + use crate::dogstatsd_contexts::DogStatsDContextDumpAPIHandler; + + #[tokio::test] + async fn dogstatsd_control_surface_registers_context_dump_on_privileged_dynamic_api() { + let address = reserve_tcp_address(); + let dogstatsd_config = DogStatsDConfiguration::from_configuration(&config_from(serde_json::json!({})).await) + .expect("default DogStatsD configuration should build"); + let surface = DogStatsDControlSurface { + stats_api_handler: DogStatsDStatisticsConfiguration::new().api_handler(), + capture_api_handler: dogstatsd_config.capture_api_handler(), + replay_api_handler: dogstatsd_config.replay_api_handler(), + context_dump_api_handler: DogStatsDContextDumpAPIHandler::new( + "configured-agent-token", + Vec::new(), + std::path::PathBuf::new(), + ) + .unwrap(), + }; + let api = surface.register_control_surfaces(DynamicAPIBuilder::new( + EndpointType::Privileged, + ListenAddress::Tcp(address), + )); + let mut supervisor = Supervisor::new("dogstatsd-control-surface-test").unwrap(); + supervisor.add_worker(api); + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let server = tokio::spawn(async move { supervisor.run_with_shutdown(shutdown_rx).await }); + + let status = post_context_dump_eventually(address).await; + + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + let _ = shutdown_tx.send(()); + server.await.unwrap().unwrap(); + } + + fn reserve_tcp_address() -> std::net::SocketAddr { + TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .expect("ephemeral TCP address should bind") + .local_addr() + .unwrap() + } + + async fn post_context_dump_eventually(address: std::net::SocketAddr) -> StatusCode { + let deadline = Instant::now() + Duration::from_secs(5); + let mut stream = loop { + if let Ok(stream) = TcpStream::connect(address).await { + break stream; + } + assert!(Instant::now() < deadline, "dynamic API did not start before deadline"); + tokio::time::sleep(Duration::from_millis(25)).await; + }; + stream + .write_all( + b"POST /dogstatsd/contexts/dump HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer \ + configured-agent-token\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + let status_digits = response + .get(9..12) + .expect("HTTP response should have a three-digit status code"); + let status = status_digits.iter().try_fold(0_u16, |status, digit| { + digit.is_ascii_digit().then(|| status * 10 + u16::from(*digit - b'0')) + }); + StatusCode::from_u16(status.expect("HTTP response status should be numeric")).unwrap() } } diff --git a/bin/agent-data-plane/src/main.rs b/bin/agent-data-plane/src/main.rs index 1bccea0a22c..be4e525a0b9 100644 --- a/bin/agent-data-plane/src/main.rs +++ b/bin/agent-data-plane/src/main.rs @@ -29,10 +29,6 @@ use crate::internal::logging::LoggingConfigurationTranslator; mod components; mod config; -#[allow( - dead_code, - reason = "wired into the DogStatsD CLI and API by subsequent feature tasks" -)] mod dogstatsd_contexts; mod internal; From 65e49d545ed1cebc98a74520b7dab2708d0e0501 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 09:12:13 -0400 Subject: [PATCH 18/44] test(adp): bound context route requests Wrap the control-surface route request write and response read in the remaining five-second test deadline. A stalled peer now fails with a clear panic instead of hanging indefinitely. --- .../src/internal/control_surfaces.rs | 46 +++++++++++++++---- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/bin/agent-data-plane/src/internal/control_surfaces.rs b/bin/agent-data-plane/src/internal/control_surfaces.rs index ea6236a35b3..4caacac5ab4 100644 --- a/bin/agent-data-plane/src/internal/control_surfaces.rs +++ b/bin/agent-data-plane/src/internal/control_surfaces.rs @@ -103,6 +103,22 @@ mod tests { server.await.unwrap().unwrap(); } + #[tokio::test] + #[should_panic(expected = "timed out waiting for DogStatsD context dump route response")] + async fn context_dump_request_exchange_times_out_when_peer_does_not_close() { + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .expect("test listener should bind"); + let address = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (_stream, _) = listener.accept().await.expect("test listener should accept"); + tokio::time::sleep(Duration::from_secs(1)).await; + }); + let stream = TcpStream::connect(address).await.expect("test client should connect"); + + exchange_context_dump_request(stream, Instant::now() + Duration::from_millis(20)).await; + } + fn reserve_tcp_address() -> std::net::SocketAddr { TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) .expect("ephemeral TCP address should bind") @@ -112,22 +128,14 @@ mod tests { async fn post_context_dump_eventually(address: std::net::SocketAddr) -> StatusCode { let deadline = Instant::now() + Duration::from_secs(5); - let mut stream = loop { + let stream = loop { if let Ok(stream) = TcpStream::connect(address).await { break stream; } assert!(Instant::now() < deadline, "dynamic API did not start before deadline"); tokio::time::sleep(Duration::from_millis(25)).await; }; - stream - .write_all( - b"POST /dogstatsd/contexts/dump HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer \ - configured-agent-token\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - ) - .await - .unwrap(); - let mut response = Vec::new(); - stream.read_to_end(&mut response).await.unwrap(); + let response = exchange_context_dump_request(stream, deadline).await; let status_digits = response .get(9..12) .expect("HTTP response should have a three-digit status code"); @@ -136,4 +144,22 @@ mod tests { }); StatusCode::from_u16(status.expect("HTTP response status should be numeric")).unwrap() } + + async fn exchange_context_dump_request(mut stream: TcpStream, deadline: Instant) -> Vec { + let remaining = deadline.saturating_duration_since(Instant::now()); + tokio::time::timeout(remaining, async { + stream + .write_all( + b"POST /dogstatsd/contexts/dump HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer \ + configured-agent-token\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await?; + let mut response = Vec::new(); + stream.read_to_end(&mut response).await?; + Ok::<_, std::io::Error>(response) + }) + .await + .expect("timed out waiting for DogStatsD context dump route response") + .expect("DogStatsD context dump route request should complete") + } } From 32d422f311e4b97889e7f080b25457b95a5f960e Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 09:25:26 -0400 Subject: [PATCH 19/44] test(adp): cover retained context lifecycle Verify snapshots follow Saluki aggregation retention, including passthrough exclusion, mixed values, ordinary flush removal, idle counters, and post-processing identity. --- bin/agent-data-plane/src/cli/run.rs | 244 +++++++++++++++++- .../src/transforms/aggregate/mod.rs | 42 ++- 2 files changed, 272 insertions(+), 14 deletions(-) diff --git a/bin/agent-data-plane/src/cli/run.rs b/bin/agent-data-plane/src/cli/run.rs index 884d7210623..986acc26420 100644 --- a/bin/agent-data-plane/src/cli/run.rs +++ b/bin/agent-data-plane/src/cli/run.rs @@ -926,28 +926,189 @@ fn write_sizing_guide(bounds: ComponentBounds) -> Result<(), GenericError> { #[cfg(test)] mod tests { - use std::{fs, path::Path}; + use std::{fs, path::Path, sync::Mutex, time::Duration}; + use async_trait::async_trait; use http::{header::AUTHORIZATION, Request, StatusCode}; use http_body_util::{BodyExt as _, Empty}; use hyper::body::Bytes; use saluki_api::{response::Response, APIHandler as _}; use saluki_components::transforms::{ - aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotEntry, AggregateMetricType, + aggregate_context_snapshot_channel_for_test, AggregateConfiguration, AggregateContextSnapshotEntry, + AggregateMetricType, ChainedConfiguration, DogStatsDMapperConfiguration, }; use saluki_config::{config_from, GenericConfiguration}; use saluki_context::Context; + use saluki_core::{ + accounting::{ComponentRegistry, MemoryBounds, MemoryBoundsBuilder, MemoryLimiter}, + components::{ + destinations::{Destination, DestinationBuilder, DestinationContext}, + sources::{Source, SourceBuilder, SourceContext}, + ComponentContext, + }, + data_model::event::{metric::Metric, Event, EventType}, + health::HealthRegistry, + runtime::Supervisor, + topology::{OutputDefinition, TopologyBlueprint}, + }; + use saluki_error::{generic_error, GenericError}; use serde_json::json; use stringtheory::MetaString; + use tokio::sync::{mpsc, oneshot}; use tower::ServiceExt as _; use super::build_dogstatsd_context_dump_api_handler; - use crate::dogstatsd_contexts::DogStatsDContextDumpAPIHandler; + use crate::{ + components::{ + dogstatsd_prefix_filter::DogStatsDPrefixFilterConfiguration, tag_filterlist::TagFilterlistConfiguration, + }, + dogstatsd_contexts::DogStatsDContextDumpAPIHandler, + }; const AUTH_TOKEN: &[u8] = b"configured-agent-token"; const CONTEXT_DUMP_FILENAME: &str = "dogstatsd_contexts.json.zstd"; const CONTEXT_DUMP_ROUTE: &str = "/dogstatsd/contexts/dump"; + #[tokio::test] + async fn retained_context_identity_follows_dogstatsd_post_processing() { + tokio::time::timeout(Duration::from_secs(5), async { + let config = config_from(json!({ + "dogstatsd_mapper_profiles": [{ + "name": "retained-context-test", + "prefix": "raw.requests.", + "mappings": [{ + "match": "raw.requests.*", + "name": "mapped.requests", + "tags": { "route": "$1" } + }] + }], + "statsd_metric_namespace": "tenant", + "statsd_metric_namespace_blocklist": [], + "metric_filterlist": ["tenant.raw.blocked"], + "metric_filterlist_match_prefix": false, + "metric_tag_filterlist": [{ + "metric_name": "tenant.mapped.requests", + "action": "exclude", + "tags": ["remove"] + }], + "aggregate_flush_interval": { "secs": 60, "nanos": 0 } + })) + .await; + + let mapper = + DogStatsDMapperConfiguration::from_configuration(&config).expect("mapper configuration should parse"); + let mapper_chain = ChainedConfiguration::default().with_transform_builder("dogstatsd_mapper", mapper); + let prefix_filter = DogStatsDPrefixFilterConfiguration::from_configuration(&config) + .expect("prefix filter configuration should parse"); + let tag_filter = + TagFilterlistConfiguration::from_configuration(&config).expect("tag filter configuration should parse"); + let aggregate = + AggregateConfiguration::from_configuration(&config).expect("aggregate configuration should parse"); + let snapshot_handle = aggregate.context_snapshot_handle(); + + let (events_tx, events_rx) = mpsc::channel(2); + let source = ControlledMetricSourceBuilder { + events: Mutex::new(Some(events_rx)), + outputs: vec![OutputDefinition::default_output(EventType::Metric)], + }; + let component_registry = ComponentRegistry::default(); + let mut blueprint = TopologyBlueprint::new("dogstatsd_retained_identity", &component_registry); + blueprint + .add_source("source", source) + .expect("controlled source should be accepted") + .add_transform("mapper", mapper_chain) + .expect("mapper should be accepted") + .add_transform("prefix_filter", prefix_filter) + .expect("prefix filter should be accepted") + .add_transform("tag_filter", tag_filter) + .expect("tag filter should be accepted") + .add_transform("aggregate", aggregate) + .expect("aggregate should be accepted") + .add_destination("destination", DrainingMetricDestinationBuilder) + .expect("draining destination should be accepted"); + blueprint + .connect_components_in_order([ + "source", + "mapper", + "prefix_filter", + "tag_filter", + "aggregate", + "destination", + ]) + .expect("DogStatsD post-processing topology should connect"); + blueprint + .with_health_registry(HealthRegistry::new()) + .with_memory_limiter(MemoryLimiter::noop()) + .with_ambient_worker_pool(); + + let mut supervisor = + Supervisor::new("dogstatsd-retained-identity").expect("test supervisor should be created"); + supervisor.add_worker(blueprint); + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let topology_task = tokio::spawn(async move { supervisor.run_with_shutdown(shutdown_rx).await }); + + events_tx + .send(Event::Metric(Metric::counter("raw.blocked", 1.0))) + .await + .expect("controlled source should accept the blocked metric"); + let input_context = Context::from_static_parts("raw.requests.checkout", &["keep:client", "remove:secret"]); + events_tx + .send(Event::Metric(Metric::counter(input_context.clone(), 1.0))) + .await + .expect("controlled source should accept the retained metric"); + + let expected_context = + Context::from_static_parts("tenant.mapped.requests", &["keep:client", "route:checkout"]); + let snapshot = loop { + let snapshot = snapshot_handle + .snapshot() + .await + .expect("running aggregate should fulfill snapshots"); + if snapshot.iter().any(|entry| entry.context() == &expected_context) { + break snapshot; + } + tokio::task::yield_now().await; + }; + + assert_eq!(snapshot.len(), 1); + assert_eq!(snapshot[0].context(), &expected_context); + assert_eq!(snapshot[0].metric_type(), AggregateMetricType::Counter); + assert_eq!(snapshot[0].context().tags().len(), 2); + assert_eq!( + snapshot[0] + .context() + .tags() + .get_single_tag("keep") + .and_then(|tag| tag.value()), + Some("client") + ); + assert_eq!( + snapshot[0] + .context() + .tags() + .get_single_tag("route") + .and_then(|tag| tag.value()), + Some("checkout") + ); + assert!(snapshot[0].context().tags().get_single_tag("remove").is_none()); + assert!(snapshot.iter().all(|entry| entry.context() != &input_context)); + assert!(snapshot + .iter() + .all(|entry| { !matches!(entry.context().name().as_ref(), "raw.blocked" | "tenant.raw.blocked") })); + + drop(events_tx); + drop(snapshot_handle); + shutdown_tx.send(()).expect("test topology should still be running"); + let topology_result = topology_task.await.expect("topology task should not panic"); + assert!( + topology_result.is_ok(), + "topology should stop cleanly: {topology_result:?}" + ); + }) + .await + .expect("DogStatsD retained identity test should complete without hanging"); + } + #[tokio::test] async fn context_dump_handler_reads_configured_credentials_and_run_path_and_uses_supplied_owner() { let run_directory = tempfile::tempdir().expect("run directory should be created"); @@ -1043,6 +1204,83 @@ mod tests { } } + struct ControlledMetricSource { + events: mpsc::Receiver, + } + + #[async_trait] + impl Source for ControlledMetricSource { + async fn run(mut self: Box, mut context: SourceContext) -> Result<(), GenericError> { + let shutdown = context.take_shutdown_handle(); + tokio::pin!(shutdown); + let mut events_open = true; + + loop { + tokio::select! { + _ = &mut shutdown => break, + maybe_event = self.events.recv(), if events_open => match maybe_event { + Some(event) => context.dispatcher().dispatch_one(event).await?, + None => events_open = false, + } + } + } + Ok(()) + } + } + + struct ControlledMetricSourceBuilder { + events: Mutex>>, + outputs: Vec>, + } + + #[async_trait] + impl SourceBuilder for ControlledMetricSourceBuilder { + fn outputs(&self) -> &[OutputDefinition] { + &self.outputs + } + + async fn build(&self, _context: ComponentContext) -> Result, GenericError> { + let events = self + .events + .lock() + .map_err(|_| generic_error!("controlled metric source receiver lock is poisoned"))? + .take() + .ok_or_else(|| generic_error!("controlled metric source receiver has already been taken"))?; + Ok(Box::new(ControlledMetricSource { events })) + } + } + + impl MemoryBounds for ControlledMetricSourceBuilder { + fn specify_bounds(&self, _builder: &mut MemoryBoundsBuilder) {} + } + + struct DrainingMetricDestination; + + #[async_trait] + impl Destination for DrainingMetricDestination { + async fn run(self: Box, mut context: DestinationContext) -> Result<(), GenericError> { + while context.events().next().await.is_some() {} + Ok(()) + } + } + + struct DrainingMetricDestinationBuilder; + + #[async_trait] + impl DestinationBuilder for DrainingMetricDestinationBuilder { + fn input_event_type(&self) -> EventType { + EventType::Metric + } + + async fn build(&self, _context: ComponentContext) -> Result, GenericError> { + Ok(Box::new(DrainingMetricDestination)) + } + } + + impl MemoryBounds for DrainingMetricDestinationBuilder { + fn specify_bounds(&self, _builder: &mut MemoryBoundsBuilder) {} + } + async fn context_dump_config(run_path: Option<&Path>, token_path: &Path) -> GenericConfiguration { let mut values = json!({ "auth_token_file_path": token_path, diff --git a/lib/saluki-components/src/transforms/aggregate/mod.rs b/lib/saluki-components/src/transforms/aggregate/mod.rs index 426f2f90ac9..6134ab101c0 100644 --- a/lib/saluki-components/src/transforms/aggregate/mod.rs +++ b/lib/saluki-components/src/transforms/aggregate/mod.rs @@ -1558,7 +1558,10 @@ mod tests { let context = Context::from_static_name("active.gauge"); assert!(state.insert(insert_ts(1), Metric::gauge(context.clone(), 1.0))); - assert_eq!(state.snapshot_contexts().len(), 1); + let active_snapshot = state.snapshot_contexts(); + assert_eq!(active_snapshot.len(), 1); + assert_eq!(active_snapshot[0].context(), &context); + assert_eq!(active_snapshot[0].metric_type(), AggregateMetricType::Gauge); let _ = get_flushed_metrics(flush_ts(1), &mut state).await; assert!(state.snapshot_contexts().is_empty()); @@ -1583,7 +1586,10 @@ mod tests { assert_eq!(first_snapshot[0].metric_type(), AggregateMetricType::Counter); let _ = get_flushed_metrics(flush_ts(2), &mut state).await; - assert_eq!(state.snapshot_contexts().len(), 1); + let second_snapshot = state.snapshot_contexts(); + assert_eq!(second_snapshot.len(), 1); + assert_eq!(second_snapshot[0].context(), &context); + assert_eq!(second_snapshot[0].metric_type(), AggregateMetricType::Counter); let _ = get_flushed_metrics(flush_ts(3), &mut state).await; assert!(state.snapshot_contexts().is_empty()); @@ -1657,28 +1663,42 @@ mod tests { let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); let topology_task = tokio::spawn(async move { supervisor.run_with_shutdown(shutdown_rx).await }); - let expected_context = Context::from_static_name("owner.loop.gauge"); + let passthrough_context = Context::from_static_name("owner.loop.timestamped.gauge"); events_tx - .send(Event::Metric(Metric::gauge(expected_context.clone(), 1.0))) + .send(Event::Metric(Metric::gauge( + passthrough_context.clone(), + (insert_ts(1), 1.0), + ))) .await - .expect("controlled source should accept an event"); + .expect("controlled source should accept a timestamped event"); + + let retained_context = Context::from_static_name("owner.loop.mixed.counter"); + let mixed_values = ScalarPoints::from_iter([(None, 2.0), (NonZeroU64::new(insert_ts(1)), 3.0)]); + events_tx + .send(Event::Metric(Metric::counter(retained_context.clone(), mixed_values))) + .await + .expect("controlled source should accept a mixed event"); let snapshot = loop { let snapshot = snapshot_handle .snapshot() .await .expect("running aggregate should fulfill snapshots"); - if snapshot.iter().any(|entry| entry.context() == &expected_context) { + if snapshot.iter().any(|entry| entry.context() == &retained_context) { break snapshot; } tokio::task::yield_now().await; }; - let entry = snapshot - .iter() - .find(|entry| entry.context() == &expected_context) - .expect("snapshot should retain the inserted context"); - assert_eq!(entry.metric_type(), AggregateMetricType::Gauge); + assert_eq!( + snapshot.len(), + 1, + "only the mixed metric's retained portion belongs in state" + ); + let entry = &snapshot[0]; + assert_eq!(entry.context(), &retained_context); + assert_eq!(entry.metric_type(), AggregateMetricType::Counter); assert_eq!(entry.unit(), None); + assert!(snapshot.iter().all(|entry| entry.context() != &passthrough_context)); drop(events_tx); drop(snapshot_handle); From 04f06835bcc3ff96a6f3c877438072318e565054 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 09:50:19 -0400 Subject: [PATCH 20/44] perf(adp): benchmark retained context dumps Measure aggregate-owner snapshot cost and streaming compressed publication through the configured one-million-context ceiling with correctness checks in every benchmark. --- Cargo.lock | 2 + bin/agent-data-plane/Cargo.toml | 7 + .../benches/dogstatsd_context_dump.rs | 125 ++++++++++++++++++ .../src/dogstatsd_contexts/artifact.rs | 14 +- lib/saluki-components/Cargo.toml | 6 + .../benches/aggregate_context_snapshot.rs | 42 ++++++ 6 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 bin/agent-data-plane/benches/dogstatsd_context_dump.rs create mode 100644 lib/saluki-components/benches/aggregate_context_snapshot.rs diff --git a/Cargo.lock b/Cargo.lock index 1a3b8ef3e0f..22dd0e7645e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,7 @@ dependencies = [ "chrono", "colored", "comfy-table", + "criterion", "datadog-agent-commons", "datadog-agent-config", "datadog-agent-config-testing", @@ -4322,6 +4323,7 @@ dependencies = [ "bytes", "bytesize", "chrono", + "criterion", "datadog-agent-commons", "datadog-agent-config", "datadog-agent-config-testing", diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index ea6d6252b64..f0e36d5cf5f 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -12,6 +12,7 @@ workspace = true default = [] fips = ["saluki-app/tls-fips", "saluki-components/fips"] antithesis = ["saluki-antithesis/antithesis", "dep:antithesis-instrumentation"] +context-dump-benchmark = [] [dependencies] agent-data-plane-config = { workspace = true } @@ -90,6 +91,7 @@ windows-sys = { workspace = true, features = [ chrono = { workspace = true } [dev-dependencies] +criterion = { workspace = true } datadog-agent-config-testing = { workspace = true } derive-where = { workspace = true } rcgen = { workspace = true, features = ["crypto", "pem"] } @@ -108,3 +110,8 @@ rcgen = { workspace = true, features = ["aws_lc_rs"] } [target.'cfg(windows)'.dev-dependencies] rcgen = { workspace = true, features = ["ring"] } + +[[bench]] +name = "dogstatsd_context_dump" +harness = false +required-features = ["context-dump-benchmark"] diff --git a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs new file mode 100644 index 00000000000..b8790ba058e --- /dev/null +++ b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs @@ -0,0 +1,125 @@ +use std::fs; +use std::hint::black_box; +use std::path::Path; + +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; +use saluki_context::{ + tags::{Tag, TagSet}, + Context, +}; +use stringtheory::MetaString; + +#[path = "../src/dogstatsd_contexts/artifact.rs"] +mod artifact; + +const CONTEXT_COUNTS: [usize; 3] = [10_000, 100_000, 1_000_000]; +const CLIENT_TAGS: [&str; 5] = [ + "env:production", + "service:payments-api", + "version:2025.03.0", + "region:us-east-1", + "availability-zone:us-east-1a", +]; +const ORIGIN_TAGS: [&str; 5] = [ + "container_id:7c9f0e2a4b1d", + "pod_name:payments-api-7d8f9c6b5f-k2m4p", + "namespace:production", + "kube_deployment:payments-api", + "image_name:registry.example/payments-api", +]; + +fn benchmark_dogstatsd_context_dump(c: &mut Criterion) { + let mut group = c.benchmark_group("dogstatsd_context_dump"); + + for context_count in CONTEXT_COUNTS { + let snapshot = build_snapshot(context_count); + assert_eq!(snapshot.len(), context_count); + + let run_directory = tempfile::tempdir().expect("benchmark run directory should be created"); + let canonical_path = run_directory.path().join(artifact::CONTEXT_DUMP_FILENAME); + let preflight_path = artifact::publish_context_dump(run_directory.path(), &snapshot) + .expect("benchmark preflight context dump should publish"); + assert_eq!(preflight_path, canonical_path); + assert!(canonical_path.is_file()); + + let mut decoded_records = 0usize; + artifact::for_each_record(&canonical_path, |record| { + assert!(!record.name.is_empty()); + assert_eq!(record.host, "benchmark-node-001.internal"); + assert!(matches!(record.metric_type.as_str(), "Counter" | "Gauge" | "Historate")); + assert_eq!(record.tagger_tags.len(), ORIGIN_TAGS.len()); + assert_eq!(record.metric_tags.len(), CLIENT_TAGS.len()); + assert!(!record.no_index); + assert_eq!(record.source, 1); + decoded_records += 1; + }) + .expect("benchmark preflight context dump should decode through the production reader"); + assert_eq!(decoded_records, context_count); + assert_only_canonical_artifact(run_directory.path()); + + let compressed_bytes = fs::metadata(&canonical_path) + .expect("benchmark preflight artifact metadata should be available") + .len(); + let benchmark_id = + BenchmarkId::from_parameter(format!("{context_count}_contexts_{compressed_bytes}_compressed_bytes")); + group.throughput(Throughput::Elements(context_count as u64)); + group.bench_function(benchmark_id, |b| { + b.iter(|| { + let published_path = artifact::publish_context_dump(run_directory.path(), &snapshot) + .expect("benchmark context dump should publish"); + assert_eq!(published_path, canonical_path); + assert!(published_path.is_file()); + black_box(published_path) + }); + }); + + assert_only_canonical_artifact(run_directory.path()); + } + + group.finish(); +} + +fn build_snapshot(context_count: usize) -> Vec { + let base_context = Context::from_parts("dogstatsd.benchmark.metric", shared_tag_set(&CLIENT_TAGS)) + .with_host(Some(MetaString::from_static("benchmark-node-001.internal"))) + .with_origin_tags(shared_tag_set(&ORIGIN_TAGS)); + + (0..context_count) + .map(|index| { + let context = base_context.with_name(format!("dogstatsd.benchmark.metric.{index:07}")); + let (metric_type, unit) = match index % 3 { + 0 => (AggregateMetricType::Counter, MetaString::empty()), + 1 => (AggregateMetricType::Gauge, MetaString::empty()), + _ => (AggregateMetricType::Histogram, MetaString::from_static("millisecond")), + }; + AggregateContextSnapshotEntry::for_test(context, metric_type, unit) + }) + .collect() +} + +fn shared_tag_set(values: &[&'static str]) -> TagSet { + let mut tags = TagSet::with_capacity(values.len()); + for value in values { + tags.insert_tag(Tag::from_static(value)); + } + tags.into_shared().to_mutable() +} + +fn assert_only_canonical_artifact(run_path: &Path) { + let mut entries: Vec<_> = fs::read_dir(run_path) + .expect("benchmark run directory should be readable") + .map(|entry| { + entry + .expect("benchmark run directory entry should be readable") + .file_name() + .to_string_lossy() + .into_owned() + }) + .collect(); + entries.sort_unstable(); + assert_eq!(entries, [artifact::CONTEXT_DUMP_FILENAME]); +} + +criterion_group!(benches, benchmark_dogstatsd_context_dump); +criterion_main!(benches); diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index 329b8f29664..737a9b080c5 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -201,7 +201,7 @@ fn open_temporary_file(path: &Path) -> io::Result { options.open(path) } -#[cfg(test)] +#[cfg(all(test, not(feature = "context-dump-benchmark")))] fn publish_open_temporary( writer: W, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, ) -> Result<(), GenericError> @@ -214,7 +214,7 @@ where }) } -#[cfg(test)] +#[cfg(all(test, not(feature = "context-dump-benchmark")))] fn publish_buffered_temporary( buffer: BufWriter, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, @@ -524,17 +524,17 @@ impl ArtifactError { Self::new(path, "decode", record_index, ArtifactDecodeError::from_serde(source)) } - #[cfg(test)] + #[cfg(all(test, not(feature = "context-dump-benchmark")))] pub(super) fn path(&self) -> &Path { &self.path } - #[cfg(test)] + #[cfg(all(test, not(feature = "context-dump-benchmark")))] pub(super) fn operation(&self) -> &'static str { self.operation } - #[cfg(test)] + #[cfg(all(test, not(feature = "context-dump-benchmark")))] pub(super) fn record_index(&self) -> usize { self.record_index } @@ -588,14 +588,14 @@ fn decode_records( Ok(()) } -#[cfg(test)] +#[cfg(all(test, not(feature = "context-dump-benchmark")))] fn collect_records(path: &Path) -> Result, ArtifactError> { let mut records = Vec::new(); for_each_record(path, |record| records.push(record))?; Ok(records) } -#[cfg(test)] +#[cfg(all(test, not(feature = "context-dump-benchmark")))] mod tests { use std::fs; use std::io::{self, BufWriter, Write as _}; diff --git a/lib/saluki-components/Cargo.toml b/lib/saluki-components/Cargo.toml index 353e888a89e..ffae1f2312b 100644 --- a/lib/saluki-components/Cargo.toml +++ b/lib/saluki-components/Cargo.toml @@ -95,6 +95,7 @@ uuid = { workspace = true, features = ["std", "v7"] } zstd = { workspace = true } [dev-dependencies] +criterion = { workspace = true } # TODO: remove after migration to typed config. Only the in-crate tests still reference the moved # `KEY_ALIASES`/`DatadogRemapper` (re-exported from `config` under `cfg(test)`). datadog-agent-config = { workspace = true } @@ -111,3 +112,8 @@ tempfile = { workspace = true } test-strategy = { workspace = true } tokio = { workspace = true, features = ["time"] } tokio-rustls = { workspace = true } + +[[bench]] +name = "aggregate_context_snapshot" +harness = false +required-features = ["test-util"] diff --git a/lib/saluki-components/benches/aggregate_context_snapshot.rs b/lib/saluki-components/benches/aggregate_context_snapshot.rs new file mode 100644 index 00000000000..8953216545e --- /dev/null +++ b/lib/saluki-components/benches/aggregate_context_snapshot.rs @@ -0,0 +1,42 @@ +use std::hint::black_box; +use std::mem::size_of; + +use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput}; +use saluki_components::transforms::{AggregateContextSnapshotBenchmarkHarness, AggregateContextSnapshotEntry}; + +const CONTEXT_COUNTS: [usize; 3] = [10_000, 100_000, 1_000_000]; + +fn benchmark_aggregate_context_snapshot(c: &mut Criterion) { + let mut group = c.benchmark_group("aggregate_context_snapshot"); + + for context_count in CONTEXT_COUNTS { + let harness = AggregateContextSnapshotBenchmarkHarness::with_contexts(context_count); + let retained_len = harness.retained_len(); + assert_eq!(retained_len, context_count); + + let preflight = harness.snapshot(); + assert_eq!(preflight.len(), retained_len); + black_box(preflight); + + let structural_bytes = context_count * size_of::(); + let benchmark_id = + BenchmarkId::from_parameter(format!("{context_count}_contexts_{structural_bytes}_structural_bytes")); + group.throughput(Throughput::Elements(context_count as u64)); + group.bench_function(benchmark_id, |b| { + b.iter_batched( + || (), + |()| { + let snapshot = harness.snapshot(); + assert_eq!(snapshot.len(), retained_len); + black_box(snapshot) + }, + BatchSize::PerIteration, + ); + }); + } + + group.finish(); +} + +criterion_group!(benches, benchmark_aggregate_context_snapshot); +criterion_main!(benches); From b000044faf0e42951e46340a4b0bb26820d4df94 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 10:09:25 -0400 Subject: [PATCH 21/44] perf(adp): stabilize context dump benchmarks Keep Criterion identities tied only to context count, move publication assertions outside timed closures, and route benchmarks through a feature-gated facade over production artifact code so unit tests remain enabled under every feature set. --- .../benches/dogstatsd_context_dump.rs | 56 +++++++------------ .../src/dogstatsd_contexts/artifact.rs | 20 +++---- .../src/dogstatsd_contexts/mod.rs | 4 +- bin/agent-data-plane/src/lib.rs | 33 +++++++++++ 4 files changed, 65 insertions(+), 48 deletions(-) create mode 100644 bin/agent-data-plane/src/lib.rs diff --git a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs index b8790ba058e..10f1e755a7d 100644 --- a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs +++ b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs @@ -2,6 +2,7 @@ use std::fs; use std::hint::black_box; use std::path::Path; +use agent_data_plane::{count_dogstatsd_context_dump_records, publish_dogstatsd_context_dump}; use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; use saluki_context::{ @@ -10,9 +11,6 @@ use saluki_context::{ }; use stringtheory::MetaString; -#[path = "../src/dogstatsd_contexts/artifact.rs"] -mod artifact; - const CONTEXT_COUNTS: [usize; 3] = [10_000, 100_000, 1_000_000]; const CLIENT_TAGS: [&str; 5] = [ "env:production", @@ -37,44 +35,36 @@ fn benchmark_dogstatsd_context_dump(c: &mut Criterion) { assert_eq!(snapshot.len(), context_count); let run_directory = tempfile::tempdir().expect("benchmark run directory should be created"); - let canonical_path = run_directory.path().join(artifact::CONTEXT_DUMP_FILENAME); - let preflight_path = artifact::publish_context_dump(run_directory.path(), &snapshot) + let canonical_path = publish_dogstatsd_context_dump(run_directory.path(), &snapshot) .expect("benchmark preflight context dump should publish"); - assert_eq!(preflight_path, canonical_path); assert!(canonical_path.is_file()); - let mut decoded_records = 0usize; - artifact::for_each_record(&canonical_path, |record| { - assert!(!record.name.is_empty()); - assert_eq!(record.host, "benchmark-node-001.internal"); - assert!(matches!(record.metric_type.as_str(), "Counter" | "Gauge" | "Historate")); - assert_eq!(record.tagger_tags.len(), ORIGIN_TAGS.len()); - assert_eq!(record.metric_tags.len(), CLIENT_TAGS.len()); - assert!(!record.no_index); - assert_eq!(record.source, 1); - decoded_records += 1; - }) - .expect("benchmark preflight context dump should decode through the production reader"); + let decoded_records = count_dogstatsd_context_dump_records(&canonical_path) + .expect("benchmark preflight context dump should decode through the production reader"); assert_eq!(decoded_records, context_count); - assert_only_canonical_artifact(run_directory.path()); + assert_only_canonical_artifact(run_directory.path(), &canonical_path); let compressed_bytes = fs::metadata(&canonical_path) .expect("benchmark preflight artifact metadata should be available") .len(); - let benchmark_id = - BenchmarkId::from_parameter(format!("{context_count}_contexts_{compressed_bytes}_compressed_bytes")); + eprintln!("dogstatsd_context_dump setup: contexts={context_count}, compressed_bytes={compressed_bytes}"); + + let benchmark_id = BenchmarkId::from_parameter(context_count); group.throughput(Throughput::Elements(context_count as u64)); group.bench_function(benchmark_id, |b| { b.iter(|| { - let published_path = artifact::publish_context_dump(run_directory.path(), &snapshot) - .expect("benchmark context dump should publish"); - assert_eq!(published_path, canonical_path); - assert!(published_path.is_file()); - black_box(published_path) + black_box( + publish_dogstatsd_context_dump(run_directory.path(), &snapshot) + .expect("benchmark context dump should publish"), + ) }); }); - assert_only_canonical_artifact(run_directory.path()); + let postflight_path = publish_dogstatsd_context_dump(run_directory.path(), &snapshot) + .expect("benchmark postflight context dump should publish"); + assert_eq!(postflight_path, canonical_path); + assert!(postflight_path.is_file()); + assert_only_canonical_artifact(run_directory.path(), &canonical_path); } group.finish(); @@ -106,19 +96,13 @@ fn shared_tag_set(values: &[&'static str]) -> TagSet { tags.into_shared().to_mutable() } -fn assert_only_canonical_artifact(run_path: &Path) { +fn assert_only_canonical_artifact(run_path: &Path, canonical_path: &Path) { let mut entries: Vec<_> = fs::read_dir(run_path) .expect("benchmark run directory should be readable") - .map(|entry| { - entry - .expect("benchmark run directory entry should be readable") - .file_name() - .to_string_lossy() - .into_owned() - }) + .map(|entry| entry.expect("benchmark run directory entry should be readable").path()) .collect(); entries.sort_unstable(); - assert_eq!(entries, [artifact::CONTEXT_DUMP_FILENAME]); + assert_eq!(entries, [canonical_path.to_path_buf()]); } criterion_group!(benches, benchmark_dogstatsd_context_dump); diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index 737a9b080c5..21335c8f692 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -201,7 +201,7 @@ fn open_temporary_file(path: &Path) -> io::Result { options.open(path) } -#[cfg(all(test, not(feature = "context-dump-benchmark")))] +#[cfg(test)] fn publish_open_temporary( writer: W, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, ) -> Result<(), GenericError> @@ -214,7 +214,7 @@ where }) } -#[cfg(all(test, not(feature = "context-dump-benchmark")))] +#[cfg(test)] fn publish_buffered_temporary( buffer: BufWriter, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, @@ -430,7 +430,7 @@ impl Drop for TemporaryPathCleanup<'_, C> { #[derive(Debug, Deserialize, Eq, PartialEq)] #[serde(rename_all = "PascalCase")] -pub(super) struct AgentContextRecord { +pub(crate) struct AgentContextRecord { pub(super) name: String, pub(super) host: String, #[serde(rename = "Type")] @@ -501,7 +501,7 @@ impl fmt::Display for ArtifactDecodeError { impl Error for ArtifactDecodeError {} #[derive(Debug)] -pub(super) struct ArtifactError { +pub(crate) struct ArtifactError { path: PathBuf, operation: &'static str, record_index: usize, @@ -524,17 +524,17 @@ impl ArtifactError { Self::new(path, "decode", record_index, ArtifactDecodeError::from_serde(source)) } - #[cfg(all(test, not(feature = "context-dump-benchmark")))] + #[cfg(test)] pub(super) fn path(&self) -> &Path { &self.path } - #[cfg(all(test, not(feature = "context-dump-benchmark")))] + #[cfg(test)] pub(super) fn operation(&self) -> &'static str { self.operation } - #[cfg(all(test, not(feature = "context-dump-benchmark")))] + #[cfg(test)] pub(super) fn record_index(&self) -> usize { self.record_index } @@ -559,7 +559,7 @@ impl Error for ArtifactError { } } -pub(super) fn for_each_record(path: &Path, mut consume: impl FnMut(AgentContextRecord)) -> Result<(), ArtifactError> { +pub(crate) fn for_each_record(path: &Path, mut consume: impl FnMut(AgentContextRecord)) -> Result<(), ArtifactError> { let file = File::open(path).map_err(|error| ArtifactError::new(path, "open", 0, error))?; let mut compressed_input = BufReader::new(file); let is_compressed = compressed_input @@ -588,14 +588,14 @@ fn decode_records( Ok(()) } -#[cfg(all(test, not(feature = "context-dump-benchmark")))] +#[cfg(test)] fn collect_records(path: &Path) -> Result, ArtifactError> { let mut records = Vec::new(); for_each_record(path, |record| records.push(record))?; Ok(records) } -#[cfg(all(test, not(feature = "context-dump-benchmark")))] +#[cfg(test)] mod tests { use std::fs; use std::io::{self, BufWriter, Write as _}; diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs index 4cbfcb0baa9..64be91a31c8 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs @@ -3,7 +3,7 @@ use std::path::Path; use saluki_error::GenericError; pub(crate) use self::api::DogStatsDContextDumpAPIHandler; -pub(crate) use self::artifact::publish_context_dump; +pub(crate) use self::artifact::{for_each_record, publish_context_dump}; pub(crate) use self::report::ContextReport; mod api; @@ -12,7 +12,7 @@ mod report; pub(crate) fn read_report(path: &Path) -> Result { let mut report = ContextReport::new(); - artifact::for_each_record(path, |record| report.ingest(record))?; + for_each_record(path, |record| report.ingest(record))?; Ok(report) } diff --git a/bin/agent-data-plane/src/lib.rs b/bin/agent-data-plane/src/lib.rs new file mode 100644 index 00000000000..8e2bbee9a7f --- /dev/null +++ b/bin/agent-data-plane/src/lib.rs @@ -0,0 +1,33 @@ +#![cfg(feature = "context-dump-benchmark")] + +//! Benchmark-only access to DogStatsD context artifact production code. + +use std::path::{Path, PathBuf}; + +use saluki_components::transforms::AggregateContextSnapshotEntry; +use saluki_error::GenericError; + +#[cfg(feature = "context-dump-benchmark")] +#[allow( + dead_code, + unused_imports, + reason = "the benchmark facade includes the complete binary-owned context dump module" +)] +#[path = "dogstatsd_contexts/mod.rs"] +mod dogstatsd_contexts; + +/// Publishes a DogStatsD context dump through the production artifact writer. +#[doc(hidden)] +pub fn publish_dogstatsd_context_dump( + run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], +) -> Result { + dogstatsd_contexts::publish_context_dump(run_path, snapshot) +} + +/// Counts DogStatsD context dump records through the production artifact reader. +#[doc(hidden)] +pub fn count_dogstatsd_context_dump_records(path: &Path) -> Result { + let mut record_count = 0; + dogstatsd_contexts::for_each_record(path, |_| record_count += 1)?; + Ok(record_count) +} From 420ac1b8775ca04daa03d8d3af0ab900cb55b875 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 10:19:50 -0400 Subject: [PATCH 22/44] docs(adp): explain DogStatsD retained context dumps Document online and offline workflows, retained-state semantics, Agent interoperability, shared-filesystem limits, sensitive-data handling, and cleanup. --- .../configuration/configuration.md | 4 +- docs/agent-data-plane/dogstatsd-top.md | 111 ++++++++++++++++++ docs/agent-data-plane/index.md | 3 + 3 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 docs/agent-data-plane/dogstatsd-top.md diff --git a/docs/agent-data-plane/configuration/configuration.md b/docs/agent-data-plane/configuration/configuration.md index 7458c916719..1e5d3344578 100644 --- a/docs/agent-data-plane/configuration/configuration.md +++ b/docs/agent-data-plane/configuration/configuration.md @@ -179,7 +179,9 @@ topology, but only collects data during a time-bounded request. To collect stati `agent-data-plane dogstatsd stats --duration-secs N` or call the privileged `/dogstatsd/stats?collection_duration_secs=N` API. The handler waits for the requested collection window, then returns count and last-seen time per metric context inline as JSON. The CLI uses the -same API and renders the result as either summary or cardinality analysis. +same API and renders the result as either summary or cardinality analysis. To inspect contexts retained by the +aggregate transform instead of samples observed during a collection window, follow the +[retained DogStatsD context dump workflow](../dogstatsd-top.md). ADP also exposes aggregate DogStatsD counters through its internal telemetry scrape endpoint. This endpoint is separate from `/dogstatsd/stats`: it does not return the per-metric count and last-seen diff --git a/docs/agent-data-plane/dogstatsd-top.md b/docs/agent-data-plane/dogstatsd-top.md new file mode 100644 index 00000000000..1d02f8f9589 --- /dev/null +++ b/docs/agent-data-plane/dogstatsd-top.md @@ -0,0 +1,111 @@ +# Inspect retained DogStatsD contexts + +Use `agent-data-plane dogstatsd top` to inspect which DogStatsD metric contexts Agent Data Plane (ADP) currently retains and to identify metric names with high cardinality. + +## Understand what `top` reports + +`top` snapshots the contexts currently retained by the `dsd_agg` aggregate transform. It does not measure packet or sample frequency, and it does not collect data over a time window. Use the separate `agent-data-plane dogstatsd stats` command when you need time-windowed sample counts and last-seen times. + +A context's full identity consists of its metric name, host, client tags, and origin tags. The report groups these contexts by metric name, so contexts that differ only by host or origin tags still increase the context count for that name. The displayed tag cardinalities include only client tags; they do not include hosts or origin tags. + +The report reflects Saluki's normal aggregate lifecycle. Contexts disappear after their aggregation state is flushed and no longer retained. Sparse counters can remain as idle counters while the aggregate transform emits zero values, then disappear according to the configured Saluki counter expiry. Do not interpret this behavior as an Agent-identical context time to live. + +## Request and analyze a dump + +Run `top` without `--path` to request a new dump and analyze it: + +```console +$ agent-data-plane dogstatsd top +Wrote /var/run/datadog/dogstatsd_contexts.json.zstd + Contexts Metric name (number of unique values for each tag) + 3 request.count (2 env, 1 service) +``` + +Online requests read the configured Agent IPC authentication token and pin the API server certificate to the configured IPC certificate. The API creates and returns only the server-local path `/dogstatsd_contexts.json.zstd`. + +The API does not return the artifact contents. The CLI reads the returned path directly, so the CLI process and ADP must see the same filesystem and path. If ADP runs in a container, run the CLI with the same mount or use the copy workflow in the next section. The API does not provide a remote download endpoint. + +Use `--num-metrics` (`-m`) and `--num-tags` (`-t`) to change the default limits of 10 metric names and 5 tag keys: + +```console +$ agent-data-plane dogstatsd top --num-metrics 20 --num-tags 8 +$ agent-data-plane dogstatsd top -m 20 -t 8 +``` + +The legacy `--mum-tags` spelling remains an alias for `--num-tags`. Do not pass both spellings in one command. Limits must be non-negative integers, and `top` rejects extra positional arguments. + +The report orders metric names by descending context count and tag keys by descending unique-value count. It resolves ties lexically. When only one item exceeds a limit, the report displays that item; when two or more exceed it, the report replaces them with one remainder row or tag summary. + +To create the artifact without rendering a report, run: + +```console +$ agent-data-plane dogstatsd dump-contexts +Wrote /var/run/datadog/dogstatsd_contexts.json.zstd +``` + +`dump-contexts` takes no additional arguments. A successful request atomically replaces any prior file at the fixed path. The completed artifact remains there until an operator removes it. + +## Analyze a copied dump offline + +Wait for `top` or `dump-contexts` to report success, then securely copy the completed artifact to the analysis host. Analyze the copy by passing its path explicitly: + +```console +$ agent-data-plane dogstatsd top --path /secure/cases/incident-123/dogstatsd_contexts.json.zstd +``` + +You can combine `--path` (`-p`) with the metric and tag limits: + +```console +$ agent-data-plane dogstatsd top -p /secure/cases/incident-123/dogstatsd_contexts.json.zstd -m 25 -t 10 +``` + +Offline mode reads only the supplied file. It does not contact ADP or read the Agent IPC authentication token or certificate. `top` requires the `--path` value and does not fall back to a file in the current working directory. + +## Interpret context and tag counts + +Each artifact record represents one full retained context. The **Contexts** column counts records with the same metric name, including contexts distinguished only by host or origin tags. + +For the parenthesized tag summary, `top` deduplicates client tags across all records for the metric name. It treats the text before the first colon as the tag key and counts unique complete tag values. It treats a tag without a colon as its own key. For example, `(3 pod_name, 2 env)` means the retained contexts contain three distinct client tags whose key is `pod_name` and two whose key is `env`. It does not mean that ADP received three or two samples. + +Use the remainder row to account for hidden contexts. For example, `17 (other 4 metrics)` means the four hidden metric names contain 17 retained contexts in total. A tag remainder such as `12 values in 3 other tags` reports the summed unique-value counts for the hidden tag keys. + +## Protect and remove dump artifacts + +> [!WARNING] +> DogStatsD context dumps can contain metric names, hosts, client tags, and origin tags that reveal tenant or workload details. Treat every dump and copy as sensitive diagnostic data. + +On Unix, ADP creates the artifact with mode `0600`, owned by the ADP process owner. Keep that access restriction when you copy or store the file, use an encrypted and access-controlled transfer method, and avoid placing it in shared temporary directories. Remove the server-side artifact and every copied artifact manually when you finish the investigation. + +A later successful dump replaces the fixed server-side file but does not remove copies elsewhere. ADP does not apply a retention or cleanup policy to the completed file. + +## Understand snapshot consistency + +ADP currently has one `dsd_agg` owner. That owner constructs an internally consistent snapshot of its retained context map and then resumes processing. ADP serializes and compresses the snapshot after the owner resumes, so file creation does not hold the owner for the serialization work. + +Snapshot construction uses memory proportional to the number of retained contexts, or O(contexts), in addition to the aggregate state itself. Serialization also retains that snapshot until publication finishes. + +If ADP uses multiple aggregate owners in the future, each owner's snapshot remains internally consistent. The combined artifact would be a rolling snapshot because owners can respond at different moments; it would not represent one atomic instant across all owners. + +## Troubleshoot errors + +Use the status or file error in the CLI output to choose a response: + +- **401 Unauthorized**: The configured Agent authentication token does not match the server token. Verify that the CLI and ADP use the same current token file. Online commands also fail before the request if they cannot read the token or certificate, or if certificate pinning rejects the server certificate. +- **404 Not Found**: The context-dump route is absent when DogStatsD is disabled. Also verify that the CLI targets the intended ADP privileged API endpoint. +- **503 Service Unavailable**: The aggregate snapshot owner is not running or stopped before it responded. Wait for the topology and `dsd_agg` to become healthy, then retry. +- **504 Gateway Timeout**: The aggregate owner did not return a snapshot within the 30-second API snapshot deadline. Check ADP health and load before retrying. +- **500 Internal Server Error**: ADP could not publish the artifact. Verify that `run_path` is configured, exists as a directory, and is writable by the ADP process. An empty `run_path` fails; ADP does not fall back to the current working directory. +- **Missing or unreadable path**: An online request returns a path on ADP's filesystem. If the request succeeds but `top` cannot open that path, give the CLI access to the same filesystem or use `dump-contexts`, copy the completed file, and run offline with `--path`. +- **Malformed or truncated artifact**: A zstd initialization error or a JSON record error usually indicates an incomplete, corrupt, or unsupported file. Copy the artifact again after dump publication completes and verify the transfer before retrying. Pass the exact file path because offline mode has no current-working-directory fallback. + +## Artifact interoperability + +ADP writes a zstd-compressed stream of newline-delimited JSON (NDJSON) without a schema version. Every record uses the Agent diagnostic schema with these exact PascalCase fields: + +```json +{"Name":"request.count","Host":"node-a","Type":"Counter","TaggerTags":["pod_name:web-a"],"MetricTags":["env:prod","service:web"],"NoIndex":false,"Source":1} +``` + +`TaggerTags` contains origin tags and `MetricTags` contains client tags. The Datadog Agent and ADP `dogstatsd top --path` command-line clients can each read artifacts produced by either implementation. Saluki detects zstd compression from the file's magic bytes rather than its filename, and it also accepts plain NDJSON with the same records. + +This schema has no version field and serves as a diagnostic interoperability contract. It is not a supported metric ingestion API; do not send these records to a DogStatsD listener or depend on them as a general-purpose storage format. diff --git a/docs/agent-data-plane/index.md b/docs/agent-data-plane/index.md index 5dd73d700c6..9ddc8c06059 100644 --- a/docs/agent-data-plane/index.md +++ b/docs/agent-data-plane/index.md @@ -9,3 +9,6 @@ This section of the documentation covers specific areas related to the developme code lives within the Saluki repository. It won't be relevant to those simply looking to contribute to or utilize Saluki for their own purposes, but is meant to live alongside the ADP codebase to ensure that all relevant information is quickly and easily accessible. + +Use [Inspect retained DogStatsD contexts](dogstatsd-top.md) to snapshot retained aggregation contexts, investigate +cardinality, and analyze sensitive dump artifacts offline. From 09584d6114a3faa520fe75c96179702007562d1d Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 10:25:02 -0400 Subject: [PATCH 23/44] docs(adp): keep generated configuration unchanged Remove the optional retained-context link from generated configuration documentation. The operator workflow remains discoverable from the handwritten Agent Data Plane index. --- docs/agent-data-plane/configuration/configuration.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/agent-data-plane/configuration/configuration.md b/docs/agent-data-plane/configuration/configuration.md index 1e5d3344578..7458c916719 100644 --- a/docs/agent-data-plane/configuration/configuration.md +++ b/docs/agent-data-plane/configuration/configuration.md @@ -179,9 +179,7 @@ topology, but only collects data during a time-bounded request. To collect stati `agent-data-plane dogstatsd stats --duration-secs N` or call the privileged `/dogstatsd/stats?collection_duration_secs=N` API. The handler waits for the requested collection window, then returns count and last-seen time per metric context inline as JSON. The CLI uses the -same API and renders the result as either summary or cardinality analysis. To inspect contexts retained by the -aggregate transform instead of samples observed during a collection window, follow the -[retained DogStatsD context dump workflow](../dogstatsd-top.md). +same API and renders the result as either summary or cardinality analysis. ADP also exposes aggregate DogStatsD counters through its internal telemetry scrape endpoint. This endpoint is separate from `/dogstatsd/stats`: it does not return the per-metric count and last-seen From f77bb2bce9a7e09db20ab0e4b85460d5136008dd Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 11:08:49 -0400 Subject: [PATCH 24/44] perf(components): skip abandoned context snapshots Check whether the one-shot requester is still present before pausing the aggregate owner to clone retained contexts and allocate the snapshot vector. Keep response delivery race-safe by continuing to ignore a send failure if cancellation occurs after the check. --- .../src/transforms/aggregate/mod.rs | 72 ++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/lib/saluki-components/src/transforms/aggregate/mod.rs b/lib/saluki-components/src/transforms/aggregate/mod.rs index 6134ab101c0..afd6862a8e0 100644 --- a/lib/saluki-components/src/transforms/aggregate/mod.rs +++ b/lib/saluki-components/src/transforms/aggregate/mod.rs @@ -156,6 +156,17 @@ fn aggregate_context_snapshot_channel() -> (AggregateContextSnapshotHandle, Aggr (AggregateContextSnapshotHandle { requests }, receiver) } +#[inline] +fn send_context_snapshot_if_open( + response: AggregateContextSnapshotRequest, build_snapshot: impl FnOnce() -> AggregateContextSnapshot, +) { + if response.is_closed() { + return; + } + + let _ = response.send(build_snapshot()); +} + /// An accepted retained-context snapshot request for test fixtures. #[cfg(any(test, feature = "test-util"))] pub struct AggregateContextSnapshotPendingResponse { @@ -606,8 +617,7 @@ impl Transform for Aggregate { snapshot_request = receive_context_snapshot_request(&mut self.context_snapshot_requests) => { match snapshot_request { Some(response) => { - let snapshot = self.state.snapshot_contexts(); - let _ = response.send(snapshot); + send_context_snapshot_if_open(response, || self.state.snapshot_contexts()); } None => self.context_snapshot_requests = None, } @@ -1226,7 +1236,7 @@ const fn is_bucket_closed( // then we run it to make sure that we are always generating sequential timestamps for data points, etc. #[cfg(test)] mod tests { - use std::mem::size_of; + use std::{cell::Cell, mem::size_of}; use float_cmp::ApproxEqRatio as _; use saluki_context::tags::{Tag, TagSet}; @@ -1595,6 +1605,62 @@ mod tests { assert!(state.snapshot_contexts().is_empty()); } + #[test] + fn canceled_snapshot_response_skips_snapshot_construction() { + let mut state = AggregationState::new( + BUCKET_WIDTH_SECS, + 10, + COUNTER_EXPIRE, + HistogramConfiguration::default(), + Telemetry::noop(), + ); + assert!(state.insert( + insert_ts(1), + Metric::gauge(Context::from_static_name("canceled.snapshot"), 1.0), + )); + let (response, receiver) = oneshot::channel(); + drop(receiver); + let snapshot_calls = Cell::new(0); + + send_context_snapshot_if_open(response, || { + snapshot_calls.set(snapshot_calls.get() + 1); + state.snapshot_contexts() + }); + + assert_eq!(snapshot_calls.get(), 0); + } + + #[test] + fn open_snapshot_response_constructs_once_and_returns_exact_entries() { + let mut state = AggregationState::new( + BUCKET_WIDTH_SECS, + 10, + COUNTER_EXPIRE, + HistogramConfiguration::default(), + Telemetry::noop(), + ); + let context = Context::from_static_name("open.snapshot"); + assert!(state.insert(insert_ts(1), Metric::gauge(context.clone(), 1.0))); + let expected = vec![AggregateContextSnapshotEntry { + context, + metric_type: AggregateMetricType::Gauge, + unit: MetaString::empty(), + }]; + let (response, mut receiver) = oneshot::channel(); + let snapshot_calls = Cell::new(0); + + send_context_snapshot_if_open(response, || { + snapshot_calls.set(snapshot_calls.get() + 1); + state.snapshot_contexts() + }); + + assert_eq!(snapshot_calls.get(), 1); + assert_eq!( + receiver.try_recv().expect("open requester should receive a snapshot"), + expected + ); + } + #[test] fn aggregate_memory_bounds_include_peak_context_snapshot() { let mut config = AggregateConfiguration::with_defaults(); From 8ff54e6cab29554180acf153e68d3fac83f0c5ca Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 11:22:00 -0400 Subject: [PATCH 25/44] fix(adp): secure Windows context dump files Create Windows staging artifacts with a protected DACL before the file becomes visible. Grant full control only through Owner Rights, LocalSystem, and Builtin Administrators while preserving CREATE_NEW atomic publication semantics. --- bin/agent-data-plane/Cargo.toml | 2 + .../src/dogstatsd_contexts/artifact.rs | 265 +++++++++++++++++- 2 files changed, 256 insertions(+), 11 deletions(-) diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index f0e36d5cf5f..1bf797dce37 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -84,6 +84,8 @@ tikv-jemallocator = { workspace = true, features = [ [target.'cfg(target_os = "windows")'.dependencies] windows-sys = { workspace = true, features = [ "Win32_Foundation", + "Win32_Security", + "Win32_Security_Authorization", "Win32_Storage_FileSystem", ] } diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index 21335c8f692..273ff04a808 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -1,6 +1,8 @@ use std::error::Error; use std::fmt; -use std::fs::{self, File, OpenOptions}; +#[cfg(not(windows))] +use std::fs::OpenOptions; +use std::fs::{self, File}; use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; @@ -12,6 +14,8 @@ use uuid::Uuid; const ZSTD_MAGIC: &[u8; 4] = b"\x28\xb5\x2f\xfd"; pub(crate) const CONTEXT_DUMP_FILENAME: &str = "dogstatsd_contexts.json.zstd"; +#[cfg(windows)] +const WINDOWS_CONTEXT_DUMP_SDDL: &str = "D:P(A;;FA;;;OW)(A;;FA;;;SY)(A;;FA;;;BA)"; struct AgentContextSnapshotRecord<'a>(&'a AggregateContextSnapshotEntry); @@ -100,7 +104,7 @@ fn publish_context_dump_with( } fn publish_context_dump_with_services( - run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, entropy: &E, permissions: &P, + run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, entropy: &E, _permissions: &P, cleanup: &C, ) -> Result where @@ -126,7 +130,8 @@ where })?; run_with_temporary_cleanup(&temporary_path, &target, cleanup, || { - permissions.normalize(&file).with_error_context(|| { + #[cfg(unix)] + _permissions.normalize(&file).with_error_context(|| { format!( "Failed to set owner-only permissions on temporary DogStatsD context dump '{}' for target '{}'.", temporary_path.display(), @@ -167,12 +172,14 @@ impl EntropySource for SystemEntropy { } trait PermissionNormalizer { + #[cfg(unix)] fn normalize(&self, file: &File) -> io::Result<()>; } struct OwnerOnlyPermissions; impl PermissionNormalizer for OwnerOnlyPermissions { + #[cfg(unix)] fn normalize(&self, file: &File) -> io::Result<()> { set_owner_only_permissions(file) } @@ -185,11 +192,7 @@ fn set_owner_only_permissions(file: &File) -> io::Result<()> { file.set_permissions(fs::Permissions::from_mode(0o600)) } -#[cfg(not(unix))] -fn set_owner_only_permissions(_file: &File) -> io::Result<()> { - Ok(()) -} - +#[cfg(not(windows))] fn open_temporary_file(path: &Path) -> io::Result { let mut options = OpenOptions::new(); options.write(true).create_new(true); @@ -201,6 +204,117 @@ fn open_temporary_file(path: &Path) -> io::Result { options.open(path) } +#[cfg(windows)] +fn open_temporary_file(path: &Path) -> io::Result { + use std::os::windows::{ + ffi::OsStrExt as _, + io::{FromRawHandle as _, OwnedHandle}, + }; + use std::ptr; + + use windows_sys::Win32::{ + Foundation::{GENERIC_WRITE, INVALID_HANDLE_VALUE}, + Storage::FileSystem::{ + CreateFileW, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, + }, + }; + + let mut wide_path: Vec<_> = path.as_os_str().encode_wide().collect(); + if wide_path.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "path contains an interior NUL", + )); + } + wide_path.push(0); + + let security_attributes = WindowsSecurityAttributes::from_sddl(WINDOWS_CONTEXT_DUMP_SDDL)?; + // SAFETY: The path is a live NUL-terminated UTF-16 buffer, the security attributes refer to a live self-relative + // descriptor, and the null template handle is valid for a new file. A successful call transfers one owned handle. + let handle = unsafe { + CreateFileW( + wide_path.as_ptr(), + GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + security_attributes.as_ptr(), + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL, + ptr::null_mut(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + + // SAFETY: `CreateFileW` returned a valid handle whose ownership has not been transferred elsewhere. + let handle = unsafe { OwnedHandle::from_raw_handle(handle) }; + Ok(File::from(handle)) +} + +#[cfg(windows)] +struct WindowsSecurityAttributes { + descriptor: *mut std::ffi::c_void, + attributes: windows_sys::Win32::Security::SECURITY_ATTRIBUTES, +} + +#[cfg(windows)] +impl WindowsSecurityAttributes { + fn from_sddl(sddl: &str) -> io::Result { + use std::ptr; + + use windows_sys::Win32::{ + Foundation::FALSE, + Security::{ + Authorization::{ConvertStringSecurityDescriptorToSecurityDescriptorW, SDDL_REVISION_1}, + SECURITY_ATTRIBUTES, + }, + }; + + let wide_sddl: Vec<_> = sddl.encode_utf16().chain(std::iter::once(0)).collect(); + let mut descriptor = ptr::null_mut(); + // SAFETY: The SDDL is a live NUL-terminated UTF-16 buffer, `descriptor` is a valid output pointer, and the + // optional descriptor-size output is null. A successful call transfers ownership of the descriptor. + let result = unsafe { + ConvertStringSecurityDescriptorToSecurityDescriptorW( + wide_sddl.as_ptr(), + SDDL_REVISION_1, + &mut descriptor, + ptr::null_mut(), + ) + }; + if result == FALSE { + return Err(io::Error::last_os_error()); + } + + Ok(Self { + descriptor, + attributes: SECURITY_ATTRIBUTES { + nLength: std::mem::size_of::() as u32, + lpSecurityDescriptor: descriptor, + bInheritHandle: FALSE, + }, + }) + } + + fn as_ptr(&self) -> *const windows_sys::Win32::Security::SECURITY_ATTRIBUTES { + &self.attributes + } +} + +#[cfg(windows)] +impl Drop for WindowsSecurityAttributes { + fn drop(&mut self) { + if !self.descriptor.is_null() { + use windows_sys::Win32::Foundation::{LocalFree, HLOCAL}; + + // SAFETY: The descriptor was allocated by the SDDL conversion API and remains owned by this value. + unsafe { + let _ = LocalFree(self.descriptor as HLOCAL); + } + } + } +} + #[cfg(test)] fn publish_open_temporary( writer: W, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, @@ -613,13 +727,17 @@ mod tests { use serde_json::json; use stringtheory::MetaString; + #[cfg(windows)] + use super::open_temporary_file; use super::{ collect_records, finish_buffered_writer, finish_zstd_encoder, publish_buffered_temporary, publish_context_dump, publish_context_dump_with, publish_context_dump_with_services, publish_open_temporary, - set_owner_only_permissions, temporary_context_dump_path, write_snapshot_records, AgentContextRecord, - AtomicReplace, EntropySource, FileSystemAtomicReplace, FileSystemCleanup, OwnerOnlyPermissions, - PermissionNormalizer, SyncAll, TemporaryFileCleanup, TemporaryPathCleanup, CONTEXT_DUMP_FILENAME, + temporary_context_dump_path, write_snapshot_records, AgentContextRecord, AtomicReplace, EntropySource, + FileSystemAtomicReplace, FileSystemCleanup, OwnerOnlyPermissions, SyncAll, TemporaryFileCleanup, + TemporaryPathCleanup, CONTEXT_DUMP_FILENAME, }; + #[cfg(unix)] + use super::{set_owner_only_permissions, PermissionNormalizer}; use crate::dogstatsd_contexts::read_report; const PLAIN_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( @@ -639,6 +757,102 @@ mod tests { "/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd" ); + #[cfg(windows)] + struct LocalAllocation(*mut std::ffi::c_void); + + #[cfg(windows)] + impl Drop for LocalAllocation { + fn drop(&mut self) { + if !self.0.is_null() { + use windows_sys::Win32::Foundation::{LocalFree, HLOCAL}; + + // SAFETY: This pointer was allocated by a Windows security API that transfers ownership to the caller. + unsafe { + let _ = LocalFree(self.0 as HLOCAL); + } + } + } + } + + #[cfg(windows)] + fn windows_file_dacl_sddl(path: &std::path::Path) -> io::Result<(u16, String)> { + use std::os::windows::ffi::OsStrExt as _; + use std::{ptr, slice}; + + use windows_sys::Win32::{ + Foundation::ERROR_SUCCESS, + Security::{ + Authorization::{ + ConvertSecurityDescriptorToStringSecurityDescriptorW, GetNamedSecurityInfoW, SDDL_REVISION_1, + SE_FILE_OBJECT, + }, + GetSecurityDescriptorControl, DACL_SECURITY_INFORMATION, + }, + }; + + let mut wide_path: Vec<_> = path.as_os_str().encode_wide().collect(); + if wide_path.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "path contains an interior NUL", + )); + } + wide_path.push(0); + + let mut descriptor = ptr::null_mut(); + // SAFETY: The path is NUL-terminated and remains alive during the call. All unused output pointers are null, + // and `descriptor` receives an allocation owned by the caller on success. + let result = unsafe { + GetNamedSecurityInfoW( + wide_path.as_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + &mut descriptor, + ) + }; + if result != ERROR_SUCCESS { + return Err(io::Error::from_raw_os_error(result as i32)); + } + let descriptor = LocalAllocation(descriptor); + + let mut control = 0; + let mut revision = 0; + // SAFETY: `descriptor` is a valid security descriptor allocation and both output pointers are valid writes. + let result = unsafe { GetSecurityDescriptorControl(descriptor.0, &mut control, &mut revision) }; + if result == 0 { + return Err(io::Error::last_os_error()); + } + + let mut sddl = ptr::null_mut(); + let mut sddl_len = 0; + // SAFETY: `descriptor` remains valid during the call, and both output pointers are valid writes. The returned + // string allocation is owned by the caller on success. + let result = unsafe { + ConvertSecurityDescriptorToStringSecurityDescriptorW( + descriptor.0, + SDDL_REVISION_1, + DACL_SECURITY_INFORMATION, + &mut sddl, + &mut sddl_len, + ) + }; + if result == 0 { + return Err(io::Error::last_os_error()); + } + let sddl_allocation = LocalAllocation(sddl.cast()); + // SAFETY: The conversion API returned `sddl_len` initialized UTF-16 code units in the live `sddl` allocation. + let wide_sddl = unsafe { slice::from_raw_parts(sddl, sddl_len as usize) }; + let wide_sddl = wide_sddl.strip_suffix(&[0]).unwrap_or(wide_sddl); + let sddl = String::from_utf16(wide_sddl).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + drop(sddl_allocation); + + Ok((control, sddl)) + } + // These fixtures and expectations mirror ContextDebugRepr and the `dogstatsd top` command at Datadog Agent // commit 9de2a8371cf8d95794f238939709491907893288. fn expected_fixture_records() -> Vec { @@ -879,6 +1093,32 @@ mod tests { } } + #[cfg(windows)] + #[test] + fn temporary_dump_is_created_with_a_protected_restrictive_dacl() { + use windows_sys::Win32::Security::SE_DACL_PROTECTED; + + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let staging_path = temporary_context_dump_path(run_directory.path(), &target, &FixedEntropy) + .expect("temporary dump path should be generated"); + let _file = open_temporary_file(&staging_path).expect("temporary dump should be created"); + + let (control, sddl) = windows_file_dacl_sddl(&staging_path).expect("temporary dump DACL should be readable"); + + assert_ne!(control & SE_DACL_PROTECTED, 0, "DACL must be protected: {sddl}"); + assert!(sddl.contains("D:P"), "DACL SDDL must carry the protected flag: {sddl}"); + for required_ace in ["(A;;FA;;;OW)", "(A;;FA;;;SY)", "(A;;FA;;;BA)"] { + assert!(sddl.contains(required_ace), "DACL is missing {required_ace}: {sddl}"); + } + assert_eq!(sddl.matches('(').count(), 3, "DACL contains an unexpected ACE: {sddl}"); + assert!(!sddl.contains(";;;WD)"), "DACL must not grant Everyone access: {sddl}"); + assert!( + !sddl.contains(";;;BU)"), + "DACL must not grant Builtin Users access: {sddl}" + ); + } + #[cfg(unix)] #[test] fn publication_normalizes_restrictive_temporary_permissions_to_exactly_0600() { @@ -942,6 +1182,7 @@ mod tests { ); } + #[cfg(unix)] #[test] fn permission_failure_closes_and_removes_the_new_temporary_file() { let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); @@ -1300,8 +1541,10 @@ mod tests { } } + #[cfg(unix)] struct FailingPermissions; + #[cfg(unix)] impl PermissionNormalizer for FailingPermissions { fn normalize(&self, _file: &fs::File) -> io::Result<()> { Err(io::Error::other("injected permission failure")) From 601a5a1d411c8df9174656e13a00927315510f5e Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 13:32:17 -0400 Subject: [PATCH 26/44] test(adp): remove root-dependent permission checks Remove chmod-based unwritable-directory assertions that are bypassed when CI runs as root. Deterministic permission, invalid-path, and injected publication failures continue to cover the same error handling paths. --- .../src/dogstatsd_contexts/api.rs | 24 ------------------- .../src/dogstatsd_contexts/artifact.rs | 19 --------------- 2 files changed, 43 deletions(-) diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs index ee2b53a43da..7db16858f7a 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs @@ -607,30 +607,6 @@ mod tests { ); } - #[cfg(unix)] - #[tokio::test] - async fn unwritable_run_path_returns_internal_error_without_an_artifact() { - use std::os::unix::fs::PermissionsExt as _; - - let run_directory = tempfile::tempdir().unwrap(); - let original_permissions = fs::metadata(run_directory.path()).unwrap().permissions(); - fs::set_permissions(run_directory.path(), fs::Permissions::from_mode(0o500)).unwrap(); - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_directory.path()).unwrap(); - let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - - let response = send(&handler, authorized_post()).await; - fs::set_permissions(run_directory.path(), original_permissions).unwrap(); - - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - assert_eq!( - response_body(response).await, - "Failed to publish DogStatsD context dump; check the configured run path and permissions." - ); - assert!(!run_directory.path().join(CONTEXT_DUMP_FILENAME).exists()); - owner.await.unwrap().unwrap(); - } - #[tokio::test] async fn publication_failure_and_blocking_task_panic_return_safe_internal_errors() { for publisher in [ diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index 273ff04a808..ae57d9818a8 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -1277,25 +1277,6 @@ mod tests { } } - #[cfg(unix)] - #[test] - fn rejects_an_unwritable_run_directory_with_target_and_operation_context() { - use std::os::unix::fs::PermissionsExt as _; - - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let original_permissions = fs::metadata(run_directory.path()).unwrap().permissions(); - fs::set_permissions(run_directory.path(), fs::Permissions::from_mode(0o500)).unwrap(); - - let result = publish_context_dump(run_directory.path(), &[]); - fs::set_permissions(run_directory.path(), original_permissions).unwrap(); - - let error = result.expect_err("unwritable run path should fail"); - let message = format!("{error:#}"); - assert!(message.contains(CONTEXT_DUMP_FILENAME), "{message}"); - assert!(message.contains("create temporary"), "{message}"); - assert_eq!(directory_entries(run_directory.path()), Vec::::new()); - } - #[test] fn injected_replace_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); From 628c0081459c67b950aa47b9cb532399100647d6 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 13:52:54 -0400 Subject: [PATCH 27/44] test(integration): verify DogStatsD top end to end Boot ADP in the Linux integration harness, send real DogStatsD contexts, exercise authenticated online and dump-only commands, validate offline analysis, and compare the Saluki artifact through the stock Agent CLI. --- .../cases/dogstatsd-top/config.yaml | 53 ++++++ .../dogstatsd-top/run_dogstatsd_top_test.py | 157 ++++++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 test/integration/cases/dogstatsd-top/config.yaml create mode 100644 test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py diff --git a/test/integration/cases/dogstatsd-top/config.yaml b/test/integration/cases/dogstatsd-top/config.yaml new file mode 100644 index 00000000000..21d8528069c --- /dev/null +++ b/test/integration/cases/dogstatsd-top/config.yaml @@ -0,0 +1,53 @@ +type: integration +name: "dogstatsd-top" +description: "Verifies retained DogStatsD contexts through the authenticated API, online CLI, and offline Agent-compatible artifact" +timeout: 180s +runtimes: [linux] + +env: + DD_API_KEY: "00000000000000000000000000000000" + DD_HOSTNAME: "integration-test-dogstatsd-top" + DD_DATA_PLANE_ENABLED: "true" + DD_DATA_PLANE_STANDALONE_MODE: "false" + DD_DATA_PLANE_DOGSTATSD_ENABLED: "true" + +container: + files: + - "run_dogstatsd_top_test.py:/dogstatsd-top/run_dogstatsd_top_test.py" + exposed_ports: + - "58125/udp" + - "55101/tcp" + +procedure: + - parallel: + - assertion: port_listening + port: 58125 + protocol: udp + timeout: 60s + - assertion: port_listening + port: 55101 + protocol: tcp + timeout: 60s + # The route is POST-only. A GET returns 405 when registration is complete and 404 when absent. + - assertion: http_check + endpoint: "https://localhost:55101/dogstatsd/contexts/dump" + status: + not_equal: 404 + insecure_skip_verify: true + timeout: 60s + - action: target_exec + command: + - "/opt/datadog-agent/embedded/bin/python3" + - "/dogstatsd-top/run_dogstatsd_top_test.py" + timeout: 90s + - assertion: file_contains + path: "/tmp/dogstatsd-top-integration-result" + pattern: "passed" + timeout: 10s + - parallel: + - assertion: process_stable_for + duration: 10s + - assertion: log_not_contains + pattern: "panic|PANIC" + regex: true + during: 10s diff --git a/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py b/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py new file mode 100644 index 00000000000..87804d0b080 --- /dev/null +++ b/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py @@ -0,0 +1,157 @@ +#!/opt/datadog-agent/embedded/bin/python3 + +import json +import shutil +import socket +import ssl +import stat +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path + +ADP = Path("/opt/datadog-agent/embedded/bin/agent-data-plane") +AGENT = Path("/opt/datadog-agent/bin/agent/agent") +CONFIG = Path("/etc/datadog-agent/datadog.yaml") +AUTH_TOKEN = Path("/etc/datadog-agent/auth_token") +API_URL = "https://127.0.0.1:55101/dogstatsd/contexts/dump" +DUMP_FILENAME = "dogstatsd_contexts.json.zstd" +COPIED_DUMP = Path("/tmp/dogstatsd-top-copied.json.zstd") +ONLINE_OUTPUT = Path("/tmp/dogstatsd-top-online-output") +OFFLINE_OUTPUT = Path("/tmp/dogstatsd-top-offline-output") +AGENT_OUTPUT = Path("/tmp/dogstatsd-top-agent-output") +RESULT = Path("/tmp/dogstatsd-top-integration-result") + +REQUEST_ROW = " 2\tintegration.top.requests\t(2 instance, 1 env, 1 service)" +QUEUE_ROW = " 1\tintegration.top.queue\t(1 queue)" + + +def run(command): + completed = subprocess.run(command, text=True, capture_output=True, timeout=30) + if completed.returncode != 0: + raise RuntimeError( + f"command failed ({completed.returncode}): {' '.join(map(str, command))}\n" + f"stdout:\n{completed.stdout}\nstderr:\n{completed.stderr}" + ) + return completed.stdout + + +def run_adp(*arguments): + return run([str(ADP), "--config", str(CONFIG), "dogstatsd", *arguments]) + + +def unverified_tls_context(): + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + + +def post_dump(authorization=None): + headers = {} + if authorization is not None: + headers["Authorization"] = f"Bearer {authorization}" + request = urllib.request.Request(API_URL, data=b"", headers=headers, method="POST") + return urllib.request.urlopen(request, context=unverified_tls_context(), timeout=10) + + +def assert_unauthorized_request(): + try: + post_dump() + except urllib.error.HTTPError as error: + body = error.read().decode("utf-8") + if error.code != 401 or body != "Authentication required.": + raise AssertionError(f"unexpected unauthenticated response: {error.code} {body!r}") + else: + raise AssertionError("context dump API accepted an unauthenticated request") + + +def send_dogstatsd_contexts(): + payload = b"\n".join( + [ + b"integration.top.requests:1|c|#host:host-a,env:prod,service:web,instance:a", + b"integration.top.requests:1|c|#host:host-b,env:prod,service:web,instance:b", + b"integration.top.queue:1|c|#queue:alpha", + ] + ) + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as client: + for _ in range(3): + client.sendto(payload, ("127.0.0.1", 58125)) + + +def wait_for_online_report(): + last_output = "" + for _ in range(40): + last_output = run_adp("top") + if REQUEST_ROW in last_output and QUEUE_ROW in last_output: + ONLINE_OUTPUT.write_text(last_output, encoding="utf-8") + return last_output + time.sleep(0.5) + raise AssertionError(f"retained contexts did not appear in online output:\n{last_output}") + + +def parse_written_path(output): + lines = output.splitlines() + if len(lines) != 1 or not lines[0].startswith("Wrote "): + raise AssertionError(f"unexpected dump-contexts output: {output!r}") + path = Path(lines[0].removeprefix("Wrote ")) + if not path.is_absolute() or path.name != DUMP_FILENAME: + raise AssertionError(f"unexpected context dump path: {path}") + if not path.is_file(): + raise AssertionError(f"context dump was not created: {path}") + return path + + +def normalize_report(output): + return output.replace("\r\n", "\n").rstrip("\n") + + +def validate_online_api(token): + with post_dump(token) as response: + body = response.read().decode("utf-8") + if response.status != 200: + raise AssertionError(f"authenticated API returned {response.status}: {body}") + if not response.headers.get_content_type() == "application/json": + raise AssertionError(f"unexpected API content type: {response.headers.get('Content-Type')}") + path = Path(json.loads(body)) + if path.name != DUMP_FILENAME or not path.is_file(): + raise AssertionError(f"authenticated API returned an invalid artifact path: {path}") + + +def validate_dump_and_offline_interoperability(): + dump_path = parse_written_path(run_adp("dump-contexts")) + if stat.S_IMODE(dump_path.stat().st_mode) != 0o600: + raise AssertionError(f"context dump mode is not 0600: {oct(stat.S_IMODE(dump_path.stat().st_mode))}") + + shutil.copyfile(dump_path, COPIED_DUMP) + offline = run_adp("top", "--path", str(COPIED_DUMP)) + OFFLINE_OUTPUT.write_text(offline, encoding="utf-8") + if REQUEST_ROW not in offline or QUEUE_ROW not in offline: + raise AssertionError(f"offline report is missing retained contexts:\n{offline}") + + agent = run([str(AGENT), "dogstatsd", "top", "--path", str(COPIED_DUMP)]) + AGENT_OUTPUT.write_text(agent, encoding="utf-8") + if normalize_report(agent) != normalize_report(offline): + raise AssertionError(f"Agent and ADP reports differ:\nADP:\n{offline}\nAgent:\n{agent}") + + +def main(): + for required_path in [ADP, AGENT, CONFIG, AUTH_TOKEN]: + if not required_path.exists(): + raise AssertionError(f"required test path does not exist: {required_path}") + + assert_unauthorized_request() + send_dogstatsd_contexts() + online = wait_for_online_report() + if not online.startswith("Wrote "): + raise AssertionError(f"online top did not print the generated artifact path:\n{online}") + + token = AUTH_TOKEN.read_text(encoding="utf-8") + validate_online_api(token) + validate_dump_and_offline_interoperability() + RESULT.write_text("passed\n", encoding="utf-8") + + +if __name__ == "__main__": + main() From d38aec8a0a9b078f6f52242ad3be0f545b0cd615 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 14:02:35 -0400 Subject: [PATCH 28/44] test(integration): verify bidirectional context dump interop Check a real Agent 7.80.3 compressed and plain context dump through both CLIs, then verify a live ADP-produced artifact renders identically through ADP and the stock Agent. --- .../agent-7.80.3-contexts.json.zstd | Bin 0 -> 217 bytes .../agent-7.80.3-contexts.ndjson | 4 ++++ .../cases/dogstatsd-top/config.yaml | 4 ++++ .../dogstatsd-top/run_dogstatsd_top_test.py | 18 +++++++++++++++++- 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 test/integration/cases/dogstatsd-top/agent-7.80.3-contexts.json.zstd create mode 100644 test/integration/cases/dogstatsd-top/agent-7.80.3-contexts.ndjson diff --git a/test/integration/cases/dogstatsd-top/agent-7.80.3-contexts.json.zstd b/test/integration/cases/dogstatsd-top/agent-7.80.3-contexts.json.zstd new file mode 100644 index 0000000000000000000000000000000000000000..40d33c4e5ebd59704521959f01aa6c0e8c30a981 GIT binary patch literal 217 zcmV;~04Dz^wJ-euScL`v(u^k|Ky8}?KQC$``+Jg1BAKLC44Q(Cpi7{PLcVNzPenPo zIf#l>%fb^nFvxr?kSGX}fySJOhch_Hb*@DK#);%Zvx%^L^6M8Nq>YgU5K zwSf$0h@(7M3m=A;JCnGY%ExFmc8|On)%FNJ4282MRN7NQ#mDWN%_{KuEJE?|g800; z41BDsZyayQ<`X7twQnk)|9}fi1QGzCkpc9QuNPq4nqecJ#K36tm}np~IaX>50XCTc T(5_O4EC%b!mxJEdymG literal 0 HcmV?d00001 diff --git a/test/integration/cases/dogstatsd-top/agent-7.80.3-contexts.ndjson b/test/integration/cases/dogstatsd-top/agent-7.80.3-contexts.ndjson new file mode 100644 index 00000000000..ff513907eb5 --- /dev/null +++ b/test/integration/cases/dogstatsd-top/agent-7.80.3-contexts.ndjson @@ -0,0 +1,4 @@ +{"Name":"interop.agent.requests","Host":"agent-b","Type":"Counter","TaggerTags":[],"MetricTags":["env:prod","instance:b"],"NoIndex":false,"Source":1} +{"Name":"interop.agent.queue","Host":"dogstatsd-interop-agent","Type":"Counter","TaggerTags":[],"MetricTags":["queue:alpha"],"NoIndex":false,"Source":1} +{"Name":"datadog.agent.started","Host":"dogstatsd-interop-agent","Type":"Count","TaggerTags":[],"MetricTags":["version:7.80.3","package_version:bin"],"NoIndex":false,"Source":0} +{"Name":"interop.agent.requests","Host":"agent-a","Type":"Counter","TaggerTags":[],"MetricTags":["env:prod","instance:a"],"NoIndex":false,"Source":1} diff --git a/test/integration/cases/dogstatsd-top/config.yaml b/test/integration/cases/dogstatsd-top/config.yaml index 21d8528069c..9305b24b550 100644 --- a/test/integration/cases/dogstatsd-top/config.yaml +++ b/test/integration/cases/dogstatsd-top/config.yaml @@ -14,6 +14,10 @@ env: container: files: - "run_dogstatsd_top_test.py:/dogstatsd-top/run_dogstatsd_top_test.py" + # Produced by Datadog Agent 7.80.3 `dogstatsd dump-contexts`; the NDJSON file is the exact + # decompressed content of the zstd artifact. + - "agent-7.80.3-contexts.json.zstd:/dogstatsd-top/agent-7.80.3-contexts.json.zstd" + - "agent-7.80.3-contexts.ndjson:/dogstatsd-top/agent-7.80.3-contexts.ndjson" exposed_ports: - "58125/udp" - "55101/tcp" diff --git a/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py b/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py index 87804d0b080..37288022df2 100644 --- a/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py +++ b/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py @@ -18,6 +18,8 @@ API_URL = "https://127.0.0.1:55101/dogstatsd/contexts/dump" DUMP_FILENAME = "dogstatsd_contexts.json.zstd" COPIED_DUMP = Path("/tmp/dogstatsd-top-copied.json.zstd") +AGENT_FIXTURE_COMPRESSED = Path("/dogstatsd-top/agent-7.80.3-contexts.json.zstd") +AGENT_FIXTURE_PLAIN = Path("/dogstatsd-top/agent-7.80.3-contexts.ndjson") ONLINE_OUTPUT = Path("/tmp/dogstatsd-top-online-output") OFFLINE_OUTPUT = Path("/tmp/dogstatsd-top-offline-output") AGENT_OUTPUT = Path("/tmp/dogstatsd-top-agent-output") @@ -119,6 +121,19 @@ def validate_online_api(token): raise AssertionError(f"authenticated API returned an invalid artifact path: {path}") +def validate_agent_fixture_interoperability(): + reports = [] + for fixture in [AGENT_FIXTURE_COMPRESSED, AGENT_FIXTURE_PLAIN]: + reports.append(run_adp("top", "--path", str(fixture))) + reports.append(run([str(AGENT), "dogstatsd", "top", "--path", str(fixture)])) + + normalized = [normalize_report(report) for report in reports] + if any(report != normalized[0] for report in normalized[1:]): + raise AssertionError(f"Agent-produced fixture reports differ: {reports}") + if "interop.agent.requests" not in normalized[0]: + raise AssertionError(f"Agent-produced fixture did not contain expected contexts: {normalized[0]}") + + def validate_dump_and_offline_interoperability(): dump_path = parse_written_path(run_adp("dump-contexts")) if stat.S_IMODE(dump_path.stat().st_mode) != 0o600: @@ -137,11 +152,12 @@ def validate_dump_and_offline_interoperability(): def main(): - for required_path in [ADP, AGENT, CONFIG, AUTH_TOKEN]: + for required_path in [ADP, AGENT, CONFIG, AUTH_TOKEN, AGENT_FIXTURE_COMPRESSED, AGENT_FIXTURE_PLAIN]: if not required_path.exists(): raise AssertionError(f"required test path does not exist: {required_path}") assert_unauthorized_request() + validate_agent_fixture_interoperability() send_dogstatsd_contexts() online = wait_for_online_report() if not online.startswith("Wrote "): From 8a59e080d7fbb4887b3e51e5fa53da19c211578b Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Wed, 22 Jul 2026 14:18:31 -0400 Subject: [PATCH 29/44] test(adp): pin NDJSON fixtures to LF Prevent Windows checkouts from translating plain NDJSON fixtures to CRLF so exact byte comparisons against compressed Agent artifacts remain portable. --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000000..a86bc779121 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# NDJSON fixtures are byte-compared with compressed streams and must not be translated to CRLF. +*.ndjson text eol=lf From 40386b69cc2b81023b35985d3322ee267a3aab6a Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Thu, 23 Jul 2026 15:01:23 -0400 Subject: [PATCH 30/44] refactor(adp): simplify context dump implementation Use tempfile for secure randomized staging instead of custom entropy and filename generation, and separate API and artifact test harnesses from production modules for focused review. --- Cargo.lock | 1 - Cargo.toml | 1 - bin/agent-data-plane/Cargo.toml | 3 +- .../src/dogstatsd_contexts/api.rs | 673 +--------- .../src/dogstatsd_contexts/api_tests.rs | 656 ++++++++++ .../src/dogstatsd_contexts/artifact.rs | 1147 +---------------- .../src/dogstatsd_contexts/artifact_tests.rs | 1075 +++++++++++++++ 7 files changed, 1767 insertions(+), 1789 deletions(-) create mode 100644 bin/agent-data-plane/src/dogstatsd_contexts/api_tests.rs create mode 100644 bin/agent-data-plane/src/dogstatsd_contexts/artifact_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 22dd0e7645e..c2dda9e781a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,7 +39,6 @@ dependencies = [ "derive-where", "foldhash 0.2.0", "futures", - "getrandom 0.4.2", "hashbrown 0.17.1", "http", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index 4075577a109..58fbb8a7ec2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -224,7 +224,6 @@ iana-time-zone = { version = "0.1", default-features = false } backon = { version = "1", default-features = false } http-serde-ext = { version = "1", default-features = false } uuid = { version = "1.23", default-features = false } -getrandom = { version = "0.4", default-features = false } rcgen = { version = "0.14", default-features = false } tikv-jemallocator = { version = "0.7", default-features = false } axum-extra = { version = "0.12", default-features = false } diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index 1bf797dce37..430dba48070 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -30,7 +30,6 @@ datadog-agent-config = { workspace = true } datadog-protos = { workspace = true } foldhash = { workspace = true } futures = { workspace = true } -getrandom = { workspace = true } hashbrown = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } @@ -58,6 +57,7 @@ serde_json = { workspace = true } serde_with = { workspace = true } stringtheory = { workspace = true } subtle = { workspace = true } +tempfile = { workspace = true } tokio = { workspace = true, features = [ "fs", "macros", @@ -103,7 +103,6 @@ saluki-config = { workspace = true, features = ["test-util"] } saluki-core = { workspace = true, features = ["test-util"] } saluki-metrics = { workspace = true, features = ["test"] } saluki-tls = { workspace = true } -tempfile = { workspace = true } tokio-rustls = { workspace = true } tower = { workspace = true, features = ["util"] } diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs index 7db16858f7a..90f67a22be3 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs @@ -15,8 +15,10 @@ use saluki_error::{generic_error, GenericError}; use subtle::ConstantTimeEq as _; use tokio::sync::Mutex; -use super::publish_context_dump; +use super::{publish_context_dump, CONTEXT_DUMP_ROUTE}; +// The Agent has no owner-request timeout. ADP uses a finite bound so a stopped topology cannot hold an HTTP request +// indefinitely; 30 seconds is several orders of magnitude above the measured one-million-context snapshot time. const PRODUCTION_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(30); const AUTHENTICATION_REQUIRED: &str = "Authentication required."; const SNAPSHOT_UNAVAILABLE: &str = @@ -237,673 +239,10 @@ impl APIHandler for DogStatsDContextDumpAPIHandler { } fn generate_routes(&self) -> Router { - Router::new().route("/dogstatsd/contexts/dump", post(Self::dump_handler)) + Router::new().route(CONTEXT_DUMP_ROUTE, post(Self::dump_handler)) } } #[cfg(test)] -mod tests { - use std::fs; - use std::path::{Path, PathBuf}; - use std::sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, - mpsc, Arc, Condvar, Mutex as StdMutex, - }; - use std::time::Duration; - - use http::{ - header::{AUTHORIZATION, CONTENT_TYPE}, - HeaderValue, Request, StatusCode, - }; - use http_body_util::{BodyExt as _, Empty}; - use hyper::body::Bytes; - use saluki_api::{response::Response, APIHandler}; - use saluki_components::transforms::{ - aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotEntry, AggregateContextSnapshotHandle, - AggregateContextSnapshotPendingResponse, AggregateMetricType, - }; - use saluki_context::Context; - use saluki_error::{generic_error, GenericError}; - use stringtheory::MetaString; - use tower::ServiceExt as _; - - use super::{ - ContextDumpPublisher, ContextSnapshotCoordinator, DogStatsDContextDumpAPIHandler, - FileSystemContextDumpPublisher, SnapshotError, - }; - use crate::dogstatsd_contexts::{ - artifact::{for_each_record, CONTEXT_DUMP_FILENAME}, - publish_context_dump, - }; - - const AUTH_TOKEN: &str = "expected-agent-token"; - const ROUTE: &str = "/dogstatsd/contexts/dump"; - - #[test] - fn constructor_rejects_empty_and_non_visible_http_tokens_without_exposing_them() { - let invalid_tokens: &[&[u8]] = &[ - b"", - b"contains space", - b"line\nbreak", - b"tab\tvalue", - b"nul\0value", - b"\x7f", - b"\xff", - ]; - - for token in invalid_tokens { - let error = match DogStatsDContextDumpAPIHandler::new(*token, Vec::new(), PathBuf::from("run")) { - Ok(_) => panic!("invalid authentication token should be rejected"), - Err(error) => error, - }; - let message = format!("{error:#}"); - if !token.is_empty() { - assert!(!message.as_bytes().windows(token.len()).any(|window| window == *token)); - } - } - } - - #[tokio::test] - async fn rejects_missing_duplicate_malformed_and_wrong_authorization_with_an_exact_safe_body() { - let publisher = Arc::new(NoOpPublisher); - let handler = test_handler(Vec::new(), PathBuf::from("unused"), publisher); - let mut duplicate = post_request(None); - duplicate - .headers_mut() - .append(AUTHORIZATION, HeaderValue::from_static("Bearer expected-agent-token")); - duplicate - .headers_mut() - .append(AUTHORIZATION, HeaderValue::from_static("Bearer expected-agent-token")); - let mut non_utf8 = post_request(None); - non_utf8.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from_bytes(b"Bearer \xff").expect("HTTP headers allow opaque non-ASCII bytes"), - ); - let cases = [ - post_request(None), - duplicate, - non_utf8, - post_request(Some("Basic expected-agent-token")), - post_request(Some("Bearer")), - post_request(Some("Bearersecret")), - post_request(Some("Bearer\texpected-agent-token")), - post_request(Some("Bearer expected-agent-token trailing")), - post_request(Some("Bearer supplied-secret")), - post_request(Some("Bearer unexpect-agent-token")), - ]; - - for request in cases { - let response = send(&handler, request).await; - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - let body = response_body(response).await; - assert_eq!(body, "Authentication required."); - assert!(!body.contains(AUTH_TOKEN)); - assert!(!body.contains("supplied-secret")); - } - } - - #[tokio::test] - async fn accepts_bearer_scheme_case_insensitively() { - for scheme in ["bearer", "BEARER", "BeArEr"] { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - - let response = send(&handler, post_request(Some(&format!("{scheme} {AUTH_TOKEN}")))).await; - - assert_eq!(response.status(), StatusCode::OK); - owner.await.unwrap().unwrap(); - } - } - - #[tokio::test] - async fn accepts_only_the_exact_bearer_credential_bytes() { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - - let response = send(&handler, post_request(Some("Bearer expected-agent-token"))).await; - - assert_eq!(response.status(), StatusCode::OK); - owner.await.unwrap().unwrap(); - } - - #[tokio::test] - async fn coordinator_reuses_the_one_owner_snapshot_allocation() { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let owner_snapshot = vec![snapshot_entry("owner.one.first"), snapshot_entry("owner.one.second")]; - let expected = owner_snapshot.clone(); - let original_pointer = owner_snapshot.as_ptr() as usize; - let original_capacity = owner_snapshot.capacity(); - let owner = tokio::spawn(async move { responder.respond(owner_snapshot).await }); - let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); - - let actual = coordinator.snapshot().await.expect("snapshot should succeed"); - - assert_eq!(actual, expected); - assert_eq!(actual.as_ptr() as usize, original_pointer); - assert_eq!(actual.capacity(), original_capacity); - owner.await.unwrap().unwrap(); - } - - #[tokio::test] - async fn coordinator_requests_all_owners_concurrently_and_flattens_each_snapshot_once_in_owner_order() { - let (first_handle, mut first_responder) = aggregate_context_snapshot_channel_for_test(); - let (second_handle, mut second_responder) = aggregate_context_snapshot_channel_for_test(); - let first_snapshot = vec![snapshot_entry("owner.one.first"), snapshot_entry("owner.one.second")]; - let second_snapshot = vec![snapshot_entry("owner.two.only")]; - let expected = [first_snapshot.clone(), second_snapshot.clone()].concat(); - let (second_responded_tx, second_responded_rx) = tokio::sync::oneshot::channel(); - let first_owner = tokio::spawn(async move { - second_responded_rx - .await - .expect("second owner should receive its request"); - first_responder.respond(first_snapshot).await - }); - let second_owner = tokio::spawn(async move { - let result = second_responder.respond(second_snapshot).await; - let _ = second_responded_tx.send(()); - result - }); - let coordinator = - ContextSnapshotCoordinator::new(vec![first_handle, second_handle], Duration::from_millis(250)); - - let actual = coordinator.snapshot().await.expect("both snapshots should succeed"); - - assert_eq!(actual, expected); - first_owner.await.unwrap().unwrap(); - second_owner.await.unwrap().unwrap(); - } - - #[tokio::test] - async fn coordinator_reports_no_handles_as_typed_unavailable() { - let coordinator = ContextSnapshotCoordinator::new(Vec::new(), Duration::from_secs(1)); - - let error = coordinator.snapshot().await.expect_err("empty owner set should fail"); - - assert_eq!(error, SnapshotError::SnapshotUnavailable); - } - - #[tokio::test] - async fn coordinator_reports_a_closed_owner_as_typed_unavailable() { - let (handle, responder) = aggregate_context_snapshot_channel_for_test(); - drop(responder); - let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); - - let error = coordinator.snapshot().await.expect_err("closed owner should fail"); - - assert_eq!(error, SnapshotError::SnapshotUnavailable); - } - - #[tokio::test] - async fn coordinator_reports_elapsed_deadline_as_typed_timeout() { - let (handle, _responder) = aggregate_context_snapshot_channel_for_test(); - let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_millis(10)); - - let error = coordinator - .snapshot() - .await - .expect_err("unresponsive owner should time out"); - - assert_eq!(error, SnapshotError::SnapshotTimedOut); - } - - #[tokio::test] - async fn canceled_coordinator_request_does_not_make_a_late_owner_response_panic_or_fail() { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); - let request = tokio::spawn(async move { coordinator.snapshot().await }); - let pending_response: AggregateContextSnapshotPendingResponse = responder - .receive() - .await - .expect("the owner should accept the snapshot request"); - - request.abort(); - let _ = request.await; - - pending_response.respond(vec![snapshot_entry("late.owner.response")]); - } - - #[tokio::test] - async fn generated_router_rejects_get_with_method_not_allowed() { - let handler = test_handler(Vec::new(), PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let request = Request::builder() - .method("GET") - .uri(ROUTE) - .body(Empty::::new()) - .unwrap(); - - let response = send(&handler, request).await; - - assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); - } - - #[tokio::test] - async fn unauthorized_post_does_not_request_a_snapshot_or_create_an_artifact() { - let run_directory = tempfile::tempdir().unwrap(); - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_directory.path().to_owned()) - .expect("handler should be valid"); - - let response = send(&handler, post_request(None)).await; - - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!(!run_directory.path().join(CONTEXT_DUMP_FILENAME).exists()); - drop(handler); - assert!(responder.respond(Vec::new()).await.is_err()); - } - - #[tokio::test] - async fn authorized_router_post_returns_only_the_json_path_to_a_complete_decodable_artifact() { - let run_directory = tempfile::tempdir().unwrap(); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let snapshot = vec![snapshot_entry("published.first"), snapshot_entry("published.second")]; - let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN.as_bytes(), vec![handle], run_directory.path()) - .expect("handler should be valid"); - let owner = tokio::spawn(async move { responder.respond(snapshot).await }); - - let response = send(&handler, post_request(Some("Bearer expected-agent-token"))).await; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response.headers().get(CONTENT_TYPE).unwrap(), "application/json"); - assert_eq!( - response_body(response).await, - serde_json::to_string(target.to_str().unwrap()).unwrap() - ); - owner.await.unwrap().unwrap(); - let mut names = Vec::new(); - for_each_record(&target, |record| names.push(record.name)).expect("published artifact should decode"); - assert_eq!(names, ["published.first", "published.second"]); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - } - - #[tokio::test] - async fn no_handles_and_closed_owner_return_service_unavailable() { - let no_handles = test_handler(Vec::new(), PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let response = send(&no_handles, authorized_post()).await; - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!( - response_body(response).await, - "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." - ); - - let (handle, responder) = aggregate_context_snapshot_channel_for_test(); - drop(responder); - let closed_owner = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let response = send(&closed_owner, authorized_post()).await; - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!( - response_body(response).await, - "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." - ); - } - - #[tokio::test] - async fn owner_stopped_after_accepting_request_returns_service_unavailable() { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let owner = tokio::spawn(async move { responder.stop_after_receiving().await }); - - let response = send(&handler, authorized_post()).await; - - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!( - response_body(response).await, - "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." - ); - owner.await.unwrap().unwrap(); - } - - #[tokio::test] - async fn snapshot_deadline_returns_gateway_timeout() { - let (handle, _responder) = aggregate_context_snapshot_channel_for_test(); - let handler = DogStatsDContextDumpAPIHandler::new_for_test( - AUTH_TOKEN, - vec![handle], - PathBuf::from("unused"), - Duration::from_millis(10), - Arc::new(NoOpPublisher), - ) - .unwrap(); - - let response = send(&handler, authorized_post()).await; - - assert_eq!(response.status(), StatusCode::GATEWAY_TIMEOUT); - assert_eq!( - response_body(response).await, - "Timed out waiting for the DogStatsD context snapshot; retry the request." - ); - } - - #[tokio::test] - async fn empty_and_non_directory_run_paths_reach_publication_and_return_safe_internal_errors() { - let temporary_directory = tempfile::tempdir().unwrap(); - let non_directory = temporary_directory.path().join("not-a-directory"); - fs::write(&non_directory, b"fixture").unwrap(); - - for run_path in [PathBuf::new(), non_directory] { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_path).unwrap(); - let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - - let response = send(&handler, authorized_post()).await; - - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - let body = response_body(response).await; - assert_eq!( - body, - "Failed to publish DogStatsD context dump; check the configured run path and permissions." - ); - assert!(!body.contains(AUTH_TOKEN)); - owner.await.unwrap().unwrap(); - } - assert_eq!( - directory_entries(temporary_directory.path()), - vec!["not-a-directory".to_owned()] - ); - } - - #[tokio::test] - async fn publication_failure_and_blocking_task_panic_return_safe_internal_errors() { - for publisher in [ - Arc::new(FailingPublisher) as Arc, - Arc::new(PanickingPublisher) as Arc, - ] { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = test_handler(vec![handle], PathBuf::from("unused"), publisher); - let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - - let response = send(&handler, authorized_post()).await; - - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); - let body = response_body(response).await; - assert!( - body == "Failed to publish DogStatsD context dump; check the configured run path and permissions." - || body == "DogStatsD context dump publication did not complete; retry the request." - ); - assert!(!body.contains("injected")); - assert!(!body.contains(AUTH_TOKEN)); - owner.await.unwrap().unwrap(); - } - } - - #[test] - fn blocking_publisher_release_guard_releases_and_notifies_on_drop() { - let release = Arc::new((StdMutex::new(false), Condvar::new())); - let (waiting_tx, waiting_rx) = mpsc::sync_channel(1); - - std::thread::scope(|scope| { - let waiter_release = release.clone(); - let waiter = scope.spawn(move || { - let (released, wake) = &*waiter_release; - let released = released.lock().unwrap(); - waiting_tx.send(()).unwrap(); - let (released, timeout) = wake - .wait_timeout_while(released, Duration::from_secs(1), |released| !*released) - .unwrap(); - assert!(!timeout.timed_out(), "release guard should notify the waiter"); - assert!(*released); - }); - waiting_rx.recv().unwrap(); - - let release_guard = BlockingPublisherReleaseGuard::new(release); - drop(release_guard); - waiter.join().unwrap(); - }); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn authorized_requests_serialize_fixed_path_publisher_executions() { - let (started_tx, started_rx) = mpsc::sync_channel(1); - let release = Arc::new((StdMutex::new(false), Condvar::new())); - let publisher = Arc::new(BlockingFirstPublisher { - active: AtomicUsize::new(0), - max_active: AtomicUsize::new(0), - calls: AtomicUsize::new(0), - first_started: started_tx, - release: release.clone(), - }); - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = test_handler(vec![handle], PathBuf::from("unused"), publisher.clone()); - let owner = tokio::spawn(async move { - responder.respond(Vec::new()).await.unwrap(); - responder.respond(Vec::new()).await.unwrap(); - }); - - let first_handler = handler.clone(); - let first = tokio::spawn(async move { send(&first_handler, authorized_post()).await }); - tokio::task::spawn_blocking(move || started_rx.recv_timeout(Duration::from_secs(1))) - .await - .unwrap() - .expect("first publisher should start"); - let release_guard = BlockingPublisherReleaseGuard::new(release); - let second_handler = handler.clone(); - let second = tokio::spawn(async move { send(&second_handler, authorized_post()).await }); - tokio::time::sleep(Duration::from_millis(25)).await; - assert_eq!(publisher.calls.load(Ordering::SeqCst), 1); - drop(release_guard); - - assert_eq!(first.await.unwrap().status(), StatusCode::OK); - assert_eq!(second.await.unwrap().status(), StatusCode::OK); - owner.await.unwrap(); - assert_eq!(publisher.calls.load(Ordering::SeqCst), 2); - assert_eq!(publisher.max_active.load(Ordering::SeqCst), 1); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn canceling_after_spawn_blocking_starts_still_completes_atomic_publication() { - let run_directory = tempfile::tempdir().unwrap(); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - fs::write(&target, b"original canonical artifact").unwrap(); - let (started_tx, started_rx) = mpsc::sync_channel(1); - let release = Arc::new((StdMutex::new(false), Condvar::new())); - let completed = Arc::new(AtomicBool::new(false)); - let publisher = Arc::new(BlockingRealPublisher { - started: started_tx, - release: release.clone(), - completed: completed.clone(), - }); - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = test_handler(vec![handle], run_directory.path().to_owned(), publisher); - let owner = tokio::spawn(async move { responder.respond(vec![snapshot_entry("after.cancel")]).await }); - let request_handler = handler.clone(); - let request = tokio::spawn(async move { send(&request_handler, authorized_post()).await }); - tokio::task::spawn_blocking(move || started_rx.recv_timeout(Duration::from_secs(1))) - .await - .unwrap() - .expect("publisher should start"); - let release_guard = BlockingPublisherReleaseGuard::new(release); - - request.abort(); - let _ = request.await; - assert_eq!(fs::read(&target).unwrap(), b"original canonical artifact"); - drop(release_guard); - tokio::time::timeout(Duration::from_secs(2), async { - while !completed.load(Ordering::SeqCst) { - tokio::time::sleep(Duration::from_millis(5)).await; - } - }) - .await - .expect("detached publication should complete"); - - owner.await.unwrap().unwrap(); - let mut names = Vec::new(); - for_each_record(&target, |record| names.push(record.name)).expect("canonical artifact should be complete"); - assert_eq!(names, ["after.cancel"]); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - } - - fn test_handler( - handles: Vec, run_path: PathBuf, publisher: Arc, - ) -> DogStatsDContextDumpAPIHandler { - DogStatsDContextDumpAPIHandler::new_for_test( - AUTH_TOKEN, - handles, - run_path, - Duration::from_millis(100), - publisher, - ) - .expect("test handler should be valid") - } - - fn post_request(authorization: Option<&str>) -> Request> { - let mut builder = Request::builder().method("POST").uri(ROUTE); - if let Some(authorization) = authorization { - builder = builder.header(AUTHORIZATION, authorization); - } - builder.body(Empty::new()).unwrap() - } - - fn authorized_post() -> Request> { - post_request(Some("Bearer expected-agent-token")) - } - - async fn send(handler: &DogStatsDContextDumpAPIHandler, request: Request>) -> Response { - handler - .generate_routes() - .with_state(handler.generate_initial_state()) - .oneshot(request) - .await - .unwrap() - } - - async fn response_body(response: Response) -> String { - String::from_utf8(response.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap() - } - - fn snapshot_entry(name: &'static str) -> AggregateContextSnapshotEntry { - AggregateContextSnapshotEntry::for_test( - Context::from_static_name(name), - AggregateMetricType::Gauge, - MetaString::empty(), - ) - } - - fn directory_entries(path: &Path) -> Vec { - let mut entries: Vec<_> = fs::read_dir(path) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - entries.sort_unstable(); - entries - } - - struct NoOpPublisher; - - impl ContextDumpPublisher for NoOpPublisher { - fn publish( - &self, run_path: PathBuf, _snapshot: Vec, - ) -> Result { - Ok(run_path.join(CONTEXT_DUMP_FILENAME)) - } - } - - struct FailingPublisher; - - impl ContextDumpPublisher for FailingPublisher { - fn publish( - &self, _run_path: PathBuf, _snapshot: Vec, - ) -> Result { - Err(generic_error!("injected publisher failure with {AUTH_TOKEN}")) - } - } - - struct PanickingPublisher; - - impl ContextDumpPublisher for PanickingPublisher { - fn publish( - &self, _run_path: PathBuf, _snapshot: Vec, - ) -> Result { - panic!("injected publisher panic") - } - } - - struct BlockingPublisherReleaseGuard { - release: Arc<(StdMutex, Condvar)>, - } - - impl BlockingPublisherReleaseGuard { - fn new(release: Arc<(StdMutex, Condvar)>) -> Self { - Self { release } - } - } - - impl Drop for BlockingPublisherReleaseGuard { - fn drop(&mut self) { - let (released, wake) = &*self.release; - let mut released = released.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); - *released = true; - wake.notify_all(); - } - } - - struct BlockingFirstPublisher { - active: AtomicUsize, - max_active: AtomicUsize, - calls: AtomicUsize, - first_started: mpsc::SyncSender<()>, - release: Arc<(StdMutex, Condvar)>, - } - - impl ContextDumpPublisher for BlockingFirstPublisher { - fn publish( - &self, run_path: PathBuf, _snapshot: Vec, - ) -> Result { - let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; - self.max_active.fetch_max(active, Ordering::SeqCst); - let call = self.calls.fetch_add(1, Ordering::SeqCst); - if call == 0 { - self.first_started.send(()).unwrap(); - let (released, wake) = &*self.release; - let mut released = released.lock().unwrap(); - while !*released { - released = wake.wait(released).unwrap(); - } - } - self.active.fetch_sub(1, Ordering::SeqCst); - Ok(run_path.join(CONTEXT_DUMP_FILENAME)) - } - } - - struct BlockingRealPublisher { - started: mpsc::SyncSender<()>, - release: Arc<(StdMutex, Condvar)>, - completed: Arc, - } - - impl ContextDumpPublisher for BlockingRealPublisher { - fn publish( - &self, run_path: PathBuf, snapshot: Vec, - ) -> Result { - self.started.send(()).unwrap(); - let (released, wake) = &*self.release; - let mut released = released.lock().unwrap(); - while !*released { - released = wake.wait(released).unwrap(); - } - drop(released); - let result = publish_context_dump(&run_path, &snapshot); - self.completed.store(true, Ordering::SeqCst); - result - } - } - - #[test] - fn production_publisher_type_uses_the_real_artifact_publisher() { - let run_directory = tempfile::tempdir().unwrap(); - let path = FileSystemContextDumpPublisher - .publish(run_directory.path().to_owned(), vec![snapshot_entry("real.publisher")]) - .unwrap(); - - let mut count = 0; - for_each_record(&path, |_| count += 1).unwrap(); - assert_eq!(count, 1); - } -} +#[path = "api_tests.rs"] +mod tests; diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api_tests.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api_tests.rs new file mode 100644 index 00000000000..305e3e1fe6b --- /dev/null +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api_tests.rs @@ -0,0 +1,656 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + mpsc, Arc, Condvar, Mutex as StdMutex, +}; +use std::time::Duration; + +use http::{ + header::{AUTHORIZATION, CONTENT_TYPE}, + HeaderValue, Request, StatusCode, +}; +use http_body_util::{BodyExt as _, Empty}; +use hyper::body::Bytes; +use saluki_api::{response::Response, APIHandler}; +use saluki_components::transforms::{ + aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotEntry, AggregateContextSnapshotHandle, + AggregateContextSnapshotPendingResponse, AggregateMetricType, +}; +use saluki_context::Context; +use saluki_error::{generic_error, GenericError}; +use stringtheory::MetaString; +use tower::ServiceExt as _; + +use super::{ + ContextDumpPublisher, ContextSnapshotCoordinator, DogStatsDContextDumpAPIHandler, FileSystemContextDumpPublisher, + SnapshotError, +}; +use crate::dogstatsd_contexts::{ + artifact::{for_each_record, CONTEXT_DUMP_FILENAME}, + publish_context_dump, +}; + +const AUTH_TOKEN: &str = "expected-agent-token"; +const ROUTE: &str = super::CONTEXT_DUMP_ROUTE; + +#[test] +fn constructor_rejects_empty_and_non_visible_http_tokens_without_exposing_them() { + let invalid_tokens: &[&[u8]] = &[ + b"", + b"contains space", + b"line\nbreak", + b"tab\tvalue", + b"nul\0value", + b"\x7f", + b"\xff", + ]; + + for token in invalid_tokens { + let error = match DogStatsDContextDumpAPIHandler::new(*token, Vec::new(), PathBuf::from("run")) { + Ok(_) => panic!("invalid authentication token should be rejected"), + Err(error) => error, + }; + let message = format!("{error:#}"); + if !token.is_empty() { + assert!(!message.as_bytes().windows(token.len()).any(|window| window == *token)); + } + } +} + +#[tokio::test] +async fn rejects_missing_duplicate_malformed_and_wrong_authorization_with_an_exact_safe_body() { + let publisher = Arc::new(NoOpPublisher); + let handler = test_handler(Vec::new(), PathBuf::from("unused"), publisher); + let mut duplicate = post_request(None); + duplicate + .headers_mut() + .append(AUTHORIZATION, HeaderValue::from_static("Bearer expected-agent-token")); + duplicate + .headers_mut() + .append(AUTHORIZATION, HeaderValue::from_static("Bearer expected-agent-token")); + let mut non_utf8 = post_request(None); + non_utf8.headers_mut().insert( + AUTHORIZATION, + HeaderValue::from_bytes(b"Bearer \xff").expect("HTTP headers allow opaque non-ASCII bytes"), + ); + let cases = [ + post_request(None), + duplicate, + non_utf8, + post_request(Some("Basic expected-agent-token")), + post_request(Some("Bearer")), + post_request(Some("Bearersecret")), + post_request(Some("Bearer\texpected-agent-token")), + post_request(Some("Bearer expected-agent-token trailing")), + post_request(Some("Bearer supplied-secret")), + post_request(Some("Bearer unexpect-agent-token")), + ]; + + for request in cases { + let response = send(&handler, request).await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + let body = response_body(response).await; + assert_eq!(body, "Authentication required."); + assert!(!body.contains(AUTH_TOKEN)); + assert!(!body.contains("supplied-secret")); + } +} + +#[tokio::test] +async fn accepts_bearer_scheme_case_insensitively() { + for scheme in ["bearer", "BEARER", "BeArEr"] { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, post_request(Some(&format!("{scheme} {AUTH_TOKEN}")))).await; + + assert_eq!(response.status(), StatusCode::OK); + owner.await.unwrap().unwrap(); + } +} + +#[tokio::test] +async fn accepts_only_the_exact_bearer_credential_bytes() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, post_request(Some("Bearer expected-agent-token"))).await; + + assert_eq!(response.status(), StatusCode::OK); + owner.await.unwrap().unwrap(); +} + +#[tokio::test] +async fn coordinator_reuses_the_one_owner_snapshot_allocation() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let owner_snapshot = vec![snapshot_entry("owner.one.first"), snapshot_entry("owner.one.second")]; + let expected = owner_snapshot.clone(); + let original_pointer = owner_snapshot.as_ptr() as usize; + let original_capacity = owner_snapshot.capacity(); + let owner = tokio::spawn(async move { responder.respond(owner_snapshot).await }); + let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); + + let actual = coordinator.snapshot().await.expect("snapshot should succeed"); + + assert_eq!(actual, expected); + assert_eq!(actual.as_ptr() as usize, original_pointer); + assert_eq!(actual.capacity(), original_capacity); + owner.await.unwrap().unwrap(); +} + +#[tokio::test] +async fn coordinator_requests_all_owners_concurrently_and_flattens_each_snapshot_once_in_owner_order() { + let (first_handle, mut first_responder) = aggregate_context_snapshot_channel_for_test(); + let (second_handle, mut second_responder) = aggregate_context_snapshot_channel_for_test(); + let first_snapshot = vec![snapshot_entry("owner.one.first"), snapshot_entry("owner.one.second")]; + let second_snapshot = vec![snapshot_entry("owner.two.only")]; + let expected = [first_snapshot.clone(), second_snapshot.clone()].concat(); + let (second_responded_tx, second_responded_rx) = tokio::sync::oneshot::channel(); + let first_owner = tokio::spawn(async move { + second_responded_rx + .await + .expect("second owner should receive its request"); + first_responder.respond(first_snapshot).await + }); + let second_owner = tokio::spawn(async move { + let result = second_responder.respond(second_snapshot).await; + let _ = second_responded_tx.send(()); + result + }); + let coordinator = ContextSnapshotCoordinator::new(vec![first_handle, second_handle], Duration::from_millis(250)); + + let actual = coordinator.snapshot().await.expect("both snapshots should succeed"); + + assert_eq!(actual, expected); + first_owner.await.unwrap().unwrap(); + second_owner.await.unwrap().unwrap(); +} + +#[tokio::test] +async fn coordinator_reports_no_handles_as_typed_unavailable() { + let coordinator = ContextSnapshotCoordinator::new(Vec::new(), Duration::from_secs(1)); + + let error = coordinator.snapshot().await.expect_err("empty owner set should fail"); + + assert_eq!(error, SnapshotError::SnapshotUnavailable); +} + +#[tokio::test] +async fn coordinator_reports_a_closed_owner_as_typed_unavailable() { + let (handle, responder) = aggregate_context_snapshot_channel_for_test(); + drop(responder); + let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); + + let error = coordinator.snapshot().await.expect_err("closed owner should fail"); + + assert_eq!(error, SnapshotError::SnapshotUnavailable); +} + +#[tokio::test] +async fn coordinator_reports_elapsed_deadline_as_typed_timeout() { + let (handle, _responder) = aggregate_context_snapshot_channel_for_test(); + let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_millis(10)); + + let error = coordinator + .snapshot() + .await + .expect_err("unresponsive owner should time out"); + + assert_eq!(error, SnapshotError::SnapshotTimedOut); +} + +#[tokio::test] +async fn canceled_coordinator_request_does_not_make_a_late_owner_response_panic_or_fail() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let coordinator = ContextSnapshotCoordinator::new(vec![handle], Duration::from_secs(1)); + let request = tokio::spawn(async move { coordinator.snapshot().await }); + let pending_response: AggregateContextSnapshotPendingResponse = responder + .receive() + .await + .expect("the owner should accept the snapshot request"); + + request.abort(); + let _ = request.await; + + pending_response.respond(vec![snapshot_entry("late.owner.response")]); +} + +#[tokio::test] +async fn generated_router_rejects_get_with_method_not_allowed() { + let handler = test_handler(Vec::new(), PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let request = Request::builder() + .method("GET") + .uri(ROUTE) + .body(Empty::::new()) + .unwrap(); + + let response = send(&handler, request).await; + + assert_eq!(response.status(), StatusCode::METHOD_NOT_ALLOWED); +} + +#[tokio::test] +async fn unauthorized_post_does_not_request_a_snapshot_or_create_an_artifact() { + let run_directory = tempfile::tempdir().unwrap(); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_directory.path().to_owned()) + .expect("handler should be valid"); + + let response = send(&handler, post_request(None)).await; + + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + assert!(!run_directory.path().join(CONTEXT_DUMP_FILENAME).exists()); + drop(handler); + assert!(responder.respond(Vec::new()).await.is_err()); +} + +#[tokio::test] +async fn authorized_router_post_returns_only_the_json_path_to_a_complete_decodable_artifact() { + let run_directory = tempfile::tempdir().unwrap(); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let snapshot = vec![snapshot_entry("published.first"), snapshot_entry("published.second")]; + let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN.as_bytes(), vec![handle], run_directory.path()) + .expect("handler should be valid"); + let owner = tokio::spawn(async move { responder.respond(snapshot).await }); + + let response = send(&handler, post_request(Some("Bearer expected-agent-token"))).await; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response.headers().get(CONTENT_TYPE).unwrap(), "application/json"); + assert_eq!( + response_body(response).await, + serde_json::to_string(target.to_str().unwrap()).unwrap() + ); + owner.await.unwrap().unwrap(); + let mut names = Vec::new(); + for_each_record(&target, |record| names.push(record.name)).expect("published artifact should decode"); + assert_eq!(names, ["published.first", "published.second"]); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); +} + +#[tokio::test] +async fn no_handles_and_closed_owner_return_service_unavailable() { + let no_handles = test_handler(Vec::new(), PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let response = send(&no_handles, authorized_post()).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response_body(response).await, + "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." + ); + + let (handle, responder) = aggregate_context_snapshot_channel_for_test(); + drop(responder); + let closed_owner = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let response = send(&closed_owner, authorized_post()).await; + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response_body(response).await, + "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." + ); +} + +#[tokio::test] +async fn owner_stopped_after_accepting_request_returns_service_unavailable() { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); + let owner = tokio::spawn(async move { responder.stop_after_receiving().await }); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + response_body(response).await, + "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running." + ); + owner.await.unwrap().unwrap(); +} + +#[tokio::test] +async fn snapshot_deadline_returns_gateway_timeout() { + let (handle, _responder) = aggregate_context_snapshot_channel_for_test(); + let handler = DogStatsDContextDumpAPIHandler::new_for_test( + AUTH_TOKEN, + vec![handle], + PathBuf::from("unused"), + Duration::from_millis(10), + Arc::new(NoOpPublisher), + ) + .unwrap(); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::GATEWAY_TIMEOUT); + assert_eq!( + response_body(response).await, + "Timed out waiting for the DogStatsD context snapshot; retry the request." + ); +} + +#[tokio::test] +async fn empty_and_non_directory_run_paths_reach_publication_and_return_safe_internal_errors() { + let temporary_directory = tempfile::tempdir().unwrap(); + let non_directory = temporary_directory.path().join("not-a-directory"); + fs::write(&non_directory, b"fixture").unwrap(); + + for run_path in [PathBuf::new(), non_directory] { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_path).unwrap(); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = response_body(response).await; + assert_eq!( + body, + "Failed to publish DogStatsD context dump; check the configured run path and permissions." + ); + assert!(!body.contains(AUTH_TOKEN)); + owner.await.unwrap().unwrap(); + } + assert_eq!( + directory_entries(temporary_directory.path()), + vec!["not-a-directory".to_owned()] + ); +} + +#[tokio::test] +async fn publication_failure_and_blocking_task_panic_return_safe_internal_errors() { + for publisher in [ + Arc::new(FailingPublisher) as Arc, + Arc::new(PanickingPublisher) as Arc, + ] { + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), publisher); + let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); + + let response = send(&handler, authorized_post()).await; + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + let body = response_body(response).await; + assert!( + body == "Failed to publish DogStatsD context dump; check the configured run path and permissions." + || body == "DogStatsD context dump publication did not complete; retry the request." + ); + assert!(!body.contains("injected")); + assert!(!body.contains(AUTH_TOKEN)); + owner.await.unwrap().unwrap(); + } +} + +#[test] +fn blocking_publisher_release_guard_releases_and_notifies_on_drop() { + let release = Arc::new((StdMutex::new(false), Condvar::new())); + let (waiting_tx, waiting_rx) = mpsc::sync_channel(1); + + std::thread::scope(|scope| { + let waiter_release = release.clone(); + let waiter = scope.spawn(move || { + let (released, wake) = &*waiter_release; + let released = released.lock().unwrap(); + waiting_tx.send(()).unwrap(); + let (released, timeout) = wake + .wait_timeout_while(released, Duration::from_secs(1), |released| !*released) + .unwrap(); + assert!(!timeout.timed_out(), "release guard should notify the waiter"); + assert!(*released); + }); + waiting_rx.recv().unwrap(); + + let release_guard = BlockingPublisherReleaseGuard::new(release); + drop(release_guard); + waiter.join().unwrap(); + }); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn authorized_requests_serialize_fixed_path_publisher_executions() { + let (started_tx, started_rx) = mpsc::sync_channel(1); + let release = Arc::new((StdMutex::new(false), Condvar::new())); + let publisher = Arc::new(BlockingFirstPublisher { + active: AtomicUsize::new(0), + max_active: AtomicUsize::new(0), + calls: AtomicUsize::new(0), + first_started: started_tx, + release: release.clone(), + }); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], PathBuf::from("unused"), publisher.clone()); + let owner = tokio::spawn(async move { + responder.respond(Vec::new()).await.unwrap(); + responder.respond(Vec::new()).await.unwrap(); + }); + + let first_handler = handler.clone(); + let first = tokio::spawn(async move { send(&first_handler, authorized_post()).await }); + tokio::task::spawn_blocking(move || started_rx.recv_timeout(Duration::from_secs(1))) + .await + .unwrap() + .expect("first publisher should start"); + let release_guard = BlockingPublisherReleaseGuard::new(release); + let second_handler = handler.clone(); + let second = tokio::spawn(async move { send(&second_handler, authorized_post()).await }); + tokio::time::sleep(Duration::from_millis(25)).await; + assert_eq!(publisher.calls.load(Ordering::SeqCst), 1); + drop(release_guard); + + assert_eq!(first.await.unwrap().status(), StatusCode::OK); + assert_eq!(second.await.unwrap().status(), StatusCode::OK); + owner.await.unwrap(); + assert_eq!(publisher.calls.load(Ordering::SeqCst), 2); + assert_eq!(publisher.max_active.load(Ordering::SeqCst), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn canceling_after_spawn_blocking_starts_still_completes_atomic_publication() { + let run_directory = tempfile::tempdir().unwrap(); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + fs::write(&target, b"original canonical artifact").unwrap(); + let (started_tx, started_rx) = mpsc::sync_channel(1); + let release = Arc::new((StdMutex::new(false), Condvar::new())); + let completed = Arc::new(AtomicBool::new(false)); + let publisher = Arc::new(BlockingRealPublisher { + started: started_tx, + release: release.clone(), + completed: completed.clone(), + }); + let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); + let handler = test_handler(vec![handle], run_directory.path().to_owned(), publisher); + let owner = tokio::spawn(async move { responder.respond(vec![snapshot_entry("after.cancel")]).await }); + let request_handler = handler.clone(); + let request = tokio::spawn(async move { send(&request_handler, authorized_post()).await }); + tokio::task::spawn_blocking(move || started_rx.recv_timeout(Duration::from_secs(1))) + .await + .unwrap() + .expect("publisher should start"); + let release_guard = BlockingPublisherReleaseGuard::new(release); + + request.abort(); + let _ = request.await; + assert_eq!(fs::read(&target).unwrap(), b"original canonical artifact"); + drop(release_guard); + tokio::time::timeout(Duration::from_secs(2), async { + while !completed.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("detached publication should complete"); + + owner.await.unwrap().unwrap(); + let mut names = Vec::new(); + for_each_record(&target, |record| names.push(record.name)).expect("canonical artifact should be complete"); + assert_eq!(names, ["after.cancel"]); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); +} + +fn test_handler( + handles: Vec, run_path: PathBuf, publisher: Arc, +) -> DogStatsDContextDumpAPIHandler { + DogStatsDContextDumpAPIHandler::new_for_test(AUTH_TOKEN, handles, run_path, Duration::from_millis(100), publisher) + .expect("test handler should be valid") +} + +fn post_request(authorization: Option<&str>) -> Request> { + let mut builder = Request::builder().method("POST").uri(ROUTE); + if let Some(authorization) = authorization { + builder = builder.header(AUTHORIZATION, authorization); + } + builder.body(Empty::new()).unwrap() +} + +fn authorized_post() -> Request> { + post_request(Some("Bearer expected-agent-token")) +} + +async fn send(handler: &DogStatsDContextDumpAPIHandler, request: Request>) -> Response { + handler + .generate_routes() + .with_state(handler.generate_initial_state()) + .oneshot(request) + .await + .unwrap() +} + +async fn response_body(response: Response) -> String { + String::from_utf8(response.into_body().collect().await.unwrap().to_bytes().to_vec()).unwrap() +} + +fn snapshot_entry(name: &'static str) -> AggregateContextSnapshotEntry { + AggregateContextSnapshotEntry::for_test( + Context::from_static_name(name), + AggregateMetricType::Gauge, + MetaString::empty(), + ) +} + +fn directory_entries(path: &Path) -> Vec { + let mut entries: Vec<_> = fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + entries.sort_unstable(); + entries +} + +struct NoOpPublisher; + +impl ContextDumpPublisher for NoOpPublisher { + fn publish( + &self, run_path: PathBuf, _snapshot: Vec, + ) -> Result { + Ok(run_path.join(CONTEXT_DUMP_FILENAME)) + } +} + +struct FailingPublisher; + +impl ContextDumpPublisher for FailingPublisher { + fn publish( + &self, _run_path: PathBuf, _snapshot: Vec, + ) -> Result { + Err(generic_error!("injected publisher failure with {AUTH_TOKEN}")) + } +} + +struct PanickingPublisher; + +impl ContextDumpPublisher for PanickingPublisher { + fn publish( + &self, _run_path: PathBuf, _snapshot: Vec, + ) -> Result { + panic!("injected publisher panic") + } +} + +struct BlockingPublisherReleaseGuard { + release: Arc<(StdMutex, Condvar)>, +} + +impl BlockingPublisherReleaseGuard { + fn new(release: Arc<(StdMutex, Condvar)>) -> Self { + Self { release } + } +} + +impl Drop for BlockingPublisherReleaseGuard { + fn drop(&mut self) { + let (released, wake) = &*self.release; + let mut released = released.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + *released = true; + wake.notify_all(); + } +} + +struct BlockingFirstPublisher { + active: AtomicUsize, + max_active: AtomicUsize, + calls: AtomicUsize, + first_started: mpsc::SyncSender<()>, + release: Arc<(StdMutex, Condvar)>, +} + +impl ContextDumpPublisher for BlockingFirstPublisher { + fn publish( + &self, run_path: PathBuf, _snapshot: Vec, + ) -> Result { + let active = self.active.fetch_add(1, Ordering::SeqCst) + 1; + self.max_active.fetch_max(active, Ordering::SeqCst); + let call = self.calls.fetch_add(1, Ordering::SeqCst); + if call == 0 { + self.first_started.send(()).unwrap(); + let (released, wake) = &*self.release; + let mut released = released.lock().unwrap(); + while !*released { + released = wake.wait(released).unwrap(); + } + } + self.active.fetch_sub(1, Ordering::SeqCst); + Ok(run_path.join(CONTEXT_DUMP_FILENAME)) + } +} + +struct BlockingRealPublisher { + started: mpsc::SyncSender<()>, + release: Arc<(StdMutex, Condvar)>, + completed: Arc, +} + +impl ContextDumpPublisher for BlockingRealPublisher { + fn publish( + &self, run_path: PathBuf, snapshot: Vec, + ) -> Result { + self.started.send(()).unwrap(); + let (released, wake) = &*self.release; + let mut released = released.lock().unwrap(); + while !*released { + released = wake.wait(released).unwrap(); + } + drop(released); + let result = publish_context_dump(&run_path, &snapshot); + self.completed.store(true, Ordering::SeqCst); + result + } +} + +#[test] +fn production_publisher_type_uses_the_real_artifact_publisher() { + let run_directory = tempfile::tempdir().unwrap(); + let path = FileSystemContextDumpPublisher + .publish(run_directory.path().to_owned(), vec![snapshot_entry("real.publisher")]) + .unwrap(); + + let mut count = 0; + for_each_record(&path, |_| count += 1).unwrap(); + assert_eq!(count, 1); +} diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index ae57d9818a8..eb7bb699ee2 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -1,7 +1,5 @@ use std::error::Error; use std::fmt; -#[cfg(not(windows))] -use std::fs::OpenOptions; use std::fs::{self, File}; use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; @@ -10,11 +8,16 @@ use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetr use saluki_context::tags::TagSet; use saluki_error::{ErrorContext as _, GenericError}; use serde::{ser::SerializeStruct as _, Deserialize, Serialize, Serializer}; -use uuid::Uuid; +use tempfile::Builder as TempFileBuilder; +// Zstandard frame magic from RFC 8878, section 3.1.1. const ZSTD_MAGIC: &[u8; 4] = b"\x28\xb5\x2f\xfd"; +// Matches the fixed Agent output path in demultiplexerendpoint/impl/endpoint.go at the pinned compatibility revision. pub(crate) const CONTEXT_DUMP_FILENAME: &str = "dogstatsd_contexts.json.zstd"; +// `MetricSourceDogstatsd` is the second value after `MetricSourceUnknown` in the Agent's metricsource.go enum. +const AGENT_METRIC_SOURCE_DOGSTATSD: u32 = 1; #[cfg(windows)] +// A protected DACL granting full control through Owner Rights, LocalSystem, and Builtin Administrators only. const WINDOWS_CONTEXT_DUMP_SDDL: &str = "D:P(A;;FA;;;OW)(A;;FA;;;SY)(A;;FA;;;BA)"; struct AgentContextSnapshotRecord<'a>(&'a AggregateContextSnapshotEntry); @@ -34,7 +37,7 @@ impl Serialize for AgentContextSnapshotRecord<'_> { record.serialize_field("TaggerTags", &BorrowedTags(context.origin_tags()))?; record.serialize_field("MetricTags", &BorrowedTags(context.tags()))?; record.serialize_field("NoIndex", &false)?; - record.serialize_field("Source", &1u32)?; + record.serialize_field("Source", &AGENT_METRIC_SOURCE_DOGSTATSD)?; record.end() } } @@ -50,6 +53,7 @@ impl Serialize for BorrowedTags<'_> { } } +// These spellings match `MetricType.String()` in the Agent's pkg/metrics/metric_sample.go. fn agent_metric_type(entry: &AggregateContextSnapshotEntry) -> &'static str { match entry.metric_type() { AggregateMetricType::Counter => "Counter", @@ -97,19 +101,19 @@ fn publish_context_dump_with( run_path, snapshot, replacer, - &SystemEntropy, + &SecureTemporaryFileFactory, &OwnerOnlyPermissions, &FileSystemCleanup, ) } -fn publish_context_dump_with_services( - run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, entropy: &E, _permissions: &P, +fn publish_context_dump_with_services( + run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, temporary_files: &F, _permissions: &P, cleanup: &C, ) -> Result where R: AtomicReplace, - E: EntropySource, + F: TemporaryFileFactory, P: PermissionNormalizer, C: TemporaryFileCleanup, { @@ -121,8 +125,7 @@ where )); } - let temporary_path = temporary_context_dump_path(run_path, &target, entropy)?; - let file = open_temporary_file(&temporary_path).with_error_context(|| { + let (file, temporary_path) = temporary_files.create(run_path).with_error_context(|| { format!( "Failed to create temporary DogStatsD context dump for target '{}'.", target.display() @@ -143,31 +146,24 @@ where Ok(target) } -fn temporary_context_dump_path( - run_path: &Path, target: &Path, entropy: &E, -) -> Result { - let mut random_bytes = [0u8; 16]; - entropy.fill(&mut random_bytes).with_error_context(|| { - format!( - "Failed to generate random temporary filename for DogStatsD context dump target '{}'.", - target.display() - ) - })?; - random_bytes[6] = (random_bytes[6] & 0x0f) | 0x40; - random_bytes[8] = (random_bytes[8] & 0x3f) | 0x80; - let identifier = Uuid::from_bytes(random_bytes); - Ok(run_path.join(format!(".{CONTEXT_DUMP_FILENAME}.{}.tmp", identifier.as_hyphenated()))) +trait TemporaryFileFactory { + fn create(&self, run_path: &Path) -> io::Result<(File, PathBuf)>; } -trait EntropySource { - fn fill(&self, bytes: &mut [u8]) -> io::Result<()>; -} +struct SecureTemporaryFileFactory; -struct SystemEntropy; +impl TemporaryFileFactory for SecureTemporaryFileFactory { + fn create(&self, run_path: &Path) -> io::Result<(File, PathBuf)> { + let prefix = format!(".{CONTEXT_DUMP_FILENAME}."); + let mut builder = TempFileBuilder::new(); + builder.prefix(&prefix).suffix(".tmp"); + + #[cfg(windows)] + let temporary = builder.make_in(run_path, open_temporary_file)?; + #[cfg(not(windows))] + let temporary = builder.tempfile_in(run_path)?; -impl EntropySource for SystemEntropy { - fn fill(&self, bytes: &mut [u8]) -> io::Result<()> { - getrandom::fill(bytes).map_err(|error| io::Error::other(error.to_string())) + temporary.keep().map_err(|error| error.error) } } @@ -192,18 +188,6 @@ fn set_owner_only_permissions(file: &File) -> io::Result<()> { file.set_permissions(fs::Permissions::from_mode(0o600)) } -#[cfg(not(windows))] -fn open_temporary_file(path: &Path) -> io::Result { - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - options.mode(0o600); - } - options.open(path) -} - #[cfg(windows)] fn open_temporary_file(path: &Path) -> io::Result { use std::os::windows::{ @@ -315,33 +299,6 @@ impl Drop for WindowsSecurityAttributes { } } -#[cfg(test)] -fn publish_open_temporary( - writer: W, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, -) -> Result<(), GenericError> -where - W: Write + SyncAll, - R: AtomicReplace, -{ - run_with_temporary_cleanup(temporary_path, target, &FileSystemCleanup, || { - publish_buffered_temporary_unmanaged(BufWriter::new(writer), temporary_path, target, snapshot, replacer) - }) -} - -#[cfg(test)] -fn publish_buffered_temporary( - buffer: BufWriter, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], - replacer: &R, -) -> Result<(), GenericError> -where - W: Write + SyncAll, - R: AtomicReplace, -{ - run_with_temporary_cleanup(temporary_path, target, &FileSystemCleanup, || { - publish_buffered_temporary_unmanaged(buffer, temporary_path, target, snapshot, replacer) - }) -} - fn publish_buffered_temporary_unmanaged( buffer: BufWriter, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, @@ -637,21 +594,6 @@ impl ArtifactError { fn decode(path: &Path, record_index: usize, source: serde_json::Error) -> Self { Self::new(path, "decode", record_index, ArtifactDecodeError::from_serde(source)) } - - #[cfg(test)] - pub(super) fn path(&self) -> &Path { - &self.path - } - - #[cfg(test)] - pub(super) fn operation(&self) -> &'static str { - self.operation - } - - #[cfg(test)] - pub(super) fn record_index(&self) -> usize { - self.record_index - } } impl fmt::Display for ArtifactError { @@ -703,1036 +645,5 @@ fn decode_records( } #[cfg(test)] -fn collect_records(path: &Path) -> Result, ArtifactError> { - let mut records = Vec::new(); - for_each_record(path, |record| records.push(record))?; - Ok(records) -} - -#[cfg(test)] -mod tests { - use std::fs; - use std::io::{self, BufWriter, Write as _}; - use std::path::PathBuf; - use std::sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, - Arc, - }; - - use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; - use saluki_context::{ - tags::{Tag, TagSet}, - Context, - }; - use serde_json::json; - use stringtheory::MetaString; - - #[cfg(windows)] - use super::open_temporary_file; - use super::{ - collect_records, finish_buffered_writer, finish_zstd_encoder, publish_buffered_temporary, publish_context_dump, - publish_context_dump_with, publish_context_dump_with_services, publish_open_temporary, - temporary_context_dump_path, write_snapshot_records, AgentContextRecord, AtomicReplace, EntropySource, - FileSystemAtomicReplace, FileSystemCleanup, OwnerOnlyPermissions, SyncAll, TemporaryFileCleanup, - TemporaryPathCleanup, CONTEXT_DUMP_FILENAME, - }; - #[cfg(unix)] - use super::{set_owner_only_permissions, PermissionNormalizer}; - use crate::dogstatsd_contexts::read_report; - - const PLAIN_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/fixtures/dogstatsd_contexts_agent.ndjson" - )); - const COMPRESSED_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd" - )); - const PLAIN_FIXTURE_PATH: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/fixtures/dogstatsd_contexts_agent.ndjson" - ); - const COMPRESSED_FIXTURE_PATH: &str = concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd" - ); - - #[cfg(windows)] - struct LocalAllocation(*mut std::ffi::c_void); - - #[cfg(windows)] - impl Drop for LocalAllocation { - fn drop(&mut self) { - if !self.0.is_null() { - use windows_sys::Win32::Foundation::{LocalFree, HLOCAL}; - - // SAFETY: This pointer was allocated by a Windows security API that transfers ownership to the caller. - unsafe { - let _ = LocalFree(self.0 as HLOCAL); - } - } - } - } - - #[cfg(windows)] - fn windows_file_dacl_sddl(path: &std::path::Path) -> io::Result<(u16, String)> { - use std::os::windows::ffi::OsStrExt as _; - use std::{ptr, slice}; - - use windows_sys::Win32::{ - Foundation::ERROR_SUCCESS, - Security::{ - Authorization::{ - ConvertSecurityDescriptorToStringSecurityDescriptorW, GetNamedSecurityInfoW, SDDL_REVISION_1, - SE_FILE_OBJECT, - }, - GetSecurityDescriptorControl, DACL_SECURITY_INFORMATION, - }, - }; - - let mut wide_path: Vec<_> = path.as_os_str().encode_wide().collect(); - if wide_path.contains(&0) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "path contains an interior NUL", - )); - } - wide_path.push(0); - - let mut descriptor = ptr::null_mut(); - // SAFETY: The path is NUL-terminated and remains alive during the call. All unused output pointers are null, - // and `descriptor` receives an allocation owned by the caller on success. - let result = unsafe { - GetNamedSecurityInfoW( - wide_path.as_ptr(), - SE_FILE_OBJECT, - DACL_SECURITY_INFORMATION, - ptr::null_mut(), - ptr::null_mut(), - ptr::null_mut(), - ptr::null_mut(), - &mut descriptor, - ) - }; - if result != ERROR_SUCCESS { - return Err(io::Error::from_raw_os_error(result as i32)); - } - let descriptor = LocalAllocation(descriptor); - - let mut control = 0; - let mut revision = 0; - // SAFETY: `descriptor` is a valid security descriptor allocation and both output pointers are valid writes. - let result = unsafe { GetSecurityDescriptorControl(descriptor.0, &mut control, &mut revision) }; - if result == 0 { - return Err(io::Error::last_os_error()); - } - - let mut sddl = ptr::null_mut(); - let mut sddl_len = 0; - // SAFETY: `descriptor` remains valid during the call, and both output pointers are valid writes. The returned - // string allocation is owned by the caller on success. - let result = unsafe { - ConvertSecurityDescriptorToStringSecurityDescriptorW( - descriptor.0, - SDDL_REVISION_1, - DACL_SECURITY_INFORMATION, - &mut sddl, - &mut sddl_len, - ) - }; - if result == 0 { - return Err(io::Error::last_os_error()); - } - let sddl_allocation = LocalAllocation(sddl.cast()); - // SAFETY: The conversion API returned `sddl_len` initialized UTF-16 code units in the live `sddl` allocation. - let wide_sddl = unsafe { slice::from_raw_parts(sddl, sddl_len as usize) }; - let wide_sddl = wide_sddl.strip_suffix(&[0]).unwrap_or(wide_sddl); - let sddl = String::from_utf16(wide_sddl).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; - drop(sddl_allocation); - - Ok((control, sddl)) - } - - // These fixtures and expectations mirror ContextDebugRepr and the `dogstatsd top` command at Datadog Agent - // commit 9de2a8371cf8d95794f238939709491907893288. - fn expected_fixture_records() -> Vec { - vec![ - AgentContextRecord { - name: "z.metric".to_owned(), - host: "node-a".to_owned(), - metric_type: "Gauge".to_owned(), - tagger_tags: vec!["pod_name:web-a".to_owned()], - metric_tags: vec![ - "env:prod".to_owned(), - "image:registry/repo:v1".to_owned(), - "bare".to_owned(), - ], - no_index: false, - source: 1, - }, - AgentContextRecord { - name: "a.metric".to_owned(), - host: "node-a".to_owned(), - metric_type: "Counter".to_owned(), - tagger_tags: vec!["pod_name:web-a".to_owned()], - metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], - no_index: false, - source: 1, - }, - AgentContextRecord { - name: "a.metric".to_owned(), - host: "node-b".to_owned(), - metric_type: "Counter".to_owned(), - tagger_tags: vec!["pod_name:web-b".to_owned()], - metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], - no_index: false, - source: 1, - }, - AgentContextRecord { - name: "a.metric".to_owned(), - host: "node-c".to_owned(), - metric_type: "Counter".to_owned(), - tagger_tags: vec!["pod_name:web-c".to_owned()], - metric_tags: vec!["env:staging".to_owned(), "service:web".to_owned()], - no_index: false, - source: 1, - }, - ] - } - - #[test] - fn streams_exact_agent_schema_for_every_metric_type() { - let cases = [ - (AggregateMetricType::Counter, None, "Counter"), - (AggregateMetricType::Rate, None, "Rate"), - (AggregateMetricType::Gauge, None, "Gauge"), - (AggregateMetricType::Set, None, "Set"), - (AggregateMetricType::Distribution, None, "Distribution"), - (AggregateMetricType::Histogram, None, "Histogram"), - (AggregateMetricType::Histogram, Some("second"), "Histogram"), - (AggregateMetricType::Histogram, Some("millisecond"), "Historate"), - ]; - let snapshot: Vec<_> = cases - .iter() - .enumerate() - .map(|(index, (metric_type, unit, _))| { - snapshot_entry( - Box::leak(format!("metric.{index}").into_boxed_str()), - (index == 0).then_some("node-a"), - *metric_type, - unit.unwrap_or_default(), - &["client:value", "bare"], - &["origin:value"], - ) - }) - .collect(); - let mut output = Vec::new(); - - write_snapshot_records(&mut output, &snapshot, PathBuf::from("plain-stream").as_path()) - .expect("snapshot should serialize"); - - let output = String::from_utf8(output).expect("serialized output should be UTF-8"); - let lines: Vec<_> = output.lines().collect(); - assert_eq!(lines.len(), cases.len()); - assert_eq!(output.bytes().filter(|byte| *byte == b'\n').count(), cases.len()); - assert!(output.ends_with('\n')); - for (index, ((_, _, expected_type), line)) in cases.iter().zip(&lines).enumerate() { - let expected_host = if index == 0 { "node-a" } else { "" }; - let expected = json!({ - "Name": format!("metric.{index}"), - "Host": expected_host, - "Type": expected_type, - "TaggerTags": ["origin:value"], - "MetricTags": ["client:value", "bare"], - "NoIndex": false, - "Source": 1, - }); - assert_eq!(serde_json::from_str::(line).unwrap(), expected); - assert_eq!( - *line, - format!( - "{{\"Name\":\"metric.{index}\",\"Host\":\"{expected_host}\",\"Type\":\"{expected_type}\",\"TaggerTags\":[\"origin:value\"],\"MetricTags\":[\"client:value\",\"bare\"],\"NoIndex\":false,\"Source\":1}}" - ) - ); - } - } - - #[test] - fn plain_stream_round_trips_through_reader_and_report() { - let snapshot = vec![ - snapshot_entry( - "shared.metric", - Some("node-a"), - AggregateMetricType::Gauge, - "", - &["env:prod", "service:web"], - &["pod_name:web-a"], - ), - snapshot_entry( - "shared.metric", - None, - AggregateMetricType::Counter, - "", - &["env:staging", "service:web"], - &["pod_name:web-b"], - ), - ]; - let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); - let mut file = artifact.reopen().expect("temporary artifact should reopen"); - - write_snapshot_records(&mut file, &snapshot, artifact.path()).expect("snapshot should serialize"); - file.flush().expect("plain stream should flush"); - - assert_eq!( - collect_records(artifact.path()).expect("plain stream should decode"), - vec![ - AgentContextRecord { - name: "shared.metric".to_owned(), - host: "node-a".to_owned(), - metric_type: "Gauge".to_owned(), - tagger_tags: vec!["pod_name:web-a".to_owned()], - metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], - no_index: false, - source: 1, - }, - AgentContextRecord { - name: "shared.metric".to_owned(), - host: String::new(), - metric_type: "Counter".to_owned(), - tagger_tags: vec!["pod_name:web-b".to_owned()], - metric_tags: vec!["env:staging".to_owned(), "service:web".to_owned()], - no_index: false, - source: 1, - }, - ] - ); - assert_eq!( - read_report(artifact.path()) - .expect("plain stream report should build") - .render(10, 10), - concat!( - " Contexts\tMetric name\t(number of unique values for each tag)\n", - " 2\tshared.metric\t(2 env, 1 service)\n", - ) - ); - } - - #[test] - fn surfaces_a_partial_writer_failure_with_record_and_path_context() { - let snapshot = [snapshot_entry( - "write.failure", - None, - AggregateMetricType::Gauge, - "", - &["env:test"], - &[], - )]; - let path = PathBuf::from("/run/test/dogstatsd_contexts.json.zstd"); - let mut writer = FailAfterBytes::new(12); - - let error = write_snapshot_records(&mut writer, &snapshot, &path).expect_err("write failure should surface"); - - assert_eq!(writer.written.len(), 12); - assert!(error.to_string().contains("serialize DogStatsD context dump record 1")); - assert!(error.to_string().contains(&path.display().to_string())); - assert!(format!("{error:#}").contains("injected write failure")); - } - - #[test] - fn publishes_a_decodable_mode_0600_dump_and_replaces_the_canonical_artifact() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - fs::write(&target, b"old canonical artifact").expect("old canonical artifact should be written"); - let snapshot = vec![ - snapshot_entry( - "published.metric", - Some("node-a"), - AggregateMetricType::Histogram, - "millisecond", - &["env:prod"], - &["pod_name:web-a"], - ), - snapshot_entry( - "published.metric", - None, - AggregateMetricType::Distribution, - "", - &["env:staging"], - &["pod_name:web-b"], - ), - ]; - - let published = publish_context_dump(run_directory.path(), &snapshot).expect("snapshot should publish"); - - assert_eq!(published, target); - assert_eq!( - collect_records(&target) - .expect("published artifact should decode") - .len(), - 2 - ); - assert_eq!( - read_report(&target) - .expect("published report should build") - .render(10, 10), - concat!( - " Contexts\tMetric name\t(number of unique values for each tag)\n", - " 2\tpublished.metric\t(2 env)\n", - ) - ); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt as _; - - assert_eq!(fs::metadata(&target).unwrap().permissions().mode() & 0o777, 0o600); - } - } - - #[cfg(windows)] - #[test] - fn temporary_dump_is_created_with_a_protected_restrictive_dacl() { - use windows_sys::Win32::Security::SE_DACL_PROTECTED; - - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let staging_path = temporary_context_dump_path(run_directory.path(), &target, &FixedEntropy) - .expect("temporary dump path should be generated"); - let _file = open_temporary_file(&staging_path).expect("temporary dump should be created"); - - let (control, sddl) = windows_file_dacl_sddl(&staging_path).expect("temporary dump DACL should be readable"); - - assert_ne!(control & SE_DACL_PROTECTED, 0, "DACL must be protected: {sddl}"); - assert!(sddl.contains("D:P"), "DACL SDDL must carry the protected flag: {sddl}"); - for required_ace in ["(A;;FA;;;OW)", "(A;;FA;;;SY)", "(A;;FA;;;BA)"] { - assert!(sddl.contains(required_ace), "DACL is missing {required_ace}: {sddl}"); - } - assert_eq!(sddl.matches('(').count(), 3, "DACL contains an unexpected ACE: {sddl}"); - assert!(!sddl.contains(";;;WD)"), "DACL must not grant Everyone access: {sddl}"); - assert!( - !sddl.contains(";;;BU)"), - "DACL must not grant Builtin Users access: {sddl}" - ); - } - - #[cfg(unix)] - #[test] - fn publication_normalizes_restrictive_temporary_permissions_to_exactly_0600() { - use std::os::unix::fs::PermissionsExt as _; - - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let normalizer = RestrictiveThenOwnerOnly::default(); - - let target = publish_context_dump_with_services( - run_directory.path(), - &[], - &FileSystemAtomicReplace, - &FixedEntropy, - &normalizer, - &FileSystemCleanup, - ) - .expect("snapshot should publish"); - - assert!(normalizer.called.load(Ordering::Relaxed)); - assert_eq!(fs::metadata(target).unwrap().permissions().mode() & 0o777, 0o600); - } - - #[test] - fn entropy_failure_is_contextual_and_creates_no_temporary_artifact() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - - let error = publish_context_dump_with_services( - run_directory.path(), - &[], - &FileSystemAtomicReplace, - &FailingEntropy, - &OwnerOnlyPermissions, - &FileSystemCleanup, - ) - .expect_err("entropy failure should surface"); - - let message = format!("{error:#}"); - assert!(message.contains("generate random temporary filename"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected entropy failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - } - - #[test] - fn temporary_name_sets_rfc_4122_version_and_variant_bits() { - let run_path = std::path::Path::new("/run/test"); - let target = run_path.join(CONTEXT_DUMP_FILENAME); - - let temporary_path = temporary_context_dump_path(run_path, &target, &FixedEntropy).unwrap(); - - assert_eq!( - temporary_path.file_name().unwrap(), - ".dogstatsd_contexts.json.zstd.11111111-1111-4111-9111-111111111111.tmp" - ); - } - - #[cfg(unix)] - #[test] - fn permission_failure_closes_and_removes_the_new_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - - let error = publish_context_dump_with_services( - run_directory.path(), - &[], - &FileSystemAtomicReplace, - &FixedEntropy, - &FailingPermissions, - &FileSystemCleanup, - ) - .expect_err("permission failure should surface"); - - let message = format!("{error:#}"); - assert!(message.contains("set owner-only permissions"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected permission failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - } - - #[test] - fn cleanup_method_surfaces_removal_failure_and_remains_armed() { - let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); - let temporary_path = temporary_directory.path().join("staged-contexts.tmp"); - fs::write(&temporary_path, b"staged").unwrap(); - let cleanup = FailingCleanup::default(); - let mut guard = TemporaryPathCleanup::new(&temporary_path, &cleanup); - - let error = guard.cleanup().expect_err("cleanup failure should surface"); - - assert!(error.to_string().contains("injected cleanup failure")); - assert!(temporary_path.exists()); - assert_eq!(cleanup.attempts.load(Ordering::Relaxed), 1); - } - - #[test] - fn reports_primary_and_cleanup_failures_without_replacing_the_canonical_artifact() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let cleanup = FailingCleanup::default(); - - let error = publish_context_dump_with_services( - run_directory.path(), - &[], - &FailingReplace, - &FixedEntropy, - &OwnerOnlyPermissions, - &cleanup, - ) - .expect_err("replacement and cleanup should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("atomically replace"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("remove temporary DogStatsD context dump"), "{message}"); - assert!(message.contains("injected cleanup failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!(directory_entries(run_directory.path()).len(), 2); - assert!(directory_entries(run_directory.path()) - .iter() - .any(|entry| entry.starts_with(&format!(".{CONTEXT_DUMP_FILENAME}.")) && entry.ends_with(".tmp"))); - assert!(cleanup.attempts.load(Ordering::Relaxed) >= 1); - } - - #[test] - fn rejects_empty_missing_and_non_directory_run_paths_with_target_and_operation_context() { - let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); - let missing = temporary_directory.path().join("missing"); - let not_directory = temporary_directory.path().join("not-a-directory"); - fs::write(¬_directory, b"file").expect("non-directory fixture should be written"); - let cases = [ - (PathBuf::new(), "validate run path"), - (missing, "create temporary"), - (not_directory, "create temporary"), - ]; - - for (run_path, expected_operation) in cases { - let error = publish_context_dump(&run_path, &[]).expect_err("invalid run path should fail"); - let message = format!("{error:#}"); - assert!(message.contains(CONTEXT_DUMP_FILENAME), "{message}"); - assert!(message.contains(expected_operation), "{message}"); - } - } - - #[test] - fn injected_replace_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let snapshot = [snapshot_entry( - "replacement.failure", - None, - AggregateMetricType::Gauge, - "", - &[], - &[], - )]; - - let error = publish_context_dump_with(run_directory.path(), &snapshot, &FailingReplace) - .expect_err("replacement should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("atomically replace"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - } - - #[test] - fn zstd_finalization_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let temporary = run_directory.path().join(".injected-zstd-failure.tmp"); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let file = fs::File::create(&temporary).expect("temporary fixture should be created"); - let writer = InjectableFile { - file, - fail_writes: true, - fail_sync: false, - }; - let buffer = BufWriter::with_capacity(0, writer); - let snapshot = [snapshot_entry( - "zstd.failure", - None, - AggregateMetricType::Gauge, - "", - &[], - &[], - )]; - - let error = publish_buffered_temporary(buffer, &temporary, &target, &snapshot, &FailingReplace) - .expect_err("zstd finalization should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("finalize zstd stream"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected write failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - } - - #[test] - fn buffered_finalization_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let temporary = run_directory.path().join(".injected-buffer-failure.tmp"); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let file = fs::File::create(&temporary).expect("temporary fixture should be created"); - let writer = InjectableFile { - file, - fail_writes: true, - fail_sync: false, - }; - let snapshot = [snapshot_entry( - "buffer.failure", - None, - AggregateMetricType::Gauge, - "", - &[], - &[], - )]; - - let error = publish_open_temporary(writer, &temporary, &target, &snapshot, &FailingReplace) - .expect_err("buffer finalization should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("finalize buffered"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected write failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - } - - #[test] - fn sync_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let temporary = run_directory.path().join(".injected-sync-failure.tmp"); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let file = fs::File::create(&temporary).expect("temporary fixture should be created"); - let writer = InjectableFile { - file, - fail_writes: false, - fail_sync: true, - }; - - let error = - publish_open_temporary(writer, &temporary, &target, &[], &FailingReplace).expect_err("sync should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("sync temporary"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected sync failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); - } - - #[test] - fn zstd_and_buffer_finalizers_surface_their_own_error_context() { - let target = PathBuf::from("/run/test/dogstatsd_contexts.json.zstd"); - let snapshot = [snapshot_entry( - "finalize.failure", - None, - AggregateMetricType::Gauge, - "", - &[], - &[], - )]; - - let zstd_failure = Arc::new(AtomicBool::new(false)); - let mut encoder = zstd::stream::write::Encoder::new(ToggleFailureWriter::new(zstd_failure.clone()), 0) - .expect("encoder should initialize"); - write_snapshot_records(&mut encoder, &snapshot, &target).expect("records should stream before failure"); - zstd_failure.store(true, Ordering::Relaxed); - let error = finish_zstd_encoder(encoder, &target).expect_err("zstd finalization should fail"); - let message = format!("{error:#}"); - assert!(message.contains("finalize zstd stream"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected finalization failure"), "{message}"); - - let buffer_failure = Arc::new(AtomicBool::new(false)); - let buffer = BufWriter::new(ToggleFailureWriter::new(buffer_failure.clone())); - let mut encoder = zstd::stream::write::Encoder::new(buffer, 0).expect("encoder should initialize"); - write_snapshot_records(&mut encoder, &snapshot, &target).expect("records should stream before failure"); - let buffer = finish_zstd_encoder(encoder, &target).expect("zstd stream should finalize"); - buffer_failure.store(true, Ordering::Relaxed); - let error = finish_buffered_writer(buffer, &target).expect_err("buffer finalization should fail"); - let message = format!("{error:#}"); - assert!(message.contains("finalize buffered"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected finalization failure"), "{message}"); - } - - struct InjectableFile { - file: fs::File, - fail_writes: bool, - fail_sync: bool, - } - - impl io::Write for InjectableFile { - fn write(&mut self, bytes: &[u8]) -> io::Result { - if self.fail_writes { - Err(io::Error::other("injected write failure")) - } else { - self.file.write(bytes) - } - } - - fn flush(&mut self) -> io::Result<()> { - if self.fail_writes { - Err(io::Error::other("injected write failure")) - } else { - self.file.flush() - } - } - } - - impl SyncAll for InjectableFile { - fn sync_all(&self) -> io::Result<()> { - if self.fail_sync { - Err(io::Error::other("injected sync failure")) - } else { - self.file.sync_all() - } - } - } - - #[derive(Debug)] - struct ToggleFailureWriter { - failure_enabled: Arc, - } - - impl ToggleFailureWriter { - fn new(failure_enabled: Arc) -> Self { - Self { failure_enabled } - } - } - - impl io::Write for ToggleFailureWriter { - fn write(&mut self, bytes: &[u8]) -> io::Result { - if self.failure_enabled.load(Ordering::Relaxed) { - Err(io::Error::other("injected finalization failure")) - } else { - Ok(bytes.len()) - } - } - - fn flush(&mut self) -> io::Result<()> { - if self.failure_enabled.load(Ordering::Relaxed) { - Err(io::Error::other("injected finalization failure")) - } else { - Ok(()) - } - } - } - - struct FixedEntropy; - - impl EntropySource for FixedEntropy { - fn fill(&self, bytes: &mut [u8]) -> io::Result<()> { - bytes.fill(0x11); - Ok(()) - } - } - - struct FailingEntropy; - - impl EntropySource for FailingEntropy { - fn fill(&self, _bytes: &mut [u8]) -> io::Result<()> { - Err(io::Error::other("injected entropy failure")) - } - } - - #[cfg(unix)] - struct FailingPermissions; - - #[cfg(unix)] - impl PermissionNormalizer for FailingPermissions { - fn normalize(&self, _file: &fs::File) -> io::Result<()> { - Err(io::Error::other("injected permission failure")) - } - } - - #[cfg(unix)] - #[derive(Default)] - struct RestrictiveThenOwnerOnly { - called: AtomicBool, - } - - #[cfg(unix)] - impl PermissionNormalizer for RestrictiveThenOwnerOnly { - fn normalize(&self, file: &fs::File) -> io::Result<()> { - use std::os::unix::fs::PermissionsExt as _; - - file.set_permissions(fs::Permissions::from_mode(0o000))?; - set_owner_only_permissions(file)?; - self.called.store(true, Ordering::Relaxed); - Ok(()) - } - } - - #[derive(Default)] - struct FailingCleanup { - attempts: AtomicUsize, - } - - impl TemporaryFileCleanup for FailingCleanup { - fn remove(&self, _path: &std::path::Path) -> io::Result<()> { - self.attempts.fetch_add(1, Ordering::Relaxed); - Err(io::Error::other("injected cleanup failure")) - } - } - - struct FailingReplace; - - impl AtomicReplace for FailingReplace { - fn replace(&self, _source: &std::path::Path, _target: &std::path::Path) -> io::Result<()> { - Err(io::Error::other("injected replacement failure")) - } - } - - fn directory_entries(path: &std::path::Path) -> Vec { - let mut entries: Vec<_> = fs::read_dir(path) - .unwrap() - .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) - .collect(); - entries.sort_unstable(); - entries - } - - struct FailAfterBytes { - remaining: usize, - written: Vec, - } - - impl FailAfterBytes { - fn new(remaining: usize) -> Self { - Self { - remaining, - written: Vec::new(), - } - } - } - - impl io::Write for FailAfterBytes { - fn write(&mut self, bytes: &[u8]) -> io::Result { - if self.remaining == 0 { - return Err(io::Error::other("injected write failure")); - } - let written = self.remaining.min(bytes.len()); - self.written.extend_from_slice(&bytes[..written]); - self.remaining -= written; - Ok(written) - } - - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - - fn snapshot_entry( - name: &'static str, host: Option<&'static str>, metric_type: AggregateMetricType, unit: &'static str, - metric_tags: &[&'static str], origin_tags: &[&'static str], - ) -> AggregateContextSnapshotEntry { - let context = Context::from_static_parts(name, metric_tags) - .with_host(host.map(MetaString::from_static)) - .with_origin_tags(origin_tags.iter().map(|tag| Tag::from_static(tag)).collect::()); - AggregateContextSnapshotEntry::for_test(context, metric_type, MetaString::from_static(unit)) - } - - #[test] - fn detects_plain_and_compressed_artifacts_by_magic_bytes() { - let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); - let compressed_without_suffix = temporary_directory.path().join("compressed-contexts"); - let plain_with_compressed_suffix = temporary_directory.path().join("plain-contexts.zstd"); - fs::write(&compressed_without_suffix, COMPRESSED_FIXTURE_BYTES).expect("compressed fixture should be copied"); - fs::write(&plain_with_compressed_suffix, PLAIN_FIXTURE_BYTES).expect("plain fixture should be copied"); - - let cases = [ - PathBuf::from(PLAIN_FIXTURE_PATH), - PathBuf::from(COMPRESSED_FIXTURE_PATH), - compressed_without_suffix, - plain_with_compressed_suffix, - ]; - let expected = expected_fixture_records(); - - for path in cases { - assert_eq!( - collect_records(&path).expect("artifact should decode"), - expected, - "{path:?}" - ); - } - } - - #[test] - fn checked_in_compressed_fixture_expands_to_the_plain_fixture_exactly() { - let decompressed = zstd::stream::decode_all(COMPRESSED_FIXTURE_BYTES) - .expect("checked-in compressed fixture should decompress"); - - assert_eq!(decompressed, PLAIN_FIXTURE_BYTES); - assert_eq!(PLAIN_FIXTURE_BYTES.last(), Some(&b'\n')); - } - - #[test] - fn rejects_invalid_artifacts_with_path_operation_and_record_index() { - let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); - let valid_record = concat!( - "{\"Name\":\"metric\",\"Host\":\"host\",\"Type\":\"Gauge\",", - "\"TaggerTags\":[],\"MetricTags\":[],\"NoIndex\":false,\"Source\":1}" - ); - let truncated_length = COMPRESSED_FIXTURE_BYTES.len() - 4; - let cases = [ - InvalidArtifactCase { - filename: "array.ndjson", - bytes: format!("[{valid_record}]").into_bytes(), - expected_record_index: 1, - }, - InvalidArtifactCase { - filename: "missing-field.ndjson", - bytes: br#"{"Name":"metric"}"#.to_vec(), - expected_record_index: 1, - }, - InvalidArtifactCase { - filename: "malformed-trailing.ndjson", - bytes: format!("{valid_record}\nnot-json").into_bytes(), - expected_record_index: 2, - }, - InvalidArtifactCase { - filename: "truncated.zstd", - bytes: COMPRESSED_FIXTURE_BYTES[..truncated_length].to_vec(), - expected_record_index: 5, - }, - ]; - - for case in cases { - let path = temporary_directory.path().join(case.filename); - fs::write(&path, case.bytes).expect("invalid artifact should be written"); - - let error = collect_records(&path).expect_err("invalid artifact should be rejected"); - - assert_eq!(error.path(), path.as_path(), "{}", case.filename); - assert_eq!(error.operation(), "decode", "{}", case.filename); - assert_eq!(error.record_index(), case.expected_record_index, "{}", case.filename); - assert!(error.to_string().contains(&path.display().to_string()), "{error}"); - assert!( - error - .to_string() - .contains(&format!("decode record {}", case.expected_record_index)), - "{error}" - ); - assert!(error.to_string().contains("JSON"), "{error}"); - assert!(error.to_string().contains("line"), "{error}"); - assert!(error.to_string().contains("column"), "{error}"); - } - } - - struct InvalidArtifactCase { - filename: &'static str, - bytes: Vec, - expected_record_index: usize, - } - - #[test] - fn ignores_unknown_fields_but_requires_every_agent_field() { - let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); - let record = concat!( - "{\"Name\":\"metric\",\"Host\":\"host\",\"Type\":\"Gauge\",", - "\"TaggerTags\":[],\"MetricTags\":[\"env:prod\"],\"NoIndex\":false,\"Source\":7,", - "\"FutureField\":\"ignored\"}\n" - ); - fs::write(artifact.path(), record).expect("artifact should be written"); - - assert_eq!( - collect_records(artifact.path()).expect("record with unknown field should decode"), - vec![AgentContextRecord { - name: "metric".to_owned(), - host: "host".to_owned(), - metric_type: "Gauge".to_owned(), - tagger_tags: Vec::new(), - metric_tags: vec!["env:prod".to_owned()], - no_index: false, - source: 7, - }] - ); - } -} +#[path = "artifact_tests.rs"] +mod tests; diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact_tests.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact_tests.rs new file mode 100644 index 00000000000..869eb493183 --- /dev/null +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact_tests.rs @@ -0,0 +1,1075 @@ +use std::fs; +#[cfg(not(windows))] +use std::fs::OpenOptions; +use std::io::{self, BufWriter, Write}; +use std::path::PathBuf; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, +}; + +use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; +use saluki_context::{ + tags::{Tag, TagSet}, + Context, +}; +use saluki_error::GenericError; +use serde_json::json; +use stringtheory::MetaString; + +use super::{ + finish_buffered_writer, finish_zstd_encoder, for_each_record, publish_buffered_temporary_unmanaged, + publish_context_dump, publish_context_dump_with, publish_context_dump_with_services, run_with_temporary_cleanup, + write_snapshot_records, AgentContextRecord, ArtifactError, AtomicReplace, FileSystemAtomicReplace, + FileSystemCleanup, OwnerOnlyPermissions, SyncAll, TemporaryFileCleanup, TemporaryFileFactory, TemporaryPathCleanup, + CONTEXT_DUMP_FILENAME, +}; +#[cfg(windows)] +use super::{open_temporary_file, SecureTemporaryFileFactory}; +#[cfg(unix)] +use super::{set_owner_only_permissions, PermissionNormalizer}; +use crate::dogstatsd_contexts::read_report; + +impl ArtifactError { + fn path(&self) -> &std::path::Path { + &self.path + } + + fn operation(&self) -> &'static str { + self.operation + } + + fn record_index(&self) -> usize { + self.record_index + } +} + +fn collect_records(path: &std::path::Path) -> Result, ArtifactError> { + let mut records = Vec::new(); + for_each_record(path, |record| records.push(record))?; + Ok(records) +} + +fn publish_open_temporary( + writer: W, temporary_path: &std::path::Path, target: &std::path::Path, snapshot: &[AggregateContextSnapshotEntry], + replacer: &R, +) -> Result<(), GenericError> +where + W: Write + SyncAll, + R: AtomicReplace, +{ + run_with_temporary_cleanup(temporary_path, target, &FileSystemCleanup, || { + publish_buffered_temporary_unmanaged(BufWriter::new(writer), temporary_path, target, snapshot, replacer) + }) +} + +fn publish_buffered_temporary( + buffer: BufWriter, temporary_path: &std::path::Path, target: &std::path::Path, + snapshot: &[AggregateContextSnapshotEntry], replacer: &R, +) -> Result<(), GenericError> +where + W: Write + SyncAll, + R: AtomicReplace, +{ + run_with_temporary_cleanup(temporary_path, target, &FileSystemCleanup, || { + publish_buffered_temporary_unmanaged(buffer, temporary_path, target, snapshot, replacer) + }) +} + +const PLAIN_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson" +)); +const COMPRESSED_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd" +)); +const PLAIN_FIXTURE_PATH: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson" +); +const COMPRESSED_FIXTURE_PATH: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd" +); + +#[cfg(windows)] +struct LocalAllocation(*mut std::ffi::c_void); + +#[cfg(windows)] +impl Drop for LocalAllocation { + fn drop(&mut self) { + if !self.0.is_null() { + use windows_sys::Win32::Foundation::{LocalFree, HLOCAL}; + + // SAFETY: This pointer was allocated by a Windows security API that transfers ownership to the caller. + unsafe { + let _ = LocalFree(self.0 as HLOCAL); + } + } + } +} + +#[cfg(windows)] +fn windows_file_dacl_sddl(path: &std::path::Path) -> io::Result<(u16, String)> { + use std::os::windows::ffi::OsStrExt as _; + use std::{ptr, slice}; + + use windows_sys::Win32::{ + Foundation::ERROR_SUCCESS, + Security::{ + Authorization::{ + ConvertSecurityDescriptorToStringSecurityDescriptorW, GetNamedSecurityInfoW, SDDL_REVISION_1, + SE_FILE_OBJECT, + }, + GetSecurityDescriptorControl, DACL_SECURITY_INFORMATION, + }, + }; + + let mut wide_path: Vec<_> = path.as_os_str().encode_wide().collect(); + if wide_path.contains(&0) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "path contains an interior NUL", + )); + } + wide_path.push(0); + + let mut descriptor = ptr::null_mut(); + // SAFETY: The path is NUL-terminated and remains alive during the call. All unused output pointers are null, + // and `descriptor` receives an allocation owned by the caller on success. + let result = unsafe { + GetNamedSecurityInfoW( + wide_path.as_ptr(), + SE_FILE_OBJECT, + DACL_SECURITY_INFORMATION, + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + ptr::null_mut(), + &mut descriptor, + ) + }; + if result != ERROR_SUCCESS { + return Err(io::Error::from_raw_os_error(result as i32)); + } + let descriptor = LocalAllocation(descriptor); + + let mut control = 0; + let mut revision = 0; + // SAFETY: `descriptor` is a valid security descriptor allocation and both output pointers are valid writes. + let result = unsafe { GetSecurityDescriptorControl(descriptor.0, &mut control, &mut revision) }; + if result == 0 { + return Err(io::Error::last_os_error()); + } + + let mut sddl = ptr::null_mut(); + let mut sddl_len = 0; + // SAFETY: `descriptor` remains valid during the call, and both output pointers are valid writes. The returned + // string allocation is owned by the caller on success. + let result = unsafe { + ConvertSecurityDescriptorToStringSecurityDescriptorW( + descriptor.0, + SDDL_REVISION_1, + DACL_SECURITY_INFORMATION, + &mut sddl, + &mut sddl_len, + ) + }; + if result == 0 { + return Err(io::Error::last_os_error()); + } + let sddl_allocation = LocalAllocation(sddl.cast()); + // SAFETY: The conversion API returned `sddl_len` initialized UTF-16 code units in the live `sddl` allocation. + let wide_sddl = unsafe { slice::from_raw_parts(sddl, sddl_len as usize) }; + let wide_sddl = wide_sddl.strip_suffix(&[0]).unwrap_or(wide_sddl); + let sddl = String::from_utf16(wide_sddl).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + drop(sddl_allocation); + + Ok((control, sddl)) +} + +// These fixtures and expectations mirror ContextDebugRepr and the `dogstatsd top` command at Datadog Agent +// commit 9de2a8371cf8d95794f238939709491907893288. +fn expected_fixture_records() -> Vec { + vec![ + AgentContextRecord { + name: "z.metric".to_owned(), + host: "node-a".to_owned(), + metric_type: "Gauge".to_owned(), + tagger_tags: vec!["pod_name:web-a".to_owned()], + metric_tags: vec![ + "env:prod".to_owned(), + "image:registry/repo:v1".to_owned(), + "bare".to_owned(), + ], + no_index: false, + source: 1, + }, + AgentContextRecord { + name: "a.metric".to_owned(), + host: "node-a".to_owned(), + metric_type: "Counter".to_owned(), + tagger_tags: vec!["pod_name:web-a".to_owned()], + metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + AgentContextRecord { + name: "a.metric".to_owned(), + host: "node-b".to_owned(), + metric_type: "Counter".to_owned(), + tagger_tags: vec!["pod_name:web-b".to_owned()], + metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + AgentContextRecord { + name: "a.metric".to_owned(), + host: "node-c".to_owned(), + metric_type: "Counter".to_owned(), + tagger_tags: vec!["pod_name:web-c".to_owned()], + metric_tags: vec!["env:staging".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + ] +} + +#[test] +fn streams_exact_agent_schema_for_every_metric_type() { + let cases = [ + (AggregateMetricType::Counter, None, "Counter"), + (AggregateMetricType::Rate, None, "Rate"), + (AggregateMetricType::Gauge, None, "Gauge"), + (AggregateMetricType::Set, None, "Set"), + (AggregateMetricType::Distribution, None, "Distribution"), + (AggregateMetricType::Histogram, None, "Histogram"), + (AggregateMetricType::Histogram, Some("second"), "Histogram"), + (AggregateMetricType::Histogram, Some("millisecond"), "Historate"), + ]; + let snapshot: Vec<_> = cases + .iter() + .enumerate() + .map(|(index, (metric_type, unit, _))| { + snapshot_entry( + Box::leak(format!("metric.{index}").into_boxed_str()), + (index == 0).then_some("node-a"), + *metric_type, + unit.unwrap_or_default(), + &["client:value", "bare"], + &["origin:value"], + ) + }) + .collect(); + let mut output = Vec::new(); + + write_snapshot_records(&mut output, &snapshot, PathBuf::from("plain-stream").as_path()) + .expect("snapshot should serialize"); + + let output = String::from_utf8(output).expect("serialized output should be UTF-8"); + let lines: Vec<_> = output.lines().collect(); + assert_eq!(lines.len(), cases.len()); + assert_eq!(output.bytes().filter(|byte| *byte == b'\n').count(), cases.len()); + assert!(output.ends_with('\n')); + for (index, ((_, _, expected_type), line)) in cases.iter().zip(&lines).enumerate() { + let expected_host = if index == 0 { "node-a" } else { "" }; + let expected = json!({ + "Name": format!("metric.{index}"), + "Host": expected_host, + "Type": expected_type, + "TaggerTags": ["origin:value"], + "MetricTags": ["client:value", "bare"], + "NoIndex": false, + "Source": 1, + }); + assert_eq!(serde_json::from_str::(line).unwrap(), expected); + assert_eq!( + *line, + format!( + "{{\"Name\":\"metric.{index}\",\"Host\":\"{expected_host}\",\"Type\":\"{expected_type}\",\"TaggerTags\":[\"origin:value\"],\"MetricTags\":[\"client:value\",\"bare\"],\"NoIndex\":false,\"Source\":1}}" + ) + ); + } +} + +#[test] +fn plain_stream_round_trips_through_reader_and_report() { + let snapshot = vec![ + snapshot_entry( + "shared.metric", + Some("node-a"), + AggregateMetricType::Gauge, + "", + &["env:prod", "service:web"], + &["pod_name:web-a"], + ), + snapshot_entry( + "shared.metric", + None, + AggregateMetricType::Counter, + "", + &["env:staging", "service:web"], + &["pod_name:web-b"], + ), + ]; + let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); + let mut file = artifact.reopen().expect("temporary artifact should reopen"); + + write_snapshot_records(&mut file, &snapshot, artifact.path()).expect("snapshot should serialize"); + file.flush().expect("plain stream should flush"); + + assert_eq!( + collect_records(artifact.path()).expect("plain stream should decode"), + vec![ + AgentContextRecord { + name: "shared.metric".to_owned(), + host: "node-a".to_owned(), + metric_type: "Gauge".to_owned(), + tagger_tags: vec!["pod_name:web-a".to_owned()], + metric_tags: vec!["env:prod".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + AgentContextRecord { + name: "shared.metric".to_owned(), + host: String::new(), + metric_type: "Counter".to_owned(), + tagger_tags: vec!["pod_name:web-b".to_owned()], + metric_tags: vec!["env:staging".to_owned(), "service:web".to_owned()], + no_index: false, + source: 1, + }, + ] + ); + assert_eq!( + read_report(artifact.path()) + .expect("plain stream report should build") + .render(10, 10), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 2\tshared.metric\t(2 env, 1 service)\n", + ) + ); +} + +#[test] +fn surfaces_a_partial_writer_failure_with_record_and_path_context() { + let snapshot = [snapshot_entry( + "write.failure", + None, + AggregateMetricType::Gauge, + "", + &["env:test"], + &[], + )]; + let path = PathBuf::from("/run/test/dogstatsd_contexts.json.zstd"); + let mut writer = FailAfterBytes::new(12); + + let error = write_snapshot_records(&mut writer, &snapshot, &path).expect_err("write failure should surface"); + + assert_eq!(writer.written.len(), 12); + assert!(error.to_string().contains("serialize DogStatsD context dump record 1")); + assert!(error.to_string().contains(&path.display().to_string())); + assert!(format!("{error:#}").contains("injected write failure")); +} + +#[test] +fn publishes_a_decodable_mode_0600_dump_and_replaces_the_canonical_artifact() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + fs::write(&target, b"old canonical artifact").expect("old canonical artifact should be written"); + let snapshot = vec![ + snapshot_entry( + "published.metric", + Some("node-a"), + AggregateMetricType::Histogram, + "millisecond", + &["env:prod"], + &["pod_name:web-a"], + ), + snapshot_entry( + "published.metric", + None, + AggregateMetricType::Distribution, + "", + &["env:staging"], + &["pod_name:web-b"], + ), + ]; + + let published = publish_context_dump(run_directory.path(), &snapshot).expect("snapshot should publish"); + + assert_eq!(published, target); + assert_eq!( + collect_records(&target) + .expect("published artifact should decode") + .len(), + 2 + ); + assert_eq!( + read_report(&target) + .expect("published report should build") + .render(10, 10), + concat!( + " Contexts\tMetric name\t(number of unique values for each tag)\n", + " 2\tpublished.metric\t(2 env)\n", + ) + ); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + + assert_eq!(fs::metadata(&target).unwrap().permissions().mode() & 0o777, 0o600); + } +} + +#[cfg(windows)] +#[test] +fn temporary_dump_is_created_with_a_protected_restrictive_dacl() { + use windows_sys::Win32::Security::SE_DACL_PROTECTED; + + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let (_file, staging_path) = SecureTemporaryFileFactory + .create(run_directory.path()) + .expect("temporary dump should be created"); + + let (control, sddl) = windows_file_dacl_sddl(&staging_path).expect("temporary dump DACL should be readable"); + + assert_ne!(control & SE_DACL_PROTECTED, 0, "DACL must be protected: {sddl}"); + assert!(sddl.contains("D:P"), "DACL SDDL must carry the protected flag: {sddl}"); + for required_ace in ["(A;;FA;;;OW)", "(A;;FA;;;SY)", "(A;;FA;;;BA)"] { + assert!(sddl.contains(required_ace), "DACL is missing {required_ace}: {sddl}"); + } + assert_eq!(sddl.matches('(').count(), 3, "DACL contains an unexpected ACE: {sddl}"); + assert!(!sddl.contains(";;;WD)"), "DACL must not grant Everyone access: {sddl}"); + assert!( + !sddl.contains(";;;BU)"), + "DACL must not grant Builtin Users access: {sddl}" + ); +} + +#[cfg(unix)] +#[test] +fn publication_normalizes_restrictive_temporary_permissions_to_exactly_0600() { + use std::os::unix::fs::PermissionsExt as _; + + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let normalizer = RestrictiveThenOwnerOnly::default(); + + let target = publish_context_dump_with_services( + run_directory.path(), + &[], + &FileSystemAtomicReplace, + &FixedTemporaryFileFactory, + &normalizer, + &FileSystemCleanup, + ) + .expect("snapshot should publish"); + + assert!(normalizer.called.load(Ordering::Relaxed)); + assert_eq!(fs::metadata(target).unwrap().permissions().mode() & 0o777, 0o600); +} + +#[test] +fn temporary_file_creation_failure_is_contextual_and_preserves_the_canonical_artifact() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + + let error = publish_context_dump_with_services( + run_directory.path(), + &[], + &FileSystemAtomicReplace, + &FailingTemporaryFileFactory, + &OwnerOnlyPermissions, + &FileSystemCleanup, + ) + .expect_err("temporary file creation failure should surface"); + + let message = format!("{error:#}"); + assert!(message.contains("create temporary"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!( + message.contains("injected temporary file creation failure"), + "{message}" + ); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); +} + +#[cfg(unix)] +#[test] +fn permission_failure_closes_and_removes_the_new_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + + let error = publish_context_dump_with_services( + run_directory.path(), + &[], + &FileSystemAtomicReplace, + &FixedTemporaryFileFactory, + &FailingPermissions, + &FileSystemCleanup, + ) + .expect_err("permission failure should surface"); + + let message = format!("{error:#}"); + assert!(message.contains("set owner-only permissions"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected permission failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); +} + +#[test] +fn cleanup_method_surfaces_removal_failure_and_remains_armed() { + let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); + let temporary_path = temporary_directory.path().join("staged-contexts.tmp"); + fs::write(&temporary_path, b"staged").unwrap(); + let cleanup = FailingCleanup::default(); + let mut guard = TemporaryPathCleanup::new(&temporary_path, &cleanup); + + let error = guard.cleanup().expect_err("cleanup failure should surface"); + + assert!(error.to_string().contains("injected cleanup failure")); + assert!(temporary_path.exists()); + assert_eq!(cleanup.attempts.load(Ordering::Relaxed), 1); +} + +#[test] +fn reports_primary_and_cleanup_failures_without_replacing_the_canonical_artifact() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let cleanup = FailingCleanup::default(); + + let error = publish_context_dump_with_services( + run_directory.path(), + &[], + &FailingReplace, + &FixedTemporaryFileFactory, + &OwnerOnlyPermissions, + &cleanup, + ) + .expect_err("replacement and cleanup should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("atomically replace"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("remove temporary DogStatsD context dump"), "{message}"); + assert!(message.contains("injected cleanup failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!(directory_entries(run_directory.path()).len(), 2); + assert!(directory_entries(run_directory.path()) + .iter() + .any(|entry| entry.starts_with(&format!(".{CONTEXT_DUMP_FILENAME}.")) && entry.ends_with(".tmp"))); + assert!(cleanup.attempts.load(Ordering::Relaxed) >= 1); +} + +#[test] +fn rejects_empty_missing_and_non_directory_run_paths_with_target_and_operation_context() { + let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); + let missing = temporary_directory.path().join("missing"); + let not_directory = temporary_directory.path().join("not-a-directory"); + fs::write(¬_directory, b"file").expect("non-directory fixture should be written"); + let cases = [ + (PathBuf::new(), "validate run path"), + (missing, "create temporary"), + (not_directory, "create temporary"), + ]; + + for (run_path, expected_operation) in cases { + let error = publish_context_dump(&run_path, &[]).expect_err("invalid run path should fail"); + let message = format!("{error:#}"); + assert!(message.contains(CONTEXT_DUMP_FILENAME), "{message}"); + assert!(message.contains(expected_operation), "{message}"); + } +} + +#[test] +fn injected_replace_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let snapshot = [snapshot_entry( + "replacement.failure", + None, + AggregateMetricType::Gauge, + "", + &[], + &[], + )]; + + let error = publish_context_dump_with(run_directory.path(), &snapshot, &FailingReplace) + .expect_err("replacement should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("atomically replace"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); +} + +#[test] +fn zstd_finalization_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let temporary = run_directory.path().join(".injected-zstd-failure.tmp"); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let file = fs::File::create(&temporary).expect("temporary fixture should be created"); + let writer = InjectableFile { + file, + fail_writes: true, + fail_sync: false, + }; + let buffer = BufWriter::with_capacity(0, writer); + let snapshot = [snapshot_entry( + "zstd.failure", + None, + AggregateMetricType::Gauge, + "", + &[], + &[], + )]; + + let error = publish_buffered_temporary(buffer, &temporary, &target, &snapshot, &FailingReplace) + .expect_err("zstd finalization should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("finalize zstd stream"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected write failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); +} + +#[test] +fn buffered_finalization_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let temporary = run_directory.path().join(".injected-buffer-failure.tmp"); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let file = fs::File::create(&temporary).expect("temporary fixture should be created"); + let writer = InjectableFile { + file, + fail_writes: true, + fail_sync: false, + }; + let snapshot = [snapshot_entry( + "buffer.failure", + None, + AggregateMetricType::Gauge, + "", + &[], + &[], + )]; + + let error = publish_open_temporary(writer, &temporary, &target, &snapshot, &FailingReplace) + .expect_err("buffer finalization should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("finalize buffered"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected write failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); +} + +#[test] +fn sync_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { + let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); + let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); + let temporary = run_directory.path().join(".injected-sync-failure.tmp"); + let original = b"original canonical artifact"; + fs::write(&target, original).expect("canonical fixture should be written"); + let file = fs::File::create(&temporary).expect("temporary fixture should be created"); + let writer = InjectableFile { + file, + fail_writes: false, + fail_sync: true, + }; + + let error = + publish_open_temporary(writer, &temporary, &target, &[], &FailingReplace).expect_err("sync should fail"); + + let message = format!("{error:#}"); + assert!(message.contains("sync temporary"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected sync failure"), "{message}"); + assert_eq!(fs::read(&target).unwrap(), original); + assert_eq!( + directory_entries(run_directory.path()), + vec![CONTEXT_DUMP_FILENAME.to_owned()] + ); +} + +#[test] +fn zstd_and_buffer_finalizers_surface_their_own_error_context() { + let target = PathBuf::from("/run/test/dogstatsd_contexts.json.zstd"); + let snapshot = [snapshot_entry( + "finalize.failure", + None, + AggregateMetricType::Gauge, + "", + &[], + &[], + )]; + + let zstd_failure = Arc::new(AtomicBool::new(false)); + let mut encoder = zstd::stream::write::Encoder::new(ToggleFailureWriter::new(zstd_failure.clone()), 0) + .expect("encoder should initialize"); + write_snapshot_records(&mut encoder, &snapshot, &target).expect("records should stream before failure"); + zstd_failure.store(true, Ordering::Relaxed); + let error = finish_zstd_encoder(encoder, &target).expect_err("zstd finalization should fail"); + let message = format!("{error:#}"); + assert!(message.contains("finalize zstd stream"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected finalization failure"), "{message}"); + + let buffer_failure = Arc::new(AtomicBool::new(false)); + let buffer = BufWriter::new(ToggleFailureWriter::new(buffer_failure.clone())); + let mut encoder = zstd::stream::write::Encoder::new(buffer, 0).expect("encoder should initialize"); + write_snapshot_records(&mut encoder, &snapshot, &target).expect("records should stream before failure"); + let buffer = finish_zstd_encoder(encoder, &target).expect("zstd stream should finalize"); + buffer_failure.store(true, Ordering::Relaxed); + let error = finish_buffered_writer(buffer, &target).expect_err("buffer finalization should fail"); + let message = format!("{error:#}"); + assert!(message.contains("finalize buffered"), "{message}"); + assert!(message.contains(&target.display().to_string()), "{message}"); + assert!(message.contains("injected finalization failure"), "{message}"); +} + +struct InjectableFile { + file: fs::File, + fail_writes: bool, + fail_sync: bool, +} + +impl io::Write for InjectableFile { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.fail_writes { + Err(io::Error::other("injected write failure")) + } else { + self.file.write(bytes) + } + } + + fn flush(&mut self) -> io::Result<()> { + if self.fail_writes { + Err(io::Error::other("injected write failure")) + } else { + self.file.flush() + } + } +} + +impl SyncAll for InjectableFile { + fn sync_all(&self) -> io::Result<()> { + if self.fail_sync { + Err(io::Error::other("injected sync failure")) + } else { + self.file.sync_all() + } + } +} + +#[derive(Debug)] +struct ToggleFailureWriter { + failure_enabled: Arc, +} + +impl ToggleFailureWriter { + fn new(failure_enabled: Arc) -> Self { + Self { failure_enabled } + } +} + +impl io::Write for ToggleFailureWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.failure_enabled.load(Ordering::Relaxed) { + Err(io::Error::other("injected finalization failure")) + } else { + Ok(bytes.len()) + } + } + + fn flush(&mut self) -> io::Result<()> { + if self.failure_enabled.load(Ordering::Relaxed) { + Err(io::Error::other("injected finalization failure")) + } else { + Ok(()) + } + } +} + +struct FixedTemporaryFileFactory; + +impl TemporaryFileFactory for FixedTemporaryFileFactory { + fn create(&self, run_path: &std::path::Path) -> io::Result<(fs::File, PathBuf)> { + let path = run_path.join(format!(".{CONTEXT_DUMP_FILENAME}.fixed.tmp")); + #[cfg(windows)] + let file = open_temporary_file(&path)?; + #[cfg(not(windows))] + let file = { + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + options.open(&path)? + }; + Ok((file, path)) + } +} + +struct FailingTemporaryFileFactory; + +impl TemporaryFileFactory for FailingTemporaryFileFactory { + fn create(&self, _run_path: &std::path::Path) -> io::Result<(fs::File, PathBuf)> { + Err(io::Error::other("injected temporary file creation failure")) + } +} + +#[cfg(unix)] +struct FailingPermissions; + +#[cfg(unix)] +impl PermissionNormalizer for FailingPermissions { + fn normalize(&self, _file: &fs::File) -> io::Result<()> { + Err(io::Error::other("injected permission failure")) + } +} + +#[cfg(unix)] +#[derive(Default)] +struct RestrictiveThenOwnerOnly { + called: AtomicBool, +} + +#[cfg(unix)] +impl PermissionNormalizer for RestrictiveThenOwnerOnly { + fn normalize(&self, file: &fs::File) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt as _; + + file.set_permissions(fs::Permissions::from_mode(0o000))?; + set_owner_only_permissions(file)?; + self.called.store(true, Ordering::Relaxed); + Ok(()) + } +} + +#[derive(Default)] +struct FailingCleanup { + attempts: AtomicUsize, +} + +impl TemporaryFileCleanup for FailingCleanup { + fn remove(&self, _path: &std::path::Path) -> io::Result<()> { + self.attempts.fetch_add(1, Ordering::Relaxed); + Err(io::Error::other("injected cleanup failure")) + } +} + +struct FailingReplace; + +impl AtomicReplace for FailingReplace { + fn replace(&self, _source: &std::path::Path, _target: &std::path::Path) -> io::Result<()> { + Err(io::Error::other("injected replacement failure")) + } +} + +fn directory_entries(path: &std::path::Path) -> Vec { + let mut entries: Vec<_> = fs::read_dir(path) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + entries.sort_unstable(); + entries +} + +struct FailAfterBytes { + remaining: usize, + written: Vec, +} + +impl FailAfterBytes { + fn new(remaining: usize) -> Self { + Self { + remaining, + written: Vec::new(), + } + } +} + +impl io::Write for FailAfterBytes { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if self.remaining == 0 { + return Err(io::Error::other("injected write failure")); + } + let written = self.remaining.min(bytes.len()); + self.written.extend_from_slice(&bytes[..written]); + self.remaining -= written; + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +fn snapshot_entry( + name: &'static str, host: Option<&'static str>, metric_type: AggregateMetricType, unit: &'static str, + metric_tags: &[&'static str], origin_tags: &[&'static str], +) -> AggregateContextSnapshotEntry { + let context = Context::from_static_parts(name, metric_tags) + .with_host(host.map(MetaString::from_static)) + .with_origin_tags(origin_tags.iter().map(|tag| Tag::from_static(tag)).collect::()); + AggregateContextSnapshotEntry::for_test(context, metric_type, MetaString::from_static(unit)) +} + +#[test] +fn detects_plain_and_compressed_artifacts_by_magic_bytes() { + let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); + let compressed_without_suffix = temporary_directory.path().join("compressed-contexts"); + let plain_with_compressed_suffix = temporary_directory.path().join("plain-contexts.zstd"); + fs::write(&compressed_without_suffix, COMPRESSED_FIXTURE_BYTES).expect("compressed fixture should be copied"); + fs::write(&plain_with_compressed_suffix, PLAIN_FIXTURE_BYTES).expect("plain fixture should be copied"); + + let cases = [ + PathBuf::from(PLAIN_FIXTURE_PATH), + PathBuf::from(COMPRESSED_FIXTURE_PATH), + compressed_without_suffix, + plain_with_compressed_suffix, + ]; + let expected = expected_fixture_records(); + + for path in cases { + assert_eq!( + collect_records(&path).expect("artifact should decode"), + expected, + "{path:?}" + ); + } +} + +#[test] +fn checked_in_compressed_fixture_expands_to_the_plain_fixture_exactly() { + let decompressed = + zstd::stream::decode_all(COMPRESSED_FIXTURE_BYTES).expect("checked-in compressed fixture should decompress"); + + assert_eq!(decompressed, PLAIN_FIXTURE_BYTES); + assert_eq!(PLAIN_FIXTURE_BYTES.last(), Some(&b'\n')); +} + +#[test] +fn rejects_invalid_artifacts_with_path_operation_and_record_index() { + let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); + let valid_record = concat!( + "{\"Name\":\"metric\",\"Host\":\"host\",\"Type\":\"Gauge\",", + "\"TaggerTags\":[],\"MetricTags\":[],\"NoIndex\":false,\"Source\":1}" + ); + let truncated_length = COMPRESSED_FIXTURE_BYTES.len() - 4; + let cases = [ + InvalidArtifactCase { + filename: "array.ndjson", + bytes: format!("[{valid_record}]").into_bytes(), + expected_record_index: 1, + }, + InvalidArtifactCase { + filename: "missing-field.ndjson", + bytes: br#"{"Name":"metric"}"#.to_vec(), + expected_record_index: 1, + }, + InvalidArtifactCase { + filename: "malformed-trailing.ndjson", + bytes: format!("{valid_record}\nnot-json").into_bytes(), + expected_record_index: 2, + }, + InvalidArtifactCase { + filename: "truncated.zstd", + bytes: COMPRESSED_FIXTURE_BYTES[..truncated_length].to_vec(), + expected_record_index: 5, + }, + ]; + + for case in cases { + let path = temporary_directory.path().join(case.filename); + fs::write(&path, case.bytes).expect("invalid artifact should be written"); + + let error = collect_records(&path).expect_err("invalid artifact should be rejected"); + + assert_eq!(error.path(), path.as_path(), "{}", case.filename); + assert_eq!(error.operation(), "decode", "{}", case.filename); + assert_eq!(error.record_index(), case.expected_record_index, "{}", case.filename); + assert!(error.to_string().contains(&path.display().to_string()), "{error}"); + assert!( + error + .to_string() + .contains(&format!("decode record {}", case.expected_record_index)), + "{error}" + ); + assert!(error.to_string().contains("JSON"), "{error}"); + assert!(error.to_string().contains("line"), "{error}"); + assert!(error.to_string().contains("column"), "{error}"); + } +} + +struct InvalidArtifactCase { + filename: &'static str, + bytes: Vec, + expected_record_index: usize, +} + +#[test] +fn ignores_unknown_fields_but_requires_every_agent_field() { + let artifact = tempfile::NamedTempFile::new().expect("temporary artifact should be created"); + let record = concat!( + "{\"Name\":\"metric\",\"Host\":\"host\",\"Type\":\"Gauge\",", + "\"TaggerTags\":[],\"MetricTags\":[\"env:prod\"],\"NoIndex\":false,\"Source\":7,", + "\"FutureField\":\"ignored\"}\n" + ); + fs::write(artifact.path(), record).expect("artifact should be written"); + + assert_eq!( + collect_records(artifact.path()).expect("record with unknown field should decode"), + vec![AgentContextRecord { + name: "metric".to_owned(), + host: "host".to_owned(), + metric_type: "Gauge".to_owned(), + tagger_tags: Vec::new(), + metric_tags: vec!["env:prod".to_owned()], + no_index: false, + source: 7, + }] + ); +} From 19ecea915339feaea3c1de5987b51cdf47a2f7bf Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Thu, 23 Jul 2026 15:03:20 -0400 Subject: [PATCH 31/44] fix(adp): align context dump API with Agent Use the Agent command API path for context dumps, centralize Data Plane client construction, and document why the authenticated client requires the Agent IPC TLS configuration. --- bin/agent-data-plane/src/cli/dogstatsd.rs | 29 +++++++++++-------- bin/agent-data-plane/src/cli/run.rs | 2 +- bin/agent-data-plane/src/cli/utils.rs | 15 ++++++---- .../src/dogstatsd_contexts/mod.rs | 3 ++ .../src/internal/control_surfaces.rs | 4 +-- lib/saluki-io/src/net/client/http/client.rs | 3 ++ .../cases/dogstatsd-top/config.yaml | 2 +- .../dogstatsd-top/run_dogstatsd_top_test.py | 2 +- 8 files changed, 38 insertions(+), 22 deletions(-) diff --git a/bin/agent-data-plane/src/cli/dogstatsd.rs b/bin/agent-data-plane/src/cli/dogstatsd.rs index a6261945aac..0dfaf0c37bf 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd.rs @@ -184,15 +184,13 @@ async fn run_dogstatsd_command( ) -> Result<(), GenericError> { match cmd.subcommand { DogstatsdSubcommand::Stats(config) => { - let mut api_client = DataPlaneAPIClient::from_config(bootstrap_config) - .error_context("Failed to create data plane API client")?; + let mut api_client = data_plane_api_client(bootstrap_config)?; handle_dogstatsd_stats(&mut api_client, config) .await .error_context("Failed to run stats subcommand") } DogstatsdSubcommand::Capture(config) => { - let mut api_client = DataPlaneAPIClient::from_config(bootstrap_config) - .error_context("Failed to create data plane API client")?; + let mut api_client = data_plane_api_client(bootstrap_config)?; handle_dogstatsd_capture(&mut api_client, config) .await .error_context("Failed to start DogStatsD capture") @@ -200,8 +198,7 @@ async fn run_dogstatsd_command( DogstatsdSubcommand::Replay(config) => { #[cfg(target_os = "linux")] { - let mut api_client = DataPlaneAPIClient::from_config(bootstrap_config) - .error_context("Failed to create data plane API client")?; + let mut api_client = data_plane_api_client(bootstrap_config)?; handle_dogstatsd_replay(&mut api_client, bootstrap_config, config) .await .error_context("Failed to replay DogStatsD traffic") @@ -225,22 +222,30 @@ async fn run_dogstatsd_command( if config.is_offline() { handle_dogstatsd_top(None, config, &mut output).await } else { - let mut api_client = DataPlaneAPIClient::from_config_with_ipc_auth(bootstrap_config) - .await - .error_context("Failed to create authenticated data plane API client for DogStatsD top.")?; + let mut api_client = authenticated_data_plane_api_client(bootstrap_config).await?; handle_dogstatsd_top(Some(&mut api_client), config, &mut output).await } } DogstatsdSubcommand::DumpContexts(_) => { - let mut api_client = DataPlaneAPIClient::from_config_with_ipc_auth(bootstrap_config) - .await - .error_context("Failed to create authenticated data plane API client for DogStatsD context dump.")?; + let mut api_client = authenticated_data_plane_api_client(bootstrap_config).await?; let mut output = std::io::stdout(); handle_dogstatsd_dump_contexts(&mut api_client, &mut output).await } } } +fn data_plane_api_client(config: &GenericConfiguration) -> Result { + DataPlaneAPIClient::from_config(config).error_context("Failed to create data plane API client") +} + +async fn authenticated_data_plane_api_client( + config: &GenericConfiguration, +) -> Result { + DataPlaneAPIClient::from_config_with_ipc_auth(config) + .await + .error_context("Failed to create authenticated data plane API client") +} + async fn handle_dogstatsd_stats(api_client: &mut DataPlaneAPIClient, cmd: StatsCommand) -> Result<(), GenericError> { // Trigger a statistics collection and wait for it to complete. info!( diff --git a/bin/agent-data-plane/src/cli/run.rs b/bin/agent-data-plane/src/cli/run.rs index 986acc26420..24e4e57ce45 100644 --- a/bin/agent-data-plane/src/cli/run.rs +++ b/bin/agent-data-plane/src/cli/run.rs @@ -967,7 +967,7 @@ mod tests { const AUTH_TOKEN: &[u8] = b"configured-agent-token"; const CONTEXT_DUMP_FILENAME: &str = "dogstatsd_contexts.json.zstd"; - const CONTEXT_DUMP_ROUTE: &str = "/dogstatsd/contexts/dump"; + const CONTEXT_DUMP_ROUTE: &str = crate::dogstatsd_contexts::CONTEXT_DUMP_ROUTE; #[tokio::test] async fn retained_context_identity_follows_dogstatsd_post_processing() { diff --git a/bin/agent-data-plane/src/cli/utils.rs b/bin/agent-data-plane/src/cli/utils.rs index 7e8a1569de0..a53e7319d51 100644 --- a/bin/agent-data-plane/src/cli/utils.rs +++ b/bin/agent-data-plane/src/cli/utils.rs @@ -26,7 +26,7 @@ use saluki_io::net::{ }; use serde::{Deserialize, Serialize}; -use crate::config::DataPlaneConfiguration; +use crate::{config::DataPlaneConfiguration, dogstatsd_contexts::CONTEXT_DUMP_ROUTE}; /// Typed API client for interacting with the APIs exposed by ADP. pub struct DataPlaneAPIClient { @@ -70,6 +70,10 @@ impl DataPlaneAPIClient { /// Creates an authenticated `DataPlaneAPIClient` for commands that use the Agent IPC credentials. /// + /// This mirrors the Agent IPC HTTP client contract: verify the configured IPC certificate and attach the Agent + /// authentication token. The complete Rustls configuration is required because it carries a custom certificate + /// verifier and client identity that the generic HTTP TLS option builder cannot express. + /// /// This client pins the privileged API server certificate to the configured IPC certificate and authenticates /// requests with the raw contents of the configured Agent authentication token file. Unlike the general-purpose /// client, it does not impose a whole-request timeout because context dump generation can legitimately take longer @@ -261,7 +265,7 @@ impl DataPlaneAPIClient { "DogStatsD context dumps require an authenticated API client; construct it with `from_config_with_ipc_auth`." ) })?; - let uri = self.build_uri("/dogstatsd/contexts/dump", None); + let uri = self.build_uri(CONTEXT_DUMP_ROUTE, None); let request = build_dogstatsd_contexts_dump_request(uri, authorization); let response = self .client @@ -557,6 +561,7 @@ mod tests { }; #[cfg(target_os = "linux")] use super::{body_when_replay_session_success, empty_when_replay_session_success}; + use crate::dogstatsd_contexts::CONTEXT_DUMP_ROUTE; #[test] fn dogstatsd_capture_failed_precondition_surfaces_server_message() { @@ -643,12 +648,12 @@ mod tests { fn dogstatsd_contexts_request_is_authenticated_empty_post() { let mut auth = HeaderValue::from_static("Bearer exact-token"); auth.set_sensitive(true); - let uri = Uri::from_static("https://127.0.0.1:5101/dogstatsd/contexts/dump"); + let uri = Uri::from_static("https://127.0.0.1:5101/agent/dogstatsd-contexts-dump"); let request = build_dogstatsd_contexts_dump_request(uri, &auth); assert_eq!(request.method(), Method::POST); - assert_eq!(request.uri().path(), "/dogstatsd/contexts/dump"); + assert_eq!(request.uri().path(), CONTEXT_DUMP_ROUTE); assert!(request.body().is_empty()); let request_auth = request.headers().get(AUTHORIZATION).expect("authorization header"); assert_eq!(request_auth, "Bearer exact-token"); @@ -779,7 +784,7 @@ mod tests { } } let request = String::from_utf8(request)?; - assert!(request.starts_with("POST /dogstatsd/contexts/dump HTTP/1.1\r\n")); + assert!(request.starts_with("POST /agent/dogstatsd-contexts-dump HTTP/1.1\r\n")); assert!( request .lines() diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs index 64be91a31c8..7ed7581e5e8 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/mod.rs @@ -10,6 +10,9 @@ mod api; mod artifact; mod report; +/// Agent-compatible command API route for requesting a DogStatsD context dump. +pub(crate) const CONTEXT_DUMP_ROUTE: &str = "/agent/dogstatsd-contexts-dump"; + pub(crate) fn read_report(path: &Path) -> Result { let mut report = ContextReport::new(); for_each_record(path, |record| report.ingest(record))?; diff --git a/bin/agent-data-plane/src/internal/control_surfaces.rs b/bin/agent-data-plane/src/internal/control_surfaces.rs index 4caacac5ab4..e87d3b74983 100644 --- a/bin/agent-data-plane/src/internal/control_surfaces.rs +++ b/bin/agent-data-plane/src/internal/control_surfaces.rs @@ -36,7 +36,7 @@ pub struct DogStatsDControlSurface { pub(crate) capture_api_handler: DogStatsDCaptureAPIHandler, /// API handler for the `/dogstatsd/replay/session` endpoints. pub(crate) replay_api_handler: DogStatsDReplayAPIHandler, - /// API handler for the `/dogstatsd/contexts/dump` endpoint. + /// API handler for the Agent-compatible `/agent/dogstatsd-contexts-dump` endpoint. pub(crate) context_dump_api_handler: DogStatsDContextDumpAPIHandler, } @@ -150,7 +150,7 @@ mod tests { tokio::time::timeout(remaining, async { stream .write_all( - b"POST /dogstatsd/contexts/dump HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer \ + b"POST /agent/dogstatsd-contexts-dump HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer \ configured-agent-token\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", ) .await?; diff --git a/lib/saluki-io/src/net/client/http/client.rs b/lib/saluki-io/src/net/client/http/client.rs index 04ea191dc8d..93944ed9163 100644 --- a/lib/saluki-io/src/net/client/http/client.rs +++ b/lib/saluki-io/src/net/client/http/client.rs @@ -287,6 +287,9 @@ impl HttpClientBuilder { /// Sets a complete client TLS configuration. /// + /// Use this for configurations with a custom server-certificate verifier or client identity, which cannot be + /// represented by the option-based TLS builder. + /// /// The supplied configuration takes precedence over all options applied through [`Self::with_tls_config`] and /// [`Self::with_min_tls_version`], regardless of call order. All supplied settings are preserved except ALPN: /// [`ClientConfig::alpn_protocols`] is cleared before connector construction so [`Self::with_http_protocol`] diff --git a/test/integration/cases/dogstatsd-top/config.yaml b/test/integration/cases/dogstatsd-top/config.yaml index 9305b24b550..70f6126e719 100644 --- a/test/integration/cases/dogstatsd-top/config.yaml +++ b/test/integration/cases/dogstatsd-top/config.yaml @@ -34,7 +34,7 @@ procedure: timeout: 60s # The route is POST-only. A GET returns 405 when registration is complete and 404 when absent. - assertion: http_check - endpoint: "https://localhost:55101/dogstatsd/contexts/dump" + endpoint: "https://localhost:55101/agent/dogstatsd-contexts-dump" status: not_equal: 404 insecure_skip_verify: true diff --git a/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py b/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py index 37288022df2..c62294d3a24 100644 --- a/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py +++ b/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py @@ -15,7 +15,7 @@ AGENT = Path("/opt/datadog-agent/bin/agent/agent") CONFIG = Path("/etc/datadog-agent/datadog.yaml") AUTH_TOKEN = Path("/etc/datadog-agent/auth_token") -API_URL = "https://127.0.0.1:55101/dogstatsd/contexts/dump" +API_URL = "https://127.0.0.1:55101/agent/dogstatsd-contexts-dump" DUMP_FILENAME = "dogstatsd_contexts.json.zstd" COPIED_DUMP = Path("/tmp/dogstatsd-top-copied.json.zstd") AGENT_FIXTURE_COMPRESSED = Path("/dogstatsd-top/agent-7.80.3-contexts.json.zstd") From f4bac43c5ba0cb022e171862394ba065d7550929 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Thu, 23 Jul 2026 15:04:23 -0400 Subject: [PATCH 32/44] docs(adp): record context dump benchmark baselines Document the Criterion commands and PR baseline measurements alongside both retained-context benchmarks, with an explicit note that the values are orientation rather than thresholds. --- bin/agent-data-plane/benches/dogstatsd_context_dump.rs | 8 ++++++++ .../benches/aggregate_context_snapshot.rs | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs index 10f1e755a7d..5221424944a 100644 --- a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs +++ b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs @@ -1,3 +1,11 @@ +//! Measures streaming zstd publication and atomic replacement of retained-context artifacts. +//! +//! Run with: +//! `cargo bench -p agent-data-plane --features context-dump-benchmark --bench dogstatsd_context_dump -- --sample-size 10` +//! +//! PR #2185's Apple Silicon baseline was approximately 15.7 ms at 10k contexts, 123 ms at 100k, and 1.13 s at one +//! million. These values provide scale context, not regression thresholds; compare runs on equivalent hardware. + use std::fs; use std::hint::black_box; use std::path::Path; diff --git a/lib/saluki-components/benches/aggregate_context_snapshot.rs b/lib/saluki-components/benches/aggregate_context_snapshot.rs index 8953216545e..a0bcc4d1433 100644 --- a/lib/saluki-components/benches/aggregate_context_snapshot.rs +++ b/lib/saluki-components/benches/aggregate_context_snapshot.rs @@ -1,3 +1,11 @@ +//! Measures the aggregate-owner pause required to clone retained context handles. +//! +//! Run with: +//! `cargo bench -p saluki-components --features test-util --bench aggregate_context_snapshot -- --sample-size 10` +//! +//! PR #2185's Apple Silicon baseline was approximately 45.8 µs at 10k contexts, 1.02 ms at 100k, and 20.8 ms at +//! one million. These values provide scale context, not regression thresholds; compare runs on equivalent hardware. + use std::hint::black_box; use std::mem::size_of; From 4ab04523d64a2972fcdb3209bef26e34a308647c Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Thu, 23 Jul 2026 15:08:33 -0400 Subject: [PATCH 33/44] test(integration): cover DogStatsD top on Windows Exercise the authenticated online and offline CLI flow on Windows and verify the published canonical artifact retains a protected Owner Rights, LocalSystem, and Administrators DACL. --- .../cases/dogstatsd-top-windows/config.yaml | 56 +++++++++ .../run_dogstatsd_top_test.ps1 | 119 ++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 test/integration/cases/dogstatsd-top-windows/config.yaml create mode 100644 test/integration/cases/dogstatsd-top-windows/run_dogstatsd_top_test.ps1 diff --git a/test/integration/cases/dogstatsd-top-windows/config.yaml b/test/integration/cases/dogstatsd-top-windows/config.yaml new file mode 100644 index 00000000000..2f4b90af96a --- /dev/null +++ b/test/integration/cases/dogstatsd-top-windows/config.yaml @@ -0,0 +1,56 @@ +type: integration +name: "dogstatsd-top-windows" +description: "Verifies the authenticated DogStatsD top workflow and protected context artifact on Windows" +timeout: 180s +runtimes: [windows] + +env: + DD_API_KEY: "00000000000000000000000000000000" + DD_HOSTNAME: "integration-test-dogstatsd-top-windows" + DD_DATA_PLANE_ENABLED: "true" + DD_DATA_PLANE_STANDALONE_MODE: "false" + DD_DATA_PLANE_DOGSTATSD_ENABLED: "true" + +container: + files: + - "run_dogstatsd_top_test.ps1:C:\\adp\\run_dogstatsd_top_test.ps1" + exposed_ports: + - "58125/udp" + - "55101/tcp" + +procedure: + - parallel: + - assertion: port_listening + port: 58125 + protocol: udp + timeout: 60s + - assertion: port_listening + port: 55101 + protocol: tcp + timeout: 60s + # The route is POST-only. A GET returns 405 when registration is complete and 404 when absent. + - assertion: http_check + endpoint: "https://localhost:55101/agent/dogstatsd-contexts-dump" + status: + not_equal: 404 + insecure_skip_verify: true + timeout: 60s + - action: target_exec + command: + - "pwsh" + - "-NoProfile" + - "-NonInteractive" + - "-File" + - "C:\\adp\\run_dogstatsd_top_test.ps1" + timeout: 90s + - assertion: file_contains + path: "C:\\dogstatsd-top-result" + pattern: "passed" + timeout: 10s + - parallel: + - assertion: process_stable_for + duration: 10s + - assertion: log_not_contains + pattern: "panic|PANIC" + regex: true + during: 10s diff --git a/test/integration/cases/dogstatsd-top-windows/run_dogstatsd_top_test.ps1 b/test/integration/cases/dogstatsd-top-windows/run_dogstatsd_top_test.ps1 new file mode 100644 index 00000000000..501e915e4e5 --- /dev/null +++ b/test/integration/cases/dogstatsd-top-windows/run_dogstatsd_top_test.ps1 @@ -0,0 +1,119 @@ +$ErrorActionPreference = "Stop" +Set-StrictMode -Version 3.0 + +$Adp = "C:\adp\agent-data-plane.exe" +$Config = "C:\ProgramData\Datadog\datadog.yaml" +$ApiUrl = "https://127.0.0.1:55101/agent/dogstatsd-contexts-dump" +$Result = "C:\dogstatsd-top-result" +$CopiedDump = "C:\dogstatsd-top-copied.json.zstd" +$RequestRow = " 2`tintegration.windows.top.requests`t(2 instance, 1 env)" +$QueueRow = " 1`tintegration.windows.top.queue`t(1 queue)" + +function Invoke-AdpDogStatsD([string[]] $Arguments) { + $Output = @(& $Adp --config $Config dogstatsd @Arguments 2>&1) + if ($LASTEXITCODE -ne 0) { + throw "ADP command failed ($LASTEXITCODE): $($Arguments -join ' ')`n$($Output -join "`n")" + } + return $Output -join "`n" +} + +function Assert-UnauthorizedRequest { + try { + Invoke-WebRequest -Uri $ApiUrl -Method Post -SkipCertificateCheck -TimeoutSec 10 | Out-Null + throw "Context dump API accepted an unauthenticated request." + } catch { + $Response = $_.Exception.Response + if ($null -eq $Response -or [int] $Response.StatusCode -ne 401) { + throw + } + } +} + +function Send-DogStatsDContexts { + $Payload = @( + "integration.windows.top.requests:1|c|#host:windows-a,env:prod,instance:a", + "integration.windows.top.requests:1|c|#host:windows-b,env:prod,instance:b", + "integration.windows.top.queue:1|c|#queue:alpha" + ) -join "`n" + $Bytes = [System.Text.Encoding]::UTF8.GetBytes($Payload) + $Client = [System.Net.Sockets.UdpClient]::new() + try { + 1..3 | ForEach-Object { [void] $Client.Send($Bytes, $Bytes.Length, "127.0.0.1", 58125) } + } finally { + $Client.Dispose() + } +} + +function Wait-ForOnlineReport { + $LastOutput = "" + foreach ($Attempt in 1..40) { + $LastOutput = Invoke-AdpDogStatsD @("top") + if ($LastOutput.Contains($RequestRow) -and $LastOutput.Contains($QueueRow)) { + return $LastOutput + } + Start-Sleep -Milliseconds 500 + } + throw "Retained contexts did not appear in online output:`n$LastOutput" +} + +function Get-DumpPath([string] $Output) { + $Lines = @($Output -split "`r?`n" | Where-Object { $_ -ne "" }) + if ($Lines.Count -ne 1 -or -not $Lines[0].StartsWith("Wrote ")) { + throw "Unexpected dump-contexts output: $Output" + } + $Path = $Lines[0].Substring("Wrote ".Length) + if (-not [System.IO.Path]::IsPathRooted($Path) -or [System.IO.Path]::GetFileName($Path) -ne "dogstatsd_contexts.json.zstd") { + throw "Unexpected context dump path: $Path" + } + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Context dump was not created: $Path" + } + return $Path +} + +function Assert-ProtectedArtifactAcl([string] $Path) { + $Acl = Get-Acl -LiteralPath $Path + if (-not $Acl.AreAccessRulesProtected) { + throw "Context dump DACL permits inherited access: $Path" + } + + $Rules = @($Acl.Access) + $Sids = @($Rules | ForEach-Object { + $_.IdentityReference.Translate([System.Security.Principal.SecurityIdentifier]).Value + }) + $ExpectedSids = @("S-1-3-4", "S-1-5-18", "S-1-5-32-544") + foreach ($ExpectedSid in $ExpectedSids) { + if ($Sids -notcontains $ExpectedSid) { + throw "Context dump DACL is missing $ExpectedSid; actual SIDs: $($Sids -join ', ')" + } + } + if (@($Sids | Sort-Object -Unique).Count -ne 3) { + throw "Context dump DACL contains unexpected trustees: $($Sids -join ', ')" + } + foreach ($Rule in $Rules) { + if ($Rule.AccessControlType -ne [System.Security.AccessControl.AccessControlType]::Allow) { + throw "Context dump DACL contains a deny rule: $Rule" + } + if (($Rule.FileSystemRights -band [System.Security.AccessControl.FileSystemRights]::FullControl) -ne + [System.Security.AccessControl.FileSystemRights]::FullControl) { + throw "Context dump DACL does not grant full control to its intended trustee: $Rule" + } + } +} + +Assert-UnauthorizedRequest +Send-DogStatsDContexts +$Online = Wait-ForOnlineReport +if (-not $Online.StartsWith("Wrote ")) { + throw "Online top did not print the generated artifact path:`n$Online" +} + +$DumpPath = Get-DumpPath (Invoke-AdpDogStatsD @("dump-contexts")) +Assert-ProtectedArtifactAcl $DumpPath +Copy-Item -LiteralPath $DumpPath -Destination $CopiedDump -Force +$Offline = Invoke-AdpDogStatsD @("top", "--path", $CopiedDump) +if (-not $Offline.Contains($RequestRow) -or -not $Offline.Contains($QueueRow)) { + throw "Offline report is missing retained contexts:`n$Offline" +} + +Set-Content -LiteralPath $Result -Value "passed" -Encoding ASCII From 60a1f81e04debcbe369ba37354cc7f32bd9af164 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Thu, 23 Jul 2026 15:31:15 -0400 Subject: [PATCH 34/44] fix(airlock): normalize Windows bind source paths Strip the Win32 verbatim namespace produced by canonicalization before passing host paths to Docker. Docker rejects those otherwise valid paths as missing, preventing Windows integration cases from mounting fixture files. --- bin/correctness/airlock/src/driver.rs | 31 ++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/bin/correctness/airlock/src/driver.rs b/bin/correctness/airlock/src/driver.rs index 62ba9b8d3c1..c17890d215e 100644 --- a/bin/correctness/airlock/src/driver.rs +++ b/bin/correctness/airlock/src/driver.rs @@ -232,7 +232,8 @@ impl DriverConfig { HP: AsRef, CP: AsRef, { - let bind_mount = format!("{}:{}", host_path.as_ref().display(), container_path.as_ref().display()); + let host_path = docker_bind_mount_host_path(host_path.as_ref()); + let bind_mount = format!("{}:{}", host_path, container_path.as_ref().display()); self.binds.push(bind_mount); self } @@ -1262,6 +1263,18 @@ fn strip_ansi_codes(input: &[u8]) -> Vec { out } +fn docker_bind_mount_host_path(path: &Path) -> String { + // Windows canonicalization uses the verbatim namespace, which the Docker bind-mount API does not accept. + let path = path.to_string_lossy(); + if let Some(path) = path.strip_prefix(r"\\?\UNC\") { + format!(r"\\{}", path) + } else if let Some(path) = path.strip_prefix(r"\\?\") { + path.to_string() + } else { + path.into_owned() + } +} + fn get_alpine_container_image() -> String { // Normally, we would just use `alpine:latest` and let Docker figure out the registry to pull it from (that is, Docker // Hub) but in CI, we don't have Docker Hub available to us, so we need to use an internal registry. @@ -1283,6 +1296,22 @@ fn get_default_airlock_labels(isolation_group_id: &str) -> HashMap Date: Thu, 23 Jul 2026 15:43:52 -0400 Subject: [PATCH 35/44] Revert "fix(airlock): normalize Windows bind source paths" This reverts commit 60a1f81e04debcbe369ba37354cc7f32bd9af164. --- bin/correctness/airlock/src/driver.rs | 31 +-------------------------- 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/bin/correctness/airlock/src/driver.rs b/bin/correctness/airlock/src/driver.rs index c17890d215e..62ba9b8d3c1 100644 --- a/bin/correctness/airlock/src/driver.rs +++ b/bin/correctness/airlock/src/driver.rs @@ -232,8 +232,7 @@ impl DriverConfig { HP: AsRef, CP: AsRef, { - let host_path = docker_bind_mount_host_path(host_path.as_ref()); - let bind_mount = format!("{}:{}", host_path, container_path.as_ref().display()); + let bind_mount = format!("{}:{}", host_path.as_ref().display(), container_path.as_ref().display()); self.binds.push(bind_mount); self } @@ -1263,18 +1262,6 @@ fn strip_ansi_codes(input: &[u8]) -> Vec { out } -fn docker_bind_mount_host_path(path: &Path) -> String { - // Windows canonicalization uses the verbatim namespace, which the Docker bind-mount API does not accept. - let path = path.to_string_lossy(); - if let Some(path) = path.strip_prefix(r"\\?\UNC\") { - format!(r"\\{}", path) - } else if let Some(path) = path.strip_prefix(r"\\?\") { - path.to_string() - } else { - path.into_owned() - } -} - fn get_alpine_container_image() -> String { // Normally, we would just use `alpine:latest` and let Docker figure out the registry to pull it from (that is, Docker // Hub) but in CI, we don't have Docker Hub available to us, so we need to use an internal registry. @@ -1296,22 +1283,6 @@ fn get_default_airlock_labels(isolation_group_id: &str) -> HashMap Date: Thu, 23 Jul 2026 15:45:04 -0400 Subject: [PATCH 36/44] test(integration): inline Windows DogStatsD top script Follow the existing Windows Panoramic pattern and pass PowerShell through target_exec. The Windows Docker daemon cannot access individual files from the CI runner checkout as bind-mount sources. --- .../cases/dogstatsd-top-windows/config.yaml | 125 +++++++++++++++++- .../run_dogstatsd_top_test.ps1 | 119 ----------------- 2 files changed, 121 insertions(+), 123 deletions(-) delete mode 100644 test/integration/cases/dogstatsd-top-windows/run_dogstatsd_top_test.ps1 diff --git a/test/integration/cases/dogstatsd-top-windows/config.yaml b/test/integration/cases/dogstatsd-top-windows/config.yaml index 2f4b90af96a..9e19587e630 100644 --- a/test/integration/cases/dogstatsd-top-windows/config.yaml +++ b/test/integration/cases/dogstatsd-top-windows/config.yaml @@ -12,8 +12,6 @@ env: DD_DATA_PLANE_DOGSTATSD_ENABLED: "true" container: - files: - - "run_dogstatsd_top_test.ps1:C:\\adp\\run_dogstatsd_top_test.ps1" exposed_ports: - "58125/udp" - "55101/tcp" @@ -40,8 +38,127 @@ procedure: - "pwsh" - "-NoProfile" - "-NonInteractive" - - "-File" - - "C:\\adp\\run_dogstatsd_top_test.ps1" + - "-Command" + - | + $ErrorActionPreference = "Stop" + Set-StrictMode -Version 3.0 + + $Adp = "C:\adp\agent-data-plane.exe" + $Config = "C:\ProgramData\Datadog\datadog.yaml" + $ApiUrl = "https://127.0.0.1:55101/agent/dogstatsd-contexts-dump" + $Result = "C:\dogstatsd-top-result" + $CopiedDump = "C:\dogstatsd-top-copied.json.zstd" + $RequestRow = " 2`tintegration.windows.top.requests`t(2 instance, 1 env)" + $QueueRow = " 1`tintegration.windows.top.queue`t(1 queue)" + + function Invoke-AdpDogStatsD([string[]] $Arguments) { + $Output = @(& $Adp --config $Config dogstatsd @Arguments 2>&1) + if ($LASTEXITCODE -ne 0) { + throw "ADP command failed ($LASTEXITCODE): $($Arguments -join ' ')`n$($Output -join "`n")" + } + return $Output -join "`n" + } + + function Assert-UnauthorizedRequest { + try { + Invoke-WebRequest -Uri $ApiUrl -Method Post -SkipCertificateCheck -TimeoutSec 10 | Out-Null + throw "Context dump API accepted an unauthenticated request." + } catch { + $Response = $_.Exception.Response + if ($null -eq $Response -or [int] $Response.StatusCode -ne 401) { + throw + } + } + } + + function Send-DogStatsDContexts { + $Payload = @( + "integration.windows.top.requests:1|c|#host:windows-a,env:prod,instance:a", + "integration.windows.top.requests:1|c|#host:windows-b,env:prod,instance:b", + "integration.windows.top.queue:1|c|#queue:alpha" + ) -join "`n" + $Bytes = [System.Text.Encoding]::UTF8.GetBytes($Payload) + $Client = [System.Net.Sockets.UdpClient]::new() + try { + 1..3 | ForEach-Object { [void] $Client.Send($Bytes, $Bytes.Length, "127.0.0.1", 58125) } + } finally { + $Client.Dispose() + } + } + + function Wait-ForOnlineReport { + $LastOutput = "" + foreach ($Attempt in 1..40) { + $LastOutput = Invoke-AdpDogStatsD @("top") + if ($LastOutput.Contains($RequestRow) -and $LastOutput.Contains($QueueRow)) { + return $LastOutput + } + Start-Sleep -Milliseconds 500 + } + throw "Retained contexts did not appear in online output:`n$LastOutput" + } + + function Get-DumpPath([string] $Output) { + $Lines = @($Output -split "`r?`n" | Where-Object { $_ -ne "" }) + if ($Lines.Count -ne 1 -or -not $Lines[0].StartsWith("Wrote ")) { + throw "Unexpected dump-contexts output: $Output" + } + $Path = $Lines[0].Substring("Wrote ".Length) + if (-not [System.IO.Path]::IsPathRooted($Path) -or [System.IO.Path]::GetFileName($Path) -ne "dogstatsd_contexts.json.zstd") { + throw "Unexpected context dump path: $Path" + } + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Context dump was not created: $Path" + } + return $Path + } + + function Assert-ProtectedArtifactAcl([string] $Path) { + $Acl = Get-Acl -LiteralPath $Path + if (-not $Acl.AreAccessRulesProtected) { + throw "Context dump DACL permits inherited access: $Path" + } + + $Rules = @($Acl.Access) + $Sids = @($Rules | ForEach-Object { + $_.IdentityReference.Translate([System.Security.Principal.SecurityIdentifier]).Value + }) + $ExpectedSids = @("S-1-3-4", "S-1-5-18", "S-1-5-32-544") + foreach ($ExpectedSid in $ExpectedSids) { + if ($Sids -notcontains $ExpectedSid) { + throw "Context dump DACL is missing $ExpectedSid; actual SIDs: $($Sids -join ', ')" + } + } + if (@($Sids | Sort-Object -Unique).Count -ne 3) { + throw "Context dump DACL contains unexpected trustees: $($Sids -join ', ')" + } + foreach ($Rule in $Rules) { + if ($Rule.AccessControlType -ne [System.Security.AccessControl.AccessControlType]::Allow) { + throw "Context dump DACL contains a deny rule: $Rule" + } + if (($Rule.FileSystemRights -band [System.Security.AccessControl.FileSystemRights]::FullControl) -ne + [System.Security.AccessControl.FileSystemRights]::FullControl) { + throw "Context dump DACL does not grant full control to its intended trustee: $Rule" + } + } + } + + Assert-UnauthorizedRequest + Send-DogStatsDContexts + $Online = Wait-ForOnlineReport + if (-not $Online.StartsWith("Wrote ")) { + throw "Online top did not print the generated artifact path:`n$Online" + } + + $DumpPath = Get-DumpPath (Invoke-AdpDogStatsD @("dump-contexts")) + Assert-ProtectedArtifactAcl $DumpPath + Copy-Item -LiteralPath $DumpPath -Destination $CopiedDump -Force + $Offline = Invoke-AdpDogStatsD @("top", "--path", $CopiedDump) + if (-not $Offline.Contains($RequestRow) -or -not $Offline.Contains($QueueRow)) { + throw "Offline report is missing retained contexts:`n$Offline" + } + + Set-Content -LiteralPath $Result -Value "passed" -Encoding ASCII timeout: 90s - assertion: file_contains path: "C:\\dogstatsd-top-result" diff --git a/test/integration/cases/dogstatsd-top-windows/run_dogstatsd_top_test.ps1 b/test/integration/cases/dogstatsd-top-windows/run_dogstatsd_top_test.ps1 deleted file mode 100644 index 501e915e4e5..00000000000 --- a/test/integration/cases/dogstatsd-top-windows/run_dogstatsd_top_test.ps1 +++ /dev/null @@ -1,119 +0,0 @@ -$ErrorActionPreference = "Stop" -Set-StrictMode -Version 3.0 - -$Adp = "C:\adp\agent-data-plane.exe" -$Config = "C:\ProgramData\Datadog\datadog.yaml" -$ApiUrl = "https://127.0.0.1:55101/agent/dogstatsd-contexts-dump" -$Result = "C:\dogstatsd-top-result" -$CopiedDump = "C:\dogstatsd-top-copied.json.zstd" -$RequestRow = " 2`tintegration.windows.top.requests`t(2 instance, 1 env)" -$QueueRow = " 1`tintegration.windows.top.queue`t(1 queue)" - -function Invoke-AdpDogStatsD([string[]] $Arguments) { - $Output = @(& $Adp --config $Config dogstatsd @Arguments 2>&1) - if ($LASTEXITCODE -ne 0) { - throw "ADP command failed ($LASTEXITCODE): $($Arguments -join ' ')`n$($Output -join "`n")" - } - return $Output -join "`n" -} - -function Assert-UnauthorizedRequest { - try { - Invoke-WebRequest -Uri $ApiUrl -Method Post -SkipCertificateCheck -TimeoutSec 10 | Out-Null - throw "Context dump API accepted an unauthenticated request." - } catch { - $Response = $_.Exception.Response - if ($null -eq $Response -or [int] $Response.StatusCode -ne 401) { - throw - } - } -} - -function Send-DogStatsDContexts { - $Payload = @( - "integration.windows.top.requests:1|c|#host:windows-a,env:prod,instance:a", - "integration.windows.top.requests:1|c|#host:windows-b,env:prod,instance:b", - "integration.windows.top.queue:1|c|#queue:alpha" - ) -join "`n" - $Bytes = [System.Text.Encoding]::UTF8.GetBytes($Payload) - $Client = [System.Net.Sockets.UdpClient]::new() - try { - 1..3 | ForEach-Object { [void] $Client.Send($Bytes, $Bytes.Length, "127.0.0.1", 58125) } - } finally { - $Client.Dispose() - } -} - -function Wait-ForOnlineReport { - $LastOutput = "" - foreach ($Attempt in 1..40) { - $LastOutput = Invoke-AdpDogStatsD @("top") - if ($LastOutput.Contains($RequestRow) -and $LastOutput.Contains($QueueRow)) { - return $LastOutput - } - Start-Sleep -Milliseconds 500 - } - throw "Retained contexts did not appear in online output:`n$LastOutput" -} - -function Get-DumpPath([string] $Output) { - $Lines = @($Output -split "`r?`n" | Where-Object { $_ -ne "" }) - if ($Lines.Count -ne 1 -or -not $Lines[0].StartsWith("Wrote ")) { - throw "Unexpected dump-contexts output: $Output" - } - $Path = $Lines[0].Substring("Wrote ".Length) - if (-not [System.IO.Path]::IsPathRooted($Path) -or [System.IO.Path]::GetFileName($Path) -ne "dogstatsd_contexts.json.zstd") { - throw "Unexpected context dump path: $Path" - } - if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { - throw "Context dump was not created: $Path" - } - return $Path -} - -function Assert-ProtectedArtifactAcl([string] $Path) { - $Acl = Get-Acl -LiteralPath $Path - if (-not $Acl.AreAccessRulesProtected) { - throw "Context dump DACL permits inherited access: $Path" - } - - $Rules = @($Acl.Access) - $Sids = @($Rules | ForEach-Object { - $_.IdentityReference.Translate([System.Security.Principal.SecurityIdentifier]).Value - }) - $ExpectedSids = @("S-1-3-4", "S-1-5-18", "S-1-5-32-544") - foreach ($ExpectedSid in $ExpectedSids) { - if ($Sids -notcontains $ExpectedSid) { - throw "Context dump DACL is missing $ExpectedSid; actual SIDs: $($Sids -join ', ')" - } - } - if (@($Sids | Sort-Object -Unique).Count -ne 3) { - throw "Context dump DACL contains unexpected trustees: $($Sids -join ', ')" - } - foreach ($Rule in $Rules) { - if ($Rule.AccessControlType -ne [System.Security.AccessControl.AccessControlType]::Allow) { - throw "Context dump DACL contains a deny rule: $Rule" - } - if (($Rule.FileSystemRights -band [System.Security.AccessControl.FileSystemRights]::FullControl) -ne - [System.Security.AccessControl.FileSystemRights]::FullControl) { - throw "Context dump DACL does not grant full control to its intended trustee: $Rule" - } - } -} - -Assert-UnauthorizedRequest -Send-DogStatsDContexts -$Online = Wait-ForOnlineReport -if (-not $Online.StartsWith("Wrote ")) { - throw "Online top did not print the generated artifact path:`n$Online" -} - -$DumpPath = Get-DumpPath (Invoke-AdpDogStatsD @("dump-contexts")) -Assert-ProtectedArtifactAcl $DumpPath -Copy-Item -LiteralPath $DumpPath -Destination $CopiedDump -Force -$Offline = Invoke-AdpDogStatsD @("top", "--path", $CopiedDump) -if (-not $Offline.Contains($RequestRow) -or -not $Offline.Contains($QueueRow)) { - throw "Offline report is missing retained contexts:`n$Offline" -} - -Set-Content -LiteralPath $Result -Value "passed" -Encoding ASCII From 825785daee858dcfecb285eca7f4b1f3a60ce0f1 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Fri, 24 Jul 2026 13:23:36 -0400 Subject: [PATCH 37/44] refactor(adp): simplify context dump publication Use NamedTempFile for same-directory creation, synchronization, cleanup, and atomic persistence. Windows artifacts now inherit the configured run_path ACL like the Datadog Agent, removing ADP-specific DACL and filesystem FFI machinery. --- Cargo.lock | 1 - bin/agent-data-plane/Cargo.toml | 8 - .../src/dogstatsd_contexts/artifact.rs | 384 ++---------- .../src/dogstatsd_contexts/artifact_tests.rs | 588 +----------------- docs/agent-data-plane/dogstatsd-top.md | 2 +- .../cases/dogstatsd-top-windows/config.yaml | 31 - 6 files changed, 41 insertions(+), 973 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2dda9e781a..1b8e24aef82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,7 +78,6 @@ dependencies = [ "tower", "tracing", "uuid", - "windows-sys 0.61.2", "zstd", ] diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index 430dba48070..aff6f617dd1 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -81,14 +81,6 @@ tikv-jemallocator = { workspace = true, features = [ "stats", ] } -[target.'cfg(target_os = "windows")'.dependencies] -windows-sys = { workspace = true, features = [ - "Win32_Foundation", - "Win32_Security", - "Win32_Security_Authorization", - "Win32_Storage_FileSystem", -] } - [build-dependencies] chrono = { workspace = true } diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs index eb7bb699ee2..9fa33571aec 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact.rs @@ -1,7 +1,11 @@ use std::error::Error; use std::fmt; -use std::fs::{self, File}; -use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; +#[cfg(unix)] +use std::fs; +use std::fs::File; +#[cfg(unix)] +use std::io; +use std::io::{BufRead, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; @@ -16,9 +20,6 @@ const ZSTD_MAGIC: &[u8; 4] = b"\x28\xb5\x2f\xfd"; pub(crate) const CONTEXT_DUMP_FILENAME: &str = "dogstatsd_contexts.json.zstd"; // `MetricSourceDogstatsd` is the second value after `MetricSourceUnknown` in the Agent's metricsource.go enum. const AGENT_METRIC_SOURCE_DOGSTATSD: u32 = 1; -#[cfg(windows)] -// A protected DACL granting full control through Owner Rights, LocalSystem, and Builtin Administrators only. -const WINDOWS_CONTEXT_DUMP_SDDL: &str = "D:P(A;;FA;;;OW)(A;;FA;;;SY)(A;;FA;;;BA)"; struct AgentContextSnapshotRecord<'a>(&'a AggregateContextSnapshotEntry); @@ -91,32 +92,6 @@ fn write_snapshot_records( pub(crate) fn publish_context_dump( run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], ) -> Result { - publish_context_dump_with(run_path, snapshot, &FileSystemAtomicReplace) -} - -fn publish_context_dump_with( - run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, -) -> Result { - publish_context_dump_with_services( - run_path, - snapshot, - replacer, - &SecureTemporaryFileFactory, - &OwnerOnlyPermissions, - &FileSystemCleanup, - ) -} - -fn publish_context_dump_with_services( - run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], replacer: &R, temporary_files: &F, _permissions: &P, - cleanup: &C, -) -> Result -where - R: AtomicReplace, - F: TemporaryFileFactory, - P: PermissionNormalizer, - C: TemporaryFileCleanup, -{ let target = run_path.join(CONTEXT_DUMP_FILENAME); if run_path.as_os_str().is_empty() { return Err(saluki_error::generic_error!( @@ -125,62 +100,47 @@ where )); } - let (file, temporary_path) = temporary_files.create(run_path).with_error_context(|| { + let prefix = format!(".{CONTEXT_DUMP_FILENAME}."); + let mut temporary = TempFileBuilder::new() + .prefix(&prefix) + .suffix(".tmp") + .tempfile_in(run_path) + .with_error_context(|| { + format!( + "Failed to create temporary DogStatsD context dump for target '{}'.", + target.display() + ) + })?; + + #[cfg(unix)] + set_owner_only_permissions(temporary.as_file()).with_error_context(|| { format!( - "Failed to create temporary DogStatsD context dump for target '{}'.", + "Failed to set owner-only permissions on temporary DogStatsD context dump '{}' for target '{}'.", + temporary.path().display(), target.display() ) })?; - run_with_temporary_cleanup(&temporary_path, &target, cleanup, || { - #[cfg(unix)] - _permissions.normalize(&file).with_error_context(|| { + let file = write_compressed_snapshot(BufWriter::new(temporary.as_file_mut()), snapshot, &target)?; + file.sync_all().with_error_context(|| { + format!( + "Failed to sync temporary DogStatsD context dump for target '{}'.", + target.display() + ) + })?; + + temporary + .persist(&target) + .map_err(|error| error.error) + .with_error_context(|| { format!( - "Failed to set owner-only permissions on temporary DogStatsD context dump '{}' for target '{}'.", - temporary_path.display(), + "Failed to atomically replace DogStatsD context dump target '{}'.", target.display() ) })?; - publish_buffered_temporary_unmanaged(BufWriter::new(file), &temporary_path, &target, snapshot, replacer) - })?; Ok(target) } -trait TemporaryFileFactory { - fn create(&self, run_path: &Path) -> io::Result<(File, PathBuf)>; -} - -struct SecureTemporaryFileFactory; - -impl TemporaryFileFactory for SecureTemporaryFileFactory { - fn create(&self, run_path: &Path) -> io::Result<(File, PathBuf)> { - let prefix = format!(".{CONTEXT_DUMP_FILENAME}."); - let mut builder = TempFileBuilder::new(); - builder.prefix(&prefix).suffix(".tmp"); - - #[cfg(windows)] - let temporary = builder.make_in(run_path, open_temporary_file)?; - #[cfg(not(windows))] - let temporary = builder.tempfile_in(run_path)?; - - temporary.keep().map_err(|error| error.error) - } -} - -trait PermissionNormalizer { - #[cfg(unix)] - fn normalize(&self, file: &File) -> io::Result<()>; -} - -struct OwnerOnlyPermissions; - -impl PermissionNormalizer for OwnerOnlyPermissions { - #[cfg(unix)] - fn normalize(&self, file: &File) -> io::Result<()> { - set_owner_only_permissions(file) - } -} - #[cfg(unix)] fn set_owner_only_permissions(file: &File) -> io::Result<()> { use std::os::unix::fs::PermissionsExt as _; @@ -188,168 +148,6 @@ fn set_owner_only_permissions(file: &File) -> io::Result<()> { file.set_permissions(fs::Permissions::from_mode(0o600)) } -#[cfg(windows)] -fn open_temporary_file(path: &Path) -> io::Result { - use std::os::windows::{ - ffi::OsStrExt as _, - io::{FromRawHandle as _, OwnedHandle}, - }; - use std::ptr; - - use windows_sys::Win32::{ - Foundation::{GENERIC_WRITE, INVALID_HANDLE_VALUE}, - Storage::FileSystem::{ - CreateFileW, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, - }, - }; - - let mut wide_path: Vec<_> = path.as_os_str().encode_wide().collect(); - if wide_path.contains(&0) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "path contains an interior NUL", - )); - } - wide_path.push(0); - - let security_attributes = WindowsSecurityAttributes::from_sddl(WINDOWS_CONTEXT_DUMP_SDDL)?; - // SAFETY: The path is a live NUL-terminated UTF-16 buffer, the security attributes refer to a live self-relative - // descriptor, and the null template handle is valid for a new file. A successful call transfers one owned handle. - let handle = unsafe { - CreateFileW( - wide_path.as_ptr(), - GENERIC_WRITE, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - security_attributes.as_ptr(), - CREATE_NEW, - FILE_ATTRIBUTE_NORMAL, - ptr::null_mut(), - ) - }; - if handle == INVALID_HANDLE_VALUE { - return Err(io::Error::last_os_error()); - } - - // SAFETY: `CreateFileW` returned a valid handle whose ownership has not been transferred elsewhere. - let handle = unsafe { OwnedHandle::from_raw_handle(handle) }; - Ok(File::from(handle)) -} - -#[cfg(windows)] -struct WindowsSecurityAttributes { - descriptor: *mut std::ffi::c_void, - attributes: windows_sys::Win32::Security::SECURITY_ATTRIBUTES, -} - -#[cfg(windows)] -impl WindowsSecurityAttributes { - fn from_sddl(sddl: &str) -> io::Result { - use std::ptr; - - use windows_sys::Win32::{ - Foundation::FALSE, - Security::{ - Authorization::{ConvertStringSecurityDescriptorToSecurityDescriptorW, SDDL_REVISION_1}, - SECURITY_ATTRIBUTES, - }, - }; - - let wide_sddl: Vec<_> = sddl.encode_utf16().chain(std::iter::once(0)).collect(); - let mut descriptor = ptr::null_mut(); - // SAFETY: The SDDL is a live NUL-terminated UTF-16 buffer, `descriptor` is a valid output pointer, and the - // optional descriptor-size output is null. A successful call transfers ownership of the descriptor. - let result = unsafe { - ConvertStringSecurityDescriptorToSecurityDescriptorW( - wide_sddl.as_ptr(), - SDDL_REVISION_1, - &mut descriptor, - ptr::null_mut(), - ) - }; - if result == FALSE { - return Err(io::Error::last_os_error()); - } - - Ok(Self { - descriptor, - attributes: SECURITY_ATTRIBUTES { - nLength: std::mem::size_of::() as u32, - lpSecurityDescriptor: descriptor, - bInheritHandle: FALSE, - }, - }) - } - - fn as_ptr(&self) -> *const windows_sys::Win32::Security::SECURITY_ATTRIBUTES { - &self.attributes - } -} - -#[cfg(windows)] -impl Drop for WindowsSecurityAttributes { - fn drop(&mut self) { - if !self.descriptor.is_null() { - use windows_sys::Win32::Foundation::{LocalFree, HLOCAL}; - - // SAFETY: The descriptor was allocated by the SDDL conversion API and remains owned by this value. - unsafe { - let _ = LocalFree(self.descriptor as HLOCAL); - } - } - } -} - -fn publish_buffered_temporary_unmanaged( - buffer: BufWriter, temporary_path: &Path, target: &Path, snapshot: &[AggregateContextSnapshotEntry], - replacer: &R, -) -> Result<(), GenericError> -where - W: Write + SyncAll, - R: AtomicReplace, -{ - let writer = write_compressed_snapshot(buffer, snapshot, target)?; - writer.sync_all().with_error_context(|| { - format!( - "Failed to sync temporary DogStatsD context dump for target '{}'.", - target.display() - ) - })?; - drop(writer); - - replacer.replace(temporary_path, target).with_error_context(|| { - format!( - "Failed to atomically replace DogStatsD context dump target '{}'.", - target.display() - ) - }) -} - -fn run_with_temporary_cleanup( - temporary_path: &Path, target: &Path, cleanup: &C, operation: impl FnOnce() -> Result, -) -> Result -where - C: TemporaryFileCleanup + ?Sized, -{ - let mut guard = TemporaryPathCleanup::new(temporary_path, cleanup); - match operation() { - Ok(value) => { - guard.disarm(); - Ok(value) - } - Err(primary_error) => match guard.cleanup() { - Ok(()) => Err(primary_error), - Err(cleanup_error) => Err(saluki_error::generic_error!( - "Failed to publish DogStatsD context dump target '{}': {:#}. Additionally, failed to remove temporary \ - DogStatsD context dump '{}': {}", - target.display(), - primary_error, - temporary_path.display(), - cleanup_error - )), - }, - } -} - fn write_compressed_snapshot( buffer: BufWriter, snapshot: &[AggregateContextSnapshotEntry], target: &Path, ) -> Result { @@ -387,118 +185,6 @@ fn finish_buffered_writer(buffer: BufWriter, target: &Path) -> Resu }) } -trait SyncAll { - fn sync_all(&self) -> io::Result<()>; -} - -impl SyncAll for File { - fn sync_all(&self) -> io::Result<()> { - File::sync_all(self) - } -} - -trait AtomicReplace { - fn replace(&self, source: &Path, target: &Path) -> io::Result<()>; -} - -struct FileSystemAtomicReplace; - -impl AtomicReplace for FileSystemAtomicReplace { - fn replace(&self, source: &Path, target: &Path) -> io::Result<()> { - #[cfg(unix)] - { - fs::rename(source, target) - } - - #[cfg(windows)] - { - move_file_replace(source, target) - } - } -} - -#[cfg(windows)] -fn move_file_replace(source: &Path, target: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt as _; - - use windows_sys::Win32::Storage::FileSystem::{MoveFileExW, MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH}; - - fn nul_terminated(path: &Path) -> io::Result> { - let mut encoded: Vec<_> = path.as_os_str().encode_wide().collect(); - if encoded.contains(&0) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "path contains an interior NUL", - )); - } - encoded.push(0); - Ok(encoded) - } - - let source = nul_terminated(source)?; - let target = nul_terminated(target)?; - // SAFETY: Both path buffers are NUL-terminated, contain no interior NULs, and remain alive for the duration of the - // call. The flags request same-filesystem replacement with write-through semantics. - let result = unsafe { - MoveFileExW( - source.as_ptr(), - target.as_ptr(), - MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, - ) - }; - if result == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } -} - -trait TemporaryFileCleanup { - fn remove(&self, path: &Path) -> io::Result<()>; -} - -struct FileSystemCleanup; - -impl TemporaryFileCleanup for FileSystemCleanup { - fn remove(&self, path: &Path) -> io::Result<()> { - fs::remove_file(path) - } -} - -struct TemporaryPathCleanup<'a, C: TemporaryFileCleanup + ?Sized> { - path: &'a Path, - cleanup: &'a C, - armed: bool, -} - -impl<'a, C: TemporaryFileCleanup + ?Sized> TemporaryPathCleanup<'a, C> { - fn new(path: &'a Path, cleanup: &'a C) -> Self { - Self { - path, - cleanup, - armed: true, - } - } - - fn cleanup(&mut self) -> io::Result<()> { - self.cleanup.remove(self.path)?; - self.disarm(); - Ok(()) - } - - fn disarm(&mut self) { - self.armed = false; - } -} - -impl Drop for TemporaryPathCleanup<'_, C> { - fn drop(&mut self) { - if self.armed { - let _ = self.cleanup.remove(self.path); - } - } -} - #[derive(Debug, Deserialize, Eq, PartialEq)] #[serde(rename_all = "PascalCase")] pub(crate) struct AgentContextRecord { diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/artifact_tests.rs b/bin/agent-data-plane/src/dogstatsd_contexts/artifact_tests.rs index 869eb493183..c6f09d5cf07 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/artifact_tests.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/artifact_tests.rs @@ -1,33 +1,19 @@ use std::fs; -#[cfg(not(windows))] -use std::fs::OpenOptions; -use std::io::{self, BufWriter, Write}; +use std::io::{self, Write}; use std::path::PathBuf; -use std::sync::{ - atomic::{AtomicBool, AtomicUsize, Ordering}, - Arc, -}; use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; use saluki_context::{ tags::{Tag, TagSet}, Context, }; -use saluki_error::GenericError; use serde_json::json; use stringtheory::MetaString; use super::{ - finish_buffered_writer, finish_zstd_encoder, for_each_record, publish_buffered_temporary_unmanaged, - publish_context_dump, publish_context_dump_with, publish_context_dump_with_services, run_with_temporary_cleanup, - write_snapshot_records, AgentContextRecord, ArtifactError, AtomicReplace, FileSystemAtomicReplace, - FileSystemCleanup, OwnerOnlyPermissions, SyncAll, TemporaryFileCleanup, TemporaryFileFactory, TemporaryPathCleanup, + for_each_record, publish_context_dump, write_snapshot_records, AgentContextRecord, ArtifactError, CONTEXT_DUMP_FILENAME, }; -#[cfg(windows)] -use super::{open_temporary_file, SecureTemporaryFileFactory}; -#[cfg(unix)] -use super::{set_owner_only_permissions, PermissionNormalizer}; use crate::dogstatsd_contexts::read_report; impl ArtifactError { @@ -50,32 +36,6 @@ fn collect_records(path: &std::path::Path) -> Result, Ar Ok(records) } -fn publish_open_temporary( - writer: W, temporary_path: &std::path::Path, target: &std::path::Path, snapshot: &[AggregateContextSnapshotEntry], - replacer: &R, -) -> Result<(), GenericError> -where - W: Write + SyncAll, - R: AtomicReplace, -{ - run_with_temporary_cleanup(temporary_path, target, &FileSystemCleanup, || { - publish_buffered_temporary_unmanaged(BufWriter::new(writer), temporary_path, target, snapshot, replacer) - }) -} - -fn publish_buffered_temporary( - buffer: BufWriter, temporary_path: &std::path::Path, target: &std::path::Path, - snapshot: &[AggregateContextSnapshotEntry], replacer: &R, -) -> Result<(), GenericError> -where - W: Write + SyncAll, - R: AtomicReplace, -{ - run_with_temporary_cleanup(temporary_path, target, &FileSystemCleanup, || { - publish_buffered_temporary_unmanaged(buffer, temporary_path, target, snapshot, replacer) - }) -} - const PLAIN_FIXTURE_BYTES: &[u8] = include_bytes!(concat!( env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/dogstatsd_contexts_agent.ndjson" @@ -93,102 +53,6 @@ const COMPRESSED_FIXTURE_PATH: &str = concat!( "/tests/fixtures/dogstatsd_contexts_agent.ndjson.zstd" ); -#[cfg(windows)] -struct LocalAllocation(*mut std::ffi::c_void); - -#[cfg(windows)] -impl Drop for LocalAllocation { - fn drop(&mut self) { - if !self.0.is_null() { - use windows_sys::Win32::Foundation::{LocalFree, HLOCAL}; - - // SAFETY: This pointer was allocated by a Windows security API that transfers ownership to the caller. - unsafe { - let _ = LocalFree(self.0 as HLOCAL); - } - } - } -} - -#[cfg(windows)] -fn windows_file_dacl_sddl(path: &std::path::Path) -> io::Result<(u16, String)> { - use std::os::windows::ffi::OsStrExt as _; - use std::{ptr, slice}; - - use windows_sys::Win32::{ - Foundation::ERROR_SUCCESS, - Security::{ - Authorization::{ - ConvertSecurityDescriptorToStringSecurityDescriptorW, GetNamedSecurityInfoW, SDDL_REVISION_1, - SE_FILE_OBJECT, - }, - GetSecurityDescriptorControl, DACL_SECURITY_INFORMATION, - }, - }; - - let mut wide_path: Vec<_> = path.as_os_str().encode_wide().collect(); - if wide_path.contains(&0) { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "path contains an interior NUL", - )); - } - wide_path.push(0); - - let mut descriptor = ptr::null_mut(); - // SAFETY: The path is NUL-terminated and remains alive during the call. All unused output pointers are null, - // and `descriptor` receives an allocation owned by the caller on success. - let result = unsafe { - GetNamedSecurityInfoW( - wide_path.as_ptr(), - SE_FILE_OBJECT, - DACL_SECURITY_INFORMATION, - ptr::null_mut(), - ptr::null_mut(), - ptr::null_mut(), - ptr::null_mut(), - &mut descriptor, - ) - }; - if result != ERROR_SUCCESS { - return Err(io::Error::from_raw_os_error(result as i32)); - } - let descriptor = LocalAllocation(descriptor); - - let mut control = 0; - let mut revision = 0; - // SAFETY: `descriptor` is a valid security descriptor allocation and both output pointers are valid writes. - let result = unsafe { GetSecurityDescriptorControl(descriptor.0, &mut control, &mut revision) }; - if result == 0 { - return Err(io::Error::last_os_error()); - } - - let mut sddl = ptr::null_mut(); - let mut sddl_len = 0; - // SAFETY: `descriptor` remains valid during the call, and both output pointers are valid writes. The returned - // string allocation is owned by the caller on success. - let result = unsafe { - ConvertSecurityDescriptorToStringSecurityDescriptorW( - descriptor.0, - SDDL_REVISION_1, - DACL_SECURITY_INFORMATION, - &mut sddl, - &mut sddl_len, - ) - }; - if result == 0 { - return Err(io::Error::last_os_error()); - } - let sddl_allocation = LocalAllocation(sddl.cast()); - // SAFETY: The conversion API returned `sddl_len` initialized UTF-16 code units in the live `sddl` allocation. - let wide_sddl = unsafe { slice::from_raw_parts(sddl, sddl_len as usize) }; - let wide_sddl = wide_sddl.strip_suffix(&[0]).unwrap_or(wide_sddl); - let sddl = String::from_utf16(wide_sddl).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; - drop(sddl_allocation); - - Ok((control, sddl)) -} - // These fixtures and expectations mirror ContextDebugRepr and the `dogstatsd top` command at Datadog Agent // commit 9de2a8371cf8d95794f238939709491907893288. fn expected_fixture_records() -> Vec { @@ -429,159 +293,6 @@ fn publishes_a_decodable_mode_0600_dump_and_replaces_the_canonical_artifact() { } } -#[cfg(windows)] -#[test] -fn temporary_dump_is_created_with_a_protected_restrictive_dacl() { - use windows_sys::Win32::Security::SE_DACL_PROTECTED; - - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let (_file, staging_path) = SecureTemporaryFileFactory - .create(run_directory.path()) - .expect("temporary dump should be created"); - - let (control, sddl) = windows_file_dacl_sddl(&staging_path).expect("temporary dump DACL should be readable"); - - assert_ne!(control & SE_DACL_PROTECTED, 0, "DACL must be protected: {sddl}"); - assert!(sddl.contains("D:P"), "DACL SDDL must carry the protected flag: {sddl}"); - for required_ace in ["(A;;FA;;;OW)", "(A;;FA;;;SY)", "(A;;FA;;;BA)"] { - assert!(sddl.contains(required_ace), "DACL is missing {required_ace}: {sddl}"); - } - assert_eq!(sddl.matches('(').count(), 3, "DACL contains an unexpected ACE: {sddl}"); - assert!(!sddl.contains(";;;WD)"), "DACL must not grant Everyone access: {sddl}"); - assert!( - !sddl.contains(";;;BU)"), - "DACL must not grant Builtin Users access: {sddl}" - ); -} - -#[cfg(unix)] -#[test] -fn publication_normalizes_restrictive_temporary_permissions_to_exactly_0600() { - use std::os::unix::fs::PermissionsExt as _; - - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let normalizer = RestrictiveThenOwnerOnly::default(); - - let target = publish_context_dump_with_services( - run_directory.path(), - &[], - &FileSystemAtomicReplace, - &FixedTemporaryFileFactory, - &normalizer, - &FileSystemCleanup, - ) - .expect("snapshot should publish"); - - assert!(normalizer.called.load(Ordering::Relaxed)); - assert_eq!(fs::metadata(target).unwrap().permissions().mode() & 0o777, 0o600); -} - -#[test] -fn temporary_file_creation_failure_is_contextual_and_preserves_the_canonical_artifact() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - - let error = publish_context_dump_with_services( - run_directory.path(), - &[], - &FileSystemAtomicReplace, - &FailingTemporaryFileFactory, - &OwnerOnlyPermissions, - &FileSystemCleanup, - ) - .expect_err("temporary file creation failure should surface"); - - let message = format!("{error:#}"); - assert!(message.contains("create temporary"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!( - message.contains("injected temporary file creation failure"), - "{message}" - ); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); -} - -#[cfg(unix)] -#[test] -fn permission_failure_closes_and_removes_the_new_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - - let error = publish_context_dump_with_services( - run_directory.path(), - &[], - &FileSystemAtomicReplace, - &FixedTemporaryFileFactory, - &FailingPermissions, - &FileSystemCleanup, - ) - .expect_err("permission failure should surface"); - - let message = format!("{error:#}"); - assert!(message.contains("set owner-only permissions"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected permission failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); -} - -#[test] -fn cleanup_method_surfaces_removal_failure_and_remains_armed() { - let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); - let temporary_path = temporary_directory.path().join("staged-contexts.tmp"); - fs::write(&temporary_path, b"staged").unwrap(); - let cleanup = FailingCleanup::default(); - let mut guard = TemporaryPathCleanup::new(&temporary_path, &cleanup); - - let error = guard.cleanup().expect_err("cleanup failure should surface"); - - assert!(error.to_string().contains("injected cleanup failure")); - assert!(temporary_path.exists()); - assert_eq!(cleanup.attempts.load(Ordering::Relaxed), 1); -} - -#[test] -fn reports_primary_and_cleanup_failures_without_replacing_the_canonical_artifact() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let cleanup = FailingCleanup::default(); - - let error = publish_context_dump_with_services( - run_directory.path(), - &[], - &FailingReplace, - &FixedTemporaryFileFactory, - &OwnerOnlyPermissions, - &cleanup, - ) - .expect_err("replacement and cleanup should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("atomically replace"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("remove temporary DogStatsD context dump"), "{message}"); - assert!(message.contains("injected cleanup failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!(directory_entries(run_directory.path()).len(), 2); - assert!(directory_entries(run_directory.path()) - .iter() - .any(|entry| entry.starts_with(&format!(".{CONTEXT_DUMP_FILENAME}.")) && entry.ends_with(".tmp"))); - assert!(cleanup.attempts.load(Ordering::Relaxed) >= 1); -} - #[test] fn rejects_empty_missing_and_non_directory_run_paths_with_target_and_operation_context() { let temporary_directory = tempfile::tempdir().expect("temporary directory should be created"); @@ -603,311 +314,22 @@ fn rejects_empty_missing_and_non_directory_run_paths_with_target_and_operation_c } #[test] -fn injected_replace_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { +fn persist_failure_is_contextual_and_removes_the_temporary_file() { let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let snapshot = [snapshot_entry( - "replacement.failure", - None, - AggregateMetricType::Gauge, - "", - &[], - &[], - )]; + fs::create_dir(&target).expect("directory should block file persistence"); - let error = publish_context_dump_with(run_directory.path(), &snapshot, &FailingReplace) - .expect_err("replacement should fail"); + let error = publish_context_dump(run_directory.path(), &[]).expect_err("persistence should fail"); let message = format!("{error:#}"); assert!(message.contains("atomically replace"), "{message}"); assert!(message.contains(&target.display().to_string()), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); assert_eq!( directory_entries(run_directory.path()), vec![CONTEXT_DUMP_FILENAME.to_owned()] ); } -#[test] -fn zstd_finalization_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let temporary = run_directory.path().join(".injected-zstd-failure.tmp"); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let file = fs::File::create(&temporary).expect("temporary fixture should be created"); - let writer = InjectableFile { - file, - fail_writes: true, - fail_sync: false, - }; - let buffer = BufWriter::with_capacity(0, writer); - let snapshot = [snapshot_entry( - "zstd.failure", - None, - AggregateMetricType::Gauge, - "", - &[], - &[], - )]; - - let error = publish_buffered_temporary(buffer, &temporary, &target, &snapshot, &FailingReplace) - .expect_err("zstd finalization should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("finalize zstd stream"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected write failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); -} - -#[test] -fn buffered_finalization_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let temporary = run_directory.path().join(".injected-buffer-failure.tmp"); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let file = fs::File::create(&temporary).expect("temporary fixture should be created"); - let writer = InjectableFile { - file, - fail_writes: true, - fail_sync: false, - }; - let snapshot = [snapshot_entry( - "buffer.failure", - None, - AggregateMetricType::Gauge, - "", - &[], - &[], - )]; - - let error = publish_open_temporary(writer, &temporary, &target, &snapshot, &FailingReplace) - .expect_err("buffer finalization should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("finalize buffered"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected write failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); -} - -#[test] -fn sync_failure_preserves_the_canonical_artifact_and_removes_the_temporary_file() { - let run_directory = tempfile::tempdir().expect("temporary run directory should be created"); - let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); - let temporary = run_directory.path().join(".injected-sync-failure.tmp"); - let original = b"original canonical artifact"; - fs::write(&target, original).expect("canonical fixture should be written"); - let file = fs::File::create(&temporary).expect("temporary fixture should be created"); - let writer = InjectableFile { - file, - fail_writes: false, - fail_sync: true, - }; - - let error = - publish_open_temporary(writer, &temporary, &target, &[], &FailingReplace).expect_err("sync should fail"); - - let message = format!("{error:#}"); - assert!(message.contains("sync temporary"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected sync failure"), "{message}"); - assert_eq!(fs::read(&target).unwrap(), original); - assert_eq!( - directory_entries(run_directory.path()), - vec![CONTEXT_DUMP_FILENAME.to_owned()] - ); -} - -#[test] -fn zstd_and_buffer_finalizers_surface_their_own_error_context() { - let target = PathBuf::from("/run/test/dogstatsd_contexts.json.zstd"); - let snapshot = [snapshot_entry( - "finalize.failure", - None, - AggregateMetricType::Gauge, - "", - &[], - &[], - )]; - - let zstd_failure = Arc::new(AtomicBool::new(false)); - let mut encoder = zstd::stream::write::Encoder::new(ToggleFailureWriter::new(zstd_failure.clone()), 0) - .expect("encoder should initialize"); - write_snapshot_records(&mut encoder, &snapshot, &target).expect("records should stream before failure"); - zstd_failure.store(true, Ordering::Relaxed); - let error = finish_zstd_encoder(encoder, &target).expect_err("zstd finalization should fail"); - let message = format!("{error:#}"); - assert!(message.contains("finalize zstd stream"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected finalization failure"), "{message}"); - - let buffer_failure = Arc::new(AtomicBool::new(false)); - let buffer = BufWriter::new(ToggleFailureWriter::new(buffer_failure.clone())); - let mut encoder = zstd::stream::write::Encoder::new(buffer, 0).expect("encoder should initialize"); - write_snapshot_records(&mut encoder, &snapshot, &target).expect("records should stream before failure"); - let buffer = finish_zstd_encoder(encoder, &target).expect("zstd stream should finalize"); - buffer_failure.store(true, Ordering::Relaxed); - let error = finish_buffered_writer(buffer, &target).expect_err("buffer finalization should fail"); - let message = format!("{error:#}"); - assert!(message.contains("finalize buffered"), "{message}"); - assert!(message.contains(&target.display().to_string()), "{message}"); - assert!(message.contains("injected finalization failure"), "{message}"); -} - -struct InjectableFile { - file: fs::File, - fail_writes: bool, - fail_sync: bool, -} - -impl io::Write for InjectableFile { - fn write(&mut self, bytes: &[u8]) -> io::Result { - if self.fail_writes { - Err(io::Error::other("injected write failure")) - } else { - self.file.write(bytes) - } - } - - fn flush(&mut self) -> io::Result<()> { - if self.fail_writes { - Err(io::Error::other("injected write failure")) - } else { - self.file.flush() - } - } -} - -impl SyncAll for InjectableFile { - fn sync_all(&self) -> io::Result<()> { - if self.fail_sync { - Err(io::Error::other("injected sync failure")) - } else { - self.file.sync_all() - } - } -} - -#[derive(Debug)] -struct ToggleFailureWriter { - failure_enabled: Arc, -} - -impl ToggleFailureWriter { - fn new(failure_enabled: Arc) -> Self { - Self { failure_enabled } - } -} - -impl io::Write for ToggleFailureWriter { - fn write(&mut self, bytes: &[u8]) -> io::Result { - if self.failure_enabled.load(Ordering::Relaxed) { - Err(io::Error::other("injected finalization failure")) - } else { - Ok(bytes.len()) - } - } - - fn flush(&mut self) -> io::Result<()> { - if self.failure_enabled.load(Ordering::Relaxed) { - Err(io::Error::other("injected finalization failure")) - } else { - Ok(()) - } - } -} - -struct FixedTemporaryFileFactory; - -impl TemporaryFileFactory for FixedTemporaryFileFactory { - fn create(&self, run_path: &std::path::Path) -> io::Result<(fs::File, PathBuf)> { - let path = run_path.join(format!(".{CONTEXT_DUMP_FILENAME}.fixed.tmp")); - #[cfg(windows)] - let file = open_temporary_file(&path)?; - #[cfg(not(windows))] - let file = { - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - options.mode(0o600); - } - options.open(&path)? - }; - Ok((file, path)) - } -} - -struct FailingTemporaryFileFactory; - -impl TemporaryFileFactory for FailingTemporaryFileFactory { - fn create(&self, _run_path: &std::path::Path) -> io::Result<(fs::File, PathBuf)> { - Err(io::Error::other("injected temporary file creation failure")) - } -} - -#[cfg(unix)] -struct FailingPermissions; - -#[cfg(unix)] -impl PermissionNormalizer for FailingPermissions { - fn normalize(&self, _file: &fs::File) -> io::Result<()> { - Err(io::Error::other("injected permission failure")) - } -} - -#[cfg(unix)] -#[derive(Default)] -struct RestrictiveThenOwnerOnly { - called: AtomicBool, -} - -#[cfg(unix)] -impl PermissionNormalizer for RestrictiveThenOwnerOnly { - fn normalize(&self, file: &fs::File) -> io::Result<()> { - use std::os::unix::fs::PermissionsExt as _; - - file.set_permissions(fs::Permissions::from_mode(0o000))?; - set_owner_only_permissions(file)?; - self.called.store(true, Ordering::Relaxed); - Ok(()) - } -} - -#[derive(Default)] -struct FailingCleanup { - attempts: AtomicUsize, -} - -impl TemporaryFileCleanup for FailingCleanup { - fn remove(&self, _path: &std::path::Path) -> io::Result<()> { - self.attempts.fetch_add(1, Ordering::Relaxed); - Err(io::Error::other("injected cleanup failure")) - } -} - -struct FailingReplace; - -impl AtomicReplace for FailingReplace { - fn replace(&self, _source: &std::path::Path, _target: &std::path::Path) -> io::Result<()> { - Err(io::Error::other("injected replacement failure")) - } -} - fn directory_entries(path: &std::path::Path) -> Vec { let mut entries: Vec<_> = fs::read_dir(path) .unwrap() diff --git a/docs/agent-data-plane/dogstatsd-top.md b/docs/agent-data-plane/dogstatsd-top.md index 1d02f8f9589..3d847c61f46 100644 --- a/docs/agent-data-plane/dogstatsd-top.md +++ b/docs/agent-data-plane/dogstatsd-top.md @@ -74,7 +74,7 @@ Use the remainder row to account for hidden contexts. For example, `17 (other 4 > [!WARNING] > DogStatsD context dumps can contain metric names, hosts, client tags, and origin tags that reveal tenant or workload details. Treat every dump and copy as sensitive diagnostic data. -On Unix, ADP creates the artifact with mode `0600`, owned by the ADP process owner. Keep that access restriction when you copy or store the file, use an encrypted and access-controlled transfer method, and avoid placing it in shared temporary directories. Remove the server-side artifact and every copied artifact manually when you finish the investigation. +On Unix, ADP creates the artifact with mode `0600`, owned by the ADP process owner. On Windows, the artifact inherits the configured `run_path` directory ACL, matching the Datadog Agent; secure a custom `run_path` for the ADP service identity and intended administrators before requesting a dump. Keep the resulting access restriction when you copy or store the file, use an encrypted and access-controlled transfer method, and avoid placing it in shared temporary directories. Remove the server-side artifact and every copied artifact manually when you finish the investigation. A later successful dump replaces the fixed server-side file but does not remove copies elsewhere. ADP does not apply a retention or cleanup policy to the completed file. diff --git a/test/integration/cases/dogstatsd-top-windows/config.yaml b/test/integration/cases/dogstatsd-top-windows/config.yaml index 9e19587e630..6df65f03474 100644 --- a/test/integration/cases/dogstatsd-top-windows/config.yaml +++ b/test/integration/cases/dogstatsd-top-windows/config.yaml @@ -113,36 +113,6 @@ procedure: return $Path } - function Assert-ProtectedArtifactAcl([string] $Path) { - $Acl = Get-Acl -LiteralPath $Path - if (-not $Acl.AreAccessRulesProtected) { - throw "Context dump DACL permits inherited access: $Path" - } - - $Rules = @($Acl.Access) - $Sids = @($Rules | ForEach-Object { - $_.IdentityReference.Translate([System.Security.Principal.SecurityIdentifier]).Value - }) - $ExpectedSids = @("S-1-3-4", "S-1-5-18", "S-1-5-32-544") - foreach ($ExpectedSid in $ExpectedSids) { - if ($Sids -notcontains $ExpectedSid) { - throw "Context dump DACL is missing $ExpectedSid; actual SIDs: $($Sids -join ', ')" - } - } - if (@($Sids | Sort-Object -Unique).Count -ne 3) { - throw "Context dump DACL contains unexpected trustees: $($Sids -join ', ')" - } - foreach ($Rule in $Rules) { - if ($Rule.AccessControlType -ne [System.Security.AccessControl.AccessControlType]::Allow) { - throw "Context dump DACL contains a deny rule: $Rule" - } - if (($Rule.FileSystemRights -band [System.Security.AccessControl.FileSystemRights]::FullControl) -ne - [System.Security.AccessControl.FileSystemRights]::FullControl) { - throw "Context dump DACL does not grant full control to its intended trustee: $Rule" - } - } - } - Assert-UnauthorizedRequest Send-DogStatsDContexts $Online = Wait-ForOnlineReport @@ -151,7 +121,6 @@ procedure: } $DumpPath = Get-DumpPath (Invoke-AdpDogStatsD @("dump-contexts")) - Assert-ProtectedArtifactAcl $DumpPath Copy-Item -LiteralPath $DumpPath -Destination $CopiedDump -Force $Offline = Invoke-AdpDogStatsD @("top", "--path", $CopiedDump) if (-not $Offline.Contains($RequestRow) -or -not $Offline.Contains($QueueRow)) { From 6454800594e550550cc8405b8247fa8d903f9a1f Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Fri, 24 Jul 2026 13:30:32 -0400 Subject: [PATCH 38/44] docs(adp): refresh context dump benchmark baseline Record the post-refactor NamedTempFile publication measurements so the benchmark comment describes the implementation under review. --- bin/agent-data-plane/benches/dogstatsd_context_dump.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs index 5221424944a..623c0dc1dff 100644 --- a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs +++ b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs @@ -3,7 +3,7 @@ //! Run with: //! `cargo bench -p agent-data-plane --features context-dump-benchmark --bench dogstatsd_context_dump -- --sample-size 10` //! -//! PR #2185's Apple Silicon baseline was approximately 15.7 ms at 10k contexts, 123 ms at 100k, and 1.13 s at one +//! PR #2185's Apple Silicon baseline was approximately 16.5 ms at 10k contexts, 123 ms at 100k, and 1.14 s at one //! million. These values provide scale context, not regression thresholds; compare runs on equivalent hardware. use std::fs; From de53306c6788cccb0f09065203cc83cca6180922 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Mon, 27 Jul 2026 15:19:20 -0400 Subject: [PATCH 39/44] refactor(adp): apply simple DogStatsD review cleanups Import the generic error macro directly in DogStatsD CLI modules and use rest-pattern destructuring for report records. These are mechanical review fixes with no behavior change. --- bin/agent-data-plane/src/cli/dogstatsd.rs | 8 ++------ bin/agent-data-plane/src/cli/dogstatsd/top.rs | 9 ++++----- bin/agent-data-plane/src/dogstatsd_contexts/report.rs | 10 +--------- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/bin/agent-data-plane/src/cli/dogstatsd.rs b/bin/agent-data-plane/src/cli/dogstatsd.rs index 0dfaf0c37bf..9fcd749febb 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd.rs @@ -17,9 +17,7 @@ use saluki_components::sources::DEFAULT_REPLAY_LOOPS; #[cfg(target_os = "linux")] use saluki_components::sources::REPLAY_CREDENTIALS_GID; use saluki_config::{DurationString, GenericConfiguration}; -#[cfg(any(target_os = "linux", test))] -use saluki_error::generic_error; -use saluki_error::{ErrorContext as _, GenericError}; +use saluki_error::{generic_error, ErrorContext as _, GenericError}; #[cfg(target_os = "linux")] use saluki_io::net::{unix::uds_sendmsg_with_creds, ProcessCredentials}; use serde::Deserialize; @@ -211,9 +209,7 @@ async fn run_dogstatsd_command( loops, } = config; let _ = (replay_file_path, loops); - Err(saluki_error::generic_error!( - "DogStatsD replay is only supported on Linux." - )) + Err(generic_error!("DogStatsD replay is only supported on Linux.")) } } DogstatsdSubcommand::Top(config) => { diff --git a/bin/agent-data-plane/src/cli/dogstatsd/top.rs b/bin/agent-data-plane/src/cli/dogstatsd/top.rs index c7472dca891..419753272aa 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd/top.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd/top.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use argh::FromArgs; use async_trait::async_trait; -use saluki_error::{ErrorContext as _, GenericError}; +use saluki_error::{generic_error, ErrorContext as _, GenericError}; use crate::cli::utils::DataPlaneAPIClient; use crate::dogstatsd_contexts::read_report; @@ -33,7 +33,7 @@ impl TopCommand { pub(super) fn validate(self) -> Result { let num_tags = match (self.num_tags, self.legacy_num_tags) { (Some(_), Some(_)) => { - return Err(saluki_error::generic_error!( + return Err(generic_error!( "Cannot use `--num-tags` and legacy `--mum-tags` together; use `--num-tags`." )); } @@ -85,9 +85,8 @@ pub(super) async fn handle_dogstatsd_top( let path = match cmd.path { Some(path) => path, None => { - let requester = requester.ok_or_else(|| { - saluki_error::generic_error!("Online DogStatsD top requires a context dump requester.") - })?; + let requester = + requester.ok_or_else(|| generic_error!("Online DogStatsD top requires a context dump requester."))?; let path = requester .request_context_dump() .await diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/report.rs b/bin/agent-data-plane/src/dogstatsd_contexts/report.rs index b2b96eed6b0..720ac1e4e10 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/report.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/report.rs @@ -16,15 +16,7 @@ impl ContextReport { } pub(super) fn ingest(&mut self, record: AgentContextRecord) { - let AgentContextRecord { - name, - host: _, - metric_type: _, - tagger_tags: _, - metric_tags, - no_index: _, - source: _, - } = record; + let AgentContextRecord { name, metric_tags, .. } = record; let summary = self.metrics.entry(name).or_default(); summary.context_count += 1; summary.metric_tags.extend(metric_tags); From c5628a41d330d1f30789f32c4e2ee58add5f3bea Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Mon, 27 Jul 2026 15:46:02 -0400 Subject: [PATCH 40/44] fix(adp): drop legacy mum-tags option Expose only the correctly spelled --num-tags option for DogStatsD top and remove the compatibility-only conflict validation and tests. --- bin/agent-data-plane/src/cli/dogstatsd.rs | 2 +- bin/agent-data-plane/src/cli/dogstatsd/top.rs | 80 +++++-------------- docs/agent-data-plane/dogstatsd-top.md | 2 +- 3 files changed, 20 insertions(+), 64 deletions(-) diff --git a/bin/agent-data-plane/src/cli/dogstatsd.rs b/bin/agent-data-plane/src/cli/dogstatsd.rs index 9fcd749febb..56b452145a8 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd.rs @@ -213,7 +213,7 @@ async fn run_dogstatsd_command( } } DogstatsdSubcommand::Top(config) => { - let config = config.validate()?; + let config = config.validate(); let mut output = std::io::stdout(); if config.is_offline() { handle_dogstatsd_top(None, config, &mut output).await diff --git a/bin/agent-data-plane/src/cli/dogstatsd/top.rs b/bin/agent-data-plane/src/cli/dogstatsd/top.rs index 419753272aa..708ffaeace1 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd/top.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd/top.rs @@ -23,29 +23,15 @@ pub(super) struct TopCommand { /// set the maximum number of tags to display for each metric #[argh(option, short = 't', long = "num-tags")] num_tags: Option, - - /// use the legacy `--mum-tags` typo for `--num-tags` - #[argh(option, long = "mum-tags")] - legacy_num_tags: Option, } impl TopCommand { - pub(super) fn validate(self) -> Result { - let num_tags = match (self.num_tags, self.legacy_num_tags) { - (Some(_), Some(_)) => { - return Err(generic_error!( - "Cannot use `--num-tags` and legacy `--mum-tags` together; use `--num-tags`." - )); - } - (Some(limit), None) | (None, Some(limit)) => limit, - (None, None) => 5, - }; - - Ok(ValidatedTopCommand { + pub(super) fn validate(self) -> ValidatedTopCommand { + ValidatedTopCommand { path: self.path, num_metrics: self.num_metrics, - num_tags, - }) + num_tags: self.num_tags.unwrap_or(5), + } } } @@ -162,8 +148,7 @@ mod tests { assert_eq!(top.path, None); assert_eq!(top.num_metrics, 10); assert_eq!(top.num_tags, None); - assert_eq!(top.legacy_num_tags, None); - let validated = top.validate().expect("defaults should pass preflight"); + let validated = top.validate(); assert_eq!(validated.path, None); assert_eq!(validated.num_metrics, 10); assert_eq!(validated.num_tags, 5); @@ -180,8 +165,7 @@ mod tests { assert_eq!(top.path, Some(PathBuf::from(PLAIN_FIXTURE))); assert_eq!(top.num_metrics, 7); assert_eq!(top.num_tags, Some(3)); - assert_eq!(top.legacy_num_tags, None); - let validated = top.validate().expect("short options should pass preflight"); + let validated = top.validate(); assert_eq!(validated.path, Some(PathBuf::from(PLAIN_FIXTURE))); assert_eq!(validated.num_metrics, 7); assert_eq!(validated.num_tags, 3); @@ -195,35 +179,12 @@ mod tests { }; assert_eq!(top.num_tags, Some(4)); - assert_eq!(top.legacy_num_tags, None); - assert_eq!(top.validate().unwrap().num_tags, 4); - } - - #[test] - fn dogstatsd_top_parses_legacy_mum_tags() { - let command = parse_dogstatsd(&["top", "--mum-tags", "6"]).expect("legacy option should parse"); - let DogstatsdSubcommand::Top(top) = command.subcommand else { - panic!("expected top subcommand"); - }; - - assert_eq!(top.num_tags, None); - assert_eq!(top.legacy_num_tags, Some(6)); - assert_eq!(top.validate().unwrap().num_tags, 6); + assert_eq!(top.validate().num_tags, 4); } #[test] - fn dogstatsd_top_preflight_rejects_both_num_tags_spellings() { - let command = parse_dogstatsd(&["top", "--num-tags", "4", "--mum-tags", "6"]) - .expect("each spelling should parse before conflict validation"); - let DogstatsdSubcommand::Top(top) = command.subcommand else { - panic!("expected top subcommand"); - }; - - let error = top.validate().expect_err("preflight should reject conflicting options"); - assert_eq!( - error.to_string(), - "Cannot use `--num-tags` and legacy `--mum-tags` together; use `--num-tags`." - ); + fn dogstatsd_top_rejects_legacy_mum_tags() { + assert!(parse_dogstatsd(&["top", "--mum-tags", "6"]).is_err()); } #[test] @@ -231,7 +192,6 @@ mod tests { for args in [ &["top", "--num-metrics", "-1"][..], &["top", "--num-tags", "-1"][..], - &["top", "--mum-tags", "-1"][..], &["top", "artifact.ndjson"][..], ] { assert!(parse_dogstatsd(args).is_err(), "arguments should be rejected: {args:?}"); @@ -253,7 +213,7 @@ mod tests { #[tokio::test] async fn dogstatsd_top_reads_an_offline_fixture_without_triggering_a_dump() { - let command = top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 10, None, None); + let command = top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 10, None); let mut requester = FakeRequester::returning_path("unused"); let mut output = RecordingWriter::default(); @@ -270,7 +230,7 @@ mod tests { let mut requester = FakeRequester::returning_path(PLAIN_FIXTURE); let mut output = RecordingWriter::default(); - handle_dogstatsd_top(Some(&mut requester), top_command(None, 10, None, None), &mut output) + handle_dogstatsd_top(Some(&mut requester), top_command(None, 10, None), &mut output) .await .expect("online top should succeed"); @@ -318,7 +278,7 @@ mod tests { let mut requester = FakeRequester::returning_path(artifact.path()); let mut output = RecordingWriter::default(); - let error = handle_dogstatsd_top(Some(&mut requester), top_command(None, 10, None, None), &mut output) + let error = handle_dogstatsd_top(Some(&mut requester), top_command(None, 10, None), &mut output) .await .expect_err("corrupt artifact should fail"); @@ -344,7 +304,7 @@ mod tests { let error = handle_dogstatsd_top( None, - top_command(Some(artifact.path().to_owned()), 10, None, None), + top_command(Some(artifact.path().to_owned()), 10, None), &mut output, ) .await @@ -368,7 +328,7 @@ mod tests { handle_dogstatsd_top( None, - top_command(Some(artifact.path().to_owned()), 10, None, None), + top_command(Some(artifact.path().to_owned()), 10, None), &mut output, ) .await @@ -384,14 +344,14 @@ mod tests { async fn dogstatsd_top_applies_custom_and_zero_report_limits() { let cases = [ ( - top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 0, Some(5), None), + top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 0, Some(5)), concat!( " Contexts\tMetric name\t(number of unique values for each tag)\n", " 4\t(other 2 metrics)\n", ), ), ( - top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 10, Some(0), None), + top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 10, Some(0)), concat!( " Contexts\tMetric name\t(number of unique values for each tag)\n", " 3\ta.metric\t(3 values in 2 other tags)\n", @@ -399,7 +359,7 @@ mod tests { ), ), ( - top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 1, Some(1), None), + top_command(Some(PathBuf::from(PLAIN_FIXTURE)), 1, Some(1)), concat!( " Contexts\tMetric name\t(number of unique values for each tag)\n", " 3\ta.metric\t(2 env, 1 service)\n", @@ -421,17 +381,13 @@ mod tests { DogstatsdCommand::from_args(&["agent-data-plane", "dogstatsd"], args) } - fn top_command( - path: Option, num_metrics: usize, num_tags: Option, legacy_num_tags: Option, - ) -> ValidatedTopCommand { + fn top_command(path: Option, num_metrics: usize, num_tags: Option) -> ValidatedTopCommand { TopCommand { path, num_metrics, num_tags, - legacy_num_tags, } .validate() - .expect("test command should pass preflight") } struct FakeRequester { diff --git a/docs/agent-data-plane/dogstatsd-top.md b/docs/agent-data-plane/dogstatsd-top.md index 3d847c61f46..8b9674a6443 100644 --- a/docs/agent-data-plane/dogstatsd-top.md +++ b/docs/agent-data-plane/dogstatsd-top.md @@ -32,7 +32,7 @@ $ agent-data-plane dogstatsd top --num-metrics 20 --num-tags 8 $ agent-data-plane dogstatsd top -m 20 -t 8 ``` -The legacy `--mum-tags` spelling remains an alias for `--num-tags`. Do not pass both spellings in one command. Limits must be non-negative integers, and `top` rejects extra positional arguments. +Limits must be non-negative integers, and `top` rejects extra positional arguments. The report orders metric names by descending context count and tag keys by descending unique-value count. It resolves ties lexically. When only one item exceeds a limit, the report displays that item; when two or more exceed it, the report replaces them with one remainder row or tag summary. From 882be00902004db6b0dc5edbd3e6e0a000c81616 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Mon, 27 Jul 2026 15:48:06 -0400 Subject: [PATCH 41/44] chore(adp): remove retained-context benchmarks The one-time performance measurements established acceptable snapshot and publication costs, but the non-gating Criterion targets required public benchmark-only facades and fixtures. Remove that scaffolding while retaining the recorded PR evidence. --- Cargo.lock | 2 - bin/agent-data-plane/Cargo.toml | 7 -- .../benches/dogstatsd_context_dump.rs | 117 ------------------ bin/agent-data-plane/src/lib.rs | 33 ----- lib/saluki-components/Cargo.toml | 6 - .../benches/aggregate_context_snapshot.rs | 50 -------- .../src/transforms/aggregate/mod.rs | 54 -------- lib/saluki-components/src/transforms/mod.rs | 4 +- 8 files changed, 2 insertions(+), 271 deletions(-) delete mode 100644 bin/agent-data-plane/benches/dogstatsd_context_dump.rs delete mode 100644 bin/agent-data-plane/src/lib.rs delete mode 100644 lib/saluki-components/benches/aggregate_context_snapshot.rs diff --git a/Cargo.lock b/Cargo.lock index 1b8e24aef82..3deb56885b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,6 @@ dependencies = [ "chrono", "colored", "comfy-table", - "criterion", "datadog-agent-commons", "datadog-agent-config", "datadog-agent-config-testing", @@ -4321,7 +4320,6 @@ dependencies = [ "bytes", "bytesize", "chrono", - "criterion", "datadog-agent-commons", "datadog-agent-config", "datadog-agent-config-testing", diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index aff6f617dd1..f9e3f78d24d 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -12,7 +12,6 @@ workspace = true default = [] fips = ["saluki-app/tls-fips", "saluki-components/fips"] antithesis = ["saluki-antithesis/antithesis", "dep:antithesis-instrumentation"] -context-dump-benchmark = [] [dependencies] agent-data-plane-config = { workspace = true } @@ -85,7 +84,6 @@ tikv-jemallocator = { workspace = true, features = [ chrono = { workspace = true } [dev-dependencies] -criterion = { workspace = true } datadog-agent-config-testing = { workspace = true } derive-where = { workspace = true } rcgen = { workspace = true, features = ["crypto", "pem"] } @@ -103,8 +101,3 @@ rcgen = { workspace = true, features = ["aws_lc_rs"] } [target.'cfg(windows)'.dev-dependencies] rcgen = { workspace = true, features = ["ring"] } - -[[bench]] -name = "dogstatsd_context_dump" -harness = false -required-features = ["context-dump-benchmark"] diff --git a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs b/bin/agent-data-plane/benches/dogstatsd_context_dump.rs deleted file mode 100644 index 623c0dc1dff..00000000000 --- a/bin/agent-data-plane/benches/dogstatsd_context_dump.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Measures streaming zstd publication and atomic replacement of retained-context artifacts. -//! -//! Run with: -//! `cargo bench -p agent-data-plane --features context-dump-benchmark --bench dogstatsd_context_dump -- --sample-size 10` -//! -//! PR #2185's Apple Silicon baseline was approximately 16.5 ms at 10k contexts, 123 ms at 100k, and 1.14 s at one -//! million. These values provide scale context, not regression thresholds; compare runs on equivalent hardware. - -use std::fs; -use std::hint::black_box; -use std::path::Path; - -use agent_data_plane::{count_dogstatsd_context_dump_records, publish_dogstatsd_context_dump}; -use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateMetricType}; -use saluki_context::{ - tags::{Tag, TagSet}, - Context, -}; -use stringtheory::MetaString; - -const CONTEXT_COUNTS: [usize; 3] = [10_000, 100_000, 1_000_000]; -const CLIENT_TAGS: [&str; 5] = [ - "env:production", - "service:payments-api", - "version:2025.03.0", - "region:us-east-1", - "availability-zone:us-east-1a", -]; -const ORIGIN_TAGS: [&str; 5] = [ - "container_id:7c9f0e2a4b1d", - "pod_name:payments-api-7d8f9c6b5f-k2m4p", - "namespace:production", - "kube_deployment:payments-api", - "image_name:registry.example/payments-api", -]; - -fn benchmark_dogstatsd_context_dump(c: &mut Criterion) { - let mut group = c.benchmark_group("dogstatsd_context_dump"); - - for context_count in CONTEXT_COUNTS { - let snapshot = build_snapshot(context_count); - assert_eq!(snapshot.len(), context_count); - - let run_directory = tempfile::tempdir().expect("benchmark run directory should be created"); - let canonical_path = publish_dogstatsd_context_dump(run_directory.path(), &snapshot) - .expect("benchmark preflight context dump should publish"); - assert!(canonical_path.is_file()); - - let decoded_records = count_dogstatsd_context_dump_records(&canonical_path) - .expect("benchmark preflight context dump should decode through the production reader"); - assert_eq!(decoded_records, context_count); - assert_only_canonical_artifact(run_directory.path(), &canonical_path); - - let compressed_bytes = fs::metadata(&canonical_path) - .expect("benchmark preflight artifact metadata should be available") - .len(); - eprintln!("dogstatsd_context_dump setup: contexts={context_count}, compressed_bytes={compressed_bytes}"); - - let benchmark_id = BenchmarkId::from_parameter(context_count); - group.throughput(Throughput::Elements(context_count as u64)); - group.bench_function(benchmark_id, |b| { - b.iter(|| { - black_box( - publish_dogstatsd_context_dump(run_directory.path(), &snapshot) - .expect("benchmark context dump should publish"), - ) - }); - }); - - let postflight_path = publish_dogstatsd_context_dump(run_directory.path(), &snapshot) - .expect("benchmark postflight context dump should publish"); - assert_eq!(postflight_path, canonical_path); - assert!(postflight_path.is_file()); - assert_only_canonical_artifact(run_directory.path(), &canonical_path); - } - - group.finish(); -} - -fn build_snapshot(context_count: usize) -> Vec { - let base_context = Context::from_parts("dogstatsd.benchmark.metric", shared_tag_set(&CLIENT_TAGS)) - .with_host(Some(MetaString::from_static("benchmark-node-001.internal"))) - .with_origin_tags(shared_tag_set(&ORIGIN_TAGS)); - - (0..context_count) - .map(|index| { - let context = base_context.with_name(format!("dogstatsd.benchmark.metric.{index:07}")); - let (metric_type, unit) = match index % 3 { - 0 => (AggregateMetricType::Counter, MetaString::empty()), - 1 => (AggregateMetricType::Gauge, MetaString::empty()), - _ => (AggregateMetricType::Histogram, MetaString::from_static("millisecond")), - }; - AggregateContextSnapshotEntry::for_test(context, metric_type, unit) - }) - .collect() -} - -fn shared_tag_set(values: &[&'static str]) -> TagSet { - let mut tags = TagSet::with_capacity(values.len()); - for value in values { - tags.insert_tag(Tag::from_static(value)); - } - tags.into_shared().to_mutable() -} - -fn assert_only_canonical_artifact(run_path: &Path, canonical_path: &Path) { - let mut entries: Vec<_> = fs::read_dir(run_path) - .expect("benchmark run directory should be readable") - .map(|entry| entry.expect("benchmark run directory entry should be readable").path()) - .collect(); - entries.sort_unstable(); - assert_eq!(entries, [canonical_path.to_path_buf()]); -} - -criterion_group!(benches, benchmark_dogstatsd_context_dump); -criterion_main!(benches); diff --git a/bin/agent-data-plane/src/lib.rs b/bin/agent-data-plane/src/lib.rs deleted file mode 100644 index 8e2bbee9a7f..00000000000 --- a/bin/agent-data-plane/src/lib.rs +++ /dev/null @@ -1,33 +0,0 @@ -#![cfg(feature = "context-dump-benchmark")] - -//! Benchmark-only access to DogStatsD context artifact production code. - -use std::path::{Path, PathBuf}; - -use saluki_components::transforms::AggregateContextSnapshotEntry; -use saluki_error::GenericError; - -#[cfg(feature = "context-dump-benchmark")] -#[allow( - dead_code, - unused_imports, - reason = "the benchmark facade includes the complete binary-owned context dump module" -)] -#[path = "dogstatsd_contexts/mod.rs"] -mod dogstatsd_contexts; - -/// Publishes a DogStatsD context dump through the production artifact writer. -#[doc(hidden)] -pub fn publish_dogstatsd_context_dump( - run_path: &Path, snapshot: &[AggregateContextSnapshotEntry], -) -> Result { - dogstatsd_contexts::publish_context_dump(run_path, snapshot) -} - -/// Counts DogStatsD context dump records through the production artifact reader. -#[doc(hidden)] -pub fn count_dogstatsd_context_dump_records(path: &Path) -> Result { - let mut record_count = 0; - dogstatsd_contexts::for_each_record(path, |_| record_count += 1)?; - Ok(record_count) -} diff --git a/lib/saluki-components/Cargo.toml b/lib/saluki-components/Cargo.toml index ffae1f2312b..353e888a89e 100644 --- a/lib/saluki-components/Cargo.toml +++ b/lib/saluki-components/Cargo.toml @@ -95,7 +95,6 @@ uuid = { workspace = true, features = ["std", "v7"] } zstd = { workspace = true } [dev-dependencies] -criterion = { workspace = true } # TODO: remove after migration to typed config. Only the in-crate tests still reference the moved # `KEY_ALIASES`/`DatadogRemapper` (re-exported from `config` under `cfg(test)`). datadog-agent-config = { workspace = true } @@ -112,8 +111,3 @@ tempfile = { workspace = true } test-strategy = { workspace = true } tokio = { workspace = true, features = ["time"] } tokio-rustls = { workspace = true } - -[[bench]] -name = "aggregate_context_snapshot" -harness = false -required-features = ["test-util"] diff --git a/lib/saluki-components/benches/aggregate_context_snapshot.rs b/lib/saluki-components/benches/aggregate_context_snapshot.rs deleted file mode 100644 index a0bcc4d1433..00000000000 --- a/lib/saluki-components/benches/aggregate_context_snapshot.rs +++ /dev/null @@ -1,50 +0,0 @@ -//! Measures the aggregate-owner pause required to clone retained context handles. -//! -//! Run with: -//! `cargo bench -p saluki-components --features test-util --bench aggregate_context_snapshot -- --sample-size 10` -//! -//! PR #2185's Apple Silicon baseline was approximately 45.8 µs at 10k contexts, 1.02 ms at 100k, and 20.8 ms at -//! one million. These values provide scale context, not regression thresholds; compare runs on equivalent hardware. - -use std::hint::black_box; -use std::mem::size_of; - -use criterion::{criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion, Throughput}; -use saluki_components::transforms::{AggregateContextSnapshotBenchmarkHarness, AggregateContextSnapshotEntry}; - -const CONTEXT_COUNTS: [usize; 3] = [10_000, 100_000, 1_000_000]; - -fn benchmark_aggregate_context_snapshot(c: &mut Criterion) { - let mut group = c.benchmark_group("aggregate_context_snapshot"); - - for context_count in CONTEXT_COUNTS { - let harness = AggregateContextSnapshotBenchmarkHarness::with_contexts(context_count); - let retained_len = harness.retained_len(); - assert_eq!(retained_len, context_count); - - let preflight = harness.snapshot(); - assert_eq!(preflight.len(), retained_len); - black_box(preflight); - - let structural_bytes = context_count * size_of::(); - let benchmark_id = - BenchmarkId::from_parameter(format!("{context_count}_contexts_{structural_bytes}_structural_bytes")); - group.throughput(Throughput::Elements(context_count as u64)); - group.bench_function(benchmark_id, |b| { - b.iter_batched( - || (), - |()| { - let snapshot = harness.snapshot(); - assert_eq!(snapshot.len(), retained_len); - black_box(snapshot) - }, - BatchSize::PerIteration, - ); - }); - } - - group.finish(); -} - -criterion_group!(benches, benchmark_aggregate_context_snapshot); -criterion_main!(benches); diff --git a/lib/saluki-components/src/transforms/aggregate/mod.rs b/lib/saluki-components/src/transforms/aggregate/mod.rs index afd6862a8e0..0e162b9f50c 100644 --- a/lib/saluki-components/src/transforms/aggregate/mod.rs +++ b/lib/saluki-components/src/transforms/aggregate/mod.rs @@ -1071,49 +1071,6 @@ impl AggregationState { } } -/// A benchmark fixture backed by a real aggregate state populated before snapshot timing begins. -#[cfg(any(test, feature = "test-util"))] -pub struct AggregateContextSnapshotBenchmarkHarness { - state: AggregationState, -} - -#[cfg(any(test, feature = "test-util"))] -impl AggregateContextSnapshotBenchmarkHarness { - /// Creates a harness retaining exactly `context_count` distinct gauge contexts. - pub fn with_contexts(context_count: usize) -> Self { - let mut state = AggregationState::new( - default_window_duration_seconds(), - context_count, - None, - HistogramConfiguration::default(), - Telemetry::new(&MetricsBuilder::default()), - ); - - for index in 0..context_count { - let context = Context::from_parts( - format!("aggregate.snapshot.benchmark.{index}"), - saluki_context::tags::TagSet::default(), - ); - assert!( - state.insert(0, Metric::gauge(context, 0.0)), - "benchmark fixture must retain every requested context" - ); - } - - Self { state } - } - - /// Takes a retained-context snapshot from the populated aggregate state. - pub fn snapshot(&self) -> Vec { - self.state.snapshot_contexts() - } - - /// Returns the exact number of contexts retained by the aggregate state. - pub fn retained_len(&self) -> usize { - self.state.contexts.len() - } -} - async fn transform_and_push_metric( context: Context, mut values: MetricValues, metadata: MetricMetadata, bucket_width_secs: NonZeroU64, hist_config: &HistogramConfiguration, dispatcher: &mut BufferedDispatcher<'_, EventsBuffer>, @@ -1872,17 +1829,6 @@ mod tests { pending_response.respond(Vec::new()); } - #[test] - fn snapshot_benchmark_harness_populates_exact_context_count() { - let empty_harness = AggregateContextSnapshotBenchmarkHarness::with_contexts(0); - assert_eq!(empty_harness.retained_len(), 0); - assert!(empty_harness.snapshot().is_empty()); - - let harness = AggregateContextSnapshotBenchmarkHarness::with_contexts(32); - assert_eq!(harness.retained_len(), 32); - assert_eq!(harness.snapshot().len(), 32); - } - #[test] fn bucket_is_closed() { // Cases are defined as: diff --git a/lib/saluki-components/src/transforms/mod.rs b/lib/saluki-components/src/transforms/mod.rs index 1899fc7f912..ebeac7c4126 100644 --- a/lib/saluki-components/src/transforms/mod.rs +++ b/lib/saluki-components/src/transforms/mod.rs @@ -6,8 +6,8 @@ pub use self::autoscaling_failover_gateway::AutoscalingFailoverGatewayConfigurat mod aggregate; #[cfg(feature = "test-util")] pub use self::aggregate::{ - aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotBenchmarkHarness, - AggregateContextSnapshotPendingResponse, AggregateContextSnapshotResponder, + aggregate_context_snapshot_channel_for_test, AggregateContextSnapshotPendingResponse, + AggregateContextSnapshotResponder, }; pub use self::aggregate::{ AggregateConfiguration, AggregateContextSnapshotEntry, AggregateContextSnapshotHandle, AggregateMetricType, From 530e9fb8350276705486f0292bcdea8fc6f14dd2 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 28 Jul 2026 13:11:12 -0400 Subject: [PATCH 42/44] test(adp): remove redundant control-surface tests Delete a trivial handler-registration test and a timeout test that exercised only test-local networking helpers rather than production behavior. --- .../src/internal/control_surfaces.rs | 114 ------------------ 1 file changed, 114 deletions(-) diff --git a/bin/agent-data-plane/src/internal/control_surfaces.rs b/bin/agent-data-plane/src/internal/control_surfaces.rs index e87d3b74983..60cc5592238 100644 --- a/bin/agent-data-plane/src/internal/control_surfaces.rs +++ b/bin/agent-data-plane/src/internal/control_surfaces.rs @@ -49,117 +49,3 @@ impl DogStatsDControlSurface { .with_handler(self.context_dump_api_handler) } } - -#[cfg(test)] -mod tests { - use std::{net::TcpListener, time::Duration}; - - use http::StatusCode; - use saluki_api::EndpointType; - use saluki_app::dynamic_api::DynamicAPIBuilder; - use saluki_components::{destinations::DogStatsDStatisticsConfiguration, sources::DogStatsDConfiguration}; - use saluki_config::config_from; - use saluki_core::runtime::Supervisor; - use saluki_io::net::ListenAddress; - use tokio::{ - io::{AsyncReadExt as _, AsyncWriteExt as _}, - net::TcpStream, - sync::oneshot, - time::Instant, - }; - - use super::DogStatsDControlSurface; - use crate::dogstatsd_contexts::DogStatsDContextDumpAPIHandler; - - #[tokio::test] - async fn dogstatsd_control_surface_registers_context_dump_on_privileged_dynamic_api() { - let address = reserve_tcp_address(); - let dogstatsd_config = DogStatsDConfiguration::from_configuration(&config_from(serde_json::json!({})).await) - .expect("default DogStatsD configuration should build"); - let surface = DogStatsDControlSurface { - stats_api_handler: DogStatsDStatisticsConfiguration::new().api_handler(), - capture_api_handler: dogstatsd_config.capture_api_handler(), - replay_api_handler: dogstatsd_config.replay_api_handler(), - context_dump_api_handler: DogStatsDContextDumpAPIHandler::new( - "configured-agent-token", - Vec::new(), - std::path::PathBuf::new(), - ) - .unwrap(), - }; - let api = surface.register_control_surfaces(DynamicAPIBuilder::new( - EndpointType::Privileged, - ListenAddress::Tcp(address), - )); - let mut supervisor = Supervisor::new("dogstatsd-control-surface-test").unwrap(); - supervisor.add_worker(api); - let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); - let server = tokio::spawn(async move { supervisor.run_with_shutdown(shutdown_rx).await }); - - let status = post_context_dump_eventually(address).await; - - assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); - let _ = shutdown_tx.send(()); - server.await.unwrap().unwrap(); - } - - #[tokio::test] - #[should_panic(expected = "timed out waiting for DogStatsD context dump route response")] - async fn context_dump_request_exchange_times_out_when_peer_does_not_close() { - let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) - .await - .expect("test listener should bind"); - let address = listener.local_addr().unwrap(); - tokio::spawn(async move { - let (_stream, _) = listener.accept().await.expect("test listener should accept"); - tokio::time::sleep(Duration::from_secs(1)).await; - }); - let stream = TcpStream::connect(address).await.expect("test client should connect"); - - exchange_context_dump_request(stream, Instant::now() + Duration::from_millis(20)).await; - } - - fn reserve_tcp_address() -> std::net::SocketAddr { - TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) - .expect("ephemeral TCP address should bind") - .local_addr() - .unwrap() - } - - async fn post_context_dump_eventually(address: std::net::SocketAddr) -> StatusCode { - let deadline = Instant::now() + Duration::from_secs(5); - let stream = loop { - if let Ok(stream) = TcpStream::connect(address).await { - break stream; - } - assert!(Instant::now() < deadline, "dynamic API did not start before deadline"); - tokio::time::sleep(Duration::from_millis(25)).await; - }; - let response = exchange_context_dump_request(stream, deadline).await; - let status_digits = response - .get(9..12) - .expect("HTTP response should have a three-digit status code"); - let status = status_digits.iter().try_fold(0_u16, |status, digit| { - digit.is_ascii_digit().then(|| status * 10 + u16::from(*digit - b'0')) - }); - StatusCode::from_u16(status.expect("HTTP response status should be numeric")).unwrap() - } - - async fn exchange_context_dump_request(mut stream: TcpStream, deadline: Instant) -> Vec { - let remaining = deadline.saturating_duration_since(Instant::now()); - tokio::time::timeout(remaining, async { - stream - .write_all( - b"POST /agent/dogstatsd-contexts-dump HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer \ - configured-agent-token\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", - ) - .await?; - let mut response = Vec::new(); - stream.read_to_end(&mut response).await?; - Ok::<_, std::io::Error>(response) - }) - .await - .expect("timed out waiting for DogStatsD context dump route response") - .expect("DogStatsD context dump route request should complete") - } -} From b95df47616c4a6f47fbeedcfda0a585fc3f9b640 Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 28 Jul 2026 14:09:21 -0400 Subject: [PATCH 43/44] refactor(adp): rely on privileged API access control Remove the context-dump route bearer token, token-file reads, Authorization header, and auth-specific tests. Keep IPC certificate pinning and client identity so a separate privileged-API mTLS change can enforce the shared access-control boundary. --- Cargo.lock | 1 - Cargo.toml | 1 - bin/agent-data-plane/Cargo.toml | 1 - bin/agent-data-plane/src/cli/dogstatsd.rs | 12 +- bin/agent-data-plane/src/cli/run.rs | 96 ++------- bin/agent-data-plane/src/cli/utils.rs | 192 +++--------------- .../src/dogstatsd_contexts/api.rs | 93 ++------- .../src/dogstatsd_contexts/api_tests.rs | 167 +++------------ docs/agent-data-plane/dogstatsd-top.md | 6 +- .../cases/dogstatsd-top-windows/config.yaml | 15 +- .../cases/dogstatsd-top/config.yaml | 2 +- .../dogstatsd-top/run_dogstatsd_top_test.py | 34 +--- 12 files changed, 101 insertions(+), 519 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3deb56885b3..c05104ca31e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,7 +67,6 @@ dependencies = [ "serde_json", "serde_with", "stringtheory", - "subtle", "tempfile", "tikv-jemallocator", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 58fbb8a7ec2..df9440893b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,7 +158,6 @@ rustls-webpki = { version = "0.103", default-features = false } schemars = { version = "0.8", default-features = false } similar-asserts = { version = "2.0", default-features = false } slab = { version = "0.4.12", default-features = false } -subtle = { version = "2.6", default-features = false } syn = { version = "2", default-features = false, features = ["full", "parsing", "visit-mut"] } tokio-util = { version = "0.7.18", default-features = false } tower = { version = "0.5", default-features = false } diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index f9e3f78d24d..bc7c7452ba6 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -55,7 +55,6 @@ serde = { workspace = true } serde_json = { workspace = true } serde_with = { workspace = true } stringtheory = { workspace = true } -subtle = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = [ "fs", diff --git a/bin/agent-data-plane/src/cli/dogstatsd.rs b/bin/agent-data-plane/src/cli/dogstatsd.rs index 56b452145a8..c79a53a2e21 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd.rs @@ -218,12 +218,12 @@ async fn run_dogstatsd_command( if config.is_offline() { handle_dogstatsd_top(None, config, &mut output).await } else { - let mut api_client = authenticated_data_plane_api_client(bootstrap_config).await?; + let mut api_client = ipc_tls_data_plane_api_client(bootstrap_config).await?; handle_dogstatsd_top(Some(&mut api_client), config, &mut output).await } } DogstatsdSubcommand::DumpContexts(_) => { - let mut api_client = authenticated_data_plane_api_client(bootstrap_config).await?; + let mut api_client = ipc_tls_data_plane_api_client(bootstrap_config).await?; let mut output = std::io::stdout(); handle_dogstatsd_dump_contexts(&mut api_client, &mut output).await } @@ -234,12 +234,10 @@ fn data_plane_api_client(config: &GenericConfiguration) -> Result Result { - DataPlaneAPIClient::from_config_with_ipc_auth(config) +async fn ipc_tls_data_plane_api_client(config: &GenericConfiguration) -> Result { + DataPlaneAPIClient::from_config_with_ipc_tls(config) .await - .error_context("Failed to create authenticated data plane API client") + .error_context("Failed to create IPC TLS data plane API client") } async fn handle_dogstatsd_stats(api_client: &mut DataPlaneAPIClient, cmd: StatsCommand) -> Result<(), GenericError> { diff --git a/bin/agent-data-plane/src/cli/run.rs b/bin/agent-data-plane/src/cli/run.rs index 24e4e57ce45..3e4102ecbf8 100644 --- a/bin/agent-data-plane/src/cli/run.rs +++ b/bin/agent-data-plane/src/cli/run.rs @@ -7,7 +7,7 @@ use std::{ use agent_data_plane_config_system::{ConfigurationSystem, EnvPrecedence, LoadedConfiguration}; use argh::FromArgs; -use datadog_agent_commons::{ipc::config::IpcAuthConfiguration, platform::PlatformSettings}; +use datadog_agent_commons::platform::PlatformSettings; use datadog_agent_config::classifier::{ConfigClassifier, Pipeline, PipelineAffinity, Severity, SupportLevel}; use saluki_app::{ accounting::{initialize_memory_bounds, MemoryBoundsConfiguration}, @@ -679,25 +679,15 @@ async fn add_baseline_traces_pipeline_to_blueprint( Ok(()) } -async fn build_dogstatsd_context_dump_api_handler( +fn build_dogstatsd_context_dump_api_handler( config: &GenericConfiguration, snapshot_handle: AggregateContextSnapshotHandle, ) -> Result { - let ipc_config = IpcAuthConfiguration::from_configuration(config)?; - let auth_token_path = ipc_config.auth_token_file_path(); - let auth_token = tokio::fs::read(&auth_token_path).await.map_err(|error| { - generic_error!( - "Failed to read Agent authentication token from file '{}' ({}).", - auth_token_path.display(), - error.kind() - ) - })?; let run_path = config .try_get_typed::("run_path") .error_context("Failed to read configured `run_path` for DogStatsD context dumps.")? .unwrap_or_default(); - DogStatsDContextDumpAPIHandler::new(auth_token, vec![snapshot_handle], run_path) - .error_context("Failed to configure DogStatsD context dump API handler.") + Ok(DogStatsDContextDumpAPIHandler::new(vec![snapshot_handle], run_path)) } async fn add_dsd_pipeline_to_blueprint( @@ -782,8 +772,7 @@ async fn add_dsd_pipeline_to_blueprint( let stats_api_handler = dsd_stats_config.api_handler(); let capture_api_handler = dsd_config.capture_api_handler(); let replay_api_handler = dsd_config.replay_api_handler(); - let context_dump_api_handler = - build_dogstatsd_context_dump_api_handler(config, dsd_context_snapshot_handle).await?; + let context_dump_api_handler = build_dogstatsd_context_dump_api_handler(config, dsd_context_snapshot_handle)?; blueprint // Components. @@ -926,10 +915,10 @@ fn write_sizing_guide(bounds: ComponentBounds) -> Result<(), GenericError> { #[cfg(test)] mod tests { - use std::{fs, path::Path, sync::Mutex, time::Duration}; + use std::{path::Path, sync::Mutex, time::Duration}; use async_trait::async_trait; - use http::{header::AUTHORIZATION, Request, StatusCode}; + use http::{Request, StatusCode}; use http_body_util::{BodyExt as _, Empty}; use hyper::body::Bytes; use saluki_api::{response::Response, APIHandler as _}; @@ -965,7 +954,6 @@ mod tests { dogstatsd_contexts::DogStatsDContextDumpAPIHandler, }; - const AUTH_TOKEN: &[u8] = b"configured-agent-token"; const CONTEXT_DUMP_FILENAME: &str = "dogstatsd_contexts.json.zstd"; const CONTEXT_DUMP_ROUTE: &str = crate::dogstatsd_contexts::CONTEXT_DUMP_ROUTE; @@ -1110,15 +1098,12 @@ mod tests { } #[tokio::test] - async fn context_dump_handler_reads_configured_credentials_and_run_path_and_uses_supplied_owner() { + async fn context_dump_handler_reads_configured_run_path_and_uses_supplied_owner() { let run_directory = tempfile::tempdir().expect("run directory should be created"); - let token_file = run_directory.path().join("auth_token"); - fs::write(&token_file, AUTH_TOKEN).expect("token should be written"); - let config = context_dump_config(Some(run_directory.path()), &token_file).await; + let config = context_dump_config(Some(run_directory.path())).await; let (snapshot_handle, mut snapshot_responder) = aggregate_context_snapshot_channel_for_test(); let handler = build_dogstatsd_context_dump_api_handler(&config, snapshot_handle) - .await .expect("configured handler should build"); let owner = tokio::spawn(async move { snapshot_responder @@ -1126,7 +1111,7 @@ mod tests { .await }); - let response = send(&handler, authorized_post()).await; + let response = send(&handler, context_dump_post()).await; assert_eq!(response.status(), StatusCode::OK); let artifact_path = run_directory.path().join(CONTEXT_DUMP_FILENAME); @@ -1139,64 +1124,18 @@ mod tests { } #[tokio::test] - async fn context_dump_handler_reports_missing_and_unreadable_token_paths_with_io_kind() { - let directory = tempfile::tempdir().expect("temporary directory should be created"); - let cases = [directory.path().join("missing-token"), directory.path().to_owned()]; - - for token_path in cases { - let expected_kind = tokio::fs::read(&token_path) - .await - .expect_err("token fixture should be unreadable") - .kind(); - let config = context_dump_config(Some(directory.path()), &token_path).await; - let (snapshot_handle, _snapshot_responder) = aggregate_context_snapshot_channel_for_test(); - - let error = match build_dogstatsd_context_dump_api_handler(&config, snapshot_handle).await { - Ok(_) => panic!("unreadable token should fail handler construction"), - Err(error) => error, - }; - let message = format!("{error:#}"); - assert!(message.contains(&token_path.display().to_string()), "{message}"); - assert!(message.contains(&expected_kind.to_string()), "{message}"); - } - } - - #[tokio::test] - async fn context_dump_handler_rejects_empty_and_invalid_raw_token_contents() { - let directory = tempfile::tempdir().expect("temporary directory should be created"); - let token_file = directory.path().join("auth_token"); - - for token in [b"".as_slice(), b"token-with-newline\n".as_slice()] { - fs::write(&token_file, token).expect("token fixture should be written"); - let config = context_dump_config(Some(directory.path()), &token_file).await; - let (snapshot_handle, _snapshot_responder) = aggregate_context_snapshot_channel_for_test(); - - if build_dogstatsd_context_dump_api_handler(&config, snapshot_handle) - .await - .is_ok() - { - panic!("invalid raw token should fail closed"); - } - } - } - - #[tokio::test] - async fn context_dump_handler_keeps_missing_and_empty_run_path_empty_until_authorized_publication() { - let directory = tempfile::tempdir().expect("temporary directory should be created"); - let token_file = directory.path().join("auth_token"); - fs::write(&token_file, AUTH_TOKEN).expect("token should be written"); + async fn context_dump_handler_keeps_missing_and_empty_run_path_empty_until_publication() { let cwd_artifact = std::env::current_dir().unwrap().join(CONTEXT_DUMP_FILENAME); assert!(!cwd_artifact.exists(), "test requires no pre-existing cwd artifact"); for run_path in [None, Some(Path::new(""))] { - let config = context_dump_config(run_path, &token_file).await; + let config = context_dump_config(run_path).await; let (snapshot_handle, mut snapshot_responder) = aggregate_context_snapshot_channel_for_test(); let handler = build_dogstatsd_context_dump_api_handler(&config, snapshot_handle) - .await - .expect("empty run path should not weaken authentication setup"); + .expect("empty run path should reach publication"); let owner = tokio::spawn(async move { snapshot_responder.respond(Vec::new()).await }); - let response = send(&handler, authorized_post()).await; + let response = send(&handler, context_dump_post()).await; assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); assert!(!cwd_artifact.exists()); @@ -1281,21 +1220,18 @@ mod tests { fn specify_bounds(&self, _builder: &mut MemoryBoundsBuilder) {} } - async fn context_dump_config(run_path: Option<&Path>, token_path: &Path) -> GenericConfiguration { - let mut values = json!({ - "auth_token_file_path": token_path, - }); + async fn context_dump_config(run_path: Option<&Path>) -> GenericConfiguration { + let mut values = json!({}); if let Some(run_path) = run_path { values["run_path"] = json!(run_path); } config_from(values).await } - fn authorized_post() -> Request> { + fn context_dump_post() -> Request> { Request::builder() .method("POST") .uri(CONTEXT_DUMP_ROUTE) - .header(AUTHORIZATION, "Bearer configured-agent-token") .body(Empty::new()) .unwrap() } diff --git a/bin/agent-data-plane/src/cli/utils.rs b/bin/agent-data-plane/src/cli/utils.rs index a53e7319d51..5354a8dbafa 100644 --- a/bin/agent-data-plane/src/cli/utils.rs +++ b/bin/agent-data-plane/src/cli/utils.rs @@ -1,15 +1,8 @@ -use std::{ - path::{Path, PathBuf}, - time::Duration, -}; +use std::{path::PathBuf, time::Duration}; use datadog_agent_commons::ipc::{config::IpcAuthConfiguration, tls::build_ipc_client_ipc_tls_config}; use futures::TryFutureExt as _; -use http::{ - header::{HeaderValue, AUTHORIZATION, CONTENT_TYPE}, - uri::PathAndQuery, - Request, Response, StatusCode, Uri, -}; +use http::{header::CONTENT_TYPE, uri::PathAndQuery, Request, Response, StatusCode, Uri}; use http_body_util::BodyExt as _; #[cfg(target_os = "linux")] use http_body_util::Full; @@ -32,7 +25,6 @@ use crate::{config::DataPlaneConfiguration, dogstatsd_contexts::CONTEXT_DUMP_ROU pub struct DataPlaneAPIClient { client: HttpClient, authority: String, - authorization: Option, } #[derive(Serialize)] @@ -65,49 +57,32 @@ impl DataPlaneAPIClient { let dp_config = DataPlaneConfiguration::from_configuration(config)?; let builder = HttpClient::builder().with_tls_config(|b| b.danger_accept_invalid_certs()); - Self::from_builder(builder, dp_config.secure_api_listen_address(), None) + Self::from_builder(builder, dp_config.secure_api_listen_address()) } - /// Creates an authenticated `DataPlaneAPIClient` for commands that use the Agent IPC credentials. - /// - /// This mirrors the Agent IPC HTTP client contract: verify the configured IPC certificate and attach the Agent - /// authentication token. The complete Rustls configuration is required because it carries a custom certificate - /// verifier and client identity that the generic HTTP TLS option builder cannot express. + /// Creates a `DataPlaneAPIClient` that uses the configured Agent IPC certificate. /// - /// This client pins the privileged API server certificate to the configured IPC certificate and authenticates - /// requests with the raw contents of the configured Agent authentication token file. Unlike the general-purpose - /// client, it does not impose a whole-request timeout because context dump generation can legitimately take longer - /// than the default timeout. + /// The complete Rustls configuration carries the custom server-certificate verifier and client identity needed by + /// Agent IPC TLS. Unlike the general-purpose client, this client does not impose a whole-request timeout because + /// context dump generation can legitimately take longer than the default timeout. /// /// # Errors /// - /// If the data plane or IPC authentication configuration is invalid, the IPC certificate or authentication token - /// cannot be read, the token cannot be represented as an HTTP header, or the HTTP client cannot be built, an error - /// is returned. - pub async fn from_config_with_ipc_auth(config: &GenericConfiguration) -> Result { + /// If the data plane or IPC TLS configuration is invalid, the IPC certificate cannot be read, or the HTTP client + /// cannot be built, an error is returned. + pub async fn from_config_with_ipc_tls(config: &GenericConfiguration) -> Result { let dp_config = DataPlaneConfiguration::from_configuration(config)?; let ipc_config = IpcAuthConfiguration::from_configuration(config)?; let tls_config = build_ipc_client_ipc_tls_config(ipc_config.ipc_cert_file_path()).await?; - let auth_token_path = ipc_config.auth_token_file_path(); - let raw_token = tokio::fs::read(&auth_token_path).await.map_err(|error| { - generic_error!( - "Failed to read Agent authentication token from file '{}' ({}).", - auth_token_path.display(), - error.kind() - ) - })?; - let authorization = authorization_from_token_file_contents(&raw_token, &auth_token_path)?; let builder = HttpClient::builder() .with_client_tls_config(tls_config) .with_connect_timeout(Duration::from_secs(20)) .without_request_timeout(); - Self::from_builder(builder, dp_config.secure_api_listen_address(), Some(authorization)) + Self::from_builder(builder, dp_config.secure_api_listen_address()) } - fn from_builder( - builder: HttpClientBuilder, listen_address: &ListenAddress, authorization: Option, - ) -> Result { + fn from_builder(builder: HttpClientBuilder, listen_address: &ListenAddress) -> Result { let (builder, authority) = match listen_address { ListenAddress::Tcp(_) => { let local_address = listen_address @@ -131,11 +106,7 @@ impl DataPlaneAPIClient { .build() .error_context("Failed to construct API client for privileged API endpoint.")?; - Ok(Self { - client, - authority, - authorization, - }) + Ok(Self { client, authority }) } fn build_uri(&self, path: &str, query: Option<&str>) -> Uri { @@ -251,22 +222,17 @@ impl DataPlaneAPIClient { .and_then(body_when_success) } - /// Requests an authenticated DogStatsD context dump and returns its path on the server. + /// Requests a DogStatsD context dump and returns its path on the server. /// /// The response contains only the server-local artifact path; this method does not download the dump contents. /// /// # Errors /// - /// If this client was not created with [`Self::from_config_with_ipc_auth`], the request fails, the server rejects - /// the request, or the successful response does not contain a JSON string path, an error is returned. + /// If the request fails, the server rejects it, or the successful response does not contain a JSON string path, an + /// error is returned. pub async fn dogstatsd_contexts_dump(&mut self) -> Result { - let authorization = self.authorization.as_ref().ok_or_else(|| { - generic_error!( - "DogStatsD context dumps require an authenticated API client; construct it with `from_config_with_ipc_auth`." - ) - })?; let uri = self.build_uri(CONTEXT_DUMP_ROUTE, None); - let request = build_dogstatsd_contexts_dump_request(uri, authorization); + let request = build_dogstatsd_contexts_dump_request(uri); let response = self .client .send(request) @@ -414,30 +380,8 @@ impl DataPlaneAPIClient { } } -fn authorization_from_token_file_contents(raw_token: &[u8], token_path: &Path) -> Result { - if raw_token.is_empty() { - return Err(generic_error!( - "Agent authentication token file '{}' is empty.", - token_path.display() - )); - } - - let mut raw_authorization = Vec::with_capacity("Bearer ".len() + raw_token.len()); - raw_authorization.extend_from_slice(b"Bearer "); - raw_authorization.extend_from_slice(raw_token); - let mut authorization = HeaderValue::from_bytes(&raw_authorization).map_err(|_| { - generic_error!( - "Agent authentication token file '{}' contains characters that are invalid in an HTTP Authorization header.", - token_path.display() - ) - })?; - authorization.set_sensitive(true); - Ok(authorization) -} - -fn build_dogstatsd_contexts_dump_request(uri: Uri, authorization: &HeaderValue) -> Request { +fn build_dogstatsd_contexts_dump_request(uri: Uri) -> Request { Request::post(uri) - .header(AUTHORIZATION, authorization.clone()) .body(String::new()) .expect("valid DogStatsD context dump request") } @@ -446,10 +390,6 @@ fn path_when_context_dump_success(resp: Response) -> Result serde_json::from_str::(resp.body()) .error_context("Failed to decode DogStatsD context dump path from response JSON."), - StatusCode::UNAUTHORIZED => Err(generic_error!( - "DogStatsD context dump authentication failed ({}); verify the configured Agent authentication token file.", - resp.status() - )), status => Err(generic_error!( "Failed to create DogStatsD context dump ({}): {}.", status, @@ -543,21 +483,17 @@ fn empty_when_replay_session_success(resp: Response) -> Result<(), Gener #[cfg(test)] mod tests { - use std::{ - path::{Path, PathBuf}, - sync::Arc, - time::Duration, - }; + use std::{path::PathBuf, sync::Arc, time::Duration}; use datadog_agent_commons::ipc::tls::{build_ipc_client_ipc_tls_config, build_ipc_server_tls_config}; - use http::{header::AUTHORIZATION, HeaderValue, Method, Response, StatusCode, Uri}; + use http::{Method, Response, StatusCode, Uri}; use rcgen::{generate_simple_self_signed, CertifiedKey}; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio_rustls::TlsAcceptor; use super::{ - authorization_from_token_file_contents, body_when_capture_success, build_dogstatsd_contexts_dump_request, - path_when_context_dump_success, DataPlaneAPIClient, + body_when_capture_success, build_dogstatsd_contexts_dump_request, path_when_context_dump_success, + DataPlaneAPIClient, }; #[cfg(target_os = "linux")] use super::{body_when_replay_session_success, empty_when_replay_session_success}; @@ -623,62 +559,15 @@ mod tests { } #[test] - fn dogstatsd_contexts_authorization_preserves_raw_token_and_is_sensitive() { - let authorization = authorization_from_token_file_contents(b" token with spaces ", Path::new("token-file")) - .expect("visible token bytes should be accepted"); - - assert_eq!(authorization, "Bearer token with spaces "); - assert!(authorization.is_sensitive()); - } - - #[test] - fn dogstatsd_contexts_authorization_rejects_empty_or_invalid_token_without_exposing_it() { - let path = Path::new("/private/auth_token"); - let empty_error = authorization_from_token_file_contents(b"", path).expect_err("empty token must fail"); - assert!(empty_error.to_string().contains("/private/auth_token")); - - let invalid_error = - authorization_from_token_file_contents(b"secret\ntoken", path).expect_err("newline in token must fail"); - let message = invalid_error.to_string(); - assert!(message.contains("/private/auth_token")); - assert!(!message.contains("secret")); - } - - #[test] - fn dogstatsd_contexts_request_is_authenticated_empty_post() { - let mut auth = HeaderValue::from_static("Bearer exact-token"); - auth.set_sensitive(true); + fn dogstatsd_contexts_request_is_empty_post_without_authorization() { let uri = Uri::from_static("https://127.0.0.1:5101/agent/dogstatsd-contexts-dump"); - let request = build_dogstatsd_contexts_dump_request(uri, &auth); + let request = build_dogstatsd_contexts_dump_request(uri); assert_eq!(request.method(), Method::POST); assert_eq!(request.uri().path(), CONTEXT_DUMP_ROUTE); assert!(request.body().is_empty()); - let request_auth = request.headers().get(AUTHORIZATION).expect("authorization header"); - assert_eq!(request_auth, "Bearer exact-token"); - assert!(request_auth.is_sensitive()); - } - - #[tokio::test] - async fn dogstatsd_contexts_unauthenticated_client_errors_before_network() { - let _ = saluki_tls::initialize_default_crypto_provider(); - let client = saluki_io::net::client::http::HttpClient::builder() - .with_tls_config(|builder| builder.danger_accept_invalid_certs()) - .build() - .expect("test client should build"); - let mut client = DataPlaneAPIClient { - client, - authority: "unresolvable.invalid:1".to_string(), - authorization: None, - }; - - let error = client - .dogstatsd_contexts_dump() - .await - .expect_err("an unauthenticated client must be rejected"); - - assert!(error.to_string().contains("authenticated")); + assert!(request.headers().is_empty()); } #[test] @@ -705,22 +594,6 @@ mod tests { assert!(error.to_string().contains("DogStatsD context dump path")); } - #[test] - fn dogstatsd_contexts_unauthorized_error_is_safe() { - let response = Response::builder() - .status(StatusCode::UNAUTHORIZED) - .body("rejected Bearer exact-token".to_string()) - .expect("valid response"); - - let error = path_when_context_dump_success(response).expect_err("unauthorized response should fail"); - let message = error.to_string(); - - assert!(message.contains("authentication")); - assert!(message.contains("401")); - assert!(!message.contains("rejected")); - assert!(!message.contains("exact-token")); - } - #[test] fn dogstatsd_contexts_server_errors_include_status_and_body() { for status in [ @@ -788,8 +661,8 @@ mod tests { assert!( request .lines() - .any(|line| line.eq_ignore_ascii_case("authorization: Bearer exact-token")), - "request must contain the exact bearer header" + .all(|line| !line.to_ascii_lowercase().starts_with("authorization:")), + "request must not contain an authorization header" ); let body = r#""/tmp/dogstatsd_contexts.json.zstd""#; @@ -806,11 +679,7 @@ mod tests { (address, task) } - fn authenticated_test_client( - address: std::net::SocketAddr, tls_config: rustls::ClientConfig, - ) -> DataPlaneAPIClient { - let mut authorization = HeaderValue::from_static("Bearer exact-token"); - authorization.set_sensitive(true); + fn ipc_tls_test_client(address: std::net::SocketAddr, tls_config: rustls::ClientConfig) -> DataPlaneAPIClient { let client = saluki_io::net::client::http::HttpClient::builder() .with_client_tls_config(tls_config) .with_connect_timeout(Duration::from_secs(2)) @@ -820,7 +689,6 @@ mod tests { DataPlaneAPIClient { client, authority: format!("localhost:{}", address.port()), - authorization: Some(authorization), } } @@ -838,7 +706,7 @@ mod tests { let matching_tls = build_ipc_client_ipc_tls_config(&certificate_a) .await .expect("matching client TLS configuration should build"); - let mut matching_client = authenticated_test_client(matching_address, matching_tls); + let mut matching_client = ipc_tls_test_client(matching_address, matching_tls); let path = matching_client .dogstatsd_contexts_dump() .await @@ -854,7 +722,7 @@ mod tests { let mismatched_tls = build_ipc_client_ipc_tls_config(&certificate_b) .await .expect("mismatched client TLS configuration should build"); - let mut mismatched_client = authenticated_test_client(mismatched_address, mismatched_tls); + let mut mismatched_client = ipc_tls_test_client(mismatched_address, mismatched_tls); let error = mismatched_client .dogstatsd_contexts_dump() .await diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs index 90f67a22be3..96191417aa9 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/api.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use std::time::Duration; use futures::future::try_join_all; -use http::{header::AUTHORIZATION, HeaderMap, StatusCode}; +use http::StatusCode; use saluki_api::{ extract::State, response::{IntoResponse as _, Response}, @@ -11,8 +11,7 @@ use saluki_api::{ APIHandler, Json, }; use saluki_components::transforms::{AggregateContextSnapshotEntry, AggregateContextSnapshotHandle}; -use saluki_error::{generic_error, GenericError}; -use subtle::ConstantTimeEq as _; +use saluki_error::GenericError; use tokio::sync::Mutex; use super::{publish_context_dump, CONTEXT_DUMP_ROUTE}; @@ -20,7 +19,6 @@ use super::{publish_context_dump, CONTEXT_DUMP_ROUTE}; // The Agent has no owner-request timeout. ADP uses a finite bound so a stopped topology cannot hold an HTTP request // indefinitely; 30 seconds is several orders of magnitude above the measured one-million-context snapshot time. const PRODUCTION_SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(30); -const AUTHENTICATION_REQUIRED: &str = "Authentication required."; const SNAPSHOT_UNAVAILABLE: &str = "DogStatsD context snapshot is unavailable; retry after the aggregate owner is running."; const SNAPSHOT_TIMED_OUT: &str = "Timed out waiting for the DogStatsD context snapshot; retry the request."; @@ -30,63 +28,6 @@ const PUBLICATION_TASK_FAILED: &str = "DogStatsD context dump publication did no const NON_UTF8_RESULT_PATH: &str = "The completed DogStatsD context dump path is not valid UTF-8; configure a UTF-8 run path."; -#[derive(Clone)] -struct AuthenticationSecret(Arc<[u8]>); - -impl AuthenticationSecret { - fn new(token: &[u8]) -> Result { - if token.is_empty() { - return Err(generic_error!("Agent authentication token must not be empty.")); - } - if !is_valid_bearer_token(token) { - return Err(generic_error!( - "Agent authentication token cannot be represented as an HTTP bearer credential." - )); - } - Ok(Self(Arc::from(token))) - } - - fn authenticates(&self, headers: &HeaderMap) -> bool { - let mut authorization_values = headers.get_all(AUTHORIZATION).iter(); - let Some(authorization) = authorization_values.next() else { - return false; - }; - if authorization_values.next().is_some() { - return false; - } - - let value = authorization.as_bytes(); - let Some(separator) = value.iter().position(|byte| *byte == b' ') else { - return false; - }; - if !value[..separator].eq_ignore_ascii_case(b"bearer") { - return false; - } - let supplied = &value[separator..]; - let Some(first_credential_byte) = supplied.iter().position(|byte| *byte != b' ') else { - return false; - }; - let supplied = &supplied[first_credential_byte..]; - if supplied.len() != self.0.len() { - return false; - } - - bool::from(supplied.ct_eq(self.0.as_ref())) - } -} - -fn is_valid_bearer_token(token: &[u8]) -> bool { - let core_length = token - .iter() - .rposition(|byte| *byte != b'=') - .map_or(0, |index| index + 1); - core_length > 0 - && token[..core_length] - .iter() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~' | b'+' | b'/')) - && token[core_length..].iter().all(|byte| *byte == b'=') -} - #[derive(Clone)] struct ContextSnapshotCoordinator { handles: Arc<[AggregateContextSnapshotHandle]>, @@ -147,7 +88,6 @@ impl ContextDumpPublisher for FileSystemContextDumpPublisher { #[derive(Clone)] pub(crate) struct DogStatsDContextDumpAPIState { - auth_secret: AuthenticationSecret, coordinator: ContextSnapshotCoordinator, run_path: PathBuf, publication_lock: Arc>, @@ -160,11 +100,8 @@ pub(crate) struct DogStatsDContextDumpAPIHandler { } impl DogStatsDContextDumpAPIHandler { - pub(crate) fn new( - auth_token: impl AsRef<[u8]>, handles: Vec, run_path: impl Into, - ) -> Result { + pub(crate) fn new(handles: Vec, run_path: impl Into) -> Self { Self::new_with_services( - auth_token.as_ref(), handles, run_path.into(), PRODUCTION_SNAPSHOT_TIMEOUT, @@ -174,33 +111,27 @@ impl DogStatsDContextDumpAPIHandler { #[cfg(test)] fn new_for_test( - auth_token: impl AsRef<[u8]>, handles: Vec, run_path: PathBuf, - snapshot_timeout: Duration, publisher: Arc, - ) -> Result { - Self::new_with_services(auth_token.as_ref(), handles, run_path, snapshot_timeout, publisher) + handles: Vec, run_path: PathBuf, snapshot_timeout: Duration, + publisher: Arc, + ) -> Self { + Self::new_with_services(handles, run_path, snapshot_timeout, publisher) } fn new_with_services( - auth_token: &[u8], handles: Vec, run_path: PathBuf, snapshot_timeout: Duration, + handles: Vec, run_path: PathBuf, snapshot_timeout: Duration, publisher: Arc, - ) -> Result { - let auth_secret = AuthenticationSecret::new(auth_token)?; - Ok(Self { + ) -> Self { + Self { state: DogStatsDContextDumpAPIState { - auth_secret, coordinator: ContextSnapshotCoordinator::new(handles, snapshot_timeout), run_path, publication_lock: Arc::new(Mutex::new(())), publisher, }, - }) - } - - async fn dump_handler(State(state): State, headers: HeaderMap) -> Response { - if !state.auth_secret.authenticates(&headers) { - return (StatusCode::UNAUTHORIZED, AUTHENTICATION_REQUIRED).into_response(); } + } + async fn dump_handler(State(state): State) -> Response { let publication_guard = state.publication_lock.clone().lock_owned().await; let snapshot = match state.coordinator.snapshot().await { Ok(snapshot) => snapshot, diff --git a/bin/agent-data-plane/src/dogstatsd_contexts/api_tests.rs b/bin/agent-data-plane/src/dogstatsd_contexts/api_tests.rs index 305e3e1fe6b..557b223feca 100644 --- a/bin/agent-data-plane/src/dogstatsd_contexts/api_tests.rs +++ b/bin/agent-data-plane/src/dogstatsd_contexts/api_tests.rs @@ -6,10 +6,7 @@ use std::sync::{ }; use std::time::Duration; -use http::{ - header::{AUTHORIZATION, CONTENT_TYPE}, - HeaderValue, Request, StatusCode, -}; +use http::{header::CONTENT_TYPE, Request, StatusCode}; use http_body_util::{BodyExt as _, Empty}; use hyper::body::Bytes; use saluki_api::{response::Response, APIHandler}; @@ -31,98 +28,9 @@ use crate::dogstatsd_contexts::{ publish_context_dump, }; -const AUTH_TOKEN: &str = "expected-agent-token"; +const SENSITIVE_PUBLISHER_DETAIL: &str = "sensitive-publisher-detail"; const ROUTE: &str = super::CONTEXT_DUMP_ROUTE; -#[test] -fn constructor_rejects_empty_and_non_visible_http_tokens_without_exposing_them() { - let invalid_tokens: &[&[u8]] = &[ - b"", - b"contains space", - b"line\nbreak", - b"tab\tvalue", - b"nul\0value", - b"\x7f", - b"\xff", - ]; - - for token in invalid_tokens { - let error = match DogStatsDContextDumpAPIHandler::new(*token, Vec::new(), PathBuf::from("run")) { - Ok(_) => panic!("invalid authentication token should be rejected"), - Err(error) => error, - }; - let message = format!("{error:#}"); - if !token.is_empty() { - assert!(!message.as_bytes().windows(token.len()).any(|window| window == *token)); - } - } -} - -#[tokio::test] -async fn rejects_missing_duplicate_malformed_and_wrong_authorization_with_an_exact_safe_body() { - let publisher = Arc::new(NoOpPublisher); - let handler = test_handler(Vec::new(), PathBuf::from("unused"), publisher); - let mut duplicate = post_request(None); - duplicate - .headers_mut() - .append(AUTHORIZATION, HeaderValue::from_static("Bearer expected-agent-token")); - duplicate - .headers_mut() - .append(AUTHORIZATION, HeaderValue::from_static("Bearer expected-agent-token")); - let mut non_utf8 = post_request(None); - non_utf8.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from_bytes(b"Bearer \xff").expect("HTTP headers allow opaque non-ASCII bytes"), - ); - let cases = [ - post_request(None), - duplicate, - non_utf8, - post_request(Some("Basic expected-agent-token")), - post_request(Some("Bearer")), - post_request(Some("Bearersecret")), - post_request(Some("Bearer\texpected-agent-token")), - post_request(Some("Bearer expected-agent-token trailing")), - post_request(Some("Bearer supplied-secret")), - post_request(Some("Bearer unexpect-agent-token")), - ]; - - for request in cases { - let response = send(&handler, request).await; - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - let body = response_body(response).await; - assert_eq!(body, "Authentication required."); - assert!(!body.contains(AUTH_TOKEN)); - assert!(!body.contains("supplied-secret")); - } -} - -#[tokio::test] -async fn accepts_bearer_scheme_case_insensitively() { - for scheme in ["bearer", "BEARER", "BeArEr"] { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - - let response = send(&handler, post_request(Some(&format!("{scheme} {AUTH_TOKEN}")))).await; - - assert_eq!(response.status(), StatusCode::OK); - owner.await.unwrap().unwrap(); - } -} - -#[tokio::test] -async fn accepts_only_the_exact_bearer_credential_bytes() { - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - - let response = send(&handler, post_request(Some("Bearer expected-agent-token"))).await; - - assert_eq!(response.status(), StatusCode::OK); - owner.await.unwrap().unwrap(); -} - #[tokio::test] async fn coordinator_reuses_the_one_owner_snapshot_allocation() { let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); @@ -233,31 +141,15 @@ async fn generated_router_rejects_get_with_method_not_allowed() { } #[tokio::test] -async fn unauthorized_post_does_not_request_a_snapshot_or_create_an_artifact() { - let run_directory = tempfile::tempdir().unwrap(); - let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_directory.path().to_owned()) - .expect("handler should be valid"); - - let response = send(&handler, post_request(None)).await; - - assert_eq!(response.status(), StatusCode::UNAUTHORIZED); - assert!(!run_directory.path().join(CONTEXT_DUMP_FILENAME).exists()); - drop(handler); - assert!(responder.respond(Vec::new()).await.is_err()); -} - -#[tokio::test] -async fn authorized_router_post_returns_only_the_json_path_to_a_complete_decodable_artifact() { +async fn router_post_returns_only_the_json_path_to_a_complete_decodable_artifact() { let run_directory = tempfile::tempdir().unwrap(); let target = run_directory.path().join(CONTEXT_DUMP_FILENAME); let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); let snapshot = vec![snapshot_entry("published.first"), snapshot_entry("published.second")]; - let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN.as_bytes(), vec![handle], run_directory.path()) - .expect("handler should be valid"); + let handler = DogStatsDContextDumpAPIHandler::new(vec![handle], run_directory.path()); let owner = tokio::spawn(async move { responder.respond(snapshot).await }); - let response = send(&handler, post_request(Some("Bearer expected-agent-token"))).await; + let response = send(&handler, context_dump_post()).await; assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.headers().get(CONTENT_TYPE).unwrap(), "application/json"); @@ -278,7 +170,7 @@ async fn authorized_router_post_returns_only_the_json_path_to_a_complete_decodab #[tokio::test] async fn no_handles_and_closed_owner_return_service_unavailable() { let no_handles = test_handler(Vec::new(), PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let response = send(&no_handles, authorized_post()).await; + let response = send(&no_handles, context_dump_post()).await; assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert_eq!( response_body(response).await, @@ -288,7 +180,7 @@ async fn no_handles_and_closed_owner_return_service_unavailable() { let (handle, responder) = aggregate_context_snapshot_channel_for_test(); drop(responder); let closed_owner = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); - let response = send(&closed_owner, authorized_post()).await; + let response = send(&closed_owner, context_dump_post()).await; assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert_eq!( response_body(response).await, @@ -302,7 +194,7 @@ async fn owner_stopped_after_accepting_request_returns_service_unavailable() { let handler = test_handler(vec![handle], PathBuf::from("unused"), Arc::new(NoOpPublisher)); let owner = tokio::spawn(async move { responder.stop_after_receiving().await }); - let response = send(&handler, authorized_post()).await; + let response = send(&handler, context_dump_post()).await; assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); assert_eq!( @@ -316,15 +208,13 @@ async fn owner_stopped_after_accepting_request_returns_service_unavailable() { async fn snapshot_deadline_returns_gateway_timeout() { let (handle, _responder) = aggregate_context_snapshot_channel_for_test(); let handler = DogStatsDContextDumpAPIHandler::new_for_test( - AUTH_TOKEN, vec![handle], PathBuf::from("unused"), Duration::from_millis(10), Arc::new(NoOpPublisher), - ) - .unwrap(); + ); - let response = send(&handler, authorized_post()).await; + let response = send(&handler, context_dump_post()).await; assert_eq!(response.status(), StatusCode::GATEWAY_TIMEOUT); assert_eq!( @@ -341,10 +231,10 @@ async fn empty_and_non_directory_run_paths_reach_publication_and_return_safe_int for run_path in [PathBuf::new(), non_directory] { let (handle, mut responder) = aggregate_context_snapshot_channel_for_test(); - let handler = DogStatsDContextDumpAPIHandler::new(AUTH_TOKEN, vec![handle], run_path).unwrap(); + let handler = DogStatsDContextDumpAPIHandler::new(vec![handle], run_path); let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - let response = send(&handler, authorized_post()).await; + let response = send(&handler, context_dump_post()).await; assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); let body = response_body(response).await; @@ -352,7 +242,7 @@ async fn empty_and_non_directory_run_paths_reach_publication_and_return_safe_int body, "Failed to publish DogStatsD context dump; check the configured run path and permissions." ); - assert!(!body.contains(AUTH_TOKEN)); + assert!(!body.contains(SENSITIVE_PUBLISHER_DETAIL)); owner.await.unwrap().unwrap(); } assert_eq!( @@ -371,7 +261,7 @@ async fn publication_failure_and_blocking_task_panic_return_safe_internal_errors let handler = test_handler(vec![handle], PathBuf::from("unused"), publisher); let owner = tokio::spawn(async move { responder.respond(Vec::new()).await }); - let response = send(&handler, authorized_post()).await; + let response = send(&handler, context_dump_post()).await; assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); let body = response_body(response).await; @@ -380,7 +270,7 @@ async fn publication_failure_and_blocking_task_panic_return_safe_internal_errors || body == "DogStatsD context dump publication did not complete; retry the request." ); assert!(!body.contains("injected")); - assert!(!body.contains(AUTH_TOKEN)); + assert!(!body.contains(SENSITIVE_PUBLISHER_DETAIL)); owner.await.unwrap().unwrap(); } } @@ -411,7 +301,7 @@ fn blocking_publisher_release_guard_releases_and_notifies_on_drop() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn authorized_requests_serialize_fixed_path_publisher_executions() { +async fn requests_serialize_fixed_path_publisher_executions() { let (started_tx, started_rx) = mpsc::sync_channel(1); let release = Arc::new((StdMutex::new(false), Condvar::new())); let publisher = Arc::new(BlockingFirstPublisher { @@ -429,14 +319,14 @@ async fn authorized_requests_serialize_fixed_path_publisher_executions() { }); let first_handler = handler.clone(); - let first = tokio::spawn(async move { send(&first_handler, authorized_post()).await }); + let first = tokio::spawn(async move { send(&first_handler, context_dump_post()).await }); tokio::task::spawn_blocking(move || started_rx.recv_timeout(Duration::from_secs(1))) .await .unwrap() .expect("first publisher should start"); let release_guard = BlockingPublisherReleaseGuard::new(release); let second_handler = handler.clone(); - let second = tokio::spawn(async move { send(&second_handler, authorized_post()).await }); + let second = tokio::spawn(async move { send(&second_handler, context_dump_post()).await }); tokio::time::sleep(Duration::from_millis(25)).await; assert_eq!(publisher.calls.load(Ordering::SeqCst), 1); drop(release_guard); @@ -465,7 +355,7 @@ async fn canceling_after_spawn_blocking_starts_still_completes_atomic_publicatio let handler = test_handler(vec![handle], run_directory.path().to_owned(), publisher); let owner = tokio::spawn(async move { responder.respond(vec![snapshot_entry("after.cancel")]).await }); let request_handler = handler.clone(); - let request = tokio::spawn(async move { send(&request_handler, authorized_post()).await }); + let request = tokio::spawn(async move { send(&request_handler, context_dump_post()).await }); tokio::task::spawn_blocking(move || started_rx.recv_timeout(Duration::from_secs(1))) .await .unwrap() @@ -497,20 +387,11 @@ async fn canceling_after_spawn_blocking_starts_still_completes_atomic_publicatio fn test_handler( handles: Vec, run_path: PathBuf, publisher: Arc, ) -> DogStatsDContextDumpAPIHandler { - DogStatsDContextDumpAPIHandler::new_for_test(AUTH_TOKEN, handles, run_path, Duration::from_millis(100), publisher) - .expect("test handler should be valid") -} - -fn post_request(authorization: Option<&str>) -> Request> { - let mut builder = Request::builder().method("POST").uri(ROUTE); - if let Some(authorization) = authorization { - builder = builder.header(AUTHORIZATION, authorization); - } - builder.body(Empty::new()).unwrap() + DogStatsDContextDumpAPIHandler::new_for_test(handles, run_path, Duration::from_millis(100), publisher) } -fn authorized_post() -> Request> { - post_request(Some("Bearer expected-agent-token")) +fn context_dump_post() -> Request> { + Request::builder().method("POST").uri(ROUTE).body(Empty::new()).unwrap() } async fn send(handler: &DogStatsDContextDumpAPIHandler, request: Request>) -> Response { @@ -559,7 +440,9 @@ impl ContextDumpPublisher for FailingPublisher { fn publish( &self, _run_path: PathBuf, _snapshot: Vec, ) -> Result { - Err(generic_error!("injected publisher failure with {AUTH_TOKEN}")) + Err(generic_error!( + "injected publisher failure with {SENSITIVE_PUBLISHER_DETAIL}" + )) } } diff --git a/docs/agent-data-plane/dogstatsd-top.md b/docs/agent-data-plane/dogstatsd-top.md index 8b9674a6443..6dddd0eabe9 100644 --- a/docs/agent-data-plane/dogstatsd-top.md +++ b/docs/agent-data-plane/dogstatsd-top.md @@ -21,7 +21,7 @@ Wrote /var/run/datadog/dogstatsd_contexts.json.zstd 3 request.count (2 env, 1 service) ``` -Online requests read the configured Agent IPC authentication token and pin the API server certificate to the configured IPC certificate. The API creates and returns only the server-local path `/dogstatsd_contexts.json.zstd`. +Online requests pin the privileged API server certificate to the configured Agent IPC certificate. The context-dump route does not apply separate bearer-token authentication; it relies on the privileged API access-control boundary. The API creates and returns only the server-local path `/dogstatsd_contexts.json.zstd`. The API does not return the artifact contents. The CLI reads the returned path directly, so the CLI process and ADP must see the same filesystem and path. If ADP runs in a container, run the CLI with the same mount or use the copy workflow in the next section. The API does not provide a remote download endpoint. @@ -59,7 +59,7 @@ You can combine `--path` (`-p`) with the metric and tag limits: $ agent-data-plane dogstatsd top -p /secure/cases/incident-123/dogstatsd_contexts.json.zstd -m 25 -t 10 ``` -Offline mode reads only the supplied file. It does not contact ADP or read the Agent IPC authentication token or certificate. `top` requires the `--path` value and does not fall back to a file in the current working directory. +Offline mode reads only the supplied file. It does not contact ADP or read the Agent IPC certificate. `top` requires the `--path` value and does not fall back to a file in the current working directory. ## Interpret context and tag counts @@ -90,7 +90,7 @@ If ADP uses multiple aggregate owners in the future, each owner's snapshot remai Use the status or file error in the CLI output to choose a response: -- **401 Unauthorized**: The configured Agent authentication token does not match the server token. Verify that the CLI and ADP use the same current token file. Online commands also fail before the request if they cannot read the token or certificate, or if certificate pinning rejects the server certificate. +- **TLS connection failure**: Online commands fail before the request if they cannot read the configured IPC certificate or if certificate pinning rejects the server certificate. - **404 Not Found**: The context-dump route is absent when DogStatsD is disabled. Also verify that the CLI targets the intended ADP privileged API endpoint. - **503 Service Unavailable**: The aggregate snapshot owner is not running or stopped before it responded. Wait for the topology and `dsd_agg` to become healthy, then retry. - **504 Gateway Timeout**: The aggregate owner did not return a snapshot within the 30-second API snapshot deadline. Check ADP health and load before retrying. diff --git a/test/integration/cases/dogstatsd-top-windows/config.yaml b/test/integration/cases/dogstatsd-top-windows/config.yaml index 6df65f03474..d69d4618ec1 100644 --- a/test/integration/cases/dogstatsd-top-windows/config.yaml +++ b/test/integration/cases/dogstatsd-top-windows/config.yaml @@ -1,6 +1,6 @@ type: integration name: "dogstatsd-top-windows" -description: "Verifies the authenticated DogStatsD top workflow and protected context artifact on Windows" +description: "Verifies the privileged DogStatsD top workflow and context artifact replacement on Windows" timeout: 180s runtimes: [windows] @@ -59,18 +59,6 @@ procedure: return $Output -join "`n" } - function Assert-UnauthorizedRequest { - try { - Invoke-WebRequest -Uri $ApiUrl -Method Post -SkipCertificateCheck -TimeoutSec 10 | Out-Null - throw "Context dump API accepted an unauthenticated request." - } catch { - $Response = $_.Exception.Response - if ($null -eq $Response -or [int] $Response.StatusCode -ne 401) { - throw - } - } - } - function Send-DogStatsDContexts { $Payload = @( "integration.windows.top.requests:1|c|#host:windows-a,env:prod,instance:a", @@ -113,7 +101,6 @@ procedure: return $Path } - Assert-UnauthorizedRequest Send-DogStatsDContexts $Online = Wait-ForOnlineReport if (-not $Online.StartsWith("Wrote ")) { diff --git a/test/integration/cases/dogstatsd-top/config.yaml b/test/integration/cases/dogstatsd-top/config.yaml index 70f6126e719..22b0ef6b0fa 100644 --- a/test/integration/cases/dogstatsd-top/config.yaml +++ b/test/integration/cases/dogstatsd-top/config.yaml @@ -1,6 +1,6 @@ type: integration name: "dogstatsd-top" -description: "Verifies retained DogStatsD contexts through the authenticated API, online CLI, and offline Agent-compatible artifact" +description: "Verifies retained DogStatsD contexts through the privileged API, online CLI, and offline Agent-compatible artifact" timeout: 180s runtimes: [linux] diff --git a/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py b/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py index c62294d3a24..2e467c4e0bf 100644 --- a/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py +++ b/test/integration/cases/dogstatsd-top/run_dogstatsd_top_test.py @@ -7,14 +7,12 @@ import stat import subprocess import time -import urllib.error import urllib.request from pathlib import Path ADP = Path("/opt/datadog-agent/embedded/bin/agent-data-plane") AGENT = Path("/opt/datadog-agent/bin/agent/agent") CONFIG = Path("/etc/datadog-agent/datadog.yaml") -AUTH_TOKEN = Path("/etc/datadog-agent/auth_token") API_URL = "https://127.0.0.1:55101/agent/dogstatsd-contexts-dump" DUMP_FILENAME = "dogstatsd_contexts.json.zstd" COPIED_DUMP = Path("/tmp/dogstatsd-top-copied.json.zstd") @@ -50,25 +48,11 @@ def unverified_tls_context(): return context -def post_dump(authorization=None): - headers = {} - if authorization is not None: - headers["Authorization"] = f"Bearer {authorization}" - request = urllib.request.Request(API_URL, data=b"", headers=headers, method="POST") +def post_dump(): + request = urllib.request.Request(API_URL, data=b"", method="POST") return urllib.request.urlopen(request, context=unverified_tls_context(), timeout=10) -def assert_unauthorized_request(): - try: - post_dump() - except urllib.error.HTTPError as error: - body = error.read().decode("utf-8") - if error.code != 401 or body != "Authentication required.": - raise AssertionError(f"unexpected unauthenticated response: {error.code} {body!r}") - else: - raise AssertionError("context dump API accepted an unauthenticated request") - - def send_dogstatsd_contexts(): payload = b"\n".join( [ @@ -109,16 +93,16 @@ def normalize_report(output): return output.replace("\r\n", "\n").rstrip("\n") -def validate_online_api(token): - with post_dump(token) as response: +def validate_online_api(): + with post_dump() as response: body = response.read().decode("utf-8") if response.status != 200: - raise AssertionError(f"authenticated API returned {response.status}: {body}") + raise AssertionError(f"context dump API returned {response.status}: {body}") if not response.headers.get_content_type() == "application/json": raise AssertionError(f"unexpected API content type: {response.headers.get('Content-Type')}") path = Path(json.loads(body)) if path.name != DUMP_FILENAME or not path.is_file(): - raise AssertionError(f"authenticated API returned an invalid artifact path: {path}") + raise AssertionError(f"context dump API returned an invalid artifact path: {path}") def validate_agent_fixture_interoperability(): @@ -152,19 +136,17 @@ def validate_dump_and_offline_interoperability(): def main(): - for required_path in [ADP, AGENT, CONFIG, AUTH_TOKEN, AGENT_FIXTURE_COMPRESSED, AGENT_FIXTURE_PLAIN]: + for required_path in [ADP, AGENT, CONFIG, AGENT_FIXTURE_COMPRESSED, AGENT_FIXTURE_PLAIN]: if not required_path.exists(): raise AssertionError(f"required test path does not exist: {required_path}") - assert_unauthorized_request() validate_agent_fixture_interoperability() send_dogstatsd_contexts() online = wait_for_online_report() if not online.startswith("Wrote "): raise AssertionError(f"online top did not print the generated artifact path:\n{online}") - token = AUTH_TOKEN.read_text(encoding="utf-8") - validate_online_api(token) + validate_online_api() validate_dump_and_offline_interoperability() RESULT.write_text("passed\n", encoding="utf-8") From dcd2a8efa2b1689b7c3b46ef60b15fba1ce21eff Mon Sep 17 00:00:00 2001 From: Travis Thieman Date: Tue, 28 Jul 2026 15:34:13 -0400 Subject: [PATCH 44/44] refactor(adp): use existing privileged API client Route DogStatsD top and dump-contexts through the same DataPlaneAPIClient path as existing privileged endpoints. Remove all mTLS preparation, certificate pinning support, TLS tests, and dependencies so the separate privileged-API mTLS PR owns that change surface. --- Cargo.lock | 4 - bin/agent-data-plane/Cargo.toml | 11 -- bin/agent-data-plane/src/cli/dogstatsd.rs | 10 +- bin/agent-data-plane/src/cli/utils.rs | 168 +------------------- docs/agent-data-plane/dogstatsd-top.md | 6 +- lib/saluki-io/src/net/client/http/client.rs | 81 +--------- 6 files changed, 15 insertions(+), 265 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c05104ca31e..dd807b985f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,8 +47,6 @@ dependencies = [ "process-memory", "prost", "prost-types", - "rcgen", - "rustls", "saluki-antithesis", "saluki-api", "saluki-app", @@ -62,7 +60,6 @@ dependencies = [ "saluki-io", "saluki-metadata", "saluki-metrics", - "saluki-tls", "serde", "serde_json", "serde_with", @@ -70,7 +67,6 @@ dependencies = [ "tempfile", "tikv-jemallocator", "tokio", - "tokio-rustls", "tokio-util", "tonic", "tower", diff --git a/bin/agent-data-plane/Cargo.toml b/bin/agent-data-plane/Cargo.toml index bc7c7452ba6..a28c6a94f03 100644 --- a/bin/agent-data-plane/Cargo.toml +++ b/bin/agent-data-plane/Cargo.toml @@ -57,7 +57,6 @@ serde_with = { workspace = true } stringtheory = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = [ - "fs", "macros", "net", "rt", @@ -85,18 +84,8 @@ chrono = { workspace = true } [dev-dependencies] datadog-agent-config-testing = { workspace = true } derive-where = { workspace = true } -rcgen = { workspace = true, features = ["crypto", "pem"] } -rustls = { workspace = true } saluki-components = { workspace = true, features = ["test-util"] } saluki-config = { workspace = true, features = ["test-util"] } saluki-core = { workspace = true, features = ["test-util"] } saluki-metrics = { workspace = true, features = ["test"] } -saluki-tls = { workspace = true } -tokio-rustls = { workspace = true } tower = { workspace = true, features = ["util"] } - -[target.'cfg(not(windows))'.dev-dependencies] -rcgen = { workspace = true, features = ["aws_lc_rs"] } - -[target.'cfg(windows)'.dev-dependencies] -rcgen = { workspace = true, features = ["ring"] } diff --git a/bin/agent-data-plane/src/cli/dogstatsd.rs b/bin/agent-data-plane/src/cli/dogstatsd.rs index c79a53a2e21..7f89f5e108e 100644 --- a/bin/agent-data-plane/src/cli/dogstatsd.rs +++ b/bin/agent-data-plane/src/cli/dogstatsd.rs @@ -218,12 +218,12 @@ async fn run_dogstatsd_command( if config.is_offline() { handle_dogstatsd_top(None, config, &mut output).await } else { - let mut api_client = ipc_tls_data_plane_api_client(bootstrap_config).await?; + let mut api_client = data_plane_api_client(bootstrap_config)?; handle_dogstatsd_top(Some(&mut api_client), config, &mut output).await } } DogstatsdSubcommand::DumpContexts(_) => { - let mut api_client = ipc_tls_data_plane_api_client(bootstrap_config).await?; + let mut api_client = data_plane_api_client(bootstrap_config)?; let mut output = std::io::stdout(); handle_dogstatsd_dump_contexts(&mut api_client, &mut output).await } @@ -234,12 +234,6 @@ fn data_plane_api_client(config: &GenericConfiguration) -> Result Result { - DataPlaneAPIClient::from_config_with_ipc_tls(config) - .await - .error_context("Failed to create IPC TLS data plane API client") -} - async fn handle_dogstatsd_stats(api_client: &mut DataPlaneAPIClient, cmd: StatsCommand) -> Result<(), GenericError> { // Trigger a statistics collection and wait for it to complete. info!( diff --git a/bin/agent-data-plane/src/cli/utils.rs b/bin/agent-data-plane/src/cli/utils.rs index 5354a8dbafa..fe5aef8f26c 100644 --- a/bin/agent-data-plane/src/cli/utils.rs +++ b/bin/agent-data-plane/src/cli/utils.rs @@ -1,6 +1,5 @@ -use std::{path::PathBuf, time::Duration}; +use std::path::PathBuf; -use datadog_agent_commons::ipc::{config::IpcAuthConfiguration, tls::build_ipc_client_ipc_tls_config}; use futures::TryFutureExt as _; use http::{header::CONTENT_TYPE, uri::PathAndQuery, Request, Response, StatusCode, Uri}; use http_body_util::BodyExt as _; @@ -60,28 +59,6 @@ impl DataPlaneAPIClient { Self::from_builder(builder, dp_config.secure_api_listen_address()) } - /// Creates a `DataPlaneAPIClient` that uses the configured Agent IPC certificate. - /// - /// The complete Rustls configuration carries the custom server-certificate verifier and client identity needed by - /// Agent IPC TLS. Unlike the general-purpose client, this client does not impose a whole-request timeout because - /// context dump generation can legitimately take longer than the default timeout. - /// - /// # Errors - /// - /// If the data plane or IPC TLS configuration is invalid, the IPC certificate cannot be read, or the HTTP client - /// cannot be built, an error is returned. - pub async fn from_config_with_ipc_tls(config: &GenericConfiguration) -> Result { - let dp_config = DataPlaneConfiguration::from_configuration(config)?; - let ipc_config = IpcAuthConfiguration::from_configuration(config)?; - let tls_config = build_ipc_client_ipc_tls_config(ipc_config.ipc_cert_file_path()).await?; - - let builder = HttpClient::builder() - .with_client_tls_config(tls_config) - .with_connect_timeout(Duration::from_secs(20)) - .without_request_timeout(); - Self::from_builder(builder, dp_config.secure_api_listen_address()) - } - fn from_builder(builder: HttpClientBuilder, listen_address: &ListenAddress) -> Result { let (builder, authority) = match listen_address { ListenAddress::Tcp(_) => { @@ -483,18 +460,11 @@ fn empty_when_replay_session_success(resp: Response) -> Result<(), Gener #[cfg(test)] mod tests { - use std::{path::PathBuf, sync::Arc, time::Duration}; + use std::path::PathBuf; - use datadog_agent_commons::ipc::tls::{build_ipc_client_ipc_tls_config, build_ipc_server_tls_config}; use http::{Method, Response, StatusCode, Uri}; - use rcgen::{generate_simple_self_signed, CertifiedKey}; - use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; - use tokio_rustls::TlsAcceptor; - - use super::{ - body_when_capture_success, build_dogstatsd_contexts_dump_request, path_when_context_dump_success, - DataPlaneAPIClient, - }; + + use super::{body_when_capture_success, build_dogstatsd_contexts_dump_request, path_when_context_dump_success}; #[cfg(target_os = "linux")] use super::{body_when_replay_session_success, empty_when_replay_session_success}; use crate::dogstatsd_contexts::CONTEXT_DUMP_ROUTE; @@ -559,7 +529,7 @@ mod tests { } #[test] - fn dogstatsd_contexts_request_is_empty_post_without_authorization() { + fn dogstatsd_contexts_request_is_empty_post() { let uri = Uri::from_static("https://127.0.0.1:5101/agent/dogstatsd-contexts-dump"); let request = build_dogstatsd_contexts_dump_request(uri); @@ -616,132 +586,4 @@ mod tests { ); } } - - fn write_generated_ipc_certificate(path: &std::path::Path) { - let CertifiedKey { cert, signing_key } = - generate_simple_self_signed(["localhost".to_string()]).expect("certificate generation should succeed"); - std::fs::write(path, format!("{}{}", cert.pem(), signing_key.serialize_pem())) - .expect("certificate file should be written"); - } - - async fn start_context_dump_tls_server( - certificate_path: &std::path::Path, - ) -> ( - std::net::SocketAddr, - tokio::task::JoinHandle, Box>>, - ) { - let server_config = build_ipc_server_tls_config(certificate_path) - .await - .expect("server TLS configuration should build"); - let acceptor = TlsAcceptor::from(Arc::new(server_config)); - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("test listener should bind"); - let address = listener.local_addr().expect("test listener should have an address"); - let task = tokio::spawn(async move { - let (stream, _) = listener.accept().await?; - let mut stream = match acceptor.accept(stream).await { - Ok(stream) => stream, - Err(error) => return Ok(Some(error.to_string())), - }; - let mut request = Vec::new(); - loop { - let mut buffer = [0_u8; 1024]; - let read = stream.read(&mut buffer).await?; - if read == 0 { - break; - } - request.extend_from_slice(&buffer[..read]); - if request.windows(4).any(|window| window == b"\r\n\r\n") { - break; - } - } - let request = String::from_utf8(request)?; - assert!(request.starts_with("POST /agent/dogstatsd-contexts-dump HTTP/1.1\r\n")); - assert!( - request - .lines() - .all(|line| !line.to_ascii_lowercase().starts_with("authorization:")), - "request must not contain an authorization header" - ); - - let body = r#""/tmp/dogstatsd_contexts.json.zstd""#; - let response = format!( - "HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", - body.len(), - body - ); - stream.write_all(response.as_bytes()).await?; - stream.shutdown().await?; - Ok(None) - }); - - (address, task) - } - - fn ipc_tls_test_client(address: std::net::SocketAddr, tls_config: rustls::ClientConfig) -> DataPlaneAPIClient { - let client = saluki_io::net::client::http::HttpClient::builder() - .with_client_tls_config(tls_config) - .with_connect_timeout(Duration::from_secs(2)) - .without_request_timeout() - .build() - .expect("test HTTP client should build"); - DataPlaneAPIClient { - client, - authority: format!("localhost:{}", address.port()), - } - } - - #[tokio::test] - async fn dogstatsd_contexts_certificate_pinning_accepts_match_and_rejects_mismatch() { - tokio::time::timeout(Duration::from_secs(10), async { - let _ = saluki_tls::initialize_default_crypto_provider(); - let directory = tempfile::tempdir().expect("temporary directory should be created"); - let certificate_a = directory.path().join("ipc-a.pem"); - let certificate_b = directory.path().join("ipc-b.pem"); - write_generated_ipc_certificate(&certificate_a); - write_generated_ipc_certificate(&certificate_b); - - let (matching_address, matching_server) = start_context_dump_tls_server(&certificate_a).await; - let matching_tls = build_ipc_client_ipc_tls_config(&certificate_a) - .await - .expect("matching client TLS configuration should build"); - let mut matching_client = ipc_tls_test_client(matching_address, matching_tls); - let path = matching_client - .dogstatsd_contexts_dump() - .await - .expect("matching pinned certificate should succeed"); - assert_eq!(path, PathBuf::from("/tmp/dogstatsd_contexts.json.zstd")); - assert!(matching_server - .await - .expect("matching server task should join") - .expect("matching server should complete") - .is_none()); - - let (mismatched_address, mismatched_server) = start_context_dump_tls_server(&certificate_a).await; - let mismatched_tls = build_ipc_client_ipc_tls_config(&certificate_b) - .await - .expect("mismatched client TLS configuration should build"); - let mut mismatched_client = ipc_tls_test_client(mismatched_address, mismatched_tls); - let error = mismatched_client - .dogstatsd_contexts_dump() - .await - .expect_err("mismatched pinned certificate must fail"); - let chain = format!("{error:#}").to_ascii_lowercase(); - assert!( - chain.contains("certificate") || chain.contains("unknownissuer") || chain.contains("unknown issuer"), - "expected a certificate verification failure, got: {error:#}" - ); - assert!( - mismatched_server - .await - .expect("mismatched server task should join") - .expect("mismatched server should complete") - .is_some(), - "the HTTP server must not receive a request after TLS verification fails" - ); - }) - .await - .expect("certificate pinning test should finish within its timeout"); - } } diff --git a/docs/agent-data-plane/dogstatsd-top.md b/docs/agent-data-plane/dogstatsd-top.md index 6dddd0eabe9..ef0b386cb77 100644 --- a/docs/agent-data-plane/dogstatsd-top.md +++ b/docs/agent-data-plane/dogstatsd-top.md @@ -21,7 +21,7 @@ Wrote /var/run/datadog/dogstatsd_contexts.json.zstd 3 request.count (2 env, 1 service) ``` -Online requests pin the privileged API server certificate to the configured Agent IPC certificate. The context-dump route does not apply separate bearer-token authentication; it relies on the privileged API access-control boundary. The API creates and returns only the server-local path `/dogstatsd_contexts.json.zstd`. +Online requests use the same privileged API client path and access-control boundary as existing ADP commands. The API creates and returns only the server-local path `/dogstatsd_contexts.json.zstd`. The API does not return the artifact contents. The CLI reads the returned path directly, so the CLI process and ADP must see the same filesystem and path. If ADP runs in a container, run the CLI with the same mount or use the copy workflow in the next section. The API does not provide a remote download endpoint. @@ -59,7 +59,7 @@ You can combine `--path` (`-p`) with the metric and tag limits: $ agent-data-plane dogstatsd top -p /secure/cases/incident-123/dogstatsd_contexts.json.zstd -m 25 -t 10 ``` -Offline mode reads only the supplied file. It does not contact ADP or read the Agent IPC certificate. `top` requires the `--path` value and does not fall back to a file in the current working directory. +Offline mode reads only the supplied file and does not contact ADP. `top` requires the `--path` value and does not fall back to a file in the current working directory. ## Interpret context and tag counts @@ -90,7 +90,7 @@ If ADP uses multiple aggregate owners in the future, each owner's snapshot remai Use the status or file error in the CLI output to choose a response: -- **TLS connection failure**: Online commands fail before the request if they cannot read the configured IPC certificate or if certificate pinning rejects the server certificate. +- **Connection failure**: Verify that the CLI can reach the configured privileged API endpoint. - **404 Not Found**: The context-dump route is absent when DogStatsD is disabled. Also verify that the CLI targets the intended ADP privileged API endpoint. - **503 Service Unavailable**: The aggregate snapshot owner is not running or stopped before it responded. Wait for the topology and `dsd_agg` to become healthy, then retry. - **504 Gateway Timeout**: The aggregate owner did not return a snapshot within the 30-second API snapshot deadline. Check ADP health and load before retrying. diff --git a/lib/saluki-io/src/net/client/http/client.rs b/lib/saluki-io/src/net/client/http/client.rs index 93944ed9163..31ad4067239 100644 --- a/lib/saluki-io/src/net/client/http/client.rs +++ b/lib/saluki-io/src/net/client/http/client.rs @@ -19,10 +19,9 @@ use hyper_util::{ }; use metrics::Counter; use pin_project::pin_project; -use rustls::ClientConfig; use saluki_error::GenericError; use saluki_metrics::MetricsBuilder; -use saluki_tls::{ensure_client_config_fips_compliant, ClientTLSConfigBuilder, TlsMinimumVersion}; +use saluki_tls::{ClientTLSConfigBuilder, TlsMinimumVersion}; use stringtheory::MetaString; use tower::{timeout::TimeoutLayer, util::BoxCloneService, BoxError, Service, ServiceBuilder, ServiceExt as _}; @@ -166,7 +165,6 @@ pub struct HttpClientBuilder { connector_builder: HttpsCapableConnectorBuilder, hyper_builder: Builder, tls_builder: ClientTLSConfigBuilder, - client_tls_config: Option, request_timeout: Option, endpoint_telemetry: Option, proxies: Option>, @@ -285,24 +283,9 @@ impl HttpClientBuilder { self } - /// Sets a complete client TLS configuration. - /// - /// Use this for configurations with a custom server-certificate verifier or client identity, which cannot be - /// represented by the option-based TLS builder. - /// - /// The supplied configuration takes precedence over all options applied through [`Self::with_tls_config`] and - /// [`Self::with_min_tls_version`], regardless of call order. All supplied settings are preserved except ALPN: - /// [`ClientConfig::alpn_protocols`] is cleared before connector construction so [`Self::with_http_protocol`] - /// selects the advertised protocols. By default, [`HttpProtocol::Auto`] advertises HTTP/2 with HTTP/1.1 fallback. - pub fn with_client_tls_config(mut self, config: ClientConfig) -> Self { - self.client_tls_config = Some(config); - self - } - /// Sets the TLS configuration. /// - /// A TLS configuration builder is provided to allow for more advanced configuration of the TLS connection. These - /// options are ignored when a complete configuration is supplied through [`Self::with_client_tls_config`]. + /// A TLS configuration builder is provided to allow for more advanced configuration of the TLS connection. pub fn with_tls_config(mut self, f: F) -> Self where F: FnOnce(ClientTLSConfigBuilder) -> ClientTLSConfigBuilder, @@ -316,8 +299,7 @@ impl HttpClientBuilder { /// Defaults to TLS 1.2. /// /// This updates the same TLS builder configured by [`Self::with_tls_config`], so call order matters when both - /// methods change the minimum TLS version. This option is ignored when a complete configuration is supplied - /// through [`Self::with_client_tls_config`]. + /// methods change the minimum TLS version. pub fn with_min_tls_version(mut self, version: TlsMinimumVersion) -> Self { self.tls_builder = self.tls_builder.with_min_tls_version(version); self @@ -350,16 +332,9 @@ impl HttpClientBuilder { /// /// # Errors /// - /// If there was an error building or validating the TLS configuration for the client, an error will be returned. + /// If there was an error building the TLS configuration for the client, an error will be returned. pub fn build(self) -> Result { - let tls_config = match self.client_tls_config { - Some(mut config) => { - ensure_client_config_fips_compliant(&config)?; - config.alpn_protocols.clear(); - config - } - None => self.tls_builder.build()?, - }; + let tls_config = self.tls_builder.build()?; let connector = self.connector_builder.build(tls_config)?; // TODO(fips): Look into updating `hyper-http-proxy` to use the provided connector for establishing the // connection to the proxy itself, even when the proxy is at an HTTPS URL, to ensure our desired TLS stack is @@ -394,7 +369,6 @@ impl Default for HttpClientBuilder { connector_builder: HttpsCapableConnectorBuilder::default(), hyper_builder, tls_builder: ClientTLSConfigBuilder::new(), - client_tls_config: None, request_timeout: Some(Duration::from_secs(20)), endpoint_telemetry: None, proxies: None, @@ -416,49 +390,4 @@ mod tests { assert_eq!(Some(5), converted.size_hint().exact()); } - - #[cfg(not(windows))] - fn test_crypto_provider() -> rustls::crypto::CryptoProvider { - rustls::crypto::aws_lc_rs::default_provider() - } - - #[cfg(windows)] - fn test_crypto_provider() -> rustls::crypto::CryptoProvider { - rustls_cng_crypto::default_provider() - } - - fn empty_client_tls_config() -> rustls::ClientConfig { - rustls::ClientConfig::builder_with_provider(test_crypto_provider().into()) - .with_safe_default_protocol_versions() - .expect("default protocol versions should be valid") - .with_root_certificates(rustls::RootCertStore::empty()) - .with_no_client_auth() - } - - #[test] - fn accepts_complete_client_tls_configuration() { - let tls_config = empty_client_tls_config(); - - HttpClient::builder() - .with_client_tls_config(tls_config.clone()) - .with_min_tls_version(TlsMinimumVersion::Tls13) - .build() - .expect("a complete TLS configuration should take precedence over later builder options"); - HttpClient::builder() - .with_min_tls_version(TlsMinimumVersion::Tls13) - .with_client_tls_config(tls_config) - .build() - .expect("a complete TLS configuration should take precedence over earlier builder options"); - } - - #[test] - fn accepts_complete_client_tls_configuration_with_alpn() { - let mut tls_config = empty_client_tls_config(); - tls_config.alpn_protocols = vec![b"h2".to_vec()]; - - HttpClient::builder() - .with_client_tls_config(tls_config) - .build() - .expect("the HTTP client should normalize ALPN from a supplied TLS configuration"); - } }