From 606aba74cf1eaa8ae88f440857ead6e0d78c4b6d Mon Sep 17 00:00:00 2001 From: Mario Savarese Date: Mon, 13 Jul 2026 10:41:10 +0100 Subject: [PATCH 1/4] feat: add richer host diagnostics --- crates/control-plane/src/configuration.rs | 10 ++++ crates/control-plane/src/health.rs | 61 ++++++++++++++++--- crates/control-plane/src/main.rs | 3 +- crates/data-plane/src/health/agent.rs | 72 +++++++++++++++-------- crates/data-plane/src/health/mod.rs | 21 +++---- crates/shared/src/server/health.rs | 66 +++++++++++++++++++++ docker-compose.yml | 20 ++++--- e2e-tests/generate-sample-ca.sh | 3 + e2e-tests/httpCustomerProcess.js | 6 +- e2e-tests/scripts/start-data-plane.sh | 13 +++- 10 files changed, 220 insertions(+), 55 deletions(-) mode change 100644 => 100755 e2e-tests/generate-sample-ca.sh diff --git a/crates/control-plane/src/configuration.rs b/crates/control-plane/src/configuration.rs index 0980b24e..b65bda1d 100644 --- a/crates/control-plane/src/configuration.rs +++ b/crates/control-plane/src/configuration.rs @@ -1,3 +1,4 @@ +use shared::server::health::EnclaveIdentity; use std::{env::VarError, str::FromStr}; use openssl::{ @@ -102,6 +103,15 @@ impl EnclaveContext { pub fn get_namespace_string(&self) -> String { format!("{}/{}", self.hyphenated_app_uuid(), self.name) } + + pub fn enclave_identity(&self) -> EnclaveIdentity { + EnclaveIdentity { + app_uuid: self.app_uuid.clone(), + team_uuid: self.team_uuid.clone(), + enclave_uuid: self.uuid.clone(), + name: self.name.clone(), + } + } } pub fn get_enclave_uuid() -> String { diff --git a/crates/control-plane/src/health.rs b/crates/control-plane/src/health.rs index 4d0d14d3..455a3170 100644 --- a/crates/control-plane/src/health.rs +++ b/crates/control-plane/src/health.rs @@ -1,11 +1,15 @@ use crate::error::ServerError; use axum::http::HeaderValue; use hyper::{Body, Request, Response}; +use log::Level; use serde::{Deserialize, Serialize}; use shared::notify_shutdown::Service; use shared::server::{ error::ServerResult, - health::{ControlPlaneState, DataPlaneState, HealthCheck, HealthCheckLog, HealthCheckVersion}, + health::{ + ControlPlaneState, DataPlaneState, EnclaveIdentity, HealthCheck, HealthCheckLog, + HealthCheckVersion, + }, tcp::TcpServer, Listener, }; @@ -14,7 +18,7 @@ use shared::{ ENCLAVE_HEALTH_CHECK_PORT, }; use std::net::SocketAddr; -use std::sync::OnceLock; +use std::sync::{Arc, OnceLock}; use tokio::sync::mpsc::Receiver; pub static IS_DRAINING: OnceLock = OnceLock::new(); @@ -30,11 +34,13 @@ pub const CONTROL_PLANE_HEALTH_CHECK_PORT: u16 = 3032; struct CombinedHealthCheckLog { control_plane: ControlPlaneState, data_plane: HealthCheckVersion, + enclave: EnclaveIdentity, } pub async fn run_ecs_health_check_service( is_draining: bool, control_plane_state: ControlPlaneState, + enclave: &EnclaveIdentity, ) -> std::result::Result, ServerError> { if is_draining { let combined_log = CombinedHealthCheckLog { @@ -43,6 +49,7 @@ pub async fn run_ecs_health_check_service( "Enclave is draining, data-plane health will not be checked".into(), ) .into(), + enclave: enclave.clone(), }; let combined_log_json = serde_json::to_string(&combined_log)?; @@ -63,13 +70,28 @@ pub async fn run_ecs_health_check_service( let combined_log = CombinedHealthCheckLog { control_plane: control_plane_state, data_plane, + enclave: enclave.clone(), }; - let combined_log_json = serde_json::to_string(&combined_log).unwrap(); + + let level = if status_to_return == 200 { + Level::Info + } else { + Level::Error + }; + log::log!( + level, + "{}", + serde_json::json!({ + "event": "health_check", + "status": status_to_return, + "result": combined_log, + }) + ); Response::builder() .status(status_to_return) .header("Content-Type", "application/json") - .body(Body::from(combined_log_json)) + .body(Body::from(serde_json::to_string(&combined_log)?)) .map_err(ServerError::from) } @@ -110,13 +132,15 @@ async fn health_check_data_plane() -> Result { pub struct HealthCheckServer { shutdown_receiver: Receiver, exited_services: Vec, + enclave_identity: Arc, } impl HealthCheckServer { - pub fn new(shutdown_receiver: Receiver) -> Self { + pub fn new(shutdown_receiver: Receiver, enclave_identity: EnclaveIdentity) -> Self { Self { shutdown_receiver, exited_services: Vec::new(), + enclave_identity: Arc::new(enclave_identity), } } @@ -138,8 +162,10 @@ impl HealthCheckServer { let service = hyper::service::service_fn({ let cp_state = cp_state.clone(); + let enclave_identity = Arc::clone(&self.enclave_identity); move |request: Request| { let cp_state = cp_state.clone(); + let enclave_identity = Arc::clone(&enclave_identity); async move { match request .headers() @@ -147,8 +173,12 @@ impl HealthCheckServer { .map(|value| value.as_bytes()) { Some(b"ECS-HealthCheck") => { - let cp_state = cp_state.clone(); - run_ecs_health_check_service(is_draining(), cp_state).await + run_ecs_health_check_service( + is_draining(), + cp_state, + &enclave_identity, + ) + .await } _ => Response::builder() .status(400) @@ -198,6 +228,15 @@ impl HealthCheckServer { mod health_check_tests { use super::*; + fn test_identity() -> EnclaveIdentity { + EnclaveIdentity { + app_uuid: "app_123".into(), + team_uuid: "team_456".into(), + enclave_uuid: "enclave_789".into(), + name: "my-enclave".into(), + } + } + async fn response_to_health_check_log(response: Response) -> CombinedHealthCheckLog { let response_body = response.into_body(); let response_body = hyper::body::to_bytes(response_body).await.unwrap(); @@ -207,13 +246,15 @@ mod health_check_tests { #[tokio::test] async fn test_enclave_health_check_service() { // the data-plane status should error, as its not running - let response = run_ecs_health_check_service(false, ControlPlaneState::Ok) + let response = run_ecs_health_check_service(false, ControlPlaneState::Ok, &test_identity()) .await .unwrap(); assert_eq!(response.status(), 500); println!("deep response: {response:?}"); let health_check_log = response_to_health_check_log(response).await; + assert_eq!(health_check_log.enclave, test_identity()); + let dp_state = match health_check_log.data_plane { HealthCheckVersion::V0(_) => panic!("Expected V1 Version"), HealthCheckVersion::V1(state) => state, @@ -226,13 +267,15 @@ mod health_check_tests { async fn test_enclave_health_check_service_with_draining_set_to_true() { // the data-plane status should error, as its not running IS_DRAINING.set(true).unwrap(); - let response = run_ecs_health_check_service(true, ControlPlaneState::Ok) + let response = run_ecs_health_check_service(true, ControlPlaneState::Ok, &test_identity()) .await .unwrap(); assert_eq!(response.status(), 500); println!("deep response: {response:?}"); let health_check_log = response_to_health_check_log(response).await; + assert_eq!(health_check_log.enclave, test_identity()); + let dp_state = match health_check_log.data_plane { HealthCheckVersion::V0(_) => panic!("Expected V1 Version"), HealthCheckVersion::V1(state) => state, diff --git a/crates/control-plane/src/main.rs b/crates/control-plane/src/main.rs index 9e052e35..7b942a94 100644 --- a/crates/control-plane/src/main.rs +++ b/crates/control-plane/src/main.rs @@ -73,7 +73,8 @@ async fn main() -> Result<()> { let (shutdown_sender, shutdown_receiver) = channel(1); - let mut health_check_server = HealthCheckServer::new(shutdown_receiver); + let enclave_identity = configuration::EnclaveContext::from_env_vars().enclave_identity(); + let mut health_check_server = HealthCheckServer::new(shutdown_receiver, enclave_identity); listen_for_shutdown_signal(); schedule_statsd_proxies(); diff --git a/crates/data-plane/src/health/agent.rs b/crates/data-plane/src/health/agent.rs index 47160837..a7232bf8 100644 --- a/crates/data-plane/src/health/agent.rs +++ b/crates/data-plane/src/health/agent.rs @@ -2,7 +2,10 @@ use crate::{ContextError, EnclaveContext}; use hyper::client::connect::Connect; use hyper::{client::HttpConnector, header, Body, Client, Method, Request}; use serde_json::Value; -use shared::{notify_shutdown::Service, server::health::UserProcessHealth}; +use shared::{ + notify_shutdown::Service, + server::health::{DataPlaneDiagnostic, DataPlaneState, UserProcessHealth}, +}; use std::collections::VecDeque; use tokio::sync::mpsc::{ channel, unbounded_channel, Receiver, Sender, UnboundedReceiver, UnboundedSender, @@ -19,11 +22,11 @@ enum HealthcheckAgentState { } pub struct HealthcheckStatusRequest { - sender: OneshotSender, + sender: OneshotSender, } impl HealthcheckStatusRequest { - pub fn new() -> (Self, OneshotReceiver) { + pub fn new() -> (Self, OneshotReceiver) { let (sender, receiver) = oneshot_channel(); (Self { sender }, receiver) } @@ -215,16 +218,16 @@ impl HealthcheckAgent { } fn serve_healthcheck_request(&mut self, request: HealthcheckStatusRequest) { - if self.buffer.is_empty() { - let _ = request.sender.send(UserProcessHealth::Unknown( - "Enclave is not initialized yet".to_string(), - )); - return; - } - - // Safety: Iterator checked to be non-empty at start of func - let max_result = self.buffer.iter().max().unwrap(); - let _ = request.sender.send(max_result.to_owned()); + let state = match self.state { + HealthcheckAgentState::Initializing => DataPlaneState::Provisioning, + HealthcheckAgentState::Ready => { + let user_process = self.buffer.iter().max().cloned().unwrap_or_else(|| { + UserProcessHealth::Unknown("No healthcheck results recorded yet".to_string()) + }); + DataPlaneState::Initialized(DataPlaneDiagnostic { user_process }) + } + }; + let _ = request.sender.send(state); } pub async fn run(mut self) { @@ -289,8 +292,8 @@ impl HealthcheckAgent { #[cfg(test)] mod test { - use super::{HealthcheckAgent, HealthcheckStatusRequest}; - use shared::server::health::UserProcessHealth; + use super::{HealthcheckAgent, HealthcheckAgentState, HealthcheckStatusRequest}; + use shared::server::health::{DataPlaneDiagnostic, DataPlaneState, UserProcessHealth}; use yup_hyper_mock::mock_connector; mock_connector!(MockHttpHealthyEmptyEndpoint { @@ -325,10 +328,26 @@ mod test { "https://127.0.0.1" => "HTTP/1.1 500 Internal Server Error\r\n\r\n{\"very-bad-state\": \"unhealthy\"}" }); + #[test] + fn validate_uninitialized_agent_reports_provisioning() { + let (mut agent, _sender, _) = + HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); + agent.buffer.push_back(UserProcessHealth::Response { + status_code: 200, + body: None, + }); + let (req, mut receiver) = HealthcheckStatusRequest::new(); + agent.serve_healthcheck_request(req); + + let result = receiver.try_recv().unwrap(); + assert!(matches!(result, DataPlaneState::Provisioning)); + } + #[test] fn validate_response_returned_from_healthy_buffer() { let (mut agent, _sender, _) = HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); + agent.state = HealthcheckAgentState::Ready; for _ in 0..5 { agent.buffer.push_back(UserProcessHealth::Response { status_code: 200, @@ -341,10 +360,13 @@ mod test { let result = receiver.try_recv().unwrap(); assert!(matches!( result, - UserProcessHealth::Response { - status_code: 200, - body: None - }, + DataPlaneState::Initialized(DataPlaneDiagnostic { + user_process: UserProcessHealth::Response { + status_code: 200, + body: None + }, + .. + }), )); } @@ -352,6 +374,7 @@ mod test { fn validate_response_returned_from_buffer_with_unknown() { let (mut agent, _sender, _) = HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); + agent.state = HealthcheckAgentState::Ready; for _ in 0..5 { agent.buffer.push_back(UserProcessHealth::Response { status_code: 200, @@ -367,10 +390,13 @@ mod test { let result = receiver.try_recv().unwrap(); assert!(matches!( result, - UserProcessHealth::Response { - status_code: 200, - body: None - } + DataPlaneState::Initialized(DataPlaneDiagnostic { + user_process: UserProcessHealth::Response { + status_code: 200, + body: None + }, + .. + }) )); } diff --git a/crates/data-plane/src/health/mod.rs b/crates/data-plane/src/health/mod.rs index 888d0f6a..35b5ac50 100644 --- a/crates/data-plane/src/health/mod.rs +++ b/crates/data-plane/src/health/mod.rs @@ -6,7 +6,7 @@ use hyper::header; use hyper::{service::service_fn, Body, Response}; use shared::bridge::{Bridge, BridgeInterface, BridgeServer, Direction}; use shared::notify_shutdown::Service; -use shared::server::health::{DataPlaneDiagnostic, DataPlaneState, UserProcessHealth}; +use shared::server::health::DataPlaneState; use shared::{server::Listener, ENCLAVE_HEALTH_CHECK_PORT}; use tokio::sync::mpsc::{Sender, UnboundedSender}; @@ -74,12 +74,7 @@ impl HealthcheckServer { let service = service_fn(move |_| { let user_process_channel = user_process_channel.clone(); async move { - let user_process_health = - check_user_process_health(&user_process_channel).await; - - let result = DataPlaneState::Initialized(DataPlaneDiagnostic { - user_process: user_process_health, - }); + let result = get_data_plane_state(&user_process_channel).await; Response::builder() .status(200) @@ -99,18 +94,18 @@ impl HealthcheckServer { } } -async fn check_user_process_health(channel: &UserProcessHealthcheckSender) -> UserProcessHealth { +async fn get_data_plane_state(channel: &UserProcessHealthcheckSender) -> DataPlaneState { let (request, receiver) = HealthcheckStatusRequest::new(); if let Err(e) = channel.send(request) { - return UserProcessHealth::Error(format!( - "Failed to send healthcheck to user process on channel {e:?}" + return DataPlaneState::Error(format!( + "Failed to send healthcheck request to agent on channel {e:?}" )); } match receiver.await { - Ok(health) => health, - Err(e) => UserProcessHealth::Error(format!( - "Failed to receive healthcheck from on channel {e:?}" + Ok(state) => state, + Err(e) => DataPlaneState::Error(format!( + "Failed to receive healthcheck from agent on channel {e:?}" )), } } diff --git a/crates/shared/src/server/health.rs b/crates/shared/src/server/health.rs index ceffa547..f1601ed7 100644 --- a/crates/shared/src/server/health.rs +++ b/crates/shared/src/server/health.rs @@ -97,6 +97,9 @@ impl HealthCheck for DataPlaneState { fn status_code(&self) -> u16 { match self { DataPlaneState::Initialized(diagnostic) if diagnostic.is_healthy() => 200, + DataPlaneState::Provisioning + | DataPlaneState::Attesting + | DataPlaneState::SourcingTlsCerts => 200, _ => 500, } } @@ -123,6 +126,14 @@ pub struct DataPlaneDiagnostic { pub user_process: UserProcessHealth, } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct EnclaveIdentity { + pub app_uuid: String, + pub team_uuid: String, + pub enclave_uuid: String, + pub name: String, +} + impl DataPlaneDiagnostic { pub fn is_healthy(&self) -> bool { match self.user_process { @@ -179,6 +190,61 @@ impl PartialOrd for UserProcessHealth { mod test { use super::*; + #[test] + fn booting_states_report_healthy_status_code() { + assert_eq!(DataPlaneState::Provisioning.status_code(), 200); + assert_eq!(DataPlaneState::Attesting.status_code(), 200); + assert_eq!(DataPlaneState::SourcingTlsCerts.status_code(), 200); + } + + #[test] + fn error_and_unknown_states_report_unhealthy_status_code() { + assert_eq!(DataPlaneState::Error("boom".into()).status_code(), 500); + assert_eq!(DataPlaneState::Unknown("unknown".into()).status_code(), 500); + } + + #[test] + fn provisioning_state_serializes_to_stable_wire_format() { + // The control plane parses this JSON off the DP<->CP boundary, so the shape matters. + let provisioning = serde_json::to_string(&DataPlaneState::Provisioning).unwrap(); + assert_eq!(provisioning, "\"Provisioning\""); + + let initialized = + serde_json::to_string(&DataPlaneState::Initialized(DataPlaneDiagnostic { + user_process: UserProcessHealth::Response { + status_code: 200, + body: None, + }, + })) + .unwrap(); + // The data plane reports only user-process health here; enclave identity is added + // by the control plane on its own envelope, not carried in this payload. + assert_eq!( + initialized, + r#"{"Initialized":{"user_process":{"Response":{"status_code":200,"body":null}}}}"# + ); + + // And it round-trips back through the same deserializer the control plane uses. + let decoded: DataPlaneState = serde_json::from_str("\"Provisioning\"").unwrap(); + assert!(matches!(decoded, DataPlaneState::Provisioning)); + } + + #[test] + fn enclave_identity_serializes_to_stable_wire_format() { + // The control plane emits this on its healthcheck envelope, so the shape matters. + let identity = serde_json::to_string(&EnclaveIdentity { + app_uuid: "app_123".into(), + team_uuid: "team_456".into(), + enclave_uuid: "enclave_789".into(), + name: "my-enclave".into(), + }) + .unwrap(); + assert_eq!( + identity, + r#"{"app_uuid":"app_123","team_uuid":"team_456","enclave_uuid":"enclave_789","name":"my-enclave"}"# + ); + } + #[tokio::test] async fn it_returns_errors_over_healthy_up_responses() { let max = [ diff --git a/docker-compose.yml b/docker-compose.yml index 2e3dd58c..d95d34de 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,15 +13,15 @@ services: - 8126:8126 networks: mynetwork: - ipv4_address: 172.20.0.6 - platform: linux/amd64 + ipv4_address: 172.20.0.6 + platform: linux/amd64 enclave: build: dockerfile: enclave.Dockerfile dns: 127.0.0.1 cap_add: - NET_ADMIN - privileged: true + privileged: true ports: - "7777:7777" - "7779:7779" @@ -30,7 +30,7 @@ services: - control-plane networks: mynetwork: - ipv4_address: 172.20.0.7 + ipv4_address: 172.20.0.7 environment: - EV_API_KEY_AUTH=${EV_API_KEY_AUTH:?No api key auth set, failing fast} - CUSTOMER_PROCESS=${CUSTOMER_PROCESS} @@ -42,11 +42,17 @@ services: ports: - "443:3031" - "3032:3032" + healthcheck: + test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "-H", "User-Agent: ECS-HealthCheck", "http://127.0.0.1:3032"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s depends_on: - statsd networks: mynetwork: - ipv4_address: 172.20.0.8 + ipv4_address: 172.20.0.8 environment: - MOCK_CRYPTO_CERT=${MOCK_CRYPTO_CERT} - MOCK_CRYPTO_KEY=${MOCK_CRYPTO_KEY} @@ -60,5 +66,5 @@ services: - ACME_ACCOUNT_HMAC_KEY_ID=${ACME_ACCOUNT_HMAC_KEY_ID} - DEREGISTRATION_TOPIC_ARN=testarn - EV_API_KEY_AUTH=${EV_API_KEY_AUTH:?No api key auth set, failing fast} - - CUSTOMER_PROCESS=${CUSTOMER_PROCESS} - - AWS_REGION=us-east-1 \ No newline at end of file + - CUSTOMER_PROCESS=${CUSTOMER_PROCESS} + - AWS_REGION=us-east-1 diff --git a/e2e-tests/generate-sample-ca.sh b/e2e-tests/generate-sample-ca.sh old mode 100644 new mode 100755 index 867cbffe..5ec75e3d --- a/e2e-tests/generate-sample-ca.sh +++ b/e2e-tests/generate-sample-ca.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Always run relative to the repo root, regardless of where the script is invoked from +cd "$(dirname "$0")/.." + # If sample-ca directory doesn't exist, create it if [ ! -d "e2e-tests/sample-ca" ]; then mkdir -p e2e-tests/sample-ca diff --git a/e2e-tests/httpCustomerProcess.js b/e2e-tests/httpCustomerProcess.js index fb92ec47..7815917e 100644 --- a/e2e-tests/httpCustomerProcess.js +++ b/e2e-tests/httpCustomerProcess.js @@ -5,7 +5,11 @@ const port = 8008 app.use(express.json()) -app.all('/hello', async (req, res) => { +app.get('/health', async (req, res) => { + res.send({ status: "ok" }) +}) + +app.all('/hello', async (req, res) => { res.send({response: "Hello from enclave", ...req.body}) }) diff --git a/e2e-tests/scripts/start-data-plane.sh b/e2e-tests/scripts/start-data-plane.sh index 4bd2ea06..fe091624 100644 --- a/e2e-tests/scripts/start-data-plane.sh +++ b/e2e-tests/scripts/start-data-plane.sh @@ -1,6 +1,17 @@ #!/bin/sh -echo {\"api_key_auth\":${EV_API_KEY_AUTH},\"egress\":{\"allow_list\": \"jsonplaceholder.typicode.com\"},\"trx_logging_enabled\":true,\"forward_proxy_protocol\":false,\"trusted_headers\": []} > /etc/dataplane-config.json +cat < /etc/dataplane-config.json +{ + "api_key_auth": ${EV_API_KEY_AUTH}, + "egress": { + "allow_list": "jsonplaceholder.typicode.com" + }, + "trx_logging_enabled": true, + "forward_proxy_protocol": false, + "trusted_headers": [], + "healthcheck": "/health" +} +EOF iptables -A OUTPUT -t nat -p tcp --dport 443 ! -d 127.0.0.1 -j DNAT --to-destination 127.0.0.1:4444 From df6b49d043f12b00ca07cb7f2353245a353b7a11 Mon Sep 17 00:00:00 2001 From: Mario Savarese Date: Mon, 13 Jul 2026 17:37:00 +0100 Subject: [PATCH 2/4] use typestate pattern for boot states --- crates/data-plane/src/health/agent.rs | 146 +++++++++++++++++++++++--- crates/data-plane/src/health/mod.rs | 25 +++-- crates/data-plane/src/main.rs | 31 ++++-- crates/shared/src/server/health.rs | 4 - 4 files changed, 167 insertions(+), 39 deletions(-) diff --git a/crates/data-plane/src/health/agent.rs b/crates/data-plane/src/health/agent.rs index a7232bf8..0704d713 100644 --- a/crates/data-plane/src/health/agent.rs +++ b/crates/data-plane/src/health/agent.rs @@ -7,6 +7,7 @@ use shared::{ server::health::{DataPlaneDiagnostic, DataPlaneState, UserProcessHealth}, }; use std::collections::VecDeque; +use std::marker::PhantomData; use tokio::sync::mpsc::{ channel, unbounded_channel, Receiver, Sender, UnboundedReceiver, UnboundedSender, }; @@ -21,6 +22,62 @@ enum HealthcheckAgentState { Ready, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum BootPhase { + #[default] + Provisioning, + Attesting, + #[cfg(feature = "tls_termination")] + SourcingTlsCerts, +} + +pub trait BootPhaseMarker {} + +/// The data plane has started but not yet begun fetching its environment. +pub struct Provisioning; +impl BootPhaseMarker for Provisioning {} + +/// The data plane is attesting to the provisioner to fetch and decrypt its environment. +pub struct Attesting; +impl BootPhaseMarker for Attesting {} + +/// The data plane is sourcing its TLS certificates from the provisioner. +pub struct SourcingTlsCerts; +impl BootPhaseMarker for SourcingTlsCerts {} + +pub struct BootProgress { + sender: UnboundedSender, + _phase: PhantomData

, +} + +impl BootProgress { + fn new(sender: UnboundedSender) -> Self { + Self { + sender, + _phase: PhantomData, + } + } + + pub fn attesting(self) -> BootProgress { + let _ = self.sender.send(BootPhase::Attesting); + BootProgress { + sender: self.sender, + _phase: PhantomData, + } + } +} + +#[cfg(feature = "tls_termination")] +impl BootProgress { + pub fn sourcing_tls_certs(self) -> BootProgress { + let _ = self.sender.send(BootPhase::SourcingTlsCerts); + BootProgress { + sender: self.sender, + _phase: PhantomData, + } + } +} + pub struct HealthcheckStatusRequest { sender: OneshotSender, } @@ -41,6 +98,9 @@ pub struct HealthcheckAgent { buffer: VecDeque, interval: std::time::Duration, state: HealthcheckAgentState, + boot_phase: BootPhase, + boot_phase_receiver: UnboundedReceiver, + awaiting_boot_phases: bool, recv: UnboundedReceiver, buffer_size_limit: usize, proto: String, @@ -59,6 +119,7 @@ impl HealthcheckAgent> { Self, UnboundedSender, Sender, + BootProgress, ) { let client = Self::build_tls_client(); Self::new( @@ -95,6 +156,7 @@ impl HealthcheckAgent { Self, UnboundedSender, Sender, + BootProgress, ) { let client = Client::builder().build_http(); Self::new( @@ -118,9 +180,11 @@ impl HealthcheckAgent { Self, UnboundedSender, Sender, + BootProgress, ) { let (sender, recv) = unbounded_channel(); let (shutdown_sender, shutdown_receiver) = channel(1); + let (boot_phase_sender, boot_phase_receiver) = unbounded_channel(); let critical_service_vec = if cfg!(feature = "network_egress") { Vec::with_capacity(5) @@ -133,6 +197,9 @@ impl HealthcheckAgent { healthcheck_path, buffer: VecDeque::with_capacity(DEFAULT_HEALTHCHECK_BUFFER_SIZE_LIMIT), state: HealthcheckAgentState::default(), + boot_phase: BootPhase::default(), + boot_phase_receiver, + awaiting_boot_phases: true, interval, recv, buffer_size_limit: DEFAULT_HEALTHCHECK_BUFFER_SIZE_LIMIT, @@ -141,7 +208,12 @@ impl HealthcheckAgent { shutdown_receiver, exited_services: critical_service_vec, }; - (healthcheck_agent, sender, shutdown_sender) + ( + healthcheck_agent, + sender, + shutdown_sender, + BootProgress::new(boot_phase_sender), + ) } fn record_result(&mut self, healthcheck_status: UserProcessHealth) { @@ -219,7 +291,12 @@ impl HealthcheckAgent { fn serve_healthcheck_request(&mut self, request: HealthcheckStatusRequest) { let state = match self.state { - HealthcheckAgentState::Initializing => DataPlaneState::Provisioning, + HealthcheckAgentState::Initializing => match self.boot_phase { + BootPhase::Provisioning => DataPlaneState::Provisioning, + BootPhase::Attesting => DataPlaneState::Attesting, + #[cfg(feature = "tls_termination")] + BootPhase::SourcingTlsCerts => DataPlaneState::SourcingTlsCerts, + }, HealthcheckAgentState::Ready => { let user_process = self.buffer.iter().max().cloned().unwrap_or_else(|| { UserProcessHealth::Unknown("No healthcheck results recorded yet".to_string()) @@ -240,6 +317,12 @@ impl HealthcheckAgent { self.serve_healthcheck_request(req); } }, + maybe_boot_phase = self.boot_phase_receiver.recv(), if self.awaiting_boot_phases => { + match maybe_boot_phase { + Some(boot_phase) => self.boot_phase = boot_phase, + None => self.awaiting_boot_phases = false, + } + }, _ = interval.tick() => self.perform_healthcheck().await } } @@ -292,7 +375,7 @@ impl HealthcheckAgent { #[cfg(test)] mod test { - use super::{HealthcheckAgent, HealthcheckAgentState, HealthcheckStatusRequest}; + use super::{BootPhase, HealthcheckAgent, HealthcheckAgentState, HealthcheckStatusRequest}; use shared::server::health::{DataPlaneDiagnostic, DataPlaneState, UserProcessHealth}; use yup_hyper_mock::mock_connector; @@ -330,7 +413,7 @@ mod test { #[test] fn validate_uninitialized_agent_reports_provisioning() { - let (mut agent, _sender, _) = + let (mut agent, _sender, _, _) = HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); agent.buffer.push_back(UserProcessHealth::Response { status_code: 200, @@ -343,9 +426,38 @@ mod test { assert!(matches!(result, DataPlaneState::Provisioning)); } + #[test] + fn validate_agent_reports_attesting_while_initializing() { + let (mut agent, _sender, _, _) = + HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); + + agent.boot_phase = BootPhase::Attesting; + let (req, mut receiver) = HealthcheckStatusRequest::new(); + agent.serve_healthcheck_request(req); + assert!(matches!( + receiver.try_recv().unwrap(), + DataPlaneState::Attesting + )); + } + + #[cfg(feature = "tls_termination")] + #[test] + fn validate_agent_reports_sourcing_tls_certs_while_initializing() { + let (mut agent, _sender, _, _) = + HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); + + agent.boot_phase = BootPhase::SourcingTlsCerts; + let (req, mut receiver) = HealthcheckStatusRequest::new(); + agent.serve_healthcheck_request(req); + assert!(matches!( + receiver.try_recv().unwrap(), + DataPlaneState::SourcingTlsCerts + )); + } + #[test] fn validate_response_returned_from_healthy_buffer() { - let (mut agent, _sender, _) = + let (mut agent, _sender, _, _) = HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); agent.state = HealthcheckAgentState::Ready; for _ in 0..5 { @@ -372,7 +484,7 @@ mod test { #[test] fn validate_response_returned_from_buffer_with_unknown() { - let (mut agent, _sender, _) = + let (mut agent, _sender, _, _) = HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); agent.state = HealthcheckAgentState::Ready; for _ in 0..5 { @@ -403,7 +515,7 @@ mod test { #[tokio::test] async fn validate_uninitialized_user_process_doesnt_return_errors() { let duration = std::time::Duration::from_secs(1); - let (mut agent, _sender, _) = HealthcheckAgent::build_agent(3000, duration, None); + let (mut agent, _sender, _, _) = HealthcheckAgent::build_agent(3000, duration, None); agent.perform_healthcheck().await; @@ -417,7 +529,7 @@ mod test { #[tokio::test] async fn validate_initialized_agent_with_no_healthcheck_path_returns_unknown() { let duration = std::time::Duration::from_secs(1); - let (mut agent, _sender, _) = HealthcheckAgent::build_agent(3000, duration, None); + let (mut agent, _sender, _, _) = HealthcheckAgent::build_agent(3000, duration, None); agent.state = super::HealthcheckAgentState::Ready; agent.perform_healthcheck().await; @@ -432,7 +544,7 @@ mod test { #[tokio::test] async fn validate_initialized_agent_with_healthy_response_returns_response() { let client = hyper::Client::builder().build(MockHttpHealthyEmptyEndpoint::default()); - let (mut agent, _sender, _) = HealthcheckAgent::new( + let (mut agent, _sender, _, _) = HealthcheckAgent::new( 3000, std::time::Duration::from_secs(1), Some("/healthz".into()), @@ -456,7 +568,7 @@ mod test { #[tokio::test] async fn validate_initialized_agent_with_healthy_response_returns_response_over_tls() { let client = hyper::Client::builder().build(MockHttpsHealthyEmptyEndpoint::default()); - let (mut agent, _sender, _) = HealthcheckAgent::new( + let (mut agent, _sender, _, _) = HealthcheckAgent::new( 3000, std::time::Duration::from_secs(1), Some("/healthz".into()), @@ -481,7 +593,7 @@ mod test { async fn validate_initialized_agent_with_unhealthy_response_returns_response_with_correct_status( ) { let client = hyper::Client::builder().build(MockHttpUnhealthyEmptyEndpoint::default()); - let (mut agent, _sender, _) = HealthcheckAgent::new( + let (mut agent, _sender, _, _) = HealthcheckAgent::new( 3000, std::time::Duration::from_secs(1), Some("/healthcheck".into()), @@ -506,7 +618,7 @@ mod test { async fn validate_initialized_agent_with_unhealthy_response_returns_response_with_correct_status_over_tls( ) { let client = hyper::Client::builder().build(MockHttpsUnhealthyEmptyEndpoint::default()); - let (mut agent, _sender, _) = HealthcheckAgent::new( + let (mut agent, _sender, _, _) = HealthcheckAgent::new( 3001, std::time::Duration::from_secs(1), Some("/healthcheck".into()), @@ -530,7 +642,7 @@ mod test { #[tokio::test] async fn it_can_parse_json_responses_from_user_process() { let client = hyper::Client::builder().build(MockHttpHealthyJsonEndpoint::default()); - let (mut agent, _sender, _) = HealthcheckAgent::new( + let (mut agent, _sender, _, _) = HealthcheckAgent::new( 3000, std::time::Duration::from_secs(1), Some("/healthcheck".into()), @@ -554,7 +666,7 @@ mod test { #[tokio::test] async fn it_can_parse_json_responses_from_user_process_over_tls() { let client = hyper::Client::builder().build(MockHttpsHealthyJsonEndpoint::default()); - let (mut agent, _sender, _) = HealthcheckAgent::new( + let (mut agent, _sender, _, _) = HealthcheckAgent::new( 3000, std::time::Duration::from_secs(1), Some("/healthcheck".into()), @@ -578,7 +690,7 @@ mod test { #[tokio::test] async fn it_can_parse_unhealthy_json_responses_from_user_process() { let client = hyper::Client::builder().build(MockHttpUnhealthyJsonEndpoint::default()); - let (mut agent, _sender, _) = HealthcheckAgent::new( + let (mut agent, _sender, _, _) = HealthcheckAgent::new( 3000, std::time::Duration::from_secs(1), Some("/healthcheck".into()), @@ -602,7 +714,7 @@ mod test { #[tokio::test] async fn it_can_parse_unhealthy_json_responses_from_user_process_over_tls() { let client = hyper::Client::builder().build(MockHttpsUnhealthyJsonEndpoint::default()); - let (mut agent, _sender, _) = HealthcheckAgent::new( + let (mut agent, _sender, _, _) = HealthcheckAgent::new( 3000, std::time::Duration::from_secs(1), Some("/healthcheck".into()), @@ -626,7 +738,7 @@ mod test { #[tokio::test] async fn it_fails_all_healthchecks_if_critical_service_has_exited() { let client = hyper::Client::builder().build(MockHttpHealthyEmptyEndpoint::default()); - let (mut agent, _sender, shutdown_channel) = HealthcheckAgent::new( + let (mut agent, _sender, shutdown_channel, _) = HealthcheckAgent::new( 3000, std::time::Duration::from_secs(1), Some("/healthcheck".into()), diff --git a/crates/data-plane/src/health/mod.rs b/crates/data-plane/src/health/mod.rs index 35b5ac50..5df70053 100644 --- a/crates/data-plane/src/health/mod.rs +++ b/crates/data-plane/src/health/mod.rs @@ -1,6 +1,7 @@ mod agent; use agent::UserProcessHealthcheckSender; +pub use agent::{Attesting, BootProgress, Provisioning, SourcingTlsCerts}; use hyper::header; use hyper::{service::service_fn, Body, Response}; @@ -16,18 +17,22 @@ fn spawn_customer_healthcheck_agent( customer_process_port: u16, healthcheck: Option, use_tls: bool, -) -> (UserProcessHealthcheckSender, Sender) { +) -> ( + UserProcessHealthcheckSender, + Sender, + BootProgress, +) { let default_interval = std::time::Duration::from_secs(1); if use_tls { - let (agent, channel, shutdown_channel) = + let (agent, channel, shutdown_channel, boot_progress) = HealthcheckAgent::build_tls_agent(customer_process_port, default_interval, healthcheck); tokio::spawn(async move { agent.run().await }); - (channel, shutdown_channel) + (channel, shutdown_channel, boot_progress) } else { - let (agent, channel, shutdown_channel) = + let (agent, channel, shutdown_channel, boot_progress) = HealthcheckAgent::build_agent(customer_process_port, default_interval, healthcheck); tokio::spawn(async move { agent.run().await }); - (channel, shutdown_channel) + (channel, shutdown_channel, boot_progress) } } @@ -35,11 +40,15 @@ pub async fn build_health_check_server( customer_process_port: u16, healthcheck: Option, use_tls: bool, -) -> shared::server::error::ServerResult<(HealthcheckServer, Sender)> { - let (user_process_healthcheck_channel, shutdown_notifier) = +) -> shared::server::error::ServerResult<( + HealthcheckServer, + Sender, + BootProgress, +)> { + let (user_process_healthcheck_channel, shutdown_notifier, boot_phase_notifier) = spawn_customer_healthcheck_agent(customer_process_port, healthcheck, use_tls); let health_check_server = HealthcheckServer::new(user_process_healthcheck_channel).await?; - Ok((health_check_server, shutdown_notifier)) + Ok((health_check_server, shutdown_notifier, boot_phase_notifier)) } pub struct HealthcheckServer { diff --git a/crates/data-plane/src/main.rs b/crates/data-plane/src/main.rs index a585bbc8..56bbc9f9 100644 --- a/crates/data-plane/src/main.rs +++ b/crates/data-plane/src/main.rs @@ -3,7 +3,7 @@ use data_plane::dns::{egressproxy::EgressProxy, enclavedns::EnclaveDnsProxy}; use data_plane::{ crypto::api::CryptoApi, env::{init_environment_loader, EnvironmentLoader}, - health::build_health_check_server, + health::{build_health_check_server, Attesting, BootProgress, Provisioning}, stats::client::StatsClient, stats::proxy::StatsProxy, time::ClockSync, @@ -88,25 +88,30 @@ fn main() { }; runtime.block_on(async move { - let Ok((health_check_server, shutdown_notifier)) = build_health_check_server( - ctx.healthcheck_port.unwrap_or(data_plane_port), - ctx.healthcheck, - ctx.healthcheck_use_tls.unwrap_or(false), - ) - .await + let Ok((health_check_server, shutdown_notifier, boot_progress)) = + build_health_check_server( + ctx.healthcheck_port.unwrap_or(data_plane_port), + ctx.healthcheck, + ctx.healthcheck_use_tls.unwrap_or(false), + ) + .await else { log::error!("Failed to launch in-Enclave healthcheck service, exiting early."); return; }; tokio::select! { - _ = start(data_plane_port, shutdown_notifier) => {}, + _ = start(data_plane_port, shutdown_notifier, boot_progress) => {}, _ = health_check_server.run() => {} }; }); } -async fn start(data_plane_port: u16, shutdown_notifier: Sender) { +async fn start( + data_plane_port: u16, + shutdown_notifier: Sender, + boot_progress: BootProgress, +) { if let Err(e) = StatsClient::init().await { log::error!("Failed to register in-Enclave stats client - {e}"); } @@ -125,6 +130,7 @@ async fn start(data_plane_port: u16, shutdown_notifier: Sender) { } let env_loader = init_environment_loader(); + let boot_progress = boot_progress.attesting(); let env_loader = match env_loader.load_env_vars().await { Ok(env_loader) => env_loader, Err(e) => { @@ -160,7 +166,7 @@ async fn start(data_plane_port: u16, shutdown_notifier: Sender) { ); } - start_data_plane(data_plane_port, context, env_loader) + start_data_plane(data_plane_port, context, env_loader, boot_progress) .notify_shutdown(Service::DataPlane, shutdown_notifier.clone()) .await; } @@ -172,6 +178,7 @@ async fn start_data_plane( data_plane_port: u16, context: FeatureContext, env_loader: EnvironmentLoader, + boot_progress: BootProgress, ) { log::info!("Data plane starting up. Forwarding traffic to {data_plane_port}"); let server = match Bridge::get_listener(ENCLAVE_CONNECT_PORT, Direction::EnclaveToHost).await { @@ -181,6 +188,8 @@ async fn start_data_plane( log::debug!("Data plane TCP server created"); log::info!("TLS Termination enabled in dataplane. Running tls server."); + // Bringing up the TLS server sources the intermediate CA cert from the provisioner. + let _ = boot_progress.sourcing_tls_certs(); if let Err(e) = data_plane::server::server::run(server, data_plane_port, context, env_loader).await { @@ -199,6 +208,8 @@ async fn start_data_plane( data_plane_port: u16, context: FeatureContext, env_loader: EnvironmentLoader, + // No TLS termination in this build, so `Attesting` is the terminal boot phase. + _boot_progress: BootProgress, ) { log::info!("Data plane starting up. Forwarding traffic to {data_plane_port}"); let mut server = diff --git a/crates/shared/src/server/health.rs b/crates/shared/src/server/health.rs index f1601ed7..454c0e98 100644 --- a/crates/shared/src/server/health.rs +++ b/crates/shared/src/server/health.rs @@ -217,21 +217,17 @@ mod test { }, })) .unwrap(); - // The data plane reports only user-process health here; enclave identity is added - // by the control plane on its own envelope, not carried in this payload. assert_eq!( initialized, r#"{"Initialized":{"user_process":{"Response":{"status_code":200,"body":null}}}}"# ); - // And it round-trips back through the same deserializer the control plane uses. let decoded: DataPlaneState = serde_json::from_str("\"Provisioning\"").unwrap(); assert!(matches!(decoded, DataPlaneState::Provisioning)); } #[test] fn enclave_identity_serializes_to_stable_wire_format() { - // The control plane emits this on its healthcheck envelope, so the shape matters. let identity = serde_json::to_string(&EnclaveIdentity { app_uuid: "app_123".into(), team_uuid: "team_456".into(), From 5e263acbcf67a73076d5f79023934fe4627a942a Mon Sep 17 00:00:00 2001 From: Mario Savarese Date: Mon, 13 Jul 2026 18:14:04 +0100 Subject: [PATCH 3/4] chore: clippy --- crates/control-plane/src/clients/cert_provisioner.rs | 8 ++++---- crates/control-plane/src/config_server.rs | 4 ++-- crates/control-plane/src/tls_proxy.rs | 4 ++-- crates/data-plane/src/base_tls_client/mod.rs | 7 ++----- crates/data-plane/src/health/agent.rs | 2 +- crates/data-plane/src/lib.rs | 6 +++--- 6 files changed, 14 insertions(+), 17 deletions(-) diff --git a/crates/control-plane/src/clients/cert_provisioner.rs b/crates/control-plane/src/clients/cert_provisioner.rs index e665def3..3f960291 100644 --- a/crates/control-plane/src/clients/cert_provisioner.rs +++ b/crates/control-plane/src/clients/cert_provisioner.rs @@ -92,8 +92,8 @@ impl CertProvisionerClient { format!( "https://{}:{}{}", configuration::get_cert_provisoner_host(), - &port, - &path + port, + path ) } @@ -129,11 +129,11 @@ impl CertProvisionerClient { .header("Content-Type", "application/json") .header( "User-Agent", - format!("Cage-Control-Plane/{}", &*CLIENT_VERSION), + format!("Cage-Control-Plane/{}", *CLIENT_VERSION), ) .header( "Accept", - format!("application/json;version={}", &*CLIENT_MAJOR_VERSION), + format!("application/json;version={}", *CLIENT_MAJOR_VERSION), ) .method(method) .body(body) diff --git a/crates/control-plane/src/config_server.rs b/crates/control-plane/src/config_server.rs index 34a84fd0..e0d12297 100644 --- a/crates/control-plane/src/config_server.rs +++ b/crates/control-plane/src/config_server.rs @@ -460,8 +460,8 @@ fn valid_order_identifiers( .map(|base_domain| { format!( "{}.{}.{}", - &enclave_context.name, - &enclave_context.hyphenated_app_uuid(), + enclave_context.name, + enclave_context.hyphenated_app_uuid(), base_domain ) }) diff --git a/crates/control-plane/src/tls_proxy.rs b/crates/control-plane/src/tls_proxy.rs index 0d614a85..88af61b8 100644 --- a/crates/control-plane/src/tls_proxy.rs +++ b/crates/control-plane/src/tls_proxy.rs @@ -73,8 +73,8 @@ impl TlsProxy { log::info!( "Running TLS proxy to {:?} on {}", - &self.targets, - &self.vsock_port + self.targets, + self.vsock_port ); loop { let (connection, target, initial_bytes) = match enclave_conn.accept().await { diff --git a/crates/data-plane/src/base_tls_client/mod.rs b/crates/data-plane/src/base_tls_client/mod.rs index 8c26dab5..70b76a70 100644 --- a/crates/data-plane/src/base_tls_client/mod.rs +++ b/crates/data-plane/src/base_tls_client/mod.rs @@ -76,13 +76,10 @@ impl BaseClient { } let mut request = request .header("Content-Type", "application/json") - .header( - "User-Agent", - format!("Cage-Data-Plane/{}", &*CLIENT_VERSION), - ) + .header("User-Agent", format!("Cage-Data-Plane/{}", *CLIENT_VERSION)) .header( "Accept", - format!("application/json;version={}", &*CLIENT_MAJOR_VERSION), + format!("application/json;version={}", *CLIENT_MAJOR_VERSION), ) .method(method) .body(payload) diff --git a/crates/data-plane/src/health/agent.rs b/crates/data-plane/src/health/agent.rs index 0704d713..8baacae3 100644 --- a/crates/data-plane/src/health/agent.rs +++ b/crates/data-plane/src/health/agent.rs @@ -331,7 +331,7 @@ impl HealthcheckAgent { fn build_healthcheck_uri(&self, healthcheck_path: &str) -> String { format!( "{}://127.0.0.1:{}{}", - &self.proto, self.customer_process_port, &healthcheck_path + self.proto, self.customer_process_port, healthcheck_path ) } diff --git a/crates/data-plane/src/lib.rs b/crates/data-plane/src/lib.rs index fac270ba..efb08792 100644 --- a/crates/data-plane/src/lib.rs +++ b/crates/data-plane/src/lib.rs @@ -107,13 +107,13 @@ impl EnclaveContext { #[cfg(not(staging))] pub fn get_cert_name(&self) -> String { - format!("{}.{}.cages.evervault.com", &self.name, &self.app_uuid) + format!("{}.{}.cages.evervault.com", self.name, self.app_uuid) } #[cfg(not(staging))] pub fn get_hyphenated_cert_name(&self) -> String { let hyphenated_app_uuid = self.app_uuid.clone().replace('_', "-"); - format!("{}.{}.cages.evervault.com", &self.name, hyphenated_app_uuid) + format!("{}.{}.cages.evervault.com", self.name, hyphenated_app_uuid) } pub fn get_trusted_cert_domains(&self) -> Vec { @@ -125,7 +125,7 @@ impl EnclaveContext { base_domains .iter() - .map(|domain| format!("{}.{}.{}", &self.name, &self.hyphenated_app_uuid(), domain)) + .map(|domain| format!("{}.{}.{}", self.name, self.hyphenated_app_uuid(), domain)) .collect() } From 0d31b46b418a0967f0b462fe91084bfd1a1f5931 Mon Sep 17 00:00:00 2001 From: Mario Savarese Date: Mon, 13 Jul 2026 18:48:56 +0100 Subject: [PATCH 4/4] test: log data plane boot state --- crates/data-plane/src/health/agent.rs | 44 ++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/crates/data-plane/src/health/agent.rs b/crates/data-plane/src/health/agent.rs index 8baacae3..95dd3c7b 100644 --- a/crates/data-plane/src/health/agent.rs +++ b/crates/data-plane/src/health/agent.rs @@ -31,6 +31,18 @@ enum BootPhase { SourcingTlsCerts, } +impl std::fmt::Display for BootPhase { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let phase = match self { + BootPhase::Provisioning => "Provisioning", + BootPhase::Attesting => "Attesting", + #[cfg(feature = "tls_termination")] + BootPhase::SourcingTlsCerts => "Sourcing TLS certificates", + }; + write!(f, "{phase}") + } +} + pub trait BootPhaseMarker {} /// The data plane has started but not yet begun fetching its environment. @@ -289,6 +301,14 @@ impl HealthcheckAgent { self.record_result(hc_result); } + fn record_boot_phase(&mut self, boot_phase: BootPhase) { + log::debug!( + "{}", + json!({"msg": "Data plane boot phase transitioned", "to": boot_phase}) + ); + self.boot_phase = boot_phase; + } + fn serve_healthcheck_request(&mut self, request: HealthcheckStatusRequest) { let state = match self.state { HealthcheckAgentState::Initializing => match self.boot_phase { @@ -319,7 +339,7 @@ impl HealthcheckAgent { }, maybe_boot_phase = self.boot_phase_receiver.recv(), if self.awaiting_boot_phases => { match maybe_boot_phase { - Some(boot_phase) => self.boot_phase = boot_phase, + Some(boot_phase) => self.record_boot_phase(boot_phase), None => self.awaiting_boot_phases = false, } }, @@ -455,6 +475,28 @@ mod test { )); } + #[test] + fn validate_boot_phase_transitions_are_recorded_and_reported() { + let (mut agent, _sender, _, boot_progress) = + HealthcheckAgent::build_agent(3000, std::time::Duration::from_secs(1), None); + + assert_eq!(agent.boot_phase, BootPhase::default()); + + let boot_progress = boot_progress.attesting(); + let phase = agent.boot_phase_receiver.try_recv().unwrap(); + agent.record_boot_phase(phase); + assert_eq!(agent.boot_phase, BootPhase::Attesting); + + let (req, mut receiver) = HealthcheckStatusRequest::new(); + agent.serve_healthcheck_request(req); + assert!(matches!( + receiver.try_recv().unwrap(), + DataPlaneState::Attesting + )); + + drop(boot_progress); + } + #[test] fn validate_response_returned_from_healthy_buffer() { let (mut agent, _sender, _, _) =