From 3f710c4c49df8e602ae86157933562d1e45d7b02 Mon Sep 17 00:00:00 2001 From: Pierre Tholoniat Date: Tue, 13 Oct 2026 14:16:35 +0000 Subject: [PATCH 001/464] fix(assets): release OCI blob stripe lock before verified copy and align lock timeout Drop the per-digest stripe `FileLock` before `verified_copy` in `BlobCache::copy_hit_with_lease` (called from `Puller::blob`) so a warm cache hit does not hold the stripe lock during verified copy, and scope the 300s `PULL_TIMEOUT` wait to `BlobCache::lease()` while keeping the 30s timeout for `materialization_lease`, `mutation_lease`, and `read_lock`. --- crates/capsem-assets/src/oci.rs | 1 + crates/capsem-assets/src/oci/cache.rs | 41 +++++++++++++++++- crates/capsem-assets/src/oci/cache/tests.rs | 48 +++++++++++++++++++++ crates/capsem-assets/src/oci/pull.rs | 12 +++--- 4 files changed, 95 insertions(+), 7 deletions(-) diff --git a/crates/capsem-assets/src/oci.rs b/crates/capsem-assets/src/oci.rs index f6d90995f..7792a75ce 100644 --- a/crates/capsem-assets/src/oci.rs +++ b/crates/capsem-assets/src/oci.rs @@ -22,6 +22,7 @@ pub use readiness::{CacheReason, CacheSnapshot, CacheState}; pub use receipts::CacheReceipt; const METADATA_LIMIT: usize = 4 * 1024 * 1024; +const PULL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); mod pull; pub use catalog::{ is_catalog_name, Catalog, CatalogEntry, CatalogVersion, Channel, CATALOG_MEDIA_TYPE, DEFAULT_CATALOG, diff --git a/crates/capsem-assets/src/oci/cache.rs b/crates/capsem-assets/src/oci/cache.rs index 4e882aa4c..9d842c7c4 100644 --- a/crates/capsem-assets/src/oci/cache.rs +++ b/crates/capsem-assets/src/oci/cache.rs @@ -179,7 +179,13 @@ impl BlobCache { /// Fixed lock stripes bound lock-file growth while coalescing equal blobs. pub(super) async fn lease(&self, digest: &str) -> Result { let hex = digest_hex(digest)?; - lock(self.root.join("locks").join(format!("{}.lock", &hex[..2]))).await + lock_with_timeout( + self.root.join("locks").join(format!("{}.lock", &hex[..2])), + LockAccess::Prepare, + LockMode::Exclusive, + lock_timeout(LockScope::BlobStripe), + ) + .await } /// Shared for the complete image/layout lifetime, separate from blob @@ -216,7 +222,18 @@ impl BlobCache { self.root.join(&self.policy.mutation_locks[0]) } + #[cfg(test)] pub(super) async fn copy_hit(&self, digest: &str, size: u64, destination: &Path) -> Result { + self.copy_hit_with_lease(digest, size, destination, &mut None).await + } + + pub(super) async fn copy_hit_with_lease( + &self, + digest: &str, + size: u64, + destination: &Path, + stripe_lease: &mut Option, + ) -> Result { let lease = self.lookup_lease().await?; let cache = self.clone(); let hex = self.entry_name(digest)?; @@ -238,6 +255,7 @@ impl BlobCache { let Some(file) = opened else { return Ok(false); }; + *stripe_lease = None; let destination = destination.to_owned(); let digest = digest.to_owned(); // Holding the opened file is enough: pruning may unlink its name but @@ -476,6 +494,8 @@ impl BlobCache { } } +const CACHE_LOCK_TIMEOUT: Duration = Duration::from_secs(30); + #[derive(Clone, Copy)] enum LockAccess { Prepare, @@ -483,6 +503,19 @@ enum LockAccess { Existing, } +#[derive(Clone, Copy)] +enum LockScope { + BlobStripe, + Cache, +} + +fn lock_timeout(scope: LockScope) -> Duration { + match scope { + LockScope::BlobStripe => super::PULL_TIMEOUT, + LockScope::Cache => CACHE_LOCK_TIMEOUT, + } +} + async fn lock(path: PathBuf) -> Result { lock_with(path, LockAccess::Prepare, LockMode::Exclusive).await } @@ -492,7 +525,11 @@ async fn read_lock(path: PathBuf) -> Result { } async fn lock_with(path: PathBuf, access: LockAccess, mode: LockMode) -> Result { - poll_until(PollOpts::new("oci-cache-lock", Duration::from_secs(30)), || { + lock_with_timeout(path, access, mode, lock_timeout(LockScope::Cache)).await +} + +async fn lock_with_timeout(path: PathBuf, access: LockAccess, mode: LockMode, timeout: Duration) -> Result { + poll_until(PollOpts::new("oci-cache-lock", timeout), || { let path = path.clone(); async move { match tokio::task::spawn_blocking(move || { diff --git a/crates/capsem-assets/src/oci/cache/tests.rs b/crates/capsem-assets/src/oci/cache/tests.rs index 4b62da36c..a0a936574 100644 --- a/crates/capsem-assets/src/oci/cache/tests.rs +++ b/crates/capsem-assets/src/oci/cache/tests.rs @@ -383,6 +383,54 @@ async fn abandoned_partial_is_reclaimed_by_publication() { assert!(!partial.exists()); } +#[tokio::test] +async fn copy_hit_with_lease_releases_stripe_lock_on_hit_and_retains_on_miss() { + let root = super::super::tests::private_dir(); + let staging = tempfile::tempdir().unwrap(); + let cache = BlobCache::at(root.path()).unwrap(); + cache.prepare().await.unwrap(); + let digest = identity(b"payload"); + let mut lease = Some(cache.lease(&digest).await.unwrap()); + assert!(!cache + .copy_hit_with_lease(&digest, 7, &staging.path().join("miss"), &mut lease) + .await + .unwrap()); + assert!(lease.is_some(), "cache miss must retain the stripe lock for download"); + let source = staging.path().join("source"); + std::fs::write(&source, b"payload").unwrap(); + cache.publish(&digest, &source).await.unwrap(); + assert!(cache + .copy_hit_with_lease(&digest, 7, &staging.path().join("hit"), &mut lease) + .await + .unwrap()); + assert!(lease.is_none(), "cache hit must release the stripe lock once opened"); +} + +#[tokio::test] +async fn stripe_lease_uses_pull_timeout_while_cache_locks_keep_thirty_seconds() { + assert_eq!(lock_timeout(LockScope::BlobStripe), super::super::PULL_TIMEOUT); + assert_eq!(lock_timeout(LockScope::BlobStripe), Duration::from_secs(300)); + assert_eq!(lock_timeout(LockScope::Cache), CACHE_LOCK_TIMEOUT); + assert_eq!(lock_timeout(LockScope::Cache), Duration::from_secs(30)); + + let root = super::super::tests::private_dir(); + let cache = BlobCache::at(root.path()).unwrap(); + cache.prepare().await.unwrap(); + let digest = identity(b"contended-stripe"); + let _held = cache.lease(&digest).await.unwrap(); + let hex = digest_hex(&digest).unwrap(); + let stripe_path = root.path().join("locks").join(format!("{}.lock", &hex[..2])); + let err = lock_with_timeout( + stripe_path, + LockAccess::Prepare, + LockMode::Exclusive, + Duration::from_millis(20), + ) + .await + .unwrap_err(); + assert!(format!("{err:#}").contains("timeout: 20ms"), "{err:#}"); +} + fn integrity(bytes: &[u8]) -> String { format!("blake3:{}", capsem_crypto::blake3::hash(bytes).to_hex()) } diff --git a/crates/capsem-assets/src/oci/pull.rs b/crates/capsem-assets/src/oci/pull.rs index 021dae0fa..96c89cae5 100644 --- a/crates/capsem-assets/src/oci/pull.rs +++ b/crates/capsem-assets/src/oci/pull.rs @@ -23,12 +23,11 @@ use super::{ catalog::{Catalog, CATALOG_MEDIA_TYPE}, digest_hex, image_reference, selector::Digest as ContentDigest, - verify_platform, CacheIdentity, ImageReference, METADATA_LIMIT, RUNTIME_CONTRACT, + verify_platform, CacheIdentity, ImageReference, METADATA_LIMIT, PULL_TIMEOUT, RUNTIME_CONTRACT, }; const LAYER_LIMIT: u64 = 2 * 1024 * 1024 * 1024; const IMAGE_LIMIT: u64 = 4 * 1024 * 1024 * 1024; -const PULL_TIMEOUT: Duration = Duration::from_secs(300); const CATALOG_LIMIT: u64 = 1024 * 1024; pub(super) fn validate_architecture(architecture: &str) -> Result<()> { @@ -583,11 +582,11 @@ impl Puller { let cache = self.cache.as_ref().map(|cache| cache.for_repository(reference)); let _lease = match &cache { Some(cache) => { - let lease = cache.lease(&descriptor.digest).await?; + let mut lease = Some(cache.lease(&descriptor.digest).await?); let hit = match usage { BlobUse::Private | BlobUse::CacheOnly => { cache - .copy_hit(&descriptor.digest, descriptor.size as u64, &destination) + .copy_hit_with_lease(&descriptor.digest, descriptor.size as u64, &destination, &mut lease) .await? } BlobUse::PublishedRoot | BlobUse::PublishedRootCacheOnly => { @@ -605,7 +604,10 @@ impl Puller { tracing::debug!(digest = %descriptor.digest, "OCI blob cache hit"); return Ok(()); } - Some(lease) + if lease.is_none() { + lease = Some(cache.lease(&descriptor.digest).await?); + } + lease } None => None, }; From 1d6fb0c1713e371e5c0c46fb883bd6196928ea1d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 7 Oct 2026 20:33:39 +0000 Subject: [PATCH 002/464] security(service): bind owner handoffs to trusted sessions and generations --- CHANGELOG.md | 81 +----------- crates/capsem-service/src/main.rs | 1 + crates/capsem-service/src/owner_handoff.rs | 68 ++++++++++ .../capsem-service/src/owner_handoff/tests.rs | 49 +++++++ .../src/router_runtime/exposures.rs | 15 ++- .../src/router_runtime/exposures/tests.rs | 121 ++++++++++++++++++ crates/capsem-service/src/switches.rs | 26 ++-- .../src/tests/network_routes.rs | 23 ++-- .../src/tests/network_routes/handoff.rs | 47 +++++++ 9 files changed, 325 insertions(+), 106 deletions(-) create mode 100644 crates/capsem-service/src/owner_handoff.rs create mode 100644 crates/capsem-service/src/owner_handoff/tests.rs create mode 100644 crates/capsem-service/src/tests/network_routes/handoff.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 54a002db9..72c2f1bc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,84 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security -- Ledger event identifiers reject NUL suffixes, and host audit-chain hashes - require binary storage. Stale declarations fail integrity readiness. - -- Ledger digest and credential-reference checks reject nonhex characters and - NUL suffixes. Readers and writers refuse older checks without rewriting the ledger. - -- Session credential grants require the original runtime's acknowledged policy - and current policy; applied-policy receipts retire with stopped sessions. - -- Credential target preflight uses the existing security rules and plugins, - including corporate ask/block and method, query and port constraints. - -- Delayed process exit cleanup fences its exact old credential runtime even - after a resumed replacement has taken over the session registry. - -- Session stop, suspend, exit and terminal cleanup fence credential grants. - Ready resume rebinds valid approval, and deletion refuses a resumed replacement. - -- Credential runtime fences preserve finite same-session approval on retained - stops and retire terminal session approvals, including recovered metadata. - -- Invalid corporate policy now refuses security admission instead of being - treated as empty constraints; the settings UI remains available for repair. - -- Credential grant issuance checks current policy and runtime ownership inside - the same admission lock as terminal retirement, preventing stale permission - checks from inserting authority after its owner has retired. - -- VM boot admission retains an exclusive generation lifetime fence. Service - restart cleanup confirms exit in the original kernel boot and PID namespace - before retiring grants and session bytes; delayed children cannot replay boot. - -- Startup retirement verifies loss of the original service parent and uses - stable native identity on Linux and Mac. Hosts without guarded signals await - the owner's parent-loss exit and refuse unconfirmed retirement. - -- Fresh and resumed VM launches carry the service's original PID and exact - spawn generation into process lifetime and boot admission. - -- VM processes verify their original service parent and trusted spawn - generation before boot, and terminate when that service disappears. - -- Linux startup pins original orphan processes and verifies fresh native - arguments before signalling. TERM/KILL and exit waits retain those handles; - uncertain identity or exit refuses startup before stale socket removal. - -- VM teardown signals and awaits its original owned child rather than a - cached PID. Shutdown, suspend and failed restore refuse uncertain exit; - service shutdown retires the child before deleting ephemeral session bytes. - -- Service startup orphan selection preserves native argument boundaries and - requires one session path directly under its own run roots. Neighbouring - root prefixes, incidental command text, duplicate options and traversal - targets cannot select unrelated workloads. - -- Managed VM expiry and authenticated close retire the original trusted - effects even when ownership storage is missing or corrupt. Failed persistence - cannot acknowledge a replacement generation as closed. - -- Generated Python requests redact nested credential validation inputs, - including raw parent inputs retained when another required field is missing. - -- Vhost-vsock validates complete descriptor and ring ranges, queue sizes, and - index-cell arithmetic before exposing guest-controlled addresses to the - kernel backend. - -- Virtio descriptor and ring metadata validate their complete guest RAM span - and address arithmetic before host access, including PCI-hole crossings. - -- OAuth disconnect joins an in-flight refresh and retires the latest rotated - token under denial, including material whose durable publication failed. - -- Disconnect admission and reconnect publication share the credential owner's - state lock, so concurrent reauthorization cannot clear a pending denial. - -- Responses WebSocket messages cross the model security boundary before - forwarding requests or returning output, including changes to rules on an - open connection. Completed turns retain model, usage and caller telemetry. +- Bind private-network and preview handoff endpoints to the coordinator's + session identity and VM-owner spawn generation. Refuse worker replies that + name another endpoint or arrive after that owner has been replaced. ### Fixed diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 7644c7f55..abe765f55 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -38,6 +38,7 @@ mod managed_routes; pub mod managed_sessions; use instance::InstanceInfo; mod network_routes; +mod owner_handoff; mod policy_mutation; use policy_mutation::{apply_policy_mutation, MutationRoute, PolicyMutation}; mod connection_routes; diff --git a/crates/capsem-service/src/owner_handoff.rs b/crates/capsem-service/src/owner_handoff.rs new file mode 100644 index 000000000..1d727f444 --- /dev/null +++ b/crates/capsem-service/src/owner_handoff.rs @@ -0,0 +1,68 @@ +//! The coordinator's session and spawn, never a worker-reported path, select +//! the endpoint to which a handoff may grant descriptors. +use super::*; + +pub(crate) struct OwnerHandoff { + id: String, + generation: uuid::Uuid, + endpoint: PathBuf, + pub(crate) uds_path: PathBuf, +} + +impl OwnerHandoff { + pub(crate) async fn acquire(state: &Arc, id: &str) -> Result { + let (id, generation, uds_path) = { + let instances = state.instances.lock().unwrap(); + let instance = instances + .get(id) + .ok_or_else(|| AppError(StatusCode::NOT_FOUND, format!("sandbox not running: {id}")))?; + let captured = (instance.id.clone(), instance.generation, instance.uds_path.clone()); + drop(instances); + captured + }; + let session_id = id.clone(); + // The stable fallback helper may create/check its private directory. + let endpoint = state + .off_worker(move |state| capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, &session_id)) + .await? + .map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("derive owner handoff: {error}"), + ) + })?; + Ok(Self { + id, + generation, + endpoint, + uds_path, + }) + } + + pub(crate) fn validate(self, state: &ServiceState, reported: &str) -> Result { + // Path equality normalizes '.' components. Authorize exactly the + // spelling we derived, without resolving any worker-controlled path. + if self.endpoint.as_os_str() != std::ffi::OsStr::new(reported) { + return Err(AppError( + StatusCode::BAD_GATEWAY, + "handoff endpoint does not belong to this VM owner".into(), + )); + } + if !state + .instances + .lock() + .unwrap() + .get(&self.id) + .is_some_and(|instance| instance.generation == self.generation) + { + return Err(AppError( + StatusCode::BAD_GATEWAY, + "VM owner changed during handoff admission".into(), + )); + } + Ok(self.endpoint) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/owner_handoff/tests.rs b/crates/capsem-service/src/owner_handoff/tests.rs new file mode 100644 index 000000000..ae0d2cb3b --- /dev/null +++ b/crates/capsem-service/src/owner_handoff/tests.rs @@ -0,0 +1,49 @@ +use super::*; +use crate::tests::{insert_fake_instance_with_session_dir, make_test_state_owned}; + +#[tokio::test] +async fn handoffs_use_the_trusted_identity_with_short_and_hashed_paths() { + for long in [false, true] { + let mut owned = make_test_state_owned(); + if long { + owned.run_dir = owned.run_dir.join("long-runtime-root-".repeat(8)); + } + let state = Arc::new(owned); + insert_fake_instance_with_session_dir(&state, "session", 1, state.run_dir.join("sessions/session")); + let expected = capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, "session").unwrap(); + let handoff = OwnerHandoff::acquire(&state, "session").await.unwrap(); + assert_eq!(handoff.validate(&state, expected.to_str().unwrap()).unwrap(), expected); + if long { + assert!( + !expected.starts_with(&state.run_dir), + "exercise the stable hashed fallback" + ); + } else { + assert!(expected.starts_with(&state.run_dir)); + } + } +} + +#[tokio::test] +async fn handoff_grants_are_revoked_when_the_owner_is_removed_or_replaced() { + for replace in [false, true] { + let state = crate::tests::make_test_state(); + insert_fake_instance_with_session_dir(&state, "session", 1, state.run_dir.join("sessions/session")); + let expected = capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, "session").unwrap(); + let handoff = OwnerHandoff::acquire(&state, "session").await.unwrap(); + state.instances.lock().unwrap().remove("session"); + if replace { + insert_fake_instance_with_session_dir(&state, "session", 1, state.run_dir.join("sessions/session")); + } + let error = handoff.validate(&state, expected.to_str().unwrap()).unwrap_err(); + assert_eq!(error.0, StatusCode::BAD_GATEWAY); + assert_eq!(error.1, "VM owner changed during handoff admission"); + } +} + +#[tokio::test] +async fn stopped_sessions_have_no_owner_handoff_grant() { + let state = crate::tests::make_test_state(); + let error = OwnerHandoff::acquire(&state, "stopped").await.err().unwrap(); + assert_eq!(error.0, StatusCode::NOT_FOUND); +} diff --git a/crates/capsem-service/src/router_runtime/exposures.rs b/crates/capsem-service/src/router_runtime/exposures.rs index 026035701..36a72986d 100644 --- a/crates/capsem-service/src/router_runtime/exposures.rs +++ b/crates/capsem-service/src/router_runtime/exposures.rs @@ -312,21 +312,23 @@ pub(crate) async fn handle_admit_preview_connection( Path((vm_id, exposure_id)): Path<(String, String)>, Json(request): Json, ) -> Result, AppError> { + let handoff = crate::owner_handoff::OwnerHandoff::acquire(&state, &vm_id).await?; let kind = match request.kind { capsem_api::PreviewAdmissionKind::Request => capsem_proto::PreviewAdmissionKind::Request, capsem_api::PreviewAdmissionKind::WebsocketUpgrade => capsem_proto::PreviewAdmissionKind::WebsocketUpgrade, }; - match ask_owner( - &state, - &vm_id, + match send_ipc_command( + &handoff.uds_path, ServiceToProcess::AdmitPreviewConnection { id: state.next_job_id(), exposure_id, session_token: request.session_token, kind, }, + Some(OWNER_TIMEOUT_SECS), ) - .await? + .await + .map_err(|error| AppError(StatusCode::BAD_GATEWAY, format!("VM owner unavailable: {error}")))? { ProcessToService::PreviewConnectionAdmitted { handoff_socket, @@ -335,7 +337,10 @@ pub(crate) async fn handle_admit_preview_connection( error: None, .. } => Ok(Json(PreviewConnectionAdmissionResponse { - handoff_socket, + handoff_socket: handoff + .validate(&state, &handoff_socket)? + .to_string_lossy() + .into_owned(), handoff_token, owner_generation: owner_generation.to_string(), })), diff --git a/crates/capsem-service/src/router_runtime/exposures/tests.rs b/crates/capsem-service/src/router_runtime/exposures/tests.rs index f5ccbd3d5..1556033d2 100644 --- a/crates/capsem-service/src/router_runtime/exposures/tests.rs +++ b/crates/capsem-service/src/router_runtime/exposures/tests.rs @@ -289,6 +289,127 @@ async fn exposures_on_a_vm_that_is_not_running_do_not_reach_any_owner() { assert!(status.is_client_error(), "{status}"); } +#[tokio::test] +async fn preview_admission_rejects_owner_supplied_foreign_handoff_paths() { + let (state, uds_path, _dir) = fixture(); + let expected = capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, "box").unwrap(); + let parent = expected.parent().unwrap(); + let forged = [ + capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, "other-box").unwrap(), + state.service_socket.clone(), + parent + .join("..") + .join(parent.file_name().unwrap()) + .join(expected.file_name().unwrap()), + parent.join(".").join(expected.file_name().unwrap()), + PathBuf::new(), + ]; + for handoff in forged { + let handoff = handoff.to_string_lossy().into_owned(); + let owner = spawn_fake_process(&uds_path, 1, move |message| { + let ServiceToProcess::AdmitPreviewConnection { id, .. } = message else { + panic!("unexpected {message:?}") + }; + let reply = ProcessToService::PreviewConnectionAdmitted { + id: *id, + handoff_socket: handoff.clone(), + handoff_token: 123, + owner_generation: 42, + error: None, + policy_refused: false, + }; + Box::pin(async move { Some(reply) }) + }); + let (status, body) = call( + &state, + axum::http::Method::POST, + "/internal/vms/box/exposures/preview-1/preview-admission", + Some(json!({"session_token": "preview-session", "kind": "request"})), + ) + .await; + owner.await.unwrap(); + assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); + assert!( + body.to_string() + .contains("handoff endpoint does not belong to this VM owner"), + "{body}" + ); + assert!( + body.get("handoff_socket").is_none(), + "never forward a forged endpoint: {body}" + ); + } +} + +#[tokio::test] +async fn preview_admission_preserves_the_coordinator_derived_handoff_endpoint() { + let (state, uds_path, _dir) = fixture(); + let expected = capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, "box").unwrap(); + let handoff = expected.to_string_lossy().into_owned(); + let owner = spawn_fake_process(&uds_path, 1, move |message| { + let ServiceToProcess::AdmitPreviewConnection { id, .. } = message else { + panic!("unexpected {message:?}") + }; + let reply = ProcessToService::PreviewConnectionAdmitted { + id: *id, + handoff_socket: handoff.clone(), + handoff_token: 123, + owner_generation: 42, + error: None, + policy_refused: false, + }; + Box::pin(async move { Some(reply) }) + }); + let (status, body) = call( + &state, + axum::http::Method::POST, + "/internal/vms/box/exposures/preview-1/preview-admission", + Some(json!({"session_token": "preview-session", "kind": "request"})), + ) + .await; + owner.await.unwrap(); + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + body, + json!({"handoff_socket": expected, "handoff_token": 123, "owner_generation": "42"}) + ); +} + +#[tokio::test] +async fn preview_admission_rejects_a_reply_from_a_replaced_owner_generation() { + let (state, uds_path, _dir) = fixture(); + let expected = capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, "box").unwrap(); + let owner_state = Arc::clone(&state); + let owner = spawn_fake_process(&uds_path, 1, move |message| { + let ServiceToProcess::AdmitPreviewConnection { id, .. } = message else { + panic!("unexpected {message:?}") + }; + owner_state.instances.lock().unwrap().get_mut("box").unwrap().generation = uuid::Uuid::new_v4(); + let reply = ProcessToService::PreviewConnectionAdmitted { + id: *id, + handoff_socket: expected.to_string_lossy().into_owned(), + handoff_token: 123, + owner_generation: 42, + error: None, + policy_refused: false, + }; + Box::pin(async move { Some(reply) }) + }); + let (status, body) = call( + &state, + axum::http::Method::POST, + "/internal/vms/box/exposures/preview-1/preview-admission", + Some(json!({"session_token": "preview-session", "kind": "request"})), + ) + .await; + owner.await.unwrap(); + assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); + assert!( + body.to_string().contains("VM owner changed during handoff admission"), + "{body}" + ); +} + /// Finding 11 (google/capsem#222): a leaked preview session could only be cut /// by deleting the whole exposure. Revoking its sessions keeps the exposure /// and ends every flow they admitted. diff --git a/crates/capsem-service/src/switches.rs b/crates/capsem-service/src/switches.rs index e4580808a..b8ad47881 100644 --- a/crates/capsem-service/src/switches.rs +++ b/crates/capsem-service/src/switches.rs @@ -366,10 +366,13 @@ fn replug_later( async fn handshake( state: &Arc, network: Uuid, - uds_path: &StdPath, + vm_id: &str, address: Ipv4Addr, generation: u32, ) -> Result<(Arc, u64, std::os::unix::net::UnixStream)> { + let handoff = crate::owner_handoff::OwnerHandoff::acquire(state, vm_id) + .await + .map_err(|error| anyhow!(error.1))?; let summary = state .networks .lock() @@ -386,7 +389,7 @@ async fn handshake( prefix: summary.subnet.prefix_len(), generation, }; - let handoff_socket = match send_ipc_command(uds_path, ask, Some(ATTACH_TIMEOUT_SECS)).await { + let handoff_socket = match send_ipc_command(&handoff.uds_path, ask, Some(ATTACH_TIMEOUT_SECS)).await { Ok(ProcessToService::LinkAttachResult { error: None, handoff_socket, @@ -396,8 +399,11 @@ async fn handshake( Ok(other) => anyhow::bail!("unexpected owner reply: {other:?}"), Err(error) => anyhow::bail!("owner unreachable: {error}"), }; + let handoff_socket = handoff + .validate(state, &handoff_socket) + .map_err(|error| anyhow!(error.1))?; let socket = std::os::unix::net::UnixStream::connect(&handoff_socket) - .with_context(|| format!("connect the owner's seat at {handoff_socket}"))?; + .with_context(|| format!("connect the owner's seat at {}", handoff_socket.display()))?; let sender = Sender::new(socket.try_clone()?)?; let receiver = Receiver::new(socket.try_clone()?)?; let token = u64::from_str_radix(&token, 16)?; @@ -425,15 +431,9 @@ async fn handshake( /// checked again, under the registry lock, once the switch has the cable: a /// member that left meanwhile, or a newer plug or unplug, unplugs it. pub(crate) async fn plug(state: &Arc, network: Uuid, vm_id: &str) -> Result<()> { - let running = state - .instances - .lock() - .unwrap() - .get(vm_id) - .map(|instance| instance.uds_path.clone()); - let Some(uds_path) = running else { + if !state.instances.lock().unwrap().contains_key(vm_id) { return Ok(()); - }; + } let (generation, address) = { let mut registry = state.networks.lock().await; let Some(address) = registry.address_of(network, vm_id) else { @@ -447,8 +447,8 @@ pub(crate) async fn plug(state: &Arc, network: Uuid, vm_id: &str) drop(registry); (generation, address) }; - let connection = capsem_crypto::uuid_v4(); - match handshake(state, network, &uds_path, address, generation).await { + let connection = Uuid::new_v4(); + match handshake(state, network, vm_id, address, generation).await { Ok((host, port, keepalive)) => { let mut registry = state.networks.lock().await; if !is_member(®istry, network, vm_id) || !state.switches.is_current(network, vm_id, generation) { diff --git a/crates/capsem-service/src/tests/network_routes.rs b/crates/capsem-service/src/tests/network_routes.rs index 429e2fafc..9a9b9b3aa 100644 --- a/crates/capsem-service/src/tests/network_routes.rs +++ b/crates/capsem-service/src/tests/network_routes.rs @@ -4,6 +4,8 @@ use super::*; use axum::http::Method; use std::time::Duration; +mod handoff; + fn app(state: &Arc) -> axum::Router { build_service_router(Arc::clone(state)) } @@ -301,6 +303,7 @@ impl FakeLinkSeat { /// `detaches` is how many LinkDetach requests the owner answers besides /// its `links` LinkAttach ones. fn fake_link_seat( + state: &ServiceState, uds: &std::path::Path, refuse: bool, links: usize, @@ -308,7 +311,7 @@ fn fake_link_seat( ) -> (Arc, tokio::task::JoinHandle>) { use capsem_foundation::unix::router_channel::{Receiver, Sender}; use std::os::fd::AsRawFd; - let handoff = uds.with_file_name("vm-b-handoff.sock"); + let handoff = capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, "vm-b").unwrap(); let _ = std::fs::remove_file(&handoff); let listener = tokio::net::UnixListener::bind(&handoff).unwrap(); let (released_tx, released_rx) = tokio::sync::mpsc::unbounded_channel(); @@ -397,7 +400,7 @@ async fn attaching_a_running_member_links_it_to_the_networks_switch_until_it_lea insert_fake_instance(&state, "vm-b", std::process::id()); let uds_b = state.instances.lock().unwrap()["vm-b"].uds_path.clone(); std::fs::create_dir_all(uds_b.parent().unwrap()).unwrap(); - let (seat, owner) = fake_link_seat(&uds_b, false, 1, 1); + let (seat, owner) = fake_link_seat(&state, &uds_b, false, 1, 1); let (_, created) = create_network(&state, "team").await; let id = created["id"].as_str().unwrap().to_string(); @@ -480,7 +483,7 @@ async fn a_rejoin_plugs_with_a_generation_newer_than_the_leave_before_it() { insert_fake_instance(&state, "vm-b", std::process::id()); let uds_b = state.instances.lock().unwrap()["vm-b"].uds_path.clone(); std::fs::create_dir_all(uds_b.parent().unwrap()).unwrap(); - let (_seat, owner) = fake_link_seat(&uds_b, false, 2, 1); + let (_seat, owner) = fake_link_seat(&state, &uds_b, false, 2, 1); let (_, created) = create_network(&state, "team").await; let id = created["id"].as_str().unwrap().to_string(); let member = format!("/networks/{id}/members/vm-b"); @@ -524,7 +527,7 @@ async fn leaving_writes_the_close_row_with_the_switchs_counters_for_the_cable() insert_fake_instance(&state, "vm-b", std::process::id()); let uds_b = state.instances.lock().unwrap()["vm-b"].uds_path.clone(); std::fs::create_dir_all(uds_b.parent().unwrap()).unwrap(); - let (_seat, _owner) = fake_link_seat(&uds_b, false, 1, 1); + let (_seat, _owner) = fake_link_seat(&state, &uds_b, false, 1, 1); let (_, created) = create_network(&state, "team").await; let id = created["id"].as_str().unwrap().to_string(); let (status, joined) = route_request(app(&state), Method::PUT, &format!("/networks/{id}/members/vm-b"), None).await; @@ -597,7 +600,7 @@ async fn a_member_links_at_start_even_when_its_owner_binds_late() { .await .unwrap(); tokio::time::sleep(Duration::from_millis(500)).await; - let (_seat, owner) = fake_link_seat(&uds_b, false, 1, 0); + let (_seat, owner) = fake_link_seat(&state, &uds_b, false, 1, 0); let mut observed = String::new(); for _ in 0..100 { @@ -618,7 +621,7 @@ async fn a_member_whose_stream_ends_is_declared_and_then_linked_again() { insert_fake_instance(&state, "vm-b", std::process::id()); let uds_b = state.instances.lock().unwrap()["vm-b"].uds_path.clone(); std::fs::create_dir_all(uds_b.parent().unwrap()).unwrap(); - let (seat, owner) = fake_link_seat(&uds_b, false, 2, 0); + let (seat, owner) = fake_link_seat(&state, &uds_b, false, 2, 0); let (_, created) = create_network(&state, "team").await; let id = created["id"].as_str().unwrap().to_string(); let (status, _) = route_request(app(&state), Method::PUT, &format!("/networks/{id}/members/vm-b"), None).await; @@ -674,7 +677,7 @@ async fn a_member_that_leaves_before_its_relink_is_not_linked_back_in() { std::fs::create_dir_all(uds_b.parent().unwrap()).unwrap(); // Room for a second plug that must never come, besides the plug and the // detach that do. - let (seat, owner) = fake_link_seat(&uds_b, false, 2, 1); + let (seat, owner) = fake_link_seat(&state, &uds_b, false, 2, 1); let (_, created) = create_network(&state, "team").await; let id = created["id"].as_str().unwrap().to_string(); let (status, _) = route_request(app(&state), Method::PUT, &format!("/networks/{id}/members/vm-b"), None).await; @@ -708,7 +711,7 @@ async fn a_member_that_leaves_mid_handshake_keeps_no_link() { insert_fake_instance(&state, "vm-b", std::process::id()); let uds_b = state.instances.lock().unwrap()["vm-b"].uds_path.clone(); std::fs::create_dir_all(uds_b.parent().unwrap()).unwrap(); - let (seat, _owner) = fake_link_seat(&uds_b, false, 1, 0); + let (seat, _owner) = fake_link_seat(&state, &uds_b, false, 1, 0); seat.answers.forget_permits(1); let (_, created) = create_network(&state, "team").await; let id = created["id"].as_str().unwrap().to_string(); @@ -753,7 +756,7 @@ async fn a_plug_that_finishes_inside_a_disconnect_leaves_no_port_behind() { insert_fake_instance(&state, "vm-b", std::process::id()); let uds_b = state.instances.lock().unwrap()["vm-b"].uds_path.clone(); std::fs::create_dir_all(uds_b.parent().unwrap()).unwrap(); - let (seat, _owner) = fake_link_seat(&uds_b, false, 1, 0); + let (seat, _owner) = fake_link_seat(&state, &uds_b, false, 1, 0); seat.answers.forget_permits(1); let (_, created) = create_network(&state, "team").await; let id = created["id"].as_str().unwrap().to_string(); @@ -816,7 +819,7 @@ async fn an_owner_that_refuses_the_link_leaves_a_failed_membership() { insert_fake_instance(&state, "vm-b", std::process::id()); let uds_b = state.instances.lock().unwrap()["vm-b"].uds_path.clone(); std::fs::create_dir_all(uds_b.parent().unwrap()).unwrap(); - let (_seat, _owner) = fake_link_seat(&uds_b, true, 1, 0); + let (_seat, _owner) = fake_link_seat(&state, &uds_b, true, 1, 0); let (_, created) = create_network(&state, "team").await; let id = created["id"].as_str().unwrap().to_string(); let (status, joined) = route_request(app(&state), Method::PUT, &format!("/networks/{id}/members/vm-b"), None).await; diff --git a/crates/capsem-service/src/tests/network_routes/handoff.rs b/crates/capsem-service/src/tests/network_routes/handoff.rs new file mode 100644 index 000000000..4cf729f62 --- /dev/null +++ b/crates/capsem-service/src/tests/network_routes/handoff.rs @@ -0,0 +1,47 @@ +use super::*; + +#[tokio::test] +async fn network_attach_rejects_a_foreign_handoff_before_connecting() { + let (state, _dir) = make_test_state_with_tempdir(); + install_test_runtime_assets(&state); + insert_fake_instance(&state, "vm-b", std::process::id()); + let uds = state.instances.lock().unwrap()["vm-b"].uds_path.clone(); + std::fs::create_dir_all(uds.parent().unwrap()).unwrap(); + let foreign = capsem_foundation::uds::private_handoff_socket_path(&state.run_dir, "vm-a").unwrap(); + let listener = std::os::unix::net::UnixListener::bind(&foreign).unwrap(); + listener.set_nonblocking(true).unwrap(); + let owner = spawn_fake_process(&uds, 1, move |message| { + let ServiceToProcess::LinkAttach { id, .. } = message else { + panic!("unexpected {message:?}") + }; + let reply = ProcessToService::LinkAttachResult { + id: *id, + handoff_socket: foreign.to_string_lossy().into_owned(), + error: None, + }; + Box::pin(async move { Some(reply) }) + }); + let (_, created) = create_network(&state, "team").await; + let network = created["id"].as_str().unwrap(); + let (status, joined) = tokio::time::timeout( + Duration::from_secs(2), + route_request( + app(&state), + Method::PUT, + &format!("/networks/{network}/members/vm-b"), + None, + ), + ) + .await + .expect("a foreign endpoint is refused before waiting for a handoff"); + owner.await.unwrap(); + assert_eq!(status, StatusCode::OK, "{joined}"); + assert_eq!(joined["members"][0]["state"], "failed", "{joined}"); + assert_eq!(listener.accept().unwrap_err().kind(), std::io::ErrorKind::WouldBlock); + let (_, logs) = route_request(app(&state), Method::GET, &format!("/networks/{network}/logs"), None).await; + assert!( + logs.to_string() + .contains("handoff endpoint does not belong to this VM owner"), + "{logs}" + ); +} From 941ed21ec7ba93b7d32d4c867c35e4c546269632 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 7 Oct 2026 20:33:57 +0000 Subject: [PATCH 003/464] docs(architecture): state current host-worker authority accurately --- skills/site-architecture/SKILL.md | 8 +-- .../references/crate-and-privilege-model.md | 51 +++++++++++-------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/skills/site-architecture/SKILL.md b/skills/site-architecture/SKILL.md index 261ede8b7..466cff9cb 100644 --- a/skills/site-architecture/SKILL.md +++ b/skills/site-architecture/SKILL.md @@ -79,9 +79,11 @@ Capsem sandboxes AI agents in air-gapped Linux VMs on macOS using Apple's Virtua - Workspace and overlay bytes are session state, never a hidden image-authoring rail; package changes go through the runtime package set (`/build-images`) or an OCI image under `images/`. -- `capsem-process` stays low privilege: a cleared allowlisted environment, - 0600 sockets, a 0700 session directory, read-only assets and guest binaries, - and only `session_dir/guest/` shared with the VM. +- `capsem-process` clears its inherited environment, uses 0600 sockets and a + 0700 session directory, attaches read-only assets and guest binaries, and + shares only `session_dir/guest/` with the VM. These controls do not confine + a compromised same-UID host worker. The VM owner still has ambient user + authority; #206 owns its OS confinement and the proxy/ledger extraction. - capsem-app contains no VM logic or `capsem-core` dependency. Gateway and tray are service-owned companions and must self-exit with their parent. diff --git a/skills/site-architecture/references/crate-and-privilege-model.md b/skills/site-architecture/references/crate-and-privilege-model.md index 80cde8bbe..8aa71ac12 100644 --- a/skills/site-architecture/references/crate-and-privilege-model.md +++ b/skills/site-architecture/references/crate-and-privilege-model.md @@ -34,9 +34,9 @@ Sharing alone is not a reason to put code in `capsem-core`. - **`capsem-logger`**: session DB schema, queries, storage, and async writer. - **`capsem-guard`**: parent-watch and singleton-flock lifecycle primitives. - **`capsem-service`**: daemon HTTP/UDS API and VM-process orchestration. -- **`capsem-process`**: low-privilege per-VM boot, vsock, IPC, and job runtime. -- **`capsem`**: CLI client; HTTP/UDS to the service, including streamed shell - and terminal operations. It never dials a per-VM process socket. +- **`capsem-process`**: per-VM boot, vsock, IPC, and job runtime; still holds + ambient host-user authority pending #206 confinement. +- **`capsem`**: CLI client; HTTP/UDS to the service, including shell streams. - **`capsem-tui`**: terminal control UI over the gateway API. - **`capsem-admin`**: runtime image build plus asset, release, and config validation. - **`@capsem/mcp`** (`mcp/typescript`): separately installed host MCP server; @@ -54,7 +54,13 @@ Sharing alone is not a reason to put code in `capsem-core`. ## Process privilege model -capsem-process is a **low-privilege** per-VM process. Security invariants: +The VM owner is a separate process, but it does not yet install an OS sandbox. +It retains the launching user's filesystem, socket and network authority. +The following controls constrain normal operation and other-user/guest access; +they do not isolate compromised sibling host workers running as the same UID. +Issue #206 owns that confinement and the proxy/ledger process extraction. + +Current controls: 1. **Minimal environment**: service uses `env_clear()` before spawn, then passes only `HOME`, `PATH`, `USER`, `TMPDIR`, `RUST_LOG`. API keys and tokens from the user's shell never reach the process. 2. **Socket permissions 0600**: Per-VM owner sockets (`{id}.sock` IPC, `{id}-handoff.sock`) are chmod 0600 after bind; no client dials them -- terminals and attach go through the service `/vms/{id}/stream` route. Only the owning user can connect. @@ -63,25 +69,30 @@ capsem-process is a **low-privilege** per-VM process. Security invariants: 5. **Gateway auth layer**: external access goes through capsem-gateway (Bearer token, rate limiting, localhost CORS). Per-VM sockets are not exposed to the network. 6. **Rootfs read-only**: runtime rootfs asset mounted read-only. Guest binaries deployed chmod 555. 7. **Guest binary security**: all injected binaries are read-only. Guest cannot modify its own agent. -8. **VirtioFS boundary**: `session_dir/guest/` is the guest-writable share and - contains the workspace. The writable overlay image stays in host-only - `session_dir/system/`, alongside the host-only ledger, logs and checkpoint. - The separate `session_dir/image/` share contains immutable image blobs and - is read-only at the device. The workspace compatibility symlink points into - `guest/`; the system image must never be reachable through that share. +8. **VirtioFS boundary**: only `session_dir/guest/` is shared via VirtioFS, including the guest workspace. The writable system overlay is in host-only `session_dir/system/`, outside the share. Ledger files and serial logs are also outside the share. The session's workspace compatibility link points into `guest/workspace/`. + +### Required authority after confinement (#206) -### What capsem-process CAN access - Its own session_dir (read-write) -- Assets dir (read-only: kernel, initrd, rootfs) -- Its own UDS sockets -- Apple VZ framework (requires `com.apple.security.virtualization` entitlement) +- Its exact verified boot assets (read-only: kernel, initrd, rootfs) +- Explicitly granted IPC and data descriptors for its session/generation +- Apple VZ framework (requires `com.apple.security.virtualization` entitlement), + or the Linux KVM resources needed for its VM + +### Limits of the current boundary + +0700 session directories and 0600 sockets prevent other users from opening +them. They do not prevent a same-UID worker from accessing another session, +the service's control socket, credentials, or persistent registry. Clearing +the spawn environment prevents inherited shell secrets, but does not deny +filesystem reads or arbitrary upstream dialing. -### What capsem-process CANNOT access -- Other VMs' session dirs (0700, different path) -- Other VMs' UDS sockets (0600) -- The service's UDS socket (filesystem permission only) -- The persistent registry or other service state -- The user's environment variables (cleared at spawn) +The existing router/switch sandbox is an effective descriptor-only OS boundary; +it cannot be copied unchanged onto the VM owner, gateway or future ledger. +Each needs its own capability policy and adversarial macOS/Linux proof. A +coordinator endpoint check prevents a worker from naming an unrelated handoff +path, but does not prevent same-UID replacement of a socket or its ancestors. +OS confinement and trusted descriptor grants must close that remaining gap. ### MITM CA key transparency The MITM proxy CA private key (`crates/capsem-core/resources/ca/capsem-ca.key`) is committed to the repo and embedded at compile time. This is intentional -- capsem's network interception exists for user visibility into what AI agents do, not for secrecy. The CA is only trusted inside capsem's own air-gapped VMs and has zero trust outside them. A public key lets anyone verify there is no hidden interception. Per-installation key generation would reduce transparency. From 353d59d99121a7b2b71179e45aabe1f675a1f03d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 7 Oct 2026 20:46:36 +0000 Subject: [PATCH 004/464] fix(gate): reconcile carried 0.7 source preflight ownership --- .../gate/tools/ci/classify_ci_scope.py | 1 + build_system/pyproject.toml | 3 ++ .../tests/gate/test_bounded_machine_lease.py | 9 ++-- codecov.yml | 1 + config/gate.toml | 11 ++--- skills/dev-testing/SKILL.md | 47 ++----------------- .../references/parallel-lifecycle-tests.md | 45 ++++++++++++++++++ tests/citadel/python_boundary_debt.toml | 2 + .../citadel/repository_surface_ownership.toml | 12 +++++ .../citadel/test_generated_output_boundary.py | 9 ++-- .../test_repository_surface_ownership.py | 2 +- .../citadel/web_benchmark_boundary_debt.toml | 5 +- tests/test_benchmark_retention_contract.py | 6 +-- web/docs/src/content/docs/usage/mcp-tools.md | 7 +++ 14 files changed, 95 insertions(+), 65 deletions(-) create mode 100644 skills/dev-testing/references/parallel-lifecycle-tests.md diff --git a/build_system/builder/gate/tools/ci/classify_ci_scope.py b/build_system/builder/gate/tools/ci/classify_ci_scope.py index bdb73e2c4..cdf5b1951 100644 --- a/build_system/builder/gate/tools/ci/classify_ci_scope.py +++ b/build_system/builder/gate/tools/ci/classify_ci_scope.py @@ -135,6 +135,7 @@ ".github", "skills", "sprints", + "integrations", "tests", "tmp", } diff --git a/build_system/pyproject.toml b/build_system/pyproject.toml index 0d56cccb6..517a18a3a 100644 --- a/build_system/pyproject.toml +++ b/build_system/pyproject.toml @@ -121,3 +121,6 @@ dev = [ "watchdog>=6.0.0", "websockets>=16.0", ] + +[tool.ty.environment] +extra-paths = ["../integrations/inspect-ai"] diff --git a/build_system/tests/gate/test_bounded_machine_lease.py b/build_system/tests/gate/test_bounded_machine_lease.py index fb293a8c5..ddb2ec9e9 100644 --- a/build_system/tests/gate/test_bounded_machine_lease.py +++ b/build_system/tests/gate/test_bounded_machine_lease.py @@ -64,12 +64,15 @@ def _environment(tmp_path: Path, **extra: str) -> dict[str, str]: @contextlib.contextmanager -def _bounded(tmp_path: Path, *command: str, machine_lease: bool = True, **extra: str) -> Iterator[subprocess.Popen[str]]: +def _bounded( + tmp_path: Path, *command: str, machine_lease: bool = True, + environment: dict[str, str] | None = None, **extra: str, +) -> Iterator[subprocess.Popen[str]]: """The real launcher around `command`; killed and drained on the way out.""" with subprocess.Popen( [sys.executable, str(BOUNDED), "--timeout-seconds", "30", *(["--machine-lease"] if machine_lease else []), "--", *command], - env=_environment(tmp_path, **extra), + env=_environment(tmp_path, **{**(environment or {}), **extra}), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -247,7 +250,7 @@ def test_cargo_inside_a_gate_run_does_not_wait_for_its_own_parent(tmp_path: Path inside_a_run = {LOCK.run_marker: "capsem-gate candidate"} with ( _gate_holds_the_machine(tmp_path), - _bounded(tmp_path, "cargo", "build", **inside_a_run) as inside, + _bounded(tmp_path, "cargo", "build", environment=inside_a_run) as inside, ): assert _finished_within(inside, 20), "a gate's own cargo deadlocked on its parent's lock" assert inside.returncode == 0 diff --git a/codecov.yml b/codecov.yml index 3281b9cbe..933d1cd0f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -62,6 +62,7 @@ flags: - build_system/release_site/scripts/** - build_system/release_site/src/** - build_system/builder/** + - integrations/inspect-ai/inspect_capsem/** carryforward: true integration: paths: diff --git a/config/gate.toml b/config/gate.toml index 1bd4183a0..a3798aeca 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -843,12 +843,9 @@ key = "version" # --------------------------------------------------------------------------- [lint] -python_roots = ["build_system/builder", "build_system/sdist_command.py", "build_system/packaging", "build_system/scripts", "build_system/tests", "tests", "guest", "images", "integrations/inspect-ai"] +python_roots = ["build_system/builder", "build_system/sdist_command.py", "build_system/packaging", "build_system/scripts", "build_system/tests", "tests", "guest", "images", "integrations"] # Already passes every rule, and must keep doing so. -strict_roots = ["build_system/builder", "build_system/sdist_command.py", "integrations/inspect-ai"] -# Resolve the separate carried namespace without treating builder/ as a -# top-level package root and shadowing its installed capsem_builder identity. -strict_ty_search_paths = ["integrations/inspect-ai"] +strict_roots = ["build_system/builder", "build_system/sdist_command.py", "integrations"] # Match the import roots exercised by the sole pytest project plus the guest # trees and script bootstrap paths that are materialized at runtime. Keeping # these explicit lets Ty inspect a conventional capsem-builder install without @@ -1064,7 +1061,7 @@ direct_concurrency = ["planrunner.py", "metadatasurvey.py"] # shrink must update or remove its entry immediately. This is debt inventory, # not an exemption list. [boundary.scripts] -roots = ["build_system/scripts", "build_system/packaging", "sdk/python", "sdk/typescript", "mcp/typescript"] +roots = ["build_system/scripts", "build_system/packaging", "sdk/python", "sdk/typescript", "mcp/typescript", "integrations"] suffixes = [".py", ".sh", ".ts", ".mjs"] max_lines = 300 @@ -2023,7 +2020,6 @@ building = ["frontend-build", "docs", "site"] # `tests/test_gate_*.py` to appear here, since by construction those need # neither. source_contract = [ - "tests/test_benchmark_routes.py", "integrations/inspect-ai/tests/containers/test_compose_fields.py", "integrations/inspect-ai/tests/containers/test_compose_inputs.py", "integrations/inspect-ai/tests/containers/test_compose_interpolation.py", @@ -2031,7 +2027,6 @@ source_contract = [ "integrations/inspect-ai/tests/containers/test_dockerfile_ca.py", "integrations/inspect-ai/tests/containers/test_dockerfile_heredoc.py", "integrations/inspect-ai/tests/containers/test_dockerfile_metadata.py", - "tests/qualification/test_candidate.py", "build_system/tests/gate/test_gate_observation.py", "build_system/tests/gate/test_gate_observation_startup.py", "build_system/tests/gate/test_gate_observation_propagation.py", diff --git a/skills/dev-testing/SKILL.md b/skills/dev-testing/SKILL.md index c19b9bb62..d0e4110b9 100644 --- a/skills/dev-testing/SKILL.md +++ b/skills/dev-testing/SKILL.md @@ -255,51 +255,10 @@ Do not add another local HTTP/MCP/OAuth/model mock server for a feature. Extend the shared mock server and its fixtures instead, then assert the route through the relevant black-box test. -## Parallel tests as dogfooding (n=4 is non-negotiable) +## Parallel VM and process lifecycle tests -`just test` runs the python suite under `pytest -n 4 --dist=loadfile`. Four real VMs boot simultaneously. **This is the canary, not just a speed-up.** We ship Capsem as a multi-VM sandbox for AI agents -- if our own test suite cannot safely boot 4 concurrent VMs, real users running an agent farm will hit the exact same bug. Treat any concurrency flake as a Capsem-side bug, not a test-tuning problem: - -- "Suspend timed out" under load -> service IPC handling is racy, not "bump the timeout" -- "Session did not become ready" -> Apple VZ resource serialization, VirtioFS lock contention, or service handling concurrent provisions; investigate, don't suppress -- Two tests both want the same VM name -> name-collision bug in `validate_vm_name` / registry, not "isolate test names better" -- Stale socket between tests -> service didn't reap a child cleanly, real production bug - -Anti-patterns when a test flakes under `-n 4`: -- Adding `time.sleep()` to "let things settle" -- masking a race -- Bumping the per-test timeout -- buying time for a real bug to manifest in prod instead of CI -- Marking the test `serial` so it runs alone -- defeating the dogfooding signal - -The exception is a true timing or benchmark probe whose assertion is the -measured number. Those tests must already be marked `serial` and `just test` -runs them immediately after the `-n 4` canary. That is not a flake escape -hatch: it prevents another benchmark file from stealing the same Apple VZ -launch budget and corrupting the number we are trying to publish. - -The host has plenty of headroom (48 GB RAM, 14 cores; 4 VMs at 2 GB / 2 CPU each = 8 GB / 8 cores). If concurrency surfaces a flake, fix the product, then re-run. Bumping `-n` higher (8, 12) is the natural follow-on once n=4 is stable -- real users will run more. - -### Orphan processes across runs are a product bug (not a test bug) - -After an interrupted test run, readiness failures, UDS connection refusals, or HTTP 500s can come from companions that survived their parent. Confirm the failing run's owned processes before diagnosing a leak. Use the repository's PID-file and owned process-tree control modules to stop only that run's exact PIDs. AGENTS.md's "Never kill by pattern" rule applies to binary-path patterns too: they can match another session's processes. - -Fix the companion lifecycle when a leak is confirmed. Every spawned companion must use `capsem-guard::install(parent_pid, lock_path)` to refuse standalone launches, enforce its singleton, and exit when its parent dies. See `/dev-rust-patterns` lesson 18. Preserve `tests/capsem-service/test_companion_lifecycle.py` and extend its regressions when adding a new companion. - -### Apple VZ lifecycle serialization is part of the product - -Apple's Virtualization.framework does not tolerate overlapping checkpoint -lifecycle operations (`saveMachineStateToURL` and `restoreMachineStateFromURL`) -on sibling VMs, and teardown must not cross those checkpoint edges. Capsem uses -`ServiceState::save_restore_lock` plus the host-wide `VzHostLock` flock: -cold starts and teardown take shared/read guards, save and restore take -exclusive/write guards. The rail holds even when pytest-xdist spawns one -`capsem-service` per worker, while independent cold starts can still run -together for the boot-latency gate. - -Do not demote suspend/resume, lifecycle, provisioning, or teardown tests to -`-n 1` to sidestep VZ races. `just test` at `-n 4` is the contract; if a -concurrent run sees restore permission errors, loop-device corruption, -connection-refused startup races, or readiness misses, fix the lifecycle rail. -Full context and failure signatures live in -`web/docs/src/content/docs/gotchas/concurrent-suspend-resume.md`. +Read [parallel-lifecycle-tests.md](references/parallel-lifecycle-tests.md) before +changing VM concurrency, companion supervision, or Apple VZ lifecycle tests. ## Adversarial testing diff --git a/skills/dev-testing/references/parallel-lifecycle-tests.md b/skills/dev-testing/references/parallel-lifecycle-tests.md new file mode 100644 index 000000000..c3c2522df --- /dev/null +++ b/skills/dev-testing/references/parallel-lifecycle-tests.md @@ -0,0 +1,45 @@ +## Parallel tests as dogfooding (n=4 is non-negotiable) + +`just test` runs the python suite under `pytest -n 4 --dist=loadfile`. Four real VMs boot simultaneously. **This is the canary, not just a speed-up.** We ship Capsem as a multi-VM sandbox for AI agents -- if our own test suite cannot safely boot 4 concurrent VMs, real users running an agent farm will hit the exact same bug. Treat any concurrency flake as a Capsem-side bug, not a test-tuning problem: + +- "Suspend timed out" under load -> service IPC handling is racy, not "bump the timeout" +- "Session did not become ready" -> Apple VZ resource serialization, VirtioFS lock contention, or service handling concurrent provisions; investigate, don't suppress +- Two tests both want the same VM name -> name-collision bug in `validate_vm_name` / registry, not "isolate test names better" +- Stale socket between tests -> service didn't reap a child cleanly, real production bug + +Anti-patterns when a test flakes under `-n 4`: +- Adding `time.sleep()` to "let things settle" -- masking a race +- Bumping the per-test timeout -- buying time for a real bug to manifest in prod instead of CI +- Marking the test `serial` so it runs alone -- defeating the dogfooding signal + +The exception is a true timing or benchmark probe whose assertion is the +measured number. Those tests must already be marked `serial` and `just test` +runs them immediately after the `-n 4` canary. That is not a flake escape +hatch: it prevents another benchmark file from stealing the same Apple VZ +launch budget and corrupting the number we are trying to publish. + +The host has plenty of headroom (48 GB RAM, 14 cores; 4 VMs at 2 GB / 2 CPU each = 8 GB / 8 cores). If concurrency surfaces a flake, fix the product, then re-run. Bumping `-n` higher (8, 12) is the natural follow-on once n=4 is stable -- real users will run more. + +### Orphan processes across runs are a product bug (not a test bug) + +After an interrupted test run, readiness failures, UDS connection refusals, or HTTP 500s can come from companions that survived their parent. Confirm the failing run's owned processes before diagnosing a leak. Use the repository's PID-file and owned process-tree control modules to stop only that run's exact PIDs. AGENTS.md's "Never kill by pattern" rule applies to binary-path patterns too: they can match another session's processes. + +Fix the companion lifecycle when a leak is confirmed. Every spawned companion must use `capsem-guard::install(parent_pid, lock_path)` to refuse standalone launches, enforce its singleton, and exit when its parent dies. See `/dev-rust-patterns` lesson 18. Preserve `tests/capsem-service/test_companion_lifecycle.py` and extend its regressions when adding a new companion. + +### Apple VZ lifecycle serialization is part of the product + +Apple's Virtualization.framework does not tolerate overlapping checkpoint +lifecycle operations (`saveMachineStateToURL` and `restoreMachineStateFromURL`) +on sibling VMs, and teardown must not cross those checkpoint edges. Capsem uses +`ServiceState::save_restore_lock` plus the host-wide `VzHostLock` flock: +cold starts and teardown take shared/read guards, save and restore take +exclusive/write guards. The rail holds even when pytest-xdist spawns one +`capsem-service` per worker, while independent cold starts can still run +together for the boot-latency gate. + +Do not demote suspend/resume, lifecycle, provisioning, or teardown tests to +`-n 1` to sidestep VZ races. `just test` at `-n 4` is the contract; if a +concurrent run sees restore permission errors, loop-device corruption, +connection-refused startup races, or readiness misses, fix the lifecycle rail. +Full context and failure signatures live in +`web/docs/src/content/docs/gotchas/concurrent-suspend-resume.md`. diff --git a/tests/citadel/python_boundary_debt.toml b/tests/citadel/python_boundary_debt.toml index 51652ad98..da53eef21 100644 --- a/tests/citadel/python_boundary_debt.toml +++ b/tests/citadel/python_boundary_debt.toml @@ -20,4 +20,6 @@ old_import_sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b785 # images holds the official image publication scripts, run through the # build_system project (`uv run --project build_system`) and type-checked as a # [lint].python_roots entry. +# integrations carries pure Inspect text transforms; the same frozen project +# owns their source-contract tests, strict type checks and coverage. python_source_roots = [".claude", "benchmarks", "build_system", "guest", "images", "integrations", "sdk", "tests"] diff --git a/tests/citadel/repository_surface_ownership.toml b/tests/citadel/repository_surface_ownership.toml index 61e895f62..91e63b504 100644 --- a/tests/citadel/repository_surface_ownership.toml +++ b/tests/citadel/repository_surface_ownership.toml @@ -196,6 +196,18 @@ coverage = "not-applicable: image recipes and publication scripts are proved by debt = "none: images/ is a new root" reason = "The official OCI images and their publication scripts; only .github/workflows/images.yaml builds and publishes them." +[[surface]] +id = "external-integrations" +state = "retained" +current = ["integrations/"] +targets = ["integrations/"] +lint = ["python.ruff", "python.ty.strict", "fast.audit.source-syntax", "fast.citadel"] +tests = ["integrations/inspect-ai/tests/containers/", "tests/citadel/test_repository_surface_ownership.py"] +ci = ["fast-gate", "test-linux", "test", "test-install", "docs-build", "site-build", "release-site-build", "pr-gate"] +coverage = "codecov unit flag; carried pure transforms join the Python coverage rail" +debt = "none: carried sources pass strict types and source ceilings; full Inspect adapter/package acceptance remains in #342" +reason = "Already-carried external integration source has one checker/test/CI owner without claiming an installed or published adapter." + [[surface]] id = "developer-skills" state = "retained" diff --git a/tests/citadel/test_generated_output_boundary.py b/tests/citadel/test_generated_output_boundary.py index d44bcc90d..1323558eb 100644 --- a/tests/citadel/test_generated_output_boundary.py +++ b/tests/citadel/test_generated_output_boundary.py @@ -525,9 +525,12 @@ def test_test_evidence_and_coverage_use_canonical_target_roots() -> None: "root erased the diagnostics of the failure that triggered teardown." ) assert workspace_evidence.parent == Path(evidence), RATIONALE - assert config["suites"]["pytest"]["coverage_flags"] == [ - "--cov=build_system/builder", - "--cov=integrations/inspect-ai/inspect_capsem", + # Surface owners choose coverage inputs; this boundary owns every report + # destination, including when another source surface joins the unit rail. + assert [ + flag for flag in config["suites"]["pytest"]["coverage_flags"] + if flag.startswith("--cov-report=") + ] == [ f"--cov-report=xml:{coverage}/python/codecov.xml", ], RATIONALE assert nextest["store"]["dir"] == f"{coverage}/nextest", RATIONALE diff --git a/tests/citadel/test_repository_surface_ownership.py b/tests/citadel/test_repository_surface_ownership.py index 8bac6a07e..1381aaf65 100644 --- a/tests/citadel/test_repository_surface_ownership.py +++ b/tests/citadel/test_repository_surface_ownership.py @@ -44,7 +44,7 @@ "crates/", "guest/", "images/", - "integrations/inspect-ai/", + "integrations/", "mcp/", "sdk/", "skills/", diff --git a/tests/citadel/web_benchmark_boundary_debt.toml b/tests/citadel/web_benchmark_boundary_debt.toml index 8ff5a2dce..683434d19 100644 --- a/tests/citadel/web_benchmark_boundary_debt.toml +++ b/tests/citadel/web_benchmark_boundary_debt.toml @@ -15,6 +15,5 @@ lint_mapping_count = 4 lint_mapping_sha256 = "ac42e4aac5f97bab38e19bf3691343d0f79934dabef4137ac501baa6f164d538" coverage_mapping_count = 4 coverage_mapping_sha256 = "6e6d023b00ea5f0dcdb1f2373c3217907ef1b14075697ad50c4a3ebf5884aee0" -legacy_reference_count = 324 -# Same 324 references; the reviewed Python root record now also names integrations. -legacy_reference_sha256 = "4017ecf37754dc6b54de93af214d39a65e208297a8a905edafa2b3c585188f45" +legacy_reference_count = 322 +legacy_reference_sha256 = "fe582d8926ad23876af1d5205e378e5cc907244591f21004a0408103cb820b27" diff --git a/tests/test_benchmark_retention_contract.py b/tests/test_benchmark_retention_contract.py index 6cbb3c595..b10b13548 100644 --- a/tests/test_benchmark_retention_contract.py +++ b/tests/test_benchmark_retention_contract.py @@ -194,13 +194,13 @@ def test_shared_host_timing_ratchets_use_the_least_contended_sample() -> None: def test_latest_benchmark_evidence_ignores_untracked_results(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) - evidence = tmp_path / PRUNE.BENCHMARKS.relative_to(PROJECT_ROOT) / BenchmarkCategory.FORK.value + evidence = tmp_path / PRUNE.BENCHMARKS.relative_to(PROJECT_ROOT) / "fork" evidence.mkdir(parents=True) (evidence / "tracked.json").write_text( json.dumps({"timestamp": 1, "identity": "tracked"}), encoding="utf-8" ) subprocess.run( - ["git", "add", "benchmarks/baselines/fork/tracked.json"], + ["git", "add", str((evidence / "tracked.json").relative_to(tmp_path))], cwd=tmp_path, check=True, ) @@ -254,7 +254,7 @@ def test_release_benchmarks_use_typed_evidence_instead_of_authored_limits() -> N def _evidence_repo(tmp_path: Path, files: dict[str, dict]) -> Path: """A tracked evidence directory, plus the config the lane names come from.""" subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) - evidence = tmp_path / "benchmarks" / "baselines" / "fork" + evidence = tmp_path / PRUNE.BENCHMARKS.relative_to(PROJECT_ROOT) / "fork" evidence.mkdir(parents=True) for name, document in files.items(): (evidence / name).write_text(json.dumps(document), encoding="utf-8") diff --git a/web/docs/src/content/docs/usage/mcp-tools.md b/web/docs/src/content/docs/usage/mcp-tools.md index ad406b072..876680932 100644 --- a/web/docs/src/content/docs/usage/mcp-tools.md +++ b/web/docs/src/content/docs/usage/mcp-tools.md @@ -63,6 +63,13 @@ create no VM and are not retried automatically. ## VM lifecycle and files +The image catalog and prefetch tools work without creating a VM: + +| Tool | Main parameters | Purpose | +| --- | --- | --- | +| `capsem_image_list` | `refresh?` | Read the admitted catalog, immutable image pins, and reported disk-cache state. Unknown state does not establish readiness. | +| `capsem_image_pull` | `image` | Prefetch an admitted image and return its resolved pin. Registry credentials stay service-owned. | + All VM-scoped tools take the immutable `vm_id` returned by `capsem_create` or `capsem_list`. From c7d553eaf66a2eed3169b5f7b5ea78cfea9fd889 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 7 Oct 2026 21:29:08 +0000 Subject: [PATCH 005/464] feat(foundation): authenticate Unix peers with kernel process identity --- crates/capsem-foundation/src/unix/mod.rs | 1 + crates/capsem-foundation/src/unix/peer.rs | 73 +++++++++++++ .../capsem-foundation/src/unix/peer/tests.rs | 102 ++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 crates/capsem-foundation/src/unix/peer.rs create mode 100644 crates/capsem-foundation/src/unix/peer/tests.rs diff --git a/crates/capsem-foundation/src/unix/mod.rs b/crates/capsem-foundation/src/unix/mod.rs index 1a6f9e02a..bc630fd86 100644 --- a/crates/capsem-foundation/src/unix/mod.rs +++ b/crates/capsem-foundation/src/unix/mod.rs @@ -8,6 +8,7 @@ pub mod fs; #[cfg(target_os = "linux")] pub mod guest_memory; pub mod lock; +pub mod peer; pub mod process; pub mod router_channel; pub mod router_sandbox; diff --git a/crates/capsem-foundation/src/unix/peer.rs b/crates/capsem-foundation/src/unix/peer.rs new file mode 100644 index 000000000..5da426bf2 --- /dev/null +++ b/crates/capsem-foundation/src/unix/peer.rs @@ -0,0 +1,73 @@ +//! Kernel identity of a connected Unix peer, independent of protocol claims. +use std::io; +use std::os::fd::BorrowedFd; + +use nix::sys::socket::{self, SockaddrLike}; + +use super::process::ProcessId; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PeerIdentity { + pub pid: ProcessId, + pub uid: u32, +} + +/// Identity recorded by the kernel when this Unix stream was connected. +/// Descriptor passing delegates that connection; it does not change this +/// identity. This proves neither executable bytes nor a live grant generation. +pub fn identity(fd: BorrowedFd<'_>) -> io::Result { + super::fd::validate_connected_stream(fd)?; + use std::os::fd::AsRawFd; + let address = socket::getpeername::(fd.as_raw_fd()).map_err(super::errno::io)?; + if address.family() != Some(socket::AddressFamily::Unix) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "peer identity requires a connected Unix stream", + )); + } + platform_identity(fd) +} + +#[cfg(target_os = "linux")] +fn platform_identity(fd: BorrowedFd<'_>) -> io::Result { + let credentials = socket::getsockopt(&fd, socket::sockopt::PeerCredentials).map_err(super::errno::io)?; + checked_identity(credentials.pid(), credentials.uid()) +} + +#[cfg(target_os = "macos")] +fn platform_identity(fd: BorrowedFd<'_>) -> io::Result { + let pid = socket::getsockopt(&fd, socket::sockopt::LocalPeerPid).map_err(super::errno::io)?; + let credentials = socket::getsockopt(&fd, socket::sockopt::LocalPeerCred).map_err(super::errno::io)?; + checked_identity(pid, credentials.uid()) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn platform_identity(_fd: BorrowedFd<'_>) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "Unix peer identity unavailable", + )) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn checked_identity(pid: i32, uid: u32) -> io::Result { + let pid = u32::try_from(pid) + .ok() + .and_then(|pid| ProcessId::try_from(pid).ok()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "kernel returned an invalid Unix peer PID"))?; + Ok(PeerIdentity { pid, uid }) +} + +/// Require the exact process and user granted by the trusted supervisor. +pub fn require(fd: BorrowedFd<'_>, expected: PeerIdentity) -> io::Result<()> { + if identity(fd)? != expected { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "Unix peer is not the granted process", + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-foundation/src/unix/peer/tests.rs b/crates/capsem-foundation/src/unix/peer/tests.rs new file mode 100644 index 000000000..27bbe37dd --- /dev/null +++ b/crates/capsem-foundation/src/unix/peer/tests.rs @@ -0,0 +1,102 @@ +use super::*; +use crate::unix::{fd, process}; +use std::io::{Read, Write}; +use std::net::{Ipv4Addr, TcpListener, TcpStream}; +use std::os::fd::AsFd; +use std::os::unix::net::{UnixDatagram, UnixListener, UnixStream}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +fn current() -> PeerIdentity { + PeerIdentity { + pid: std::process::id().try_into().unwrap(), + uid: process::current_uid(), + } +} + +#[test] +fn connected_unix_pairs_have_kernel_identity_and_remain_owned_by_the_caller() { + let (mut left, mut right) = UnixStream::pair().unwrap(); + assert_eq!(identity(left.as_fd()).unwrap(), current()); + assert_eq!(identity(right.as_fd()).unwrap(), current()); + require(left.as_fd(), current()).unwrap(); + left.write_all(b"still open").unwrap(); + let mut bytes = [0; 10]; + right.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"still open"); +} + +#[test] +fn a_same_uid_child_cannot_claim_the_parent_process_identity() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("peer.sock"); + let listener = UnixListener::bind(&path).unwrap(); + let mut child = Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "unix::peer::tests::connecting_child", "--nocapture"]) + .env_clear() + .env("CAPSEM_PEER_TEST_SOCKET", &path) + .stdin(Stdio::null()) + .spawn() + .unwrap(); + assert!(fd::wait_readable(listener.as_fd(), Duration::from_secs(5)).unwrap()); + let (mut socket, _) = listener.accept().unwrap(); + let child_identity = PeerIdentity { + pid: child.id().try_into().unwrap(), + uid: process::current_uid(), + }; + let observed = identity(socket.as_fd()); + let refused = require(socket.as_fd(), current()); + let allowed = require(socket.as_fd(), child_identity); + socket.write_all(b"done").unwrap(); + assert!(child.wait().unwrap().success()); + assert_eq!(observed.unwrap(), child_identity); + assert_eq!(refused.unwrap_err().kind(), io::ErrorKind::PermissionDenied); + allowed.unwrap(); +} + +#[test] +fn connecting_child() { + let Some(path) = std::env::var_os("CAPSEM_PEER_TEST_SOCKET") else { + return; + }; + let mut socket = UnixStream::connect(path).unwrap(); + socket.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut reply = [0; 4]; + socket.read_exact(&mut reply).unwrap(); + assert_eq!(&reply, b"done"); +} + +#[test] +fn mismatched_uid_is_refused_even_when_the_pid_matches() { + let (socket, _) = UnixStream::pair().unwrap(); + let mut wrong = current(); + wrong.uid ^= 1; + assert_eq!( + require(socket.as_fd(), wrong).unwrap_err().kind(), + io::ErrorKind::PermissionDenied + ); +} + +#[test] +fn non_unix_streams_and_non_sockets_cannot_supply_identity() { + let file = tempfile::tempfile().unwrap(); + let error = identity(file.as_fd()).unwrap_err(); + assert_eq!(error.raw_os_error(), Some(nix::errno::Errno::ENOTSOCK as i32)); + let (datagram, _) = UnixDatagram::pair().unwrap(); + assert_eq!( + identity(datagram.as_fd()).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + let dir = tempfile::tempdir().unwrap(); + let unconnected = UnixListener::bind(dir.path().join("listener.sock")).unwrap(); + assert_eq!( + identity(unconnected.as_fd()).unwrap_err().raw_os_error(), + Some(nix::errno::Errno::ENOTCONN as i32) + ); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let socket = TcpStream::connect(listener.local_addr().unwrap()).unwrap(); + assert_eq!( + identity(socket.as_fd()).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); +} From b2973aacb911c1100f9201c3d1a16c1b858135a4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 7 Oct 2026 21:42:41 +0000 Subject: [PATCH 006/464] security(process): admit control only from the coordinator parent --- CHANGELOG.md | 4 ++ crates/capsem-process/src/ipc.rs | 14 ++++- crates/capsem-process/src/ipc/tests.rs | 13 ++++- .../src/ipc/tests/authorization.rs | 53 +++++++++++++++++++ crates/capsem-process/src/main.rs | 29 ++++++---- 5 files changed, 98 insertions(+), 15 deletions(-) create mode 100644 crates/capsem-process/src/ipc/tests/authorization.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 72c2f1bc9..2617b2928 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Authenticate VM-owner control connections with the kernel's process and user + identity before reading protocol claims. Owners retain a session singleton + and exit when their launching coordinator dies. + - Bind private-network and preview handoff endpoints to the coordinator's session identity and VM-owner spawn generation. Refuse worker replies that name another endpoint or arrive after that owner has been replaced. diff --git a/crates/capsem-process/src/ipc.rs b/crates/capsem-process/src/ipc.rs index af7f6b909..cee6442e8 100644 --- a/crates/capsem-process/src/ipc.rs +++ b/crates/capsem-process/src/ipc.rs @@ -1,5 +1,6 @@ use anyhow::{Context, Result}; use capsem_foundation::ipc_channel::{channel_from_std, Receiver, Sender}; +use capsem_foundation::unix::peer::PeerIdentity; use capsem_proto::ipc::{ProcessToService, ServiceToProcess}; use capsem_proto::HostToGuest; use std::collections::HashMap; @@ -57,7 +58,15 @@ const GUEST_FILE_WRITE_MODE: u32 = 0o644; /// Negotiate the synchronous Hello side-channel away from Tokio's worker /// threads, then hand the verified socket to the typed async IPC transport. /// A peer mismatch is a refused connection, not a process-fatal error. -async fn open_ipc_channel(stream: tokio::net::UnixStream) -> Result> { +async fn open_ipc_channel( + stream: tokio::net::UnixStream, + controller: PeerIdentity, +) -> Result> { + use std::os::fd::AsFd; + if let Err(error) = capsem_foundation::unix::peer::require(stream.as_fd(), controller) { + warn!(target: "ipc", %error, "unauthorized IPC peer; refusing connection before Hello"); + return Ok(None); + } let std_stream = stream.into_std()?; let traceparent = capsem_foundation::telemetry::current_parent_traceparent(); let (std_stream, handshake) = tokio::task::spawn_blocking(move || { @@ -122,6 +131,7 @@ async fn record_guest_write( #[allow(clippy::too_many_arguments)] pub(crate) async fn handle_ipc_connection( stream: tokio::net::UnixStream, + controller: PeerIdentity, ctrl_tx: mpsc::Sender, ipc_tx: broadcast::Sender, term_relay: Arc, @@ -136,7 +146,7 @@ pub(crate) async fn handle_ipc_connection( // First frame on every IPC connection is a Hello -- detect cross-version // mixes (capsem-service built before X, capsem-process built after) in // ~1s with a structured log line instead of a 30s silent timeout. - let Some(((tx, rx), stream_role)) = open_ipc_channel(stream).await? else { + let Some(((tx, rx), stream_role)) = open_ipc_channel(stream, controller).await? else { return Ok(()); }; diff --git a/crates/capsem-process/src/ipc/tests.rs b/crates/capsem-process/src/ipc/tests.rs index 143700fb9..8b2300a31 100644 --- a/crates/capsem-process/src/ipc/tests.rs +++ b/crates/capsem-process/src/ipc/tests.rs @@ -3,6 +3,8 @@ use std::time::Duration; use super::*; +mod authorization; +use authorization::controller_identity; mod classification; mod publications; mod reload; @@ -62,7 +64,10 @@ async fn negotiated_channel_carries_typed_messages_in_both_directions() { channel }); - let ((process_tx, process_rx), stream_role) = open_ipc_channel(process_stream).await.unwrap().unwrap(); + let ((process_tx, process_rx), stream_role) = open_ipc_channel(process_stream, controller_identity()) + .await + .unwrap() + .unwrap(); assert!(!stream_role, "a service connection is a command connection"); let (service_tx, service_rx) = service.await.unwrap(); @@ -78,7 +83,10 @@ async fn malformed_handshake_is_refused_before_typed_ipc_starts() { let (process_stream, mut peer) = tokio::net::UnixStream::pair().unwrap(); peer.write_all(&0_u32.to_be_bytes()).await.unwrap(); - assert!(open_ipc_channel(process_stream).await.unwrap().is_none()); + assert!(open_ipc_channel(process_stream, controller_identity()) + .await + .unwrap() + .is_none()); } /// The owner's IPC dispatcher as `main.rs` wires it, on a fixture policy @@ -202,6 +210,7 @@ impl Dispatcher { pub(in crate::ipc) fn connection(&self, stream: tokio::net::UnixStream) -> tokio::task::JoinHandle> { tokio::spawn(handle_ipc_connection( stream, + controller_identity(), self.ctrl_tx.clone(), self.events_tx.clone(), Arc::clone(&self.term_relay), diff --git a/crates/capsem-process/src/ipc/tests/authorization.rs b/crates/capsem-process/src/ipc/tests/authorization.rs new file mode 100644 index 000000000..8224d267c --- /dev/null +++ b/crates/capsem-process/src/ipc/tests/authorization.rs @@ -0,0 +1,53 @@ +use super::*; + +pub(super) fn controller_identity() -> PeerIdentity { + PeerIdentity { + pid: std::process::id().try_into().unwrap(), + uid: capsem_foundation::unix::process::current_uid(), + } +} + +#[tokio::test] +async fn claimed_service_and_stream_roles_do_not_authorize_a_wrong_process() { + for role in [ + "capsem-service", + capsem_proto::handshake::STREAM_PEER_ID, + "other-worker", + ] { + let (owner, peer) = tokio::net::UnixStream::pair().unwrap(); + let mut wrong = controller_identity(); + wrong.pid = (wrong.pid.get() + 1).try_into().unwrap(); + let hello = tokio::task::spawn_blocking(move || { + let mut peer = peer.into_std().unwrap(); + capsem_foundation::ipc_handshake::negotiate_initiator(&mut peer, role, "") + }); + let admitted = open_ipc_channel(owner, wrong).await.unwrap(); + let claimed = hello.await.unwrap(); + assert!(admitted.is_none(), "worker claiming {role} was admitted"); + assert!(claimed.is_err(), "unauthorized worker received a successful Hello"); + } +} + +#[tokio::test] +async fn unauthorized_idle_peers_are_refused_without_waiting_for_hello() { + let (owner, _idle) = tokio::net::UnixStream::pair().unwrap(); + let mut wrong = controller_identity(); + wrong.pid = (wrong.pid.get() + 1).try_into().unwrap(); + let admitted = tokio::time::timeout(Duration::from_millis(100), open_ipc_channel(owner, wrong)) + .await + .expect("identity is checked before an untrusted peer may spend the Hello deadline") + .unwrap(); + assert!(admitted.is_none()); +} + +#[tokio::test] +async fn a_matching_pid_with_a_wrong_uid_is_refused_before_hello() { + let (owner, _idle) = tokio::net::UnixStream::pair().unwrap(); + let mut wrong = controller_identity(); + wrong.uid ^= 1; + let admitted = tokio::time::timeout(Duration::from_millis(100), open_ipc_channel(owner, wrong)) + .await + .expect("wrong user must be refused before reading Hello") + .unwrap(); + assert!(admitted.is_none()); +} diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index b6bba9a54..3a7c444e7 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -215,9 +215,10 @@ fn main() -> Result<()> { default_filter: "info", })?; let args = Args::parse(); - // Declared before VM/runtime resources, so normal teardown releases the - // generation lifetime fence last. SIGKILL still needs native exit proof. - let _spawn_owner = owner::enroll(&args)?; + let controller = capsem_foundation::unix::peer::PeerIdentity { + pid: capsem_foundation::unix::process::parent_process_id().context("missing coordinator parent")?, + uid: capsem_foundation::unix::process::current_uid(), + }; // Root span shared across the whole capsem-process run: every // subsequent log line inherits `vm_id` and `trace_id` as structured @@ -226,17 +227,22 @@ fn main() -> Result<()> { let root_span = tracing::info_span!("vm", vm_id = %args.id, trace_id = %trace_id); let _root_span_guard = root_span.enter(); - let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; - - info!(id = %args.id, "capsem-sandbox-process starting"); - // Held until the process exits: dropping it flushes the last measurements. - let _metric_export = metric_export::install(&args.id, args.metric_endpoint.as_deref()); - std::fs::create_dir_all(&args.session_dir)?; let mut session_dir = args.session_dir.clone(); if let Ok(resolved) = session_dir.canonicalize() { session_dir = resolved; } + // The kernel parent is the coordinator that launched this owner. Keep + // that identity for every IPC check and die if it changes, so an orphan + // cannot accept a later process that reuses the old parent's PID. + let _owner_guards = capsem_guard::install(Some(controller.pid.get()), &session_dir.join("process.lock"))? + .context("this session already has a running VM owner")?; + + let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + + info!(id = %args.id, "capsem-sandbox-process starting"); + // Held until the process exits: dropping it flushes the last measurements. + let _metric_export = metric_export::install(&args.id, args.metric_endpoint.as_deref()); let guest_dir = prepare_session_layout(&session_dir, args.scratch_disk_size_gb)?; // The image share is attached to every session, read-only at the device: @@ -324,9 +330,9 @@ fn main() -> Result<()> { rt.spawn(async move { if let Err(e) = run_async_main_loop( args, + controller, vm_arc, vsock_rx, - sm, trace_id_for_loop, session_dir_for_loop, shutdown_for_loop, @@ -394,9 +400,9 @@ fn main() -> Result<()> { async fn run_async_main_loop( args: Args, + controller: capsem_foundation::unix::peer::PeerIdentity, vm: Arc>>, vsock_rx: mpsc::UnboundedReceiver, - _sm: capsem_core::host_state::HostStateMachine, trace_id: String, session_dir: std::path::PathBuf, shutdown: Arc>, @@ -749,6 +755,7 @@ async fn run_async_main_loop( tokio::spawn(async move { if let Err(e) = ipc::handle_ipc_connection( stream, + controller, tx_c, ipc_tx_pass, term_c, From ea3fd9995d38a3fa1acd5b76e702eb5fb3b6017a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 7 Oct 2026 22:13:58 +0000 Subject: [PATCH 007/464] security(service): authenticate registered owner connections --- CHANGELOG.md | 5 + crates/capsem-service/src/active_policy.rs | 2 +- crates/capsem-service/src/container_setup.rs | 3 +- .../src/container_setup/relaunch.rs | 1 + .../src/container_setup/tests.rs | 2 +- crates/capsem-service/src/main.rs | 1 + crates/capsem-service/src/mcp_routes.rs | 4 +- crates/capsem-service/src/owner_connection.rs | 97 ++++++++++ .../src/owner_connection/tests.rs | 165 ++++++++++++++++++ crates/capsem-service/src/owner_handoff.rs | 14 +- .../src/router_runtime/exposures.rs | 7 +- .../src/router_runtime/exposures/tests.rs | 4 +- .../src/router_runtime/streams.rs | 70 +++++--- .../src/router_runtime/streams/tests.rs | 2 +- .../src/router_runtime/streams/tests/flow.rs | 32 ++++ crates/capsem-service/src/switches.rs | 15 +- crates/capsem-service/src/tests/files_api.rs | 2 +- .../capsem-service/src/tests/ipc_command.rs | 156 ++++++++++++++++- crates/capsem-service/src/vm_files.rs | 2 +- crates/capsem-service/src/vm_files/exec.rs | 1 + .../src/vm_files/file_boundary.rs | 1 + crates/capsem-service/src/vm_files/fork.rs | 2 +- .../src/vm_files/ipc_command.rs | 28 ++- .../src/vm_files/provision/tests.rs | 92 ++++++++-- crates/capsem-service/src/vm_lifecycle.rs | 83 ++++----- 25 files changed, 649 insertions(+), 142 deletions(-) create mode 100644 crates/capsem-service/src/owner_connection.rs create mode 100644 crates/capsem-service/src/owner_connection/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 2617b2928..c7209ff92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Authenticate coordinator connections against each registered VM owner's kernel + process and user identity before sending protocol bytes. Refuse stale owner + generations for commands, streams and private-network descriptor handoffs; + cancel streaming commands when their authority is revoked. + - Authenticate VM-owner control connections with the kernel's process and user identity before reading protocol claims. Owners retain a session singleton and exit when their launching coordinator dies. diff --git a/crates/capsem-service/src/active_policy.rs b/crates/capsem-service/src/active_policy.rs index 9b2577338..41be1a9d7 100644 --- a/crates/capsem-service/src/active_policy.rs +++ b/crates/capsem-service/src/active_policy.rs @@ -123,7 +123,7 @@ pub(crate) async fn push_policy_to_running_instances( let request = ServiceToProcess::ReloadConfig { id: state.next_job_id(), }; - match send_ipc_command(uds_path, request, Some(5)).await { + match send_ipc_command(state, uds_path, request, Some(5)).await { Ok(ProcessToService::ConfigReloadResult { active_policy_digest: Some(applied), error: None, diff --git a/crates/capsem-service/src/container_setup.rs b/crates/capsem-service/src/container_setup.rs index 6083e4551..191b9578f 100644 --- a/crates/capsem-service/src/container_setup.rs +++ b/crates/capsem-service/src/container_setup.rs @@ -353,7 +353,7 @@ async fn run(state: &Arc, id: &str, generation: u64, spec: Contain LaunchWait::Crashed => return Err("container owner exited before launch".into()), LaunchWait::TimedOut => return Err("container owner did not launch within 30s".into()), } - match send_ipc_command(&uds_path, admission, Some(5)).await? { + match send_ipc_command(state, &uds_path, admission, Some(5)).await? { ProcessToService::ContainerPullAdmission { error: None, .. } => {} ProcessToService::ContainerPullAdmission { error: Some(error), @@ -491,6 +491,7 @@ async fn run(state: &Arc, id: &str, generation: u64, spec: Contain let uds_path = running_uds_path(state, id).map_err(|e| e.1)?; let started = std::time::Instant::now(); let reply = send_ipc_command( + state, &uds_path, ServiceToProcess::Exec { id: state.next_job_id(), diff --git a/crates/capsem-service/src/container_setup/relaunch.rs b/crates/capsem-service/src/container_setup/relaunch.rs index 7b80b5f55..04cc701ec 100644 --- a/crates/capsem-service/src/container_setup/relaunch.rs +++ b/crates/capsem-service/src/container_setup/relaunch.rs @@ -144,6 +144,7 @@ fn relaunch_in_background(state: &Arc, id: &str, generation: u64) tokio::spawn(async move { let reply = tokio::select! { reply = send_ipc_command( + &state, &uds_path, ServiceToProcess::Exec { id: state.next_job_id(), diff --git a/crates/capsem-service/src/container_setup/tests.rs b/crates/capsem-service/src/container_setup/tests.rs index aabfb61d3..be16d323a 100644 --- a/crates/capsem-service/src/container_setup/tests.rs +++ b/crates/capsem-service/src/container_setup/tests.rs @@ -137,7 +137,7 @@ fn fixture(images: FixtureImages) -> Fixture { let session_dir = dir.path().join("session"); let workspace = session_dir.join("guest/workspace"); std::fs::create_dir_all(&workspace).unwrap(); - insert_fake_instance_with_session_dir(&state, "box", 1, session_dir); + insert_fake_instance_with_session_dir(&state, "box", std::process::id(), session_dir); let uds_path = state.instances.lock().unwrap()["box"].uds_path.clone(); std::fs::write(uds_path.with_extension("ready"), b"1\n").unwrap(); Fixture { diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index abe765f55..d31a50ebb 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -38,6 +38,7 @@ mod managed_routes; pub mod managed_sessions; use instance::InstanceInfo; mod network_routes; +mod owner_connection; mod owner_handoff; mod policy_mutation; use policy_mutation::{apply_policy_mutation, MutationRoute, PolicyMutation}; diff --git a/crates/capsem-service/src/mcp_routes.rs b/crates/capsem-service/src/mcp_routes.rs index d7f3ff236..03c9aa8b5 100644 --- a/crates/capsem-service/src/mcp_routes.rs +++ b/crates/capsem-service/src/mcp_routes.rs @@ -182,7 +182,7 @@ pub(super) async fn handle_mcp_server_refresh( let mut refreshed = 0; for (vm_id, uds_path) in &targets { let id = state.next_job_id(); - match send_ipc_command(uds_path, ServiceToProcess::McpRefreshTools { id }, Some(30)).await { + match send_ipc_command(&state, uds_path, ServiceToProcess::McpRefreshTools { id }, Some(30)).await { Ok(_) => refreshed += 1, Err(error) => warn!(vm_id = %vm_id, server_id = %server_id, %error, "MCP tool refresh failed"), } @@ -270,7 +270,7 @@ pub(super) async fn handle_mcp_tool_call( namespaced_name, arguments_json, }; - match send_ipc_command(&uds_path, msg, Some(60)) + match send_ipc_command(&state, &uds_path, msg, Some(60)) .await .map_err(|e| AppError(StatusCode::BAD_GATEWAY, e))? { diff --git a/crates/capsem-service/src/owner_connection.rs b/crates/capsem-service/src/owner_connection.rs new file mode 100644 index 000000000..a513b9bf9 --- /dev/null +++ b/crates/capsem-service/src/owner_connection.rs @@ -0,0 +1,97 @@ +//! A coordinator-owned spawn grants IPC authority; a pathname or Hello does not. +use super::*; +use capsem_foundation::unix::{peer, process}; +use std::os::fd::AsFd; + +pub(crate) struct OwnerConnection { + id: String, + generation: uuid::Uuid, + identity: peer::PeerIdentity, + pub(crate) uds_path: PathBuf, +} + +impl OwnerConnection { + pub(crate) fn pid(&self) -> u32 { + self.identity.pid.get() + } + + pub(crate) fn authenticate( + &self, + state: &ServiceState, + socket: &std::os::unix::net::UnixStream, + ) -> Result<(), String> { + peer::require(socket.as_fd(), self.identity).map_err(|e| format!("VM owner authentication: {e}"))?; + self.validate(state, false) + } + + pub(crate) fn capture(instance: &InstanceInfo) -> Result { + Ok(Self { + id: instance.id.clone(), + generation: instance.generation, + identity: peer::PeerIdentity { + pid: process::ProcessId::try_from(instance.pid).map_err(|e| e.to_string())?, + uid: process::current_uid(), + }, + uds_path: instance.uds_path.clone(), + }) + } + + pub(crate) fn acquire(state: &ServiceState, path: &StdPath) -> Result { + let instances = state.instances.lock().unwrap(); + let mut matching = instances + .values() + .filter(|instance| instance.uds_path.as_os_str() == path.as_os_str()); + let owner = Self::capture(matching.next().ok_or("VM owner is not registered")?)?; + if matching.next().is_some() { + return Err("VM owner endpoint is ambiguous".into()); + } + drop(instances); + Ok(owner) + } + + pub(crate) fn validate(&self, state: &ServiceState, retiring: bool) -> Result<(), String> { + let instances = state.instances.lock().unwrap(); + let current = match instances.get(&self.id) { + Some(instance) => { + instance.generation == self.generation + && instance.pid == self.identity.pid.get() + && instance.uds_path.as_os_str() == self.uds_path.as_os_str() + } + // Only the lifecycle owner that captured and claimed this spawn + // may finish its shutdown after removing its registry entry. + None => retiring, + }; + drop(instances); + if !current { + return Err("VM owner changed during IPC admission".into()); + } + Ok(()) + } + + pub(crate) async fn open( + &self, + state: &ServiceState, + role: &str, + retiring: bool, + ) -> Result<(Sender, Receiver), String> { + self.validate(state, retiring)?; + let socket = tokio::net::UnixStream::connect(&self.uds_path) + .await + .and_then(|socket| socket.into_std()) + .map_err(|e| format!("VM owner unavailable: {e}"))?; + peer::require(socket.as_fd(), self.identity).map_err(|e| format!("VM owner authentication: {e}"))?; + self.validate(state, retiring)?; + let (socket, _) = capsem_foundation::ipc_handshake::negotiate_initiator_off_worker( + socket, + role, + capsem_foundation::telemetry::current_parent_traceparent(), + ) + .await + .map_err(|e| format!("VM owner handshake: {e}"))?; + self.validate(state, retiring)?; + channel_from_std(socket).map_err(|e| format!("VM owner channel: {e}")) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/owner_connection/tests.rs b/crates/capsem-service/src/owner_connection/tests.rs new file mode 100644 index 000000000..1a800049b --- /dev/null +++ b/crates/capsem-service/src/owner_connection/tests.rs @@ -0,0 +1,165 @@ +use super::*; +use crate::tests::{insert_fake_instance_with_session_dir, make_test_state}; + +fn fixture() -> (Arc, OwnerConnection) { + let state = make_test_state(); + let session = state.run_dir.join("sessions/owner"); + std::fs::create_dir_all(&session).unwrap(); + insert_fake_instance_with_session_dir(&state, "owner", std::process::id(), session); + let grant = OwnerConnection::capture(&state.instances.lock().unwrap()["owner"]).unwrap(); + (state, grant) +} + +#[tokio::test] +async fn replaced_generation_is_rejected_before_connect() { + let (state, grant) = fixture(); + let listener = tokio::net::UnixListener::bind(&grant.uds_path).unwrap(); + state.instances.lock().unwrap().get_mut("owner").unwrap().generation = uuid::Uuid::new_v4(); + assert!(grant.open(&state, "capsem-service", false).await.is_err()); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), listener.accept()) + .await + .is_err() + ); +} + +#[tokio::test] +async fn generation_changed_during_hello_cannot_receive_a_request() { + use std::os::fd::AsFd; + let (state, grant) = fixture(); + let listener = tokio::net::UnixListener::bind(&grant.uds_path).unwrap(); + let peer_state = Arc::clone(&state); + let server = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = socket.into_std().unwrap(); + tokio::task::spawn_blocking(move || { + assert!( + capsem_foundation::unix::fd::wait_readable(socket.as_fd(), std::time::Duration::from_secs(5)).unwrap() + ); + peer_state + .instances + .lock() + .unwrap() + .get_mut("owner") + .unwrap() + .generation = uuid::Uuid::new_v4(); + capsem_foundation::ipc_handshake::negotiate_responder(&mut socket, "capsem-process", "").unwrap(); + socket + }) + .await + .unwrap() + }); + assert!(grant.open(&state, "capsem-service", false).await.is_err()); + let mut socket = server.await.unwrap(); + tokio::task::spawn_blocking(move || { + use std::io::Read; + socket.set_nonblocking(false).unwrap(); + socket + .set_read_timeout(Some(std::time::Duration::from_secs(5))) + .unwrap(); + assert_eq!(socket.read(&mut [0; 1]).unwrap(), 0); + }) + .await + .unwrap(); +} + +#[test] +fn shutdown_grant_accepts_only_the_claimed_generation_or_an_empty_slot() { + let (state, grant) = fixture(); + let mut instance = state.instances.lock().unwrap().remove("owner").unwrap(); + assert!(grant.validate(&state, false).is_err()); + assert!(grant.validate(&state, true).is_ok()); + instance.generation = uuid::Uuid::new_v4(); + state.instances.lock().unwrap().insert("owner".into(), instance); + assert!(grant.validate(&state, true).is_err()); +} + +#[tokio::test] +async fn suspending_owner_child() { + let Some(path) = std::env::var_os("CAPSEM_SUSPENDING_OWNER") else { + return; + }; + use std::io::Write; + use std::os::fd::AsFd; + let listener = tokio::net::UnixListener::bind(path).unwrap(); + let mut control = + std::os::unix::net::UnixStream::from(capsem_foundation::unix::fd::duplicate(std::io::stdin().as_fd()).unwrap()); + control.write_all(b"ready").unwrap(); + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = socket.into_std().unwrap(); + let socket = tokio::task::spawn_blocking(move || { + capsem_foundation::ipc_handshake::negotiate_responder(&mut socket, "capsem-process-test", "").unwrap(); + socket + }) + .await + .unwrap(); + let (tx, rx) = channel_from_std::(socket).unwrap(); + let ServiceToProcess::Suspend { checkpoint_path } = rx.recv().await.unwrap() else { + panic!("expected suspend") + }; + assert_eq!(checkpoint_path, RESUME_CHECKPOINT_NAME); + tx.send(ProcessToService::StateChanged { + id: "owner".into(), + state: "Suspended".into(), + trigger: "test".into(), + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn authenticated_suspend_accepts_its_original_childs_confirmation_and_exit() { + use tokio::io::AsyncReadExt; + let (state, owner) = fixture(); + let (control, child_control) = std::os::unix::net::UnixStream::pair().unwrap(); + control.set_nonblocking(true).unwrap(); + let mut control = tokio::net::UnixStream::from_std(control).unwrap(); + let child = tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "owner_connection::tests::suspending_owner_child", + "--nocapture", + ]) + .env_clear() + .env("CAPSEM_SUSPENDING_OWNER", &owner.uds_path) + .stdin(std::process::Stdio::from(std::os::fd::OwnedFd::from(child_control))) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut ready = [0; 5]; + tokio::time::timeout(std::time::Duration::from_secs(5), control.read_exact(&mut ready)) + .await + .unwrap() + .unwrap(); + assert_eq!(&ready, b"ready"); + let (generation, session_dir) = { + let mut instances = state.instances.lock().unwrap(); + let instance = instances.get_mut("owner").unwrap(); + instance.pid = child.id().unwrap(); + instance.persistent = true; + let captured = (instance.generation, instance.session_dir.clone()); + drop(instances); + captured + }; + let retirement = state.retirements.register("owner", generation).unwrap(); + let reaper = crate::instance_reaper::spawn_exit_reaper( + child, + "owner".into(), + "owner".into(), + Arc::clone(&state), + owner.uds_path, + session_dir, + retirement, + ); + let result = tokio::time::timeout( + std::time::Duration::from_secs(10), + handle_suspend(State(Arc::clone(&state)), Path("owner".into())), + ) + .await + .unwrap(); + reaper.await.unwrap(); + assert!(result.unwrap().0.success); + assert!(!state.instances.lock().unwrap().contains_key("owner")); +} diff --git a/crates/capsem-service/src/owner_handoff.rs b/crates/capsem-service/src/owner_handoff.rs index 1d727f444..5dfa21298 100644 --- a/crates/capsem-service/src/owner_handoff.rs +++ b/crates/capsem-service/src/owner_handoff.rs @@ -6,17 +6,19 @@ pub(crate) struct OwnerHandoff { id: String, generation: uuid::Uuid, endpoint: PathBuf, - pub(crate) uds_path: PathBuf, + pub(crate) owner: crate::owner_connection::OwnerConnection, } impl OwnerHandoff { pub(crate) async fn acquire(state: &Arc, id: &str) -> Result { - let (id, generation, uds_path) = { + let (id, generation, owner) = { let instances = state.instances.lock().unwrap(); let instance = instances .get(id) .ok_or_else(|| AppError(StatusCode::NOT_FOUND, format!("sandbox not running: {id}")))?; - let captured = (instance.id.clone(), instance.generation, instance.uds_path.clone()); + let owner = crate::owner_connection::OwnerConnection::capture(instance) + .map_err(|e| AppError(StatusCode::BAD_GATEWAY, e))?; + let captured = (instance.id.clone(), instance.generation, owner); drop(instances); captured }; @@ -35,11 +37,11 @@ impl OwnerHandoff { id, generation, endpoint, - uds_path, + owner, }) } - pub(crate) fn validate(self, state: &ServiceState, reported: &str) -> Result { + pub(crate) fn validate(&self, state: &ServiceState, reported: &str) -> Result { // Path equality normalizes '.' components. Authorize exactly the // spelling we derived, without resolving any worker-controlled path. if self.endpoint.as_os_str() != std::ffi::OsStr::new(reported) { @@ -60,7 +62,7 @@ impl OwnerHandoff { "VM owner changed during handoff admission".into(), )); } - Ok(self.endpoint) + Ok(self.endpoint.clone()) } } diff --git a/crates/capsem-service/src/router_runtime/exposures.rs b/crates/capsem-service/src/router_runtime/exposures.rs index 36a72986d..4ee3d1597 100644 --- a/crates/capsem-service/src/router_runtime/exposures.rs +++ b/crates/capsem-service/src/router_runtime/exposures.rs @@ -46,7 +46,7 @@ fn api_publication(publication: capsem_proto::ipc::PublicationInfo) -> ExposureI async fn ask_owner(state: &ServiceState, id: &str, request: ServiceToProcess) -> Result { let uds_path = running_uds_path(state, id)?; - send_ipc_command(&uds_path, request, Some(OWNER_TIMEOUT_SECS)) + send_ipc_command(state, &uds_path, request, Some(OWNER_TIMEOUT_SECS)) .await .map_err(|e| AppError(StatusCode::BAD_GATEWAY, format!("VM owner unavailable: {e}"))) } @@ -317,8 +317,9 @@ pub(crate) async fn handle_admit_preview_connection( capsem_api::PreviewAdmissionKind::Request => capsem_proto::PreviewAdmissionKind::Request, capsem_api::PreviewAdmissionKind::WebsocketUpgrade => capsem_proto::PreviewAdmissionKind::WebsocketUpgrade, }; - match send_ipc_command( - &handoff.uds_path, + match send_owner_command( + &state, + &handoff.owner, ServiceToProcess::AdmitPreviewConnection { id: state.next_job_id(), exposure_id, diff --git a/crates/capsem-service/src/router_runtime/exposures/tests.rs b/crates/capsem-service/src/router_runtime/exposures/tests.rs index 1556033d2..e051d1253 100644 --- a/crates/capsem-service/src/router_runtime/exposures/tests.rs +++ b/crates/capsem-service/src/router_runtime/exposures/tests.rs @@ -7,7 +7,7 @@ fn fixture() -> (Arc, PathBuf, tempfile::TempDir) { let session_dir = dir.path().join("session"); std::fs::create_dir_all(session_dir.join("guest/workspace")).unwrap(); std::fs::write(state.run_dir.join("preview.port"), "19444").unwrap(); - insert_fake_instance_with_session_dir(&state, "box", 1, session_dir); + insert_fake_instance_with_session_dir(&state, "box", std::process::id(), session_dir); let uds_path = state.instances.lock().unwrap()["box"].uds_path.clone(); (state, uds_path, dir) } @@ -405,7 +405,7 @@ async fn preview_admission_rejects_a_reply_from_a_replaced_owner_generation() { owner.await.unwrap(); assert_eq!(status, StatusCode::BAD_GATEWAY, "{body}"); assert!( - body.to_string().contains("VM owner changed during handoff admission"), + body.to_string().contains("VM owner changed during IPC admission"), "{body}" ); } diff --git a/crates/capsem-service/src/router_runtime/streams.rs b/crates/capsem-service/src/router_runtime/streams.rs index 615a9befc..6d22aac23 100644 --- a/crates/capsem-service/src/router_runtime/streams.rs +++ b/crates/capsem-service/src/router_runtime/streams.rs @@ -42,12 +42,19 @@ pub(crate) async fn handle_stream( )); } let uds_path = running_uds_path(&state, &id)?; + let owner = crate::owner_connection::OwnerConnection::acquire(&state, &uds_path) + .map_err(|e| AppError(StatusCode::BAD_GATEWAY, e))?; Ok(upgrade .max_message_size(stream::MAX_STREAM_FRAME_BYTES) - .on_upgrade(move |socket| session(state, id, uds_path, socket))) + .on_upgrade(move |socket| session(state, id, owner, socket))) } -async fn session(state: Arc, id: String, uds_path: PathBuf, socket: WebSocket) { +async fn session( + state: Arc, + id: String, + grant: crate::owner_connection::OwnerConnection, + socket: WebSocket, +) { let (mut client_tx, mut client_rx) = socket.split(); let outcome = async { let (kind, command) = tokio::time::timeout(START_TIMEOUT, first_start(&mut client_rx)) @@ -60,12 +67,12 @@ async fn session(state: Arc, id: String, uds_path: PathBuf, socket StreamKind::Terminal | StreamKind::Exec => None, }; let owner = async { - wait_for_vm_ready(&uds_path, 30, Some(&state), Some(&id)).await?; - open_owner(&uds_path).await + wait_for_vm_ready(&grant.uds_path, 30, Some(&state), Some(&id)).await?; + grant.open(&state, capsem_proto::handshake::STREAM_PEER_ID, false).await } .await; match (kind, claim) { - (StreamKind::Terminal, _) => terminal(owner?, &mut client_tx, &mut client_rx).await, + (StreamKind::Terminal, _) => terminal(&state, &grant, owner?, &mut client_tx, &mut client_rx).await, (StreamKind::Exec, _) => { let command = command.expect("decoder requires an exec command"); // The stream protocol names no target yet, so a streamed exec @@ -73,6 +80,7 @@ async fn session(state: Arc, id: String, uds_path: PathBuf, socket let target = capsem_api::exec_target(None, container_setup::runs_container(&state, &id).await)?; exec( &state, + &grant, owner?, (command, proto_exec_target(target)), &mut client_tx, @@ -94,7 +102,7 @@ async fn session(state: Arc, id: String, uds_path: PathBuf, socket // The launcher is the VM's: it is what starts the workload. let command = capsem_core::container::LAUNCH_COMMAND.to_string(); let job = (command, capsem_proto::ipc::ExecTarget::Vm); - exec(&state, owner, job, &mut client_tx, &mut client_rx).await + exec(&state, &grant, owner, job, &mut client_tx, &mut client_rx).await } Err(error) => Err(error), }; @@ -157,21 +165,6 @@ async fn next_client_frame( } } -async fn open_owner(uds_path: &StdPath) -> Result { - let socket = tokio::net::UnixStream::connect(uds_path) - .await - .and_then(|socket| socket.into_std()) - .map_err(|e| format!("VM owner unavailable: {e}"))?; - let (socket, _) = capsem_foundation::ipc_handshake::negotiate_initiator_off_worker( - socket, - capsem_proto::handshake::STREAM_PEER_ID, - capsem_foundation::telemetry::current_parent_traceparent(), - ) - .await - .map_err(|e| format!("VM owner handshake: {e}"))?; - capsem_foundation::ipc_channel::channel_from_std(socket).map_err(|e| format!("VM owner channel: {e}")) -} - async fn send_data( client_tx: &mut futures::stream::SplitSink, channel: StreamChannel, @@ -194,20 +187,25 @@ async fn send_status( } async fn terminal( + state: &ServiceState, + grant: &crate::owner_connection::OwnerConnection, (owner_tx, owner_rx): OwnerChannel, client_tx: &mut futures::stream::SplitSink, client_rx: &mut futures::stream::SplitStream, ) -> Result<(), String> { let owner_closed = |e: std::io::Error| format!("VM owner closed: {e}"); + grant.validate(state, false)?; owner_tx .send(ServiceToProcess::StartTerminalStream) .await .map_err(owner_closed)?; - send_status(client_tx, &StreamStatus::Started).await?; let result = async { + send_status(client_tx, &StreamStatus::Started).await?; loop { tokio::select! { - frame = next_client_frame(client_rx) => match frame? { + frame = next_client_frame(client_rx) => { + grant.validate(state, false)?; + match frame? { None => break Ok(()), Some(ClientControl::Stdin(data)) => { owner_tx.send(ServiceToProcess::TerminalInput { data }).await.map_err(owner_closed)?; @@ -217,11 +215,15 @@ async fn terminal( } Some(ClientControl::Control(StreamControl::CloseStdin)) => {} Some(ClientControl::Control(StreamControl::Start { .. })) => break Err("stream already started".into()), + } }, - message = owner_rx.recv() => match message.map_err(owner_closed)? { + message = owner_rx.recv() => { + grant.validate(state, false)?; + match message.map_err(owner_closed)? { ProcessToService::TerminalOutput { data } => send_data(client_tx, StreamChannel::Stdout, &data).await?, ProcessToService::TerminalStreamEnded { reason } => break Err(reason), _ => {} + } }, } } @@ -236,6 +238,7 @@ async fn terminal( /// client left first and the command was cancelled. async fn exec( state: &ServiceState, + grant: &crate::owner_connection::OwnerConnection, (owner_tx, owner_rx): OwnerChannel, (command, target): (String, capsem_proto::ipc::ExecTarget), client_tx: &mut futures::stream::SplitSink, @@ -243,6 +246,7 @@ async fn exec( ) -> Result, String> { let owner_closed = |e: std::io::Error| format!("VM owner closed: {e}"); let job = state.next_job_id(); + grant.validate(state, false)?; owner_tx .send(ServiceToProcess::ExecStream { id: job, @@ -251,7 +255,6 @@ async fn exec( }) .await .map_err(owner_closed)?; - send_status(client_tx, &StreamStatus::Started).await?; let mut stdin_closed = false; // Stdin frames the owner can still queue. Client frames are read only // while there is credit, so the owner's read loop never waits on a full @@ -259,9 +262,13 @@ async fn exec( let mut credit = capsem_proto::EXEC_STDIN_WINDOW; let mut ping = tokio::time::interval(CREDIT_WAIT_PING); ping.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); - let result = loop { + let result = async { + send_status(client_tx, &StreamStatus::Started).await?; + loop { tokio::select! { - frame = next_client_frame(client_rx), if credit > 0 => match frame? { + frame = next_client_frame(client_rx), if credit > 0 => { + grant.validate(state, false)?; + match frame? { None => break Ok(None), Some(ClientControl::Stdin(_)) if stdin_closed => break Err("exec stdin is already closed".into()), Some(ClientControl::Stdin(data)) => { @@ -276,13 +283,16 @@ async fn exec( Some(ClientControl::Control(StreamControl::CloseStdin)) => {} Some(ClientControl::Control(StreamControl::Start { .. })) => break Err("stream already started".into()), Some(ClientControl::Control(StreamControl::Resize { .. })) => {} + } }, _ = ping.tick(), if credit == 0 => { if client_tx.send(Message::Ping(Default::default())).await.is_err() { break Ok(None); } } - message = owner_rx.recv() => match message.map_err(owner_closed)? { + message = owner_rx.recv() => { + grant.validate(state, false)?; + match message.map_err(owner_closed)? { ProcessToService::ExecInputConsumed { id } if id == job => { credit = (credit + 1).min(capsem_proto::EXEC_STDIN_WINDOW); } @@ -301,9 +311,11 @@ async fn exec( break Ok(Some(exit_code)); } _ => {} + } }, } - }; + } + }.await; if !matches!(result, Ok(Some(_))) { let _ = owner_tx.send(ServiceToProcess::CancelExec { id: job }).await; } diff --git a/crates/capsem-service/src/router_runtime/streams/tests.rs b/crates/capsem-service/src/router_runtime/streams/tests.rs index a6caa0bce..6a8b8e792 100644 --- a/crates/capsem-service/src/router_runtime/streams/tests.rs +++ b/crates/capsem-service/src/router_runtime/streams/tests.rs @@ -19,7 +19,7 @@ async fn fixture() -> Fixture { let state = make_test_state(); let session_dir = dir.path().join("session"); std::fs::create_dir_all(session_dir.join("guest/workspace")).unwrap(); - insert_fake_instance_with_session_dir(&state, "box", 1, session_dir); + insert_fake_instance_with_session_dir(&state, "box", std::process::id(), session_dir); let uds_path = state.instances.lock().unwrap()["box"].uds_path.clone(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); diff --git a/crates/capsem-service/src/router_runtime/streams/tests/flow.rs b/crates/capsem-service/src/router_runtime/streams/tests/flow.rs index 6d353c037..eb2f151fc 100644 --- a/crates/capsem-service/src/router_runtime/streams/tests/flow.rs +++ b/crates/capsem-service/src/router_runtime/streams/tests/flow.rs @@ -119,3 +119,35 @@ async fn a_client_leaving_while_out_of_credit_cancels_the_exec() { drop(client); owner.await.unwrap(); } + +#[tokio::test] +async fn replaced_generation_output_is_rejected_and_the_old_exec_is_cancelled() { + let fx = fixture().await; + let state = Arc::clone(&fx.state); + let owner = owner(&fx.uds_path, move |tx, rx| async move { + let ServiceToProcess::ExecStream { id, .. } = rx.recv().await.unwrap() else { + panic!("expected ExecStream") + }; + state.instances.lock().unwrap().get_mut("box").unwrap().generation = uuid::Uuid::new_v4(); + tx.send(ProcessToService::ExecOutput { + id, + channel: capsem_proto::ExecOutputChannel::Stdout, + data: b"revoked output".to_vec(), + }) + .await + .unwrap(); + let message = tokio::time::timeout(Duration::from_secs(5), rx.recv()) + .await + .unwrap() + .unwrap(); + assert!( + matches!(message, ServiceToProcess::CancelExec { id: cancelled } if cancelled == id), + "{message:?}" + ); + }); + let mut client = connect(fx.address, Some(stream::STREAM_SUBPROTOCOL)).await.unwrap(); + start_exec(&mut client, "sleep 600").await; + let result = status(&next_server_frame(&mut client).await.unwrap().1); + assert!(matches!(result, StreamStatus::Error { message } if message.contains("VM owner changed"))); + owner.await.unwrap(); +} diff --git a/crates/capsem-service/src/switches.rs b/crates/capsem-service/src/switches.rs index b8ad47881..ab6a7c8ab 100644 --- a/crates/capsem-service/src/switches.rs +++ b/crates/capsem-service/src/switches.rs @@ -389,7 +389,7 @@ async fn handshake( prefix: summary.subnet.prefix_len(), generation, }; - let handoff_socket = match send_ipc_command(&handoff.uds_path, ask, Some(ATTACH_TIMEOUT_SECS)).await { + let handoff_socket = match send_owner_command(state, &handoff.owner, ask, Some(ATTACH_TIMEOUT_SECS)).await { Ok(ProcessToService::LinkAttachResult { error: None, handoff_socket, @@ -402,8 +402,11 @@ async fn handshake( let handoff_socket = handoff .validate(state, &handoff_socket) .map_err(|error| anyhow!(error.1))?; - let socket = std::os::unix::net::UnixStream::connect(&handoff_socket) + let socket = tokio::net::UnixStream::connect(&handoff_socket) + .await + .and_then(|socket| socket.into_std()) .with_context(|| format!("connect the owner's seat at {}", handoff_socket.display()))?; + handoff.owner.authenticate(state, &socket).map_err(|e| anyhow!(e))?; let sender = Sender::new(socket.try_clone()?)?; let receiver = Receiver::new(socket.try_clone()?)?; let token = u64::from_str_radix(&token, 16)?; @@ -421,6 +424,7 @@ async fn handshake( answer.fds.len() ); let stream = answer.fds.into_iter().next().unwrap(); + handoff.owner.validate(state, false).map_err(|e| anyhow!(e))?; let host = host(state, network).await?; let port = host.plug(generation, address, stream.as_fd()).await?; Ok((host, port, socket)) @@ -607,16 +611,17 @@ fn take_cable_down(state: &Arc, network: Uuid, vm_id: &str, genera .lock() .unwrap() .get(vm_id) - .map(|instance| instance.uds_path.clone()); - let Some(uds_path) = running else { return }; + .map(owner_connection::OwnerConnection::capture); + let Some(Ok(owner)) = running else { return }; let vm_id = vm_id.to_string(); + let state = Arc::clone(state); tokio::spawn(async move { let ask = ServiceToProcess::LinkDetach { id: capsem_crypto::uuid_v4().as_u128() as u64, network: network.to_string(), generation, }; - match send_ipc_command(&uds_path, ask, Some(ATTACH_TIMEOUT_SECS)).await { + match send_owner_command(&state, &owner, ask, Some(ATTACH_TIMEOUT_SECS)).await { Ok(ProcessToService::LinkDetachResult { error: None, .. }) => {} Ok(ProcessToService::LinkDetachResult { error: Some(error), .. }) => { warn!(%network, vm_id, %error, "owner did not take the cable down") diff --git a/crates/capsem-service/src/tests/files_api.rs b/crates/capsem-service/src/tests/files_api.rs index 9b1e885e8..ec2f3f6a6 100644 --- a/crates/capsem-service/src/tests/files_api.rs +++ b/crates/capsem-service/src/tests/files_api.rs @@ -87,7 +87,7 @@ pub(super) fn setup_vm_with_workspace_and_uds( id: vm_id.into(), name: vm_id.into(), asset_pins: test_asset_pins(), - pid: 1, + pid: std::process::id(), uds_path, session_dir, ram_mb: 2048, diff --git a/crates/capsem-service/src/tests/ipc_command.rs b/crates/capsem-service/src/tests/ipc_command.rs index 49e88cb80..baebe3f2c 100644 --- a/crates/capsem-service/src/tests/ipc_command.rs +++ b/crates/capsem-service/src/tests/ipc_command.rs @@ -1,13 +1,164 @@ use super::*; +#[test] +fn socket_replacement_victim_child() { + let Some(path) = std::env::var_os("CAPSEM_OWNER_SOCKET_TEST") else { + return; + }; + use std::io::{Read, Write}; + use std::os::fd::AsFd; + let _listener = std::os::unix::net::UnixListener::bind(path).unwrap(); + let mut control = + std::os::unix::net::UnixStream::from(capsem_foundation::unix::fd::duplicate(std::io::stdin().as_fd()).unwrap()); + control + .set_read_timeout(Some(std::time::Duration::from_secs(10))) + .unwrap(); + control.write_all(b"ready").unwrap(); + let mut done = [0; 1]; + control.read_exact(&mut done).unwrap(); +} + +#[tokio::test] +async fn replaced_owner_socket_cannot_receive_another_sessions_commands() { + use std::io::{Read, Write}; + use std::os::fd::AsFd; + use std::process::{Command, Stdio}; + let state = make_test_state(); + let session_dir = state.run_dir.join("sessions/victim"); + std::fs::create_dir_all(&session_dir).unwrap(); + let socket_path = session_dir.join("process.sock"); + let (mut control, child_control) = std::os::unix::net::UnixStream::pair().unwrap(); + control + .set_read_timeout(Some(std::time::Duration::from_secs(5))) + .unwrap(); + let mut victim = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "tests::ipc_command::socket_replacement_victim_child", + "--nocapture", + ]) + .env_clear() + .env("CAPSEM_OWNER_SOCKET_TEST", &socket_path) + .stdin(Stdio::from(std::os::fd::OwnedFd::from(child_control))) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let mut ready = [0; 5]; + control.read_exact(&mut ready).unwrap(); + assert_eq!(&ready, b"ready"); + insert_fake_instance_with_session_dir(&state, "victim", victim.id(), session_dir); + + // The victim still owns its listener, but this same-UID process replaces + // the filesystem entry with its own listener and a valid protocol peer. + std::fs::remove_file(&socket_path).unwrap(); + let listener = tokio::net::UnixListener::bind(&socket_path).unwrap(); + std::fs::write(socket_path.with_extension("ready"), b"ready").unwrap(); + let attacker = tokio::spawn(async move { + let (socket, _) = listener.accept().await.unwrap(); + let socket = socket.into_std().unwrap(); + let (socket, hello, queued) = tokio::task::spawn_blocking(move || { + let mut socket = socket; + assert!( + capsem_foundation::unix::fd::wait_readable(socket.as_fd(), std::time::Duration::from_secs(5)).unwrap() + ); + let queued = capsem_foundation::unix::fd::stream_queues(socket.as_fd()).unread; + let hello = capsem_foundation::ipc_handshake::negotiate_responder(&mut socket, "capsem-process", ""); + (socket, hello, queued) + }) + .await + .unwrap(); + if hello.is_err() { + return (queued, false); + } + let (tx, rx): (Sender, Receiver) = channel_from_std(socket).unwrap(); + let ServiceToProcess::Exec { id, command, .. } = rx.recv().await.unwrap() else { + panic!("expected the victim's exec") + }; + assert_eq!(command, "echo protected-command"); + tx.send(ProcessToService::ExecResult { + id, + stdout: b"forged owner reply".to_vec(), + stderr: Vec::new(), + exit_code: 0, + truncated: false, + }) + .await + .unwrap(); + (queued, true) + }); + let (status, body) = route_request( + build_service_router(Arc::clone(&state)), + axum::http::Method::POST, + "/vms/victim/exec", + Some(json!({"command": "echo protected-command", "timeout_secs": 5})), + ) + .await; + let (protocol_bytes, received_command) = attacker.await.unwrap(); + control.write_all(b"x").unwrap(); + assert!(victim.wait().unwrap().success()); + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}"); + assert_eq!(protocol_bytes, Some(0), "authenticate before sending even Hello"); + assert!( + !received_command, + "the replacement process received another session's command" + ); +} + /// Every IPC connection to a VM owner also receives its lifecycle broadcasts. /// `send_ipc_command` took the first non-filtered message as the reply, so a /// `ShutdownRequested` that raced an exec became "unexpected IPC response for /// exec". The reply is the message carrying the request's id. +#[tokio::test] +async fn replaced_generation_cannot_supply_an_accepted_command_reply() { + let state = make_test_state(); + let session = state.run_dir.join("sessions/replaced"); + std::fs::create_dir_all(&session).unwrap(); + insert_fake_instance_with_session_dir(&state, "replaced", std::process::id(), session); + let path = state.instances.lock().unwrap()["replaced"].uds_path.clone(); + let peer_state = Arc::clone(&state); + let owner = spawn_fake_process(&path, 1, move |message| { + let ServiceToProcess::Exec { id, .. } = message else { + panic!("expected exec") + }; + peer_state + .instances + .lock() + .unwrap() + .get_mut("replaced") + .unwrap() + .generation = uuid::Uuid::new_v4(); + let reply = ProcessToService::ExecResult { + id: *id, + stdout: b"old generation".to_vec(), + stderr: vec![], + exit_code: 0, + truncated: false, + }; + Box::pin(async move { Some(reply) }) + }); + let result = send_ipc_command( + &state, + &path, + ServiceToProcess::Exec { + id: 42, + command: "true".into(), + target: capsem_proto::ipc::ExecTarget::Vm, + }, + Some(5), + ) + .await; + owner.await.unwrap(); + assert!(result.unwrap_err().contains("VM owner changed")); +} + #[tokio::test] async fn send_ipc_command_ignores_lifecycle_broadcasts() { - let dir = tempfile::tempdir().unwrap(); - let uds_path = dir.path().join("vm.sock"); + let state = make_test_state(); + let session_dir = state.run_dir.join("sessions/vm"); + std::fs::create_dir_all(&session_dir).unwrap(); + insert_fake_instance_with_session_dir(&state, "vm", std::process::id(), session_dir); + let uds_path = state.instances.lock().unwrap()["vm"].uds_path.clone(); let listener = tokio::net::UnixListener::bind(&uds_path).unwrap(); let owner = tokio::spawn(async move { let (stream, _) = listener.accept().await.unwrap(); @@ -59,6 +210,7 @@ async fn send_ipc_command_ignores_lifecycle_broadcasts() { }); let reply = send_ipc_command( + &state, &uds_path, ServiceToProcess::Exec { id: 41, diff --git a/crates/capsem-service/src/vm_files.rs b/crates/capsem-service/src/vm_files.rs index a16c239ec..50a888a57 100644 --- a/crates/capsem-service/src/vm_files.rs +++ b/crates/capsem-service/src/vm_files.rs @@ -19,7 +19,7 @@ pub(crate) use diagnostics::{handle_host_logs, handle_logs, handle_panics, handl pub(crate) use diagnostics::{session_db_triage, session_triage_statements}; pub(super) use exec::{handle_exec, proto_exec_target}; pub(crate) use fork::{clone_session_state, handle_fork}; -pub(super) use ipc_command::send_ipc_command; +pub(super) use ipc_command::{send_ipc_command, send_owner_command}; pub(super) fn gib(bytes: u64) -> u64 { bytes / 1024 / 1024 / 1024 diff --git a/crates/capsem-service/src/vm_files/exec.rs b/crates/capsem-service/src/vm_files/exec.rs index ba063e55b..7197cf27b 100644 --- a/crates/capsem-service/src/vm_files/exec.rs +++ b/crates/capsem-service/src/vm_files/exec.rs @@ -28,6 +28,7 @@ pub(crate) async fn handle_exec( .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e))?; let res = send_ipc_command( + &state, &uds_path, ServiceToProcess::Exec { id: state.next_job_id(), diff --git a/crates/capsem-service/src/vm_files/file_boundary.rs b/crates/capsem-service/src/vm_files/file_boundary.rs index 6ef1834a7..2ec081a45 100644 --- a/crates/capsem-service/src/vm_files/file_boundary.rs +++ b/crates/capsem-service/src/vm_files/file_boundary.rs @@ -32,6 +32,7 @@ pub(crate) async fn log_file_boundary_on_owner( ) -> Result>, AppError> { let id = state.next_job_id(); let res = send_ipc_command( + state, uds_path, ServiceToProcess::LogFileBoundary { id, diff --git a/crates/capsem-service/src/vm_files/fork.rs b/crates/capsem-service/src/vm_files/fork.rs index ce85c9220..389e199fc 100644 --- a/crates/capsem-service/src/vm_files/fork.rs +++ b/crates/capsem-service/src/vm_files/fork.rs @@ -149,7 +149,7 @@ async fn clone_guest_state( id, destination: destination.to_string_lossy().into_owned(), }; - let result = match send_ipc_command(uds_path, request, Some(CLONE_STATE_REPLY_SECS)).await { + let result = match send_ipc_command(state, uds_path, request, Some(CLONE_STATE_REPLY_SECS)).await { Ok(ProcessToService::CloneStateResult { size_bytes: Some(size), error: None, diff --git a/crates/capsem-service/src/vm_files/ipc_command.rs b/crates/capsem-service/src/vm_files/ipc_command.rs index 9101084c8..e562365c4 100644 --- a/crates/capsem-service/src/vm_files/ipc_command.rs +++ b/crates/capsem-service/src/vm_files/ipc_command.rs @@ -3,25 +3,22 @@ use super::*; #[tracing::instrument(skip_all, fields(cmd = ?std::mem::discriminant(&cmd), timeout_secs = ?timeout_secs))] pub(crate) async fn send_ipc_command( + state: &ServiceState, uds_path: &std::path::Path, cmd: ServiceToProcess, timeout_secs: Option, ) -> Result { - let stream = tokio::net::UnixStream::connect(uds_path) - .await - .map_err(|e| format!("failed to connect to sandbox: {e}"))?; - let std_stream = stream - .into_std() - .map_err(|e| format!("failed to convert stream: {e}"))?; - let (std_stream, _) = capsem_foundation::ipc_handshake::negotiate_initiator_off_worker( - std_stream, - "capsem-service", - capsem_foundation::telemetry::current_parent_traceparent(), - ) - .await - .map_err(|e| format!("IPC handshake failed: {e}"))?; - let (tx, rx): (Sender, Receiver) = - channel_from_std(std_stream).map_err(|e| format!("failed to create IPC channel: {e}"))?; + let owner = owner_connection::OwnerConnection::acquire(state, uds_path)?; + send_owner_command(state, &owner, cmd, timeout_secs).await +} + +pub(crate) async fn send_owner_command( + state: &ServiceState, + owner: &owner_connection::OwnerConnection, + cmd: ServiceToProcess, + timeout_secs: Option, +) -> Result { + let (tx, rx) = owner.open(state, "capsem-service", false).await?; tx.send(cmd.clone()) .await @@ -58,6 +55,7 @@ pub(crate) async fn send_ipc_command( None => matches!(msg, ProcessToService::Pong) && matches!(cmd, ServiceToProcess::Ping), }; if answers { + owner.validate(state, false)?; return Ok(msg); } } diff --git a/crates/capsem-service/src/vm_files/provision/tests.rs b/crates/capsem-service/src/vm_files/provision/tests.rs index cc3b3c533..33db12973 100644 --- a/crates/capsem-service/src/vm_files/provision/tests.rs +++ b/crates/capsem-service/src/vm_files/provision/tests.rs @@ -1,6 +1,41 @@ use super::*; use crate::container_setup::{ContainerSetups, ImageFetch, ImageSource, PullFuture}; -use crate::tests::{insert_fake_instance_with_session_dir, spawn_fake_process}; +use crate::tests::insert_fake_instance_with_session_dir; + +/// Re-executed owner stays alive until the create's real teardown kills it. +#[tokio::test] +async fn refused_image_owner_child() { + let Some(path) = std::env::var_os("CAPSEM_REFUSED_IMAGE_OWNER") else { + return; + }; + use std::io::Write; + use std::os::fd::AsFd; + let listener = tokio::net::UnixListener::bind(&path).unwrap(); + std::fs::write(PathBuf::from(&path).with_extension("ready"), b"ready").unwrap(); + let mut control = + std::os::unix::net::UnixStream::from(capsem_foundation::unix::fd::duplicate(std::io::stdin().as_fd()).unwrap()); + control.write_all(b"ready").unwrap(); + let (socket, _) = listener.accept().await.unwrap(); + let mut socket = socket.into_std().unwrap(); + let socket = tokio::task::spawn_blocking(move || { + capsem_foundation::ipc_handshake::negotiate_responder(&mut socket, "capsem-process-test", "").unwrap(); + socket + }) + .await + .unwrap(); + let (tx, rx) = channel_from_std::(socket).unwrap(); + let ServiceToProcess::AdmitContainerPull { id, .. } = rx.recv().await.unwrap() else { + panic!("expected admission") + }; + tx.send(ProcessToService::ContainerPullAdmission { + id, + error: None, + policy_refused: false, + }) + .await + .unwrap(); + std::future::pending::<()>().await; +} /// An image source whose registry refuses every pull. Its policy grants that /// registry, so the refusal is the registry's: the default policy reads the @@ -44,7 +79,6 @@ impl ImageSource for RefusingImages { /// kept for post-mortem the way any failed session is. #[tokio::test] async fn a_failed_container_create_discards_the_vm_and_keeps_its_ledger() { - let dir = tempfile::tempdir().unwrap(); let mut state = crate::tests::make_test_state_owned(); state.containers = ContainerSetups::with_source(Box::new(RefusingImages)); let state = Arc::new(state); @@ -53,8 +87,43 @@ async fn a_failed_container_create_discards_the_vm_and_keeps_its_ledger() { // Not SQLite: its counters cannot be read at stop, and the ledger must be kept anyway. std::fs::write(session_dir.join("session.db"), b"ledger").unwrap(); std::fs::write(session_dir.join("process.log"), b"log").unwrap(); - // pid 0: teardown must not signal a real process. - insert_fake_instance_with_session_dir(&state, "box", 0, session_dir.clone()); + use tokio::io::AsyncReadExt; + let uds_path = session_dir.join("process.sock"); + let (control, child_control) = std::os::unix::net::UnixStream::pair().unwrap(); + control.set_nonblocking(true).unwrap(); + let mut control = tokio::net::UnixStream::from_std(control).unwrap(); + let child = tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "vm_files::provision::tests::refused_image_owner_child", + "--nocapture", + ]) + .env_clear() + .env("CAPSEM_REFUSED_IMAGE_OWNER", &uds_path) + .stdin(std::process::Stdio::from(std::os::fd::OwnedFd::from(child_control))) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut ready = [0; 5]; + tokio::time::timeout(std::time::Duration::from_secs(5), control.read_exact(&mut ready)) + .await + .unwrap() + .unwrap(); + assert_eq!(&ready, b"ready"); + insert_fake_instance_with_session_dir(&state, "box", child.id().unwrap(), session_dir.clone()); + let generation = state.instances.lock().unwrap()["box"].generation; + let retirement = state.retirements.register("box", generation).unwrap(); + let reaper = crate::instance_reaper::spawn_exit_reaper( + child, + "box".into(), + "named-box".into(), + Arc::clone(&state), + uds_path, + session_dir.clone(), + retirement, + ); let mut entry = crate::tests::test_persistent_entry("named-box", session_dir.clone()); entry.id = "box".into(); state @@ -64,18 +133,6 @@ async fn a_failed_container_create_discards_the_vm_and_keeps_its_ledger() { .data .vms .insert("named-box".into(), entry); - let uds_path = state.instances.lock().unwrap()["box"].uds_path.clone(); - let owner = spawn_fake_process(&uds_path, 1, |message| { - let reply = match message { - ServiceToProcess::AdmitContainerPull { id, .. } => Some(ProcessToService::ContainerPullAdmission { - id: *id, - error: None, - policy_refused: false, - }), - other => panic!("unexpected owner message: {other:?}"), - }; - Box::pin(async move { reply }) - }); let spec = api::ContainerSpec { image: "registry.example/app:1".into(), args: vec![], @@ -85,7 +142,7 @@ async fn a_failed_container_create_discards_the_vm_and_keeps_its_ledger() { }; let error = finish_create(&state, "box", &[], Some(spec)).await.unwrap_err(); - owner.await.unwrap(); + reaper.await.unwrap(); assert_eq!(error.0, StatusCode::INTERNAL_SERVER_ERROR, "{}", error.1); assert!(error.1.contains("registry refused the image"), "{}", error.1); assert!( @@ -106,5 +163,4 @@ async fn a_failed_container_create_discards_the_vm_and_keeps_its_ledger() { let kept = find_failed_session_dir(&state.run_dir, "box").expect("the failed create's ledger and logs are kept"); assert_eq!(std::fs::read(kept.join("session.db")).unwrap(), b"ledger"); assert_eq!(std::fs::read(kept.join("process.log")).unwrap(), b"log"); - drop(dir); } diff --git a/crates/capsem-service/src/vm_lifecycle.rs b/crates/capsem-service/src/vm_lifecycle.rs index 737034f21..66bccab28 100644 --- a/crates/capsem-service/src/vm_lifecycle.rs +++ b/crates/capsem-service/src/vm_lifecycle.rs @@ -292,12 +292,13 @@ pub(super) async fn shutdown_vm_process( i.pid, i.persistent, i.generation, + owner_connection::OwnerConnection::capture(i), ) }); drop(instances); result }; - let Some((uds_path, session_dir, pid, persistent, generation)) = snapshot else { + let Some((uds_path, session_dir, pid, persistent, generation, owner)) = snapshot else { drop(_shutdown_guard); drop(_vz_host_guard); drop(_vz_guard); @@ -343,23 +344,14 @@ pub(super) async fn shutdown_vm_process( if mode.retains_state() { // Send shutdown command via IPC (or SIGTERM as fallback). - let stream_res = tokio::net::UnixStream::connect(&uds_path).await; - if let Ok(stream) = stream_res { - if let Ok(std_stream) = stream.into_std() { - if let Ok((std_stream, _)) = capsem_foundation::ipc_handshake::negotiate_initiator_off_worker( - std_stream, - "capsem-service", - capsem_foundation::telemetry::current_parent_traceparent(), - ) - .await - { - if let Ok((tx, _)) = channel_from_std::(std_stream) { - capsem_core::try_send!("ipc_graceful_shutdown", tx.send(ServiceToProcess::Shutdown).await); - } - } - } - } else { - signal_vm_process(state, id, generation, process_control::Signal::Terminate).await?; + let channel = match owner { + Ok(owner) => owner.open(state, "capsem-service", true).await, + Err(error) => Err(error), + }; + if let Ok((tx, _)) = channel { + capsem_core::try_send!("ipc_graceful_shutdown", tx.send(ServiceToProcess::Shutdown).await); + } else if pid > 0 { + process_control::send_or_log(pid, process_control::Signal::Terminate, "vm-ipc-shutdown-fallback"); } } else { // Destructive delete has no state to flush. SIGKILL also prevents the @@ -425,7 +417,7 @@ pub(super) async fn handle_suspend( // The host-wide flock also serializes pytest-xdist service processes. let _vz_host_guard = acquire_vz_host_lock(startup::VzHostLockMode::Exclusive).await?; - let (uds_path, generation) = { + let owner = { let mut instances = state.instances.lock().unwrap(); let i = instances .get_mut(&id) @@ -436,36 +428,17 @@ pub(super) async fn handle_suspend( "ephemeral VMs cannot be suspended (persist first)".into(), )); } - let result = (i.uds_path.clone(), i.generation); + let result = owner_connection::OwnerConnection::capture(i); drop(instances); result - }; - - let stream = tokio::net::UnixStream::connect(&uds_path).await.map_err(|e| { - AppError( - StatusCode::INTERNAL_SERVER_ERROR, - format!("failed to connect to VM IPC: {e}"), - ) - })?; - let std_stream = stream.into_std().map_err(|e| { - AppError( - StatusCode::INTERNAL_SERVER_ERROR, - format!("failed to convert stream: {e}"), - ) - })?; - let (std_stream, _) = capsem_foundation::ipc_handshake::negotiate_initiator_off_worker( - std_stream, - "capsem-service", - capsem_foundation::telemetry::current_parent_traceparent(), - ) - .await - .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, format!("IPC handshake failed: {e}")))?; - let (tx, rx) = channel_from_std::(std_stream).map_err(|e| { - AppError( - StatusCode::INTERNAL_SERVER_ERROR, - format!("failed to create IPC channel: {e}"), - ) - })?; + } + .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e))?; + let uds_path = owner.uds_path.clone(); + let pid = owner.pid(); + let (tx, rx) = owner + .open(&state, "capsem-service", false) + .await + .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e))?; let checkpoint_path = RESUME_CHECKPOINT_NAME.to_string(); tx.send(ServiceToProcess::Suspend { checkpoint_path }) @@ -504,6 +477,11 @@ pub(super) async fn handle_suspend( Ok(confirmation) => confirmation, Err(_) => SuspendConfirmation::TimedOut, }; + // A normal owner exit may already have removed its registry entry; a + // replacement must never inherit the old suspend's cleanup or result. + owner + .validate(&state, true) + .map_err(|e| AppError(StatusCode::BAD_GATEWAY, e))?; if let Some((outcome, error)) = suspend_failure(confirmation) { // The guest never acknowledged suspend. Leaving the process alive @@ -523,12 +501,10 @@ pub(super) async fn handle_suspend( // released its VZ checkpoint and VirtioFS share before resume can spawn a // replacement. The helper bounds natural exit, then SIGKILLs and waits for // reaping instead of guessing that a fixed post-kill sleep was sufficient. - wait_for_process_exit(&state, &id, generation, std::time::Duration::from_millis(500)).await?; - - state - .connections - .stop_runtime(&id, generation, capsem_credentials::GrantStopPolicy::Preserve) - .await?; + wait_for_process_exit(pid, std::time::Duration::from_millis(500)).await; + owner + .validate(&state, true) + .map_err(|e| AppError(StatusCode::BAD_GATEWAY, e))?; tracing::warn!(id, "handle_suspend (success) removing instance"); state.instances.lock().unwrap().remove(&id); @@ -953,6 +929,7 @@ pub(super) async fn handle_run( // 2. Execute command. let job_id = state.next_job_id(); let exec_result = send_ipc_command( + &state, &uds_path, ServiceToProcess::Exec { id: job_id, From 30760045fd5178d0f1c1a8e138fdb0296cb84c60 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 15:43:23 +0000 Subject: [PATCH 008/464] security(api): require preview owner identity --- crates/capsem-api/src/exposures.rs | 2 ++ crates/capsem-api/src/exposures/tests.rs | 26 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/crates/capsem-api/src/exposures.rs b/crates/capsem-api/src/exposures.rs index 21e908d89..6c993f0ee 100644 --- a/crates/capsem-api/src/exposures.rs +++ b/crates/capsem-api/src/exposures.rs @@ -140,6 +140,8 @@ pub struct PreviewConnectionAdmissionResponse { pub handoff_socket: String, pub handoff_token: u64, pub owner_generation: String, + pub owner_pid: u32, + pub owner_uid: u32, } #[cfg(test)] diff --git a/crates/capsem-api/src/exposures/tests.rs b/crates/capsem-api/src/exposures/tests.rs index 025798c8b..dfba839d5 100644 --- a/crates/capsem-api/src/exposures/tests.rs +++ b/crates/capsem-api/src/exposures/tests.rs @@ -58,3 +58,29 @@ fn browser_previews_have_no_unauthenticated_loopback_listener() { assert!(!session.url.contains(&session.bootstrap_token)); assert!(!session.url.contains('?')); } + +#[test] +fn preview_handoff_grants_require_explicit_owner_process_identity() { + let wire = json!({ + "handoff_socket": "/private/owner.sock", "handoff_token": 123, + "owner_generation": "42", "owner_pid": 2000, "owner_uid": 1000, + }); + let admitted: PreviewConnectionAdmissionResponse = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(admitted).unwrap(), wire); + for missing in ["owner_pid", "owner_uid"] { + let mut incomplete = wire.clone(); + incomplete.as_object_mut().unwrap().remove(missing); + assert!( + serde_json::from_value::(incomplete).is_err(), + "{missing} is required" + ); + } + for (field, value) in [("owner_pid", json!("2000")), ("owner_uid", json!(-1))] { + let mut malformed = wire.clone(); + malformed[field] = value; + assert!( + serde_json::from_value::(malformed).is_err(), + "{field} must be a kernel identity value" + ); + } +} From 1a19b651339baa9be25e48a6c1ac59434aacf2fb Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 15:45:19 +0000 Subject: [PATCH 009/464] security(service): mint preview peer identity --- crates/capsem-service/src/owner_connection.rs | 4 ++++ crates/capsem-service/src/router_runtime/exposures.rs | 2 ++ .../capsem-service/src/router_runtime/exposures/tests.rs | 8 +++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/capsem-service/src/owner_connection.rs b/crates/capsem-service/src/owner_connection.rs index a513b9bf9..65fe32698 100644 --- a/crates/capsem-service/src/owner_connection.rs +++ b/crates/capsem-service/src/owner_connection.rs @@ -15,6 +15,10 @@ impl OwnerConnection { self.identity.pid.get() } + pub(crate) fn uid(&self) -> u32 { + self.identity.uid + } + pub(crate) fn authenticate( &self, state: &ServiceState, diff --git a/crates/capsem-service/src/router_runtime/exposures.rs b/crates/capsem-service/src/router_runtime/exposures.rs index 4ee3d1597..17aa29b53 100644 --- a/crates/capsem-service/src/router_runtime/exposures.rs +++ b/crates/capsem-service/src/router_runtime/exposures.rs @@ -344,6 +344,8 @@ pub(crate) async fn handle_admit_preview_connection( .into_owned(), handoff_token, owner_generation: owner_generation.to_string(), + owner_pid: handoff.owner.pid(), + owner_uid: handoff.owner.uid(), })), ProcessToService::PreviewConnectionAdmitted { error: Some(error), .. } => { Err(AppError(StatusCode::UNAUTHORIZED, error)) diff --git a/crates/capsem-service/src/router_runtime/exposures/tests.rs b/crates/capsem-service/src/router_runtime/exposures/tests.rs index e051d1253..bfe0e15ff 100644 --- a/crates/capsem-service/src/router_runtime/exposures/tests.rs +++ b/crates/capsem-service/src/router_runtime/exposures/tests.rs @@ -371,7 +371,13 @@ async fn preview_admission_preserves_the_coordinator_derived_handoff_endpoint() assert_eq!(status, StatusCode::OK, "{body}"); assert_eq!( body, - json!({"handoff_socket": expected, "handoff_token": 123, "owner_generation": "42"}) + json!({ + "handoff_socket": expected, + "handoff_token": 123, + "owner_generation": "42", + "owner_pid": std::process::id(), + "owner_uid": capsem_foundation::unix::process::current_uid(), + }) ); } From 133473d4fbcd1432d71f40231948a69e68c627c7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 15:48:32 +0000 Subject: [PATCH 010/464] security(gateway): authenticate preview owner handoffs --- CHANGELOG.md | 4 +- crates/capsem-gateway/src/preview.rs | 11 ++- .../src/preview/tests/admission.rs | 76 +++++++++++++++++++ 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7209ff92..3f4a2a79d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Bind private-network and preview handoff endpoints to the coordinator's session identity and VM-owner spawn generation. Refuse worker replies that - name another endpoint or arrive after that owner has been replaced. + name another endpoint or arrive after that owner has been replaced, and + authenticate the preview owner's kernel identity before transferring a + browser connection descriptor. ### Fixed diff --git a/crates/capsem-gateway/src/preview.rs b/crates/capsem-gateway/src/preview.rs index 69db889fd..597cbf51e 100644 --- a/crates/capsem-gateway/src/preview.rs +++ b/crates/capsem-gateway/src/preview.rs @@ -17,7 +17,7 @@ use capsem_foundation::unix::router_channel::{Receiver, Sender}; use capsem_proto::privatelink::{decode_seat_frame, seat_frame, SEAT_PREVIEW}; use http_body_util::BodyExt; use std::collections::HashMap; -use std::os::fd::AsRawFd; +use std::os::fd::{AsFd, AsRawFd}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -409,7 +409,14 @@ fn validate_bootstrap_head(head: &Head) -> anyhow::Result<()> { async fn handoff(stream: tokio::net::TcpStream, admitted: PreviewConnectionAdmissionResponse) -> anyhow::Result<()> { let source = stream.into_std()?; - let seat = std::os::unix::net::UnixStream::connect(&admitted.handoff_socket)?; + let seat = tokio::net::UnixStream::connect(&admitted.handoff_socket) + .await? + .into_std()?; + let owner = capsem_foundation::unix::peer::PeerIdentity { + pid: capsem_foundation::unix::process::ProcessId::try_from(admitted.owner_pid)?, + uid: admitted.owner_uid, + }; + capsem_foundation::unix::peer::require(seat.as_fd(), owner).context("authenticate preview owner")?; let sender = Sender::new(seat.try_clone()?)?; sender .send(&seat_frame(SEAT_PREVIEW, admitted.handoff_token), &[source.as_raw_fd()]) diff --git a/crates/capsem-gateway/src/preview/tests/admission.rs b/crates/capsem-gateway/src/preview/tests/admission.rs index 97d4458bd..75110c15d 100644 --- a/crates/capsem-gateway/src/preview/tests/admission.rs +++ b/crates/capsem-gateway/src/preview/tests/admission.rs @@ -4,6 +4,82 @@ use std::sync::atomic::Ordering; use std::time::Instant; use tokio::io::{AsyncReadExt, AsyncWriteExt}; +async fn tcp_pair() -> (tokio::net::TcpStream, tokio::net::TcpStream) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let connect = tokio::net::TcpStream::connect(listener.local_addr().unwrap()); + let (server, client) = tokio::join!(listener.accept(), connect); + (server.unwrap().0, client.unwrap()) +} + +#[tokio::test] +async fn a_wrong_process_at_the_granted_path_receives_no_seat_or_browser_descriptor() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("replaced.sock"); + let listener = tokio::net::UnixListener::bind(&path).unwrap(); + let (browser, _client) = tcp_pair().await; + let current = std::process::id(); + let wrong = if current == 1 { 2 } else { 1 }; + let admission = PreviewConnectionAdmissionResponse { + handoff_socket: path.to_string_lossy().into_owned(), + handoff_token: 77, + owner_generation: "trusted-generation".into(), + owner_pid: wrong, + owner_uid: capsem_foundation::unix::process::current_uid(), + }; + + let attempted = tokio::spawn(handoff(browser, admission)); + let (seat, _) = listener.accept().await.unwrap(); + let receiver = Receiver::new(seat.into_std().unwrap()).unwrap(); + let error = match receiver.recv().await { + Ok(_) => panic!("replacement received a preview seat or descriptor"), + Err(error) => error, + }; + assert_eq!(error.kind(), std::io::ErrorKind::UnexpectedEof); + let error = attempted.await.unwrap().unwrap_err(); + assert!(format!("{error:#}").contains("granted process"), "{error:#}"); +} + +#[tokio::test] +async fn the_authenticated_owner_receives_the_exact_browser_descriptor() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("owner.sock"); + let listener = tokio::net::UnixListener::bind(&path).unwrap(); + let owner = tokio::spawn(async move { + let (seat, _) = listener.accept().await.unwrap(); + let seat = seat.into_std().unwrap(); + let receiver = Receiver::new(seat.try_clone().unwrap()).unwrap(); + let frame = receiver.recv().await.unwrap(); + let (kind, token) = decode_seat_frame(&frame.bytes).unwrap(); + assert_eq!((kind, token, frame.fds.len()), (SEAT_PREVIEW, 78, 1)); + let browser = std::net::TcpStream::from(frame.fds.into_iter().next().unwrap()); + browser.set_nonblocking(true).unwrap(); + let mut browser = tokio::net::TcpStream::from_std(browser).unwrap(); + let mut byte = [0]; + browser.read_exact(&mut byte).await.unwrap(); + assert_eq!(byte, [b'x']); + Sender::new(seat) + .unwrap() + .send(&seat_frame(SEAT_PREVIEW, token), &[]) + .await + .unwrap(); + }); + let (browser, mut client) = tcp_pair().await; + client.write_all(b"x").await.unwrap(); + handoff( + browser, + PreviewConnectionAdmissionResponse { + handoff_socket: path.to_string_lossy().into_owned(), + handoff_token: 78, + owner_generation: "trusted-generation".into(), + owner_pid: std::process::id(), + owner_uid: capsem_foundation::unix::process::current_uid(), + }, + ) + .await + .unwrap(); + owner.await.unwrap(); +} + /// A stalled partial request used to spin: `peek` leaves the socket readable, /// so waiting on readability returned at once and the loop ran until the /// header deadline, burning a core share per connection. From 9499d6ab93b1783fff8a93f2ccaa2495d58f2538 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 15:53:00 +0000 Subject: [PATCH 011/464] security(foundation): retain child identity until explicit reap --- crates/capsem-foundation/src/unix/process.rs | 27 +++++++++++++------ .../src/unix/process/tests.rs | 17 ++++++++++++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/crates/capsem-foundation/src/unix/process.rs b/crates/capsem-foundation/src/unix/process.rs index 134e7c7b9..d1ba6aafd 100644 --- a/crates/capsem-foundation/src/unix/process.rs +++ b/crates/capsem-foundation/src/unix/process.rs @@ -159,18 +159,29 @@ fn lone_exited_child_group(leader: ProcessId) -> bool { /// Observe an owned child's exit without reaping it. Keeping the zombie until /// group cleanup reserves its PID even if its descendants change groups. pub fn child_has_exited(pid: ProcessId) -> io::Result { + observe_child_exit(pid, true) +} + +/// Wait for an owned child to exit without reaping it. +/// +/// The caller retains custody of the child's PID and exit status until its +/// process handle is explicitly waited. This call blocks and therefore belongs +/// on a blocking worker when used from an async runtime. +pub fn wait_for_child_exit(pid: ProcessId) -> io::Result<()> { + if observe_child_exit(pid, false)? { + Ok(()) + } else { + Err(io::Error::other("blocking child exit observation returned no status")) + } +} + +fn observe_child_exit(pid: ProcessId, nonblocking: bool) -> io::Result { loop { // SAFETY: zero is a valid initial siginfo_t; waitid writes this owned // buffer, and P_PID names only the validated positive child identifier. let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; - let result = unsafe { - libc::waitid( - libc::P_PID, - pid.get() as libc::id_t, - &mut info, - libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, - ) - }; + let options = libc::WEXITED | libc::WNOWAIT | if nonblocking { libc::WNOHANG } else { 0 }; + let result = unsafe { libc::waitid(libc::P_PID, pid.get() as libc::id_t, &mut info, options) }; if result == 0 { // SAFETY: waitid initialized the child-status fields; an unchanged // zero PID means no status was available with WNOHANG. diff --git a/crates/capsem-foundation/src/unix/process/tests.rs b/crates/capsem-foundation/src/unix/process/tests.rs index 603609629..94f33345f 100644 --- a/crates/capsem-foundation/src/unix/process/tests.rs +++ b/crates/capsem-foundation/src/unix/process/tests.rs @@ -102,6 +102,23 @@ fn exit_observation_keeps_the_child_waitable_and_its_pid_reserved() { assert_eq!(child.wait().unwrap().code(), Some(7)); } +#[test] +fn blocking_exit_observation_keeps_the_status_waitable() { + let mut child = std::process::Command::new("sh") + .args(["-c", "sleep 0.05; exit 9"]) + .spawn() + .unwrap(); + let pid = ProcessId::try_from(child.id()).unwrap(); + + wait_for_child_exit(pid).unwrap(); + + assert!( + child_has_exited(pid).unwrap(), + "blocking observation must leave the exited child unreaped" + ); + assert_eq!(child.wait().unwrap().code(), Some(9)); +} + #[test] fn an_exited_group_leader_does_not_hide_a_live_member() { use std::os::unix::process::{CommandExt, ExitStatusExt}; From 3aabc4eb63a3f23bd88e94a3b9646c5a9129c36d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 15:59:48 +0000 Subject: [PATCH 012/464] security(service): revoke worker authority before reap --- CHANGELOG.md | 4 + crates/capsem-service/src/instance_reaper.rs | 61 ++++++-- .../src/tests/instance_reaper.rs | 144 ++++-------------- 3 files changed, 84 insertions(+), 125 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f4a2a79d..f8ec92f75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Keep an exited VM owner's process identity under coordinator custody until + its registered generation has been revoked, preventing PID reuse from + inheriting worker and broker authority. + - Authenticate coordinator connections against each registered VM owner's kernel process and user identity before sending protocol bytes. Refuse stale owner generations for commands, streams and private-network descriptor handoffs; diff --git a/crates/capsem-service/src/instance_reaper.rs b/crates/capsem-service/src/instance_reaper.rs index eb7b96be6..01302604f 100644 --- a/crates/capsem-service/src/instance_reaper.rs +++ b/crates/capsem-service/src/instance_reaper.rs @@ -4,10 +4,12 @@ use super::*; mod retirement; pub(crate) use retirement::{Registration, Retirements}; -pub(super) fn kill_and_reap( - mut child: tokio::process::Child, - mut retirement: Registration, -) -> tokio::task::JoinHandle<()> { +pub(super) enum ExitedGeneration { + Current(Option>), + Replaced, +} + +pub(super) fn kill_and_reap(mut child: tokio::process::Child) { let _ = child.start_kill(); tokio::spawn(async move { if retirement.wait_child(&mut child).await.is_ok() { @@ -35,7 +37,8 @@ pub(super) fn spawn_exit_reaper( // task can be delayed across an exit/replacement or PID reuse. let generation = retirement.generation(); tokio::spawn(async move { - let Ok(exit_status) = retirement.wait_child(&mut child).await else { + let registration = revoke_exited_generation(&child, &id, &state, generation).await; + let Ok(exit_status) = child.wait().await else { return; }; info!(id, ?exit_status, "capsem-process exited, cleaning up"); @@ -67,12 +70,9 @@ pub(super) fn spawn_exit_reaper( // child exited without an explicit service-side shutdown removing it. // A clean process exit is a graceful shutdown regardless of whether // the guest or service initiated it; anything else is a crash. - let removed = { - let mut instances = state.instances.lock().unwrap(); - if instances - .get(&id) - .is_some_and(|instance| Some(instance.pid) != pid || instance.generation != generation) - { + let removed = match registration { + ExitedGeneration::Current(removed) => removed, + ExitedGeneration::Replaced => { // A cold fallback can replace a failed restore while holding // the exclusive guard. Its registry, DB and sockets are not // owned by this delayed reaper. @@ -84,7 +84,6 @@ pub(super) fn spawn_exit_reaper( retirement.complete(); return; } - instances.remove(&id) }; if let Some(info) = &removed { // The VM is gone: a container setup for it has nothing left to do. @@ -189,3 +188,41 @@ pub(super) fn spawn_exit_reaper( retirement.complete(); }) } + +pub(super) async fn revoke_exited_generation( + child: &tokio::process::Child, + id: &str, + state: &ServiceState, + generation: uuid::Uuid, +) -> ExitedGeneration { + let pid = child.id(); + if let Some(raw_pid) = pid { + match capsem_foundation::unix::process::ProcessId::try_from(raw_pid) { + Ok(pid) => { + match tokio::task::spawn_blocking(move || capsem_foundation::unix::process::wait_for_child_exit(pid)) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => { + error!(id, ?pid, error = %error, "failed to retain child exit identity") + } + Err(error) => { + error!(id, ?pid, error = %error, "child exit observer failed") + } + } + } + Err(error) => error!(id, ?pid, error = %error, "child has invalid process identity"), + } + } else { + error!(id, "child process identity disappeared before exit observation"); + } + + let mut instances = state.instances.lock().unwrap(); + if instances + .get(id) + .is_some_and(|instance| Some(instance.pid) != pid || instance.generation != generation) + { + return ExitedGeneration::Replaced; + } + ExitedGeneration::Current(instances.remove(id).map(Box::new)) +} diff --git a/crates/capsem-service/src/tests/instance_reaper.rs b/crates/capsem-service/src/tests/instance_reaper.rs index 732dd397d..50a70e259 100644 --- a/crates/capsem-service/src/tests/instance_reaper.rs +++ b/crates/capsem-service/src/tests/instance_reaper.rs @@ -1,128 +1,28 @@ use super::*; #[tokio::test] -async fn vm_shutdown_targets_owned_child_even_when_cached_pid_names_an_unrelated_process() { +async fn exited_generation_is_revoked_while_its_pid_is_still_reserved() { let (state, _dir) = make_test_state_with_tempdir(); - let id = "owned-signal-target"; + let id = "unreaped-owner"; let session = state.run_dir.join("sessions").join(id); std::fs::create_dir_all(&session).unwrap(); - let child = tokio::process::Command::new("sleep") - .arg("30") - .kill_on_drop(true) + let mut child = tokio::process::Command::new("sh") + .args(["-c", "sleep 0.05; exit 11"]) .spawn() .unwrap(); - let mut unrelated = tokio::process::Command::new("sleep") - .arg("30") - .kill_on_drop(true) - .spawn() - .unwrap(); - insert_fake_instance_with_session_dir(&state, id, unrelated.id().unwrap(), session.clone()); - let generation = state.instances.lock().unwrap().get(id).unwrap().generation; - let registration = state.retirements.register(id, generation).unwrap(); - let reaper = crate::instance_reaper::spawn_exit_reaper( - child, - id.into(), - id.into(), - state.clone(), - state.instance_socket_path(id).unwrap(), - session, - registration, - ); - let result = tokio::time::timeout( - std::time::Duration::from_secs(3), - shutdown_vm_process(&state, id, ShutdownMode::Discard, Some(generation)), - ) - .await - .unwrap() - .unwrap(); - assert!(result.is_some()); - assert!( - unrelated.try_wait().unwrap().is_none(), - "shutdown signaled a cached/recycled PID" - ); - reaper.await.unwrap(); - assert!(state.retirements.wait(id, generation).await.unwrap()); - unrelated.kill().await.unwrap(); -} + let pid = child.id().unwrap(); + insert_fake_instance_with_session_dir(&state, id, pid, session); + let generation = state.instances.lock().unwrap()[id].generation; -#[tokio::test] -async fn vm_shutdown_without_original_child_authority_cannot_acknowledge_exit() { - let (state, _dir) = make_test_state_with_tempdir(); - let id = "missing-child-authority"; - let session = state.run_dir.join("sessions").join(id); - std::fs::create_dir_all(&session).unwrap(); - let mut unrelated = tokio::process::Command::new("sleep") - .arg("30") - .kill_on_drop(true) - .spawn() - .unwrap(); - insert_fake_instance_with_session_dir(&state, id, unrelated.id().unwrap(), session.clone()); - let generation = state.instances.lock().unwrap().get(id).unwrap().generation; - let result = shutdown_vm_process(&state, id, ShutdownMode::Discard, Some(generation)).await; - assert!(matches!(result, Err(AppError(StatusCode::SERVICE_UNAVAILABLE, _)))); - assert!(unrelated.try_wait().unwrap().is_none()); - assert!(session.exists(), "unknown exit must not remove owned session bytes"); - assert!( - state.instances.lock().unwrap().contains_key(id), - "unknown owner cannot claim the instance" - ); - unrelated.kill().await.unwrap(); -} + crate::instance_reaper::revoke_exited_generation(&child, id, &state, generation).await; -#[tokio::test] -async fn service_shutdown_uses_original_child_owner_before_removing_session_bytes() { - let (state, _dir) = make_test_state_with_tempdir(); - let launch = state.lifecycle.admit().unwrap(); - let id = "service-owned-shutdown"; - let session = state.run_dir.join("sessions").join(id); - std::fs::create_dir_all(&session).unwrap(); - std::fs::write(session.join("marker"), b"original").unwrap(); - let child = tokio::process::Command::new("sleep") - .arg("30") - .kill_on_drop(true) - .spawn() - .unwrap(); - let mut unrelated = tokio::process::Command::new("sleep") - .arg("30") - .kill_on_drop(true) - .spawn() - .unwrap(); - let shutdown = { - let state = state.clone(); - tokio::spawn(async move { kill_all_vm_processes(&state).await }) - }; - capsem_foundation::poll::poll_until( - capsem_foundation::poll::PollOpts::new("shutdown-seals-admission", std::time::Duration::from_secs(1)), - || std::future::ready(state.lifecycle.admit().is_err().then_some(())), - ) - .await - .unwrap(); - assert!(!shutdown.is_finished(), "shutdown missed the admitted original worker"); - insert_fake_instance_with_session_dir(&state, id, unrelated.id().unwrap(), session.clone()); - let generation = state.instances.lock().unwrap().get(id).unwrap().generation; - let registration = state.retirements.register(id, generation).unwrap(); - let reaper = crate::instance_reaper::spawn_exit_reaper( - child, - id.into(), - id.into(), - state.clone(), - state.instance_socket_path(id).unwrap(), - session.clone(), - registration, - ); - drop(launch); - tokio::time::timeout(std::time::Duration::from_secs(4), shutdown) - .await - .unwrap() - .unwrap(); + assert!(!state.instances.lock().unwrap().contains_key(id)); + let pid = capsem_foundation::unix::process::ProcessId::try_from(pid).unwrap(); assert!( - unrelated.try_wait().unwrap().is_none(), - "service stop targeted unrelated cached PID" + capsem_foundation::unix::process::child_has_exited(pid).unwrap(), + "generation revocation must happen before the child is reaped" ); - reaper.await.unwrap(); - assert!(state.retirements.wait(id, generation).await.unwrap()); - assert!(!session.exists()); - unrelated.kill().await.unwrap(); + assert_eq!(child.wait().await.unwrap().code(), Some(11)); } #[tokio::test] @@ -280,6 +180,24 @@ fn child_reapers_start_after_instance_registration() { } } +#[test] +fn child_exit_authority_is_revoked_before_reaping() { + let source = include_str!("../instance_reaper.rs") + .split_once("pub(super) fn spawn_exit_reaper") + .unwrap() + .1; + let revoke = source + .find("revoke_exited_generation(&child") + .expect("the reaper observes exit and revokes the registered generation"); + let reap = source + .find("child.wait().await") + .expect("the reaper eventually consumes the child exit status"); + assert!( + revoke < reap, + "the kernel must still reserve the worker PID when its authority is revoked" + ); +} + /// There is one child reaper. The resume path had its own, which removed the /// instance and nothing else: no session-index stop, no checkpoint or defunct /// bookkeeping in the persistent registry, no crash evidence. A resumed From 2573504ec9776b002151fcbf2262e59c721f6af4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:04:15 +0000 Subject: [PATCH 013/464] security(foundation): centralize inherited descriptor closure --- crates/capsem-foundation/src/unix/fd.rs | 49 +++++++++- crates/capsem-foundation/src/unix/fd/tests.rs | 95 +++++++++---------- .../src/unix/router_sandbox.rs | 34 +------ 3 files changed, 96 insertions(+), 82 deletions(-) diff --git a/crates/capsem-foundation/src/unix/fd.rs b/crates/capsem-foundation/src/unix/fd.rs index ab6320337..62e0dcae1 100644 --- a/crates/capsem-foundation/src/unix/fd.rs +++ b/crates/capsem-foundation/src/unix/fd.rs @@ -1,7 +1,7 @@ //! Owned descriptor operations with atomic inheritance guarantees. use std::io; -use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, OwnedFd}; +use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, OwnedFd, RawFd}; use std::time::Duration; use nix::errno::Errno; @@ -10,6 +10,53 @@ use nix::sys::socket::{self, Shutdown}; use super::errno; +/// Close every inherited descriptor except standard streams. +/// +/// # Safety +/// Call only at process entry, before any other code owns descriptors above 2 +/// or another thread can open and reuse them. +pub unsafe fn close_inherited_descriptors() -> io::Result<()> { + unsafe { close_inherited_descriptors_except(&[]) } +} + +/// Close ambient inherited descriptors while retaining explicit grants. +/// +/// # Safety +/// Call only at process entry, before any other code owns descriptors above 2 +/// or another thread can open and reuse them. Every descriptor in `preserved` +/// must be an intentional grant whose ownership the caller establishes next. +pub unsafe fn close_inherited_descriptors_except(preserved: &[RawFd]) -> io::Result<()> { + #[cfg(target_os = "macos")] + let directory = "/dev/fd"; + #[cfg(not(target_os = "macos"))] + let directory = "/proc/self/fd"; + let mut descriptors = Vec::new(); + for entry in std::fs::read_dir(directory)? { + let name = entry?.file_name(); + let fd: RawFd = name + .to_str() + .ok_or_else(|| io::Error::other("invalid descriptor name"))? + .parse() + .map_err(io::Error::other)?; + if should_close_inherited_descriptor(fd, preserved) { + descriptors.push(fd); + } + } + for fd in descriptors { + // The caller guarantees no live Rust owner or fd reuse. The directory + // iterator itself has closed; EBADF for that fd is expected. + match nix::unistd::close(fd) { + Ok(()) | Err(Errno::EBADF) => {} + Err(error) => return Err(errno::io(error)), + } + } + Ok(()) +} + +fn should_close_inherited_descriptor(fd: RawFd, preserved: &[RawFd]) -> bool { + fd > 2 && !preserved.contains(&fd) +} + /// Which half of a connected socket to close. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SocketShutdown { diff --git a/crates/capsem-foundation/src/unix/fd/tests.rs b/crates/capsem-foundation/src/unix/fd/tests.rs index d2ebce343..70110866f 100644 --- a/crates/capsem-foundation/src/unix/fd/tests.rs +++ b/crates/capsem-foundation/src/unix/fd/tests.rs @@ -1,5 +1,5 @@ use std::io::{Read, Write}; -use std::os::fd::{AsFd, AsRawFd}; +use std::os::fd::{AsFd, AsRawFd, FromRawFd}; use std::os::unix::net::UnixStream; use nix::fcntl::{fcntl, FcntlArg, FdFlag, OFlag}; @@ -8,57 +8,56 @@ use super::{duplicate, retry_eintr, set_nonblocking, shutdown, wait_readable, So use nix::errno::Errno; #[test] -#[cfg(target_os = "linux")] -fn counter_signals_are_owned_and_accumulate_completions() { - use nix::sys::eventfd::{EfdFlags, EventFd}; - let counter = EventFd::from_flags(EfdFlags::EFD_CLOEXEC | EfdFlags::EFD_NONBLOCK).unwrap(); - super::signal_counter(counter.as_fd()).unwrap(); - assert_eq!(counter.read().unwrap(), 1); - super::signal_counter(counter.as_fd()).unwrap(); - super::signal_counter(counter.as_fd()).unwrap(); - assert_eq!(counter.read().unwrap(), 2); +fn inherited_descriptor_policy_keeps_only_stdio_and_explicit_grants() { + for descriptor in 0..=2 { + assert!(!super::should_close_inherited_descriptor(descriptor, &[])); + } + assert!(super::should_close_inherited_descriptor(7, &[])); + assert!(!super::should_close_inherited_descriptor(7, &[7, 11])); + assert!(super::should_close_inherited_descriptor(9, &[7, 11])); } #[test] -fn positioned_io_preserves_cursor_and_reports_short_reads_and_eof() { - use std::io::{Seek, SeekFrom}; - let mut file = tempfile::tempfile().unwrap(); - file.write_all(b"abcdef").unwrap(); - assert_eq!(super::regular_file_size(file.as_fd()).unwrap(), 6); - file.seek(SeekFrom::Start(2)).unwrap(); - assert_eq!(super::write_at(file.as_fd(), b"XY", 4).unwrap(), 2); - assert_eq!(file.stream_position().unwrap(), 2); - let mut bytes = [0; 8]; - assert_eq!(super::read_at(file.as_fd(), &mut bytes, 3).unwrap(), 3); - assert_eq!(&bytes[..3], b"dXY"); - assert_eq!(super::read_at(file.as_fd(), &mut bytes, 6).unwrap(), 0); - assert_eq!(file.stream_position().unwrap(), 2); -} +fn isolated_exec_child_closes_ambient_descriptor_and_keeps_declared_grant() { + const AMBIENT: &str = "CAPSEM_TEST_AMBIENT_FD"; + const GRANT: &str = "CAPSEM_TEST_GRANTED_FD"; + if let (Ok(ambient), Ok(grant)) = (std::env::var(AMBIENT), std::env::var(GRANT)) { + let ambient: i32 = ambient.parse().unwrap(); + let grant: i32 = grant.parse().unwrap(); + // SAFETY: this isolated subprocess runs exactly this one test and exits + // immediately after taking ownership of its declared report grant. + unsafe { super::close_inherited_descriptors_except(&[grant]).unwrap() }; + let ambient_closed = fcntl(ambient, FcntlArg::F_GETFD) == Err(Errno::EBADF); + let grant_open = fcntl(grant, FcntlArg::F_GETFD).is_ok(); + // SAFETY: the descriptor was inherited solely for this child report. + let mut report = unsafe { UnixStream::from_raw_fd(grant) }; + report.write_all(&[u8::from(ambient_closed && grant_open)]).unwrap(); + std::process::exit(i32::from(!(ambient_closed && grant_open))); + } -#[test] -fn positioned_io_rejects_unrepresentable_offsets_and_preserves_os_errors() { - let file = tempfile::tempfile().unwrap(); - assert_eq!( - super::read_at(file.as_fd(), &mut [0], u64::MAX) - .unwrap_err() - .raw_os_error(), - Some(22) - ); - assert_eq!( - super::write_at(file.as_fd(), b"x", u64::MAX) - .unwrap_err() - .raw_os_error(), - Some(22) - ); - let (socket, _peer) = UnixStream::pair().unwrap(); - assert_eq!( - super::regular_file_size(socket.as_fd()).unwrap_err().kind(), - std::io::ErrorKind::InvalidInput - ); - assert_eq!( - super::read_at(socket.as_fd(), &mut [0], 0).unwrap_err().kind(), - std::io::ErrorKind::NotSeekable - ); + let (_ambient_parent, ambient_child) = UnixStream::pair().unwrap(); + let (mut report_parent, report_child) = UnixStream::pair().unwrap(); + for descriptor in [&ambient_child, &report_child] { + fcntl(descriptor.as_raw_fd(), FcntlArg::F_SETFD(FdFlag::empty())).unwrap(); + } + let executable = std::env::current_exe().unwrap(); + let mut child = std::process::Command::new(executable) + .args([ + "--exact", + "unix::fd::tests::isolated_exec_child_closes_ambient_descriptor_and_keeps_declared_grant", + "--test-threads=1", + ]) + .env(AMBIENT, ambient_child.as_raw_fd().to_string()) + .env(GRANT, report_child.as_raw_fd().to_string()) + .spawn() + .unwrap(); + drop(ambient_child); + drop(report_child); + + let mut result = [0]; + report_parent.read_exact(&mut result).unwrap(); + assert_eq!(result, [1]); + assert!(child.wait().unwrap().success()); } #[test] diff --git a/crates/capsem-foundation/src/unix/router_sandbox.rs b/crates/capsem-foundation/src/unix/router_sandbox.rs index ce505fd76..39aa97901 100644 --- a/crates/capsem-foundation/src/unix/router_sandbox.rs +++ b/crates/capsem-foundation/src/unix/router_sandbox.rs @@ -2,39 +2,7 @@ use std::io; -/// Close every inherited descriptor except the three explicitly assigned -/// standard streams before creating runtime or parent-watch threads. -/// -/// # Safety -/// Call only at process entry, before any other code owns descriptors above 2 -/// or another thread can open/reuse them. -pub unsafe fn close_inherited_descriptors() -> io::Result<()> { - #[cfg(target_os = "macos")] - let directory = "/dev/fd"; - #[cfg(not(target_os = "macos"))] - let directory = "/proc/self/fd"; - let mut descriptors = Vec::new(); - for entry in std::fs::read_dir(directory)? { - let name = entry?.file_name(); - let fd: i32 = name - .to_str() - .ok_or_else(|| io::Error::other("invalid descriptor name"))? - .parse() - .map_err(io::Error::other)?; - if fd > 2 { - descriptors.push(fd); - } - } - for fd in descriptors { - // The caller guarantees no live Rust owner or fd reuse. The directory - // iterator itself has closed; EBADF for that fd is expected. - match nix::unistd::close(fd) { - Ok(()) | Err(nix::errno::Errno::EBADF) => {} - Err(error) => return Err(super::errno::io(error)), - } - } - Ok(()) -} +pub use super::fd::close_inherited_descriptors; /// Restrict the whole process after runtime setup and descriptor inheritance. /// Only connected descriptors are granted; no listener permission is needed. From 3d648f8552593c692702281874878e3a8aea6667 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:07:10 +0000 Subject: [PATCH 014/464] security(process): close ambient descriptors at entry --- CHANGELOG.md | 3 +++ crates/capsem-process/src/main.rs | 3 +++ crates/capsem-process/src/tests.rs | 12 ++++++++++++ 3 files changed, 18 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ec92f75..30273981a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Close ambient inherited file descriptors before each VM owner initializes, + preventing service resources from leaking across the worker exec boundary. + - Keep an exited VM owner's process identity under coordinator custody until its registered generation has been revoked, preventing PID reuse from inheriting worker and broker authority. diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 3a7c444e7..2bb556045 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -209,6 +209,9 @@ fn prepare_session_layout(session_dir: &Path, scratch_disk_size_gb: u32) -> Resu } fn main() -> Result<()> { + // SAFETY: process entry precedes argument parsing, telemetry, descriptor + // owners and runtime threads. Broker grants will be named here explicitly. + unsafe { capsem_foundation::unix::fd::close_inherited_descriptors()? }; let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { service: "capsem-process", sink: capsem_foundation::telemetry::LogSink::Stderr, diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index fd0d69310..5ca0a7275 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -1,6 +1,18 @@ use super::*; use clap::Parser; +#[test] +fn entry_closes_ambient_descriptors_before_runtime_initialization() { + let source = include_str!("main.rs"); + let entry = source.split_once("fn main() -> Result<()> {").unwrap().1; + let close = entry + .find("close_inherited_descriptors()") + .expect("VM owner closes ambient descriptors at process entry"); + let telemetry = entry.find("telemetry::init").unwrap(); + let arguments = entry.find("Args::parse()").unwrap(); + assert!(close < telemetry && close < arguments); +} + // ----------------------------------------------------------------------- // Args parsing // ----------------------------------------------------------------------- From ae2e01a25e47c2af1fed616fc6c89f12de853aa1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:08:57 +0000 Subject: [PATCH 015/464] security(gateway): close ambient descriptors before runtime --- CHANGELOG.md | 5 +++-- crates/capsem-gateway/src/main.rs | 11 +++++++++-- crates/capsem-gateway/src/tests.rs | 12 ++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30273981a..75eb94062 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security -- Close ambient inherited file descriptors before each VM owner initializes, - preventing service resources from leaking across the worker exec boundary. +- Close ambient inherited file descriptors before each VM owner or gateway + initializes, preventing service resources from leaking across worker exec + boundaries. - Keep an exited VM owner's process identity under coordinator custody until its registered generation has been revoked, preventing PID reuse from diff --git a/crates/capsem-gateway/src/main.rs b/crates/capsem-gateway/src/main.rs index 76e02a4bf..a527970ec 100644 --- a/crates/capsem-gateway/src/main.rs +++ b/crates/capsem-gateway/src/main.rs @@ -70,8 +70,15 @@ pub struct AppState { pub previews: preview::PreviewState, } -#[tokio::main] -async fn main() -> Result<()> { +fn main() -> Result<()> { + // SAFETY: process entry precedes argument parsing, telemetry, descriptor + // owners and runtime threads. Broker grants will be named here explicitly. + unsafe { capsem_foundation::unix::fd::close_inherited_descriptors()? }; + let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + runtime.block_on(run()) +} + +async fn run() -> Result<()> { let args = Args::parse(); let run_dir = gateway_run_dir(&args); let _ = std::fs::create_dir_all(&run_dir); diff --git a/crates/capsem-gateway/src/tests.rs b/crates/capsem-gateway/src/tests.rs index 72aec4dc3..58ebb2992 100644 --- a/crates/capsem-gateway/src/tests.rs +++ b/crates/capsem-gateway/src/tests.rs @@ -8,6 +8,18 @@ use crate::status::StatusCache; mod hostile; mod route_forwarding; +#[test] +fn entry_closes_ambient_descriptors_before_runtime_construction() { + let source = include_str!("main.rs"); + let entry = source.split_once("fn main() -> Result<()> {").unwrap().1; + let close = entry + .find("close_inherited_descriptors()") + .expect("gateway closes ambient descriptors at process entry"); + let runtime = entry.find("tokio::runtime::Builder").unwrap(); + let run = entry.find("runtime.block_on(run())").unwrap(); + assert!(close < runtime && runtime < run); +} + struct EnvGuard { key: &'static str, prev: Option, From 02d8225840f652cb75c71111c1290f165ba6b77f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:20:10 +0000 Subject: [PATCH 016/464] security(service): authenticate private DNS by peer identity --- CHANGELOG.md | 4 + crates/capsem-service/src/api.rs | 1 - crates/capsem-service/src/instance.rs | 4 - crates/capsem-service/src/main.rs | 2 + crates/capsem-service/src/owner_connection.rs | 11 +++ crates/capsem-service/src/private_routes.rs | 57 ++++--------- crates/capsem-service/src/service_peer.rs | 15 ++++ .../src/service_runtime/shutdown.rs | 6 +- crates/capsem-service/src/tests.rs | 1 - crates/capsem-service/src/tests/files_api.rs | 1 - .../src/tests/network_routes.rs | 83 ++++++++++++++++--- .../src/tests/settings_files.rs | 2 - .../src/vm_lifecycle/provision.rs | 2 - .../src/vm_lifecycle/resume_process.rs | 2 - 14 files changed, 125 insertions(+), 66 deletions(-) create mode 100644 crates/capsem-service/src/service_peer.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 75eb94062..f6996aa05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Authenticate private-network name lookups with the registered VM owner's + kernel Unix identity, eliminating the reusable bearer secret previously + written into each session directory. + - Close ambient inherited file descriptors before each VM owner or gateway initializes, preventing service resources from leaking across worker exec boundaries. diff --git a/crates/capsem-service/src/api.rs b/crates/capsem-service/src/api.rs index aa6a66d76..70eee123a 100644 --- a/crates/capsem-service/src/api.rs +++ b/crates/capsem-service/src/api.rs @@ -7,7 +7,6 @@ use std::net::Ipv4Addr; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct PrivateResolveRequest { pub source_vm: String, - pub owner_secret: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/capsem-service/src/instance.rs b/crates/capsem-service/src/instance.rs index 573c2a9c6..1b9c5772f 100644 --- a/crates/capsem-service/src/instance.rs +++ b/crates/capsem-service/src/instance.rs @@ -23,10 +23,6 @@ pub(crate) struct InstanceInfo { pub(crate) env: Option>, /// Sandbox this VM was cloned from, if any pub(crate) forked_from: Option, - /// What the VM owner shows when it asks the service about private names - /// on the VM's behalf: minted at spawn, written to the session directory - /// for the owner alone, matched here. Never reaches the guest. - pub(crate) owner_secret: String, } #[cfg(test)] diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index d31a50ebb..5cd8117cd 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -57,6 +57,7 @@ mod session_housekeeping; use session_cleanup::{finalize_one_shot_session, handle_preserve_failure, preserve_failed_run_shutdown_result}; mod ledger_routes; mod router_runtime; +mod service_peer; mod service_runtime; mod settings_routes; mod shutdown_policy; @@ -69,6 +70,7 @@ mod vm_lifecycle; use asset_routes::*; use ledger_routes::*; use router_runtime::*; +use service_peer::ServicePeer; use service_runtime::*; use settings_routes::*; use shutdown_policy::*; diff --git a/crates/capsem-service/src/owner_connection.rs b/crates/capsem-service/src/owner_connection.rs index 65fe32698..fbcdaa50f 100644 --- a/crates/capsem-service/src/owner_connection.rs +++ b/crates/capsem-service/src/owner_connection.rs @@ -28,6 +28,17 @@ impl OwnerConnection { self.validate(state, false) } + pub(crate) fn authenticate_identity( + &self, + state: &ServiceState, + identity: peer::PeerIdentity, + ) -> Result<(), String> { + if identity != self.identity { + return Err("Unix peer is not the granted VM owner".into()); + } + self.validate(state, false) + } + pub(crate) fn capture(instance: &InstanceInfo) -> Result { Ok(Self { id: instance.id.clone(), diff --git a/crates/capsem-service/src/private_routes.rs b/crates/capsem-service/src/private_routes.rs index 3972f9e65..bba09b2f2 100644 --- a/crates/capsem-service/src/private_routes.rs +++ b/crates/capsem-service/src/private_routes.rs @@ -3,54 +3,26 @@ //! with. Traffic never comes here: every protocol rides each member's cable //! to the network's switch (`switches`). //! -//! The owner proves itself with the secret minted for its VM at spawn. +//! The owner proves itself with the kernel identity of its service connection. use super::*; -const OWNER_SECRET_FILE: &str = "owner-secret"; - -/// Mint the secret an owner will show, and leave it in its session directory -/// for that owner alone. -pub(super) fn mint_owner_secret(session_dir: &std::path::Path) -> Result { - let secret = format!( - "{}{}", - capsem_crypto::uuid_v4().simple(), - capsem_crypto::uuid_v4().simple() - ); - let path = session_dir.join(OWNER_SECRET_FILE); - let mut file = std::fs::OpenOptions::new(); - file.write(true).create(true).truncate(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - file.mode(0o600); - } - std::io::Write::write_all( - &mut file.open(&path).with_context(|| format!("create {}", path.display()))?, - secret.as_bytes(), - ) - .with_context(|| format!("write {}", path.display()))?; - Ok(secret) -} - -fn secrets_match(presented: &str, expected: &str) -> bool { - capsem_crypto::secrets::constant_time_eq(presented.as_bytes(), expected.as_bytes()) -} - /// The asker is the running owner of `vm`, or nobody. -fn owner_of(state: &ServiceState, vm: &str, secret: &str) -> Result<(), AppError> { - let expected = state +fn owner_of(state: &ServiceState, vm: &str, peer: ServicePeer) -> Result<(), AppError> { + let owner = state .instances .lock() .unwrap() .get(vm) - .map(|instance| instance.owner_secret.clone()); - match expected { - Some(minted) if secrets_match(secret, &minted) => Ok(()), - _ => Err(AppError( - StatusCode::FORBIDDEN, - format!("VM {vm} has no running owner presenting this secret"), - )), - } + .map(owner_connection::OwnerConnection::capture) + .transpose() + .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; + let identity = peer + .0 + .ok_or_else(|| AppError(StatusCode::FORBIDDEN, "service peer identity unavailable".into()))?; + owner + .ok_or_else(|| AppError(StatusCode::FORBIDDEN, format!("VM {vm} has no running owner")))? + .authenticate_identity(state, identity) + .map_err(|error| AppError(StatusCode::FORBIDDEN, error)) } /// What a VM is called on the private zone: its display name. @@ -110,9 +82,10 @@ async fn visible_to(state: &ServiceState, source_vm: &str) -> Vec { /// member across the asker's networks has that name. pub(super) async fn handle_private_resolve( State(state): State>, + axum::extract::ConnectInfo(peer): axum::extract::ConnectInfo, Json(request): Json, ) -> Result, AppError> { - owner_of(&state, &request.source_vm, &request.owner_secret)?; + owner_of(&state, &request.source_vm, peer)?; let visible = visible_to(&state, &request.source_vm).await; let found = match (&request.name, request.address) { (Some(name), None) => { diff --git a/crates/capsem-service/src/service_peer.rs b/crates/capsem-service/src/service_peer.rs new file mode 100644 index 000000000..6d87bfec5 --- /dev/null +++ b/crates/capsem-service/src/service_peer.rs @@ -0,0 +1,15 @@ +//! Kernel identity attached to every request accepted on the service UDS. + +use axum::extract::connect_info::Connected; +use axum::serve::IncomingStream; +use std::os::fd::AsFd; +use tokio::net::UnixListener; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct ServicePeer(pub(crate) Option); + +impl Connected> for ServicePeer { + fn connect_info(stream: IncomingStream<'_, UnixListener>) -> Self { + Self(capsem_foundation::unix::peer::identity(stream.io().as_fd()).ok()) + } +} diff --git a/crates/capsem-service/src/service_runtime/shutdown.rs b/crates/capsem-service/src/service_runtime/shutdown.rs index 8a994274e..82e6ef07b 100644 --- a/crates/capsem-service/src/service_runtime/shutdown.rs +++ b/crates/capsem-service/src/service_runtime/shutdown.rs @@ -22,7 +22,11 @@ async fn drain( timeout: Duration, ) -> io::Result<()> { let (started, stopped) = oneshot::channel(); - let server = axum::serve(listener, app).with_graceful_shutdown(async move { + let server = axum::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { shutdown.await; let _ = started.send(()); }); diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 10946c1ec..2cc3f6981 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -152,7 +152,6 @@ pub(crate) fn test_instance() -> InstanceInfo { persistent: false, env: None, forked_from: None, - owner_secret: String::new(), } } diff --git a/crates/capsem-service/src/tests/files_api.rs b/crates/capsem-service/src/tests/files_api.rs index ec2f3f6a6..09995920d 100644 --- a/crates/capsem-service/src/tests/files_api.rs +++ b/crates/capsem-service/src/tests/files_api.rs @@ -97,7 +97,6 @@ pub(super) fn setup_vm_with_workspace_and_uds( persistent: false, env: None, forked_from: None, - owner_secret: String::new(), }, ); } diff --git a/crates/capsem-service/src/tests/network_routes.rs b/crates/capsem-service/src/tests/network_routes.rs index 9a9b9b3aa..21c3e91ae 100644 --- a/crates/capsem-service/src/tests/network_routes.rs +++ b/crates/capsem-service/src/tests/network_routes.rs @@ -274,10 +274,6 @@ async fn network_logs_page_with_a_cursor_and_refuse_a_foreign_one() { assert_eq!(status, StatusCode::BAD_REQUEST); } -fn owner_secret(state: &ServiceState, vm: &str, secret: &str) { - state.instances.lock().unwrap().get_mut(vm).unwrap().owner_secret = secret.into(); -} - /// The TCP admission rows of a logs page; a running member's link rows /// (these fake owners have no seat, so theirs say `block`) sit beside them. /// A fake owner's link seat: answers LinkAttach with its handoff socket, @@ -838,7 +834,73 @@ async fn an_owner_that_refuses_the_link_leaves_a_failed_membership() { } async fn private_resolve(state: &Arc, request: serde_json::Value) -> (StatusCode, serde_json::Value) { - route_request(app(state), Method::POST, "/networks/private/resolve", Some(request)).await + private_resolve_from(state, request, std::process::id()).await +} + +async fn private_resolve_from( + state: &Arc, + request: serde_json::Value, + pid: u32, +) -> (StatusCode, serde_json::Value) { + let peer = ServicePeer(Some(capsem_foundation::unix::peer::PeerIdentity { + pid: capsem_foundation::unix::process::ProcessId::try_from(pid).unwrap(), + uid: capsem_foundation::unix::process::current_uid(), + })); + let app = app(state).layer(axum::extract::connect_info::MockConnectInfo(peer)); + route_request(app, Method::POST, "/networks/private/resolve", Some(request)).await +} + +#[test] +fn private_name_wrong_process_client() { + let Ok(socket) = std::env::var("CAPSEM_TEST_PRIVATE_NAME_SOCKET") else { + return; + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (status, _) = runtime + .block_on(capsem_core::service_uds::post_json( + std::path::Path::new(&socket), + "/networks/private/resolve", + &json!({"source_vm": "vm-a", "name": "vm-a"}), + )) + .unwrap(); + assert_eq!(status, StatusCode::FORBIDDEN.as_u16()); +} + +#[tokio::test] +async fn private_names_reject_a_real_same_uid_wrong_process() { + let (state, dir) = make_test_state_with_tempdir(); + insert_fake_instance(&state, "vm-a", std::process::id()); + let socket = dir.path().join("private-name-peer.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let router = app(&state).into_make_service_with_connect_info::(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let (owner_status, _) = capsem_core::service_uds::post_json( + &socket, + "/networks/private/resolve", + &json!({"source_vm": "vm-a", "name": "nobody"}), + ) + .await + .unwrap(); + assert_eq!( + owner_status, + StatusCode::NOT_FOUND.as_u16(), + "the registered owner passes authentication before name lookup" + ); + let status = tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "tests::network_routes::private_name_wrong_process_client", + "--nocapture", + ]) + .env("CAPSEM_TEST_PRIVATE_NAME_SOCKET", &socket) + .status() + .await + .unwrap(); + server.abort(); + assert!(status.success(), "wrong-process probe failed: {status}"); } #[tokio::test] @@ -855,7 +917,6 @@ async fn private_names_resolve_only_to_members_the_asker_shares_a_network_with() insert_fake_instance(&state, id, std::process::id()); state.instances.lock().unwrap().get_mut(id).unwrap().name = name.into(); } - owner_secret(&state, "vm-a", "secret-a"); // Each VM but vm-a is in one network, so its lease there is its address. let mut addresses: HashMap<&str, String> = HashMap::new(); let (_, team) = create_network(&state, "team").await; @@ -892,7 +953,7 @@ async fn private_names_resolve_only_to_members_the_asker_shares_a_network_with() addresses.insert(vm, lease); } let ask = |name: Option<&str>, address: Option<&str>| { - let mut request = json!({ "source_vm": "vm-a", "owner_secret": "secret-a" }); + let mut request = json!({ "source_vm": "vm-a" }); if let Some(name) = name { request["name"] = json!(name); } @@ -938,9 +999,11 @@ async fn private_names_resolve_only_to_members_the_asker_shares_a_network_with() let (status, _) = private_resolve(&state, ask(Some("beta.team"), Some(&addresses["vm-b"]))).await; assert_eq!(status, StatusCode::BAD_REQUEST); - let mut forged = ask(Some("beta.team"), None); - forged["owner_secret"] = json!("wrong"); - assert_eq!(private_resolve(&state, forged).await.0, StatusCode::FORBIDDEN); + assert_eq!( + private_resolve_from(&state, ask(Some("beta.team"), None), 1).await.0, + StatusCode::FORBIDDEN, + "a same-UID process outside the registered owner PID is not authority" + ); // Leaving takes the name with it at once. let (status, _) = route_request( diff --git a/crates/capsem-service/src/tests/settings_files.rs b/crates/capsem-service/src/tests/settings_files.rs index fb0e40ac0..79aebfaf3 100644 --- a/crates/capsem-service/src/tests/settings_files.rs +++ b/crates/capsem-service/src/tests/settings_files.rs @@ -201,7 +201,6 @@ fn resolve_rejects_symlink_escape() { persistent: false, env: None, forked_from: None, - owner_secret: String::new(), }, ); @@ -235,7 +234,6 @@ fn resolve_valid_path_inside_workspace() { persistent: false, env: None, forked_from: None, - owner_secret: String::new(), }, ); diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index 4577f247b..07a670dfc 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -168,7 +168,6 @@ impl ServiceState { } let process_log_path = session_dir.join("process.log"); - let owner_secret = private_routes::mint_owner_secret(&session_dir)?; let process_log_file = std::fs::OpenOptions::new() .create(true) .append(true) @@ -346,7 +345,6 @@ impl ServiceState { persistent, env, forked_from: from_name, - owner_secret, }, ); drop(instances); diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index c06e50427..82d2c43eb 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -71,7 +71,6 @@ impl ServiceState { self.validate_pinned_asset_files(&resolved, &entry.asset_pins)?; let process_log_path = entry.session_dir.join("process.log"); - let owner_secret = private_routes::mint_owner_secret(&entry.session_dir)?; let process_log_file = std::fs::OpenOptions::new() .create(true) .append(true) @@ -242,7 +241,6 @@ impl ServiceState { persistent: true, env: None, forked_from: entry.forked_from, - owner_secret, }, ); drop(instances); From 6e5058d5ceaabbfc7a4451bb0373ba1f4c489764 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:22:22 +0000 Subject: [PATCH 017/464] security(process): remove private DNS bearer material --- CHANGELOG.md | 2 +- crates/capsem-process/src/main.rs | 2 -- crates/capsem-process/src/private_names.rs | 10 ++------ .../capsem-process/src/private_names/tests.rs | 25 +++++++++++-------- crates/capsem-process/src/private_seats.rs | 13 ++-------- crates/capsem-process/src/tests.rs | 8 ++++++ 6 files changed, 27 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6996aa05..08bdc6ee8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Authenticate private-network name lookups with the registered VM owner's kernel Unix identity, eliminating the reusable bearer secret previously - written into each session directory. + written into each session directory and carried in lookup requests. - Close ambient inherited file descriptors before each VM owner or gateway initializes, preventing service resources from leaking across worker exec diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 2bb556045..e1d2a9972 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -467,7 +467,6 @@ async fn run_async_main_loop( service_socket: args.service_socket.as_deref(), uds_path: &args.uds_path, run_dir: args.run_dir.as_deref(), - session_dir: &session_dir, }, &job_store, ctrl_tx.clone(), @@ -617,7 +616,6 @@ async fn run_async_main_loop( // The private zone is the service's to answer, for this VM's networks. let private_names = Arc::new(private_names::ServicePrivateNames::new( seats.service_socket, - seats.owner_secret, args.id.clone(), )); let dns_handler = Arc::new( diff --git a/crates/capsem-process/src/private_names.rs b/crates/capsem-process/src/private_names.rs index 894eeac49..18d3b56a2 100644 --- a/crates/capsem-process/src/private_names.rs +++ b/crates/capsem-process/src/private_names.rs @@ -8,23 +8,17 @@ use std::path::PathBuf; pub(crate) struct ServicePrivateNames { service_socket: PathBuf, - owner_secret: String, vm_id: String, } impl ServicePrivateNames { - pub(crate) fn new(service_socket: PathBuf, owner_secret: String, vm_id: String) -> Self { - Self { - service_socket, - owner_secret, - vm_id, - } + pub(crate) fn new(service_socket: PathBuf, vm_id: String) -> Self { + Self { service_socket, vm_id } } async fn ask(&self, question: serde_json::Value) -> Option { let mut request = question; request["source_vm"] = serde_json::Value::String(self.vm_id.clone()); - request["owner_secret"] = serde_json::Value::String(self.owner_secret.clone()); match capsem_core::service_uds::post_json(&self.service_socket, "/networks/private/resolve", &request).await { Ok((200, answer)) => Some(answer), Ok((404, _)) => None, diff --git a/crates/capsem-process/src/private_names/tests.rs b/crates/capsem-process/src/private_names/tests.rs index 453fc172e..7988298ee 100644 --- a/crates/capsem-process/src/private_names/tests.rs +++ b/crates/capsem-process/src/private_names/tests.rs @@ -11,9 +11,6 @@ async fn fake_service(socket: PathBuf, seen: Arc>>) let seen = Arc::clone(&seen); async move { seen.lock().unwrap().push(body.clone()); - if body["owner_secret"] != "secret-a" { - return (axum::http::StatusCode::FORBIDDEN, axum::Json(serde_json::json!({}))); - } match (body["name"].as_str(), body["address"].as_str()) { (Some("beta.team"), None) => ( axum::http::StatusCode::OK, @@ -44,7 +41,7 @@ async fn names_and_addresses_are_the_services_answers_and_misses_are_none() { let socket = dir.path().join("service.sock"); let seen = Arc::new(Mutex::new(Vec::new())); fake_service(socket.clone(), Arc::clone(&seen)).await; - let names = ServicePrivateNames::new(socket, "secret-a".into(), "vm-a".into()); + let names = ServicePrivateNames::new(socket, "vm-a".into()); assert_eq!(names.address_of("beta.team").await, Some(Ipv4Addr::new(10, 128, 0, 3))); assert_eq!(names.address_of("nobody.team").await, None); assert_eq!( @@ -54,9 +51,8 @@ async fn names_and_addresses_are_the_services_answers_and_misses_are_none() { assert_eq!(names.name_of(Ipv4Addr::new(10, 128, 0, 9)).await, None); let asked = seen.lock().unwrap().clone(); assert_eq!(asked.len(), 4); - assert!(asked - .iter() - .all(|ask| ask["source_vm"] == "vm-a" && ask["owner_secret"] == "secret-a")); + assert!(asked.iter().all(|ask| ask["source_vm"] == "vm-a")); + assert!(asked.iter().all(|ask| ask.get("owner_secret").is_none())); assert_eq!(asked[0]["name"], "beta.team"); assert_eq!(asked[2]["address"], "10.128.0.3"); } @@ -65,9 +61,16 @@ async fn names_and_addresses_are_the_services_answers_and_misses_are_none() { async fn a_refusal_or_an_absent_service_is_a_miss_not_a_failure() { let dir = tempfile::tempdir().unwrap(); let socket = dir.path().join("service.sock"); - fake_service(socket.clone(), Arc::new(Mutex::new(Vec::new()))).await; - let wrong = ServicePrivateNames::new(socket, "wrong".into(), "vm-a".into()); - assert_eq!(wrong.address_of("beta.team").await, None); - let absent = ServicePrivateNames::new(dir.path().join("nobody.sock"), "secret-a".into(), "vm-a".into()); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + tokio::spawn(async move { + let app = axum::Router::new().route( + "/networks/private/resolve", + axum::routing::post(|| async { axum::http::StatusCode::FORBIDDEN }), + ); + let _ = axum::serve(listener, app).await; + }); + let refused = ServicePrivateNames::new(socket, "vm-a".into()); + assert_eq!(refused.address_of("beta.team").await, None); + let absent = ServicePrivateNames::new(dir.path().join("nobody.sock"), "vm-a".into()); assert_eq!(absent.address_of("beta.team").await, None); } diff --git a/crates/capsem-process/src/private_seats.rs b/crates/capsem-process/src/private_seats.rs index f7655e9a5..6e63a6a8f 100644 --- a/crates/capsem-process/src/private_seats.rs +++ b/crates/capsem-process/src/private_seats.rs @@ -1,8 +1,7 @@ //! Binding this owner's private seats at start: the cables holding each //! network's guest stream, and the socket the service asks a cable's stream //! on. -//! It needs the service's socket, the run directory the service named and -//! the secret it minted for this VM. +//! It needs the service's socket and the run directory the service named. use crate::job_store::JobStore; use anyhow::{Context, Result}; use capsem_proto::ipc::ServiceToProcess; @@ -16,7 +15,6 @@ pub(crate) struct Seats<'a> { pub service_socket: Option<&'a Path>, pub uds_path: &'a Path, pub run_dir: Option<&'a Path>, - pub session_dir: &'a Path, } fn bound(path: std::path::PathBuf, what: &str) -> Result<(std::path::PathBuf, UnixListener)> { @@ -32,7 +30,6 @@ fn bound(path: std::path::PathBuf, what: &str) -> Result<(std::path::PathBuf, Un /// the service on its VM's behalf. pub(crate) struct Bound { pub service_socket: std::path::PathBuf, - pub owner_secret: String, } pub(crate) fn bind( @@ -49,9 +46,6 @@ pub(crate) fn bind( .unwrap_or_else(|| Path::new("/tmp")) .to_path_buf(); let run_dir = seats.run_dir.map(Path::to_path_buf).unwrap_or(walked_up); - let owner_secret = std::fs::read_to_string(seats.session_dir.join("owner-secret")) - .map(|secret| secret.trim().to_string()) - .unwrap_or_default(); let service_socket = seats .service_socket .map(Path::to_path_buf) @@ -65,8 +59,5 @@ pub(crate) fn bind( )?; let _ = job_store.cable_seat.set(seat_path); tokio::spawn(cables.serve_seat(seat_listener)); - Ok(Bound { - service_socket, - owner_secret, - }) + Ok(Bound { service_socket }) } diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index 5ca0a7275..82df692f4 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -13,6 +13,14 @@ fn entry_closes_ambient_descriptors_before_runtime_initialization() { assert!(close < telemetry && close < arguments); } +#[test] +fn private_name_broker_carries_no_session_bearer() { + for source in [include_str!("private_names.rs"), include_str!("private_seats.rs")] { + assert!(!source.contains("owner_secret")); + assert!(!source.contains("owner-secret")); + } +} + // ----------------------------------------------------------------------- // Args parsing // ----------------------------------------------------------------------- From 3fb41c56bfceeb75439d8e25299116696519b630 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:27:28 +0000 Subject: [PATCH 018/464] refactor(telemetry): allow brokered metric transport --- crates/capsem-telemetry/src/export.rs | 51 +++++++++++++++++++-- crates/capsem-telemetry/src/export/tests.rs | 47 +++++++++++++++++++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/crates/capsem-telemetry/src/export.rs b/crates/capsem-telemetry/src/export.rs index 6b0328b02..e627c1e56 100644 --- a/crates/capsem-telemetry/src/export.rs +++ b/crates/capsem-telemetry/src/export.rs @@ -12,9 +12,9 @@ //! on the workspace reqwest (rustls with ring): no gRPC stack and no second //! TLS provider come with it. //! -//! Every process that records metrics installs this itself -- the service -//! and each `capsem-process` -- tagged with its own resource attributes, so -//! nothing has to relay one process's measurements through another. +//! Each process installs its own recorder, tagged with its resource +//! attributes. The service uses the bounded reqwest transport below; a +//! confined worker can supply a local broker transport instead. use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; @@ -34,6 +34,11 @@ use opentelemetry_sdk::Resource; /// How long one export request may take before it is abandoned. const EXPORT_TIMEOUT: Duration = Duration::from_secs(10); +/// A syntactically valid endpoint used only to let the OTLP encoder construct +/// requests for a caller-supplied transport. The transport owns delivery and +/// must not resolve or dial this reserved name. +const BROKER_REQUEST_ENDPOINT: &str = "http://capsem-metric-broker.invalid/v1/metrics"; + /// The environment variables that turn export on without a corp config. const OTEL_ENDPOINT_ENV: [&str; 2] = ["OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"]; @@ -140,6 +145,25 @@ pub fn install( attributes: Vec, ) -> Result { let provider = provider(destination, service_name, attributes)?; + install_provider(provider) +} + +/// Install OTLP export with a caller-owned HTTP transport. +/// +/// The exporter still owns protobuf encoding, resource attributes, periodic +/// collection, and its request deadline. The supplied client owns delivery; +/// this crate gives it a request addressed to a reserved, non-routable name so +/// no collector destination crosses into the caller. +pub fn install_with_http_client( + client: impl opentelemetry_http::HttpClient + 'static, + service_name: &'static str, + attributes: Vec, +) -> Result { + let provider = provider_with_http_client(client, service_name, attributes)?; + install_provider(provider) +} + +fn install_provider(provider: SdkMeterProvider) -> Result { let recorder = OtelRecorder::new(provider.meter("capsem")); metrics::set_global_recorder(recorder).map_err(|_| InstallError::RecorderAlreadySet)?; crate::describe_all(); @@ -159,6 +183,27 @@ fn provider( if let Destination::Corp(base) = destination { builder = builder.with_endpoint(format!("{base}/v1/metrics")); } + build_provider(builder, service_name, attributes) +} + +fn provider_with_http_client( + client: impl opentelemetry_http::HttpClient + 'static, + service_name: &'static str, + attributes: Vec, +) -> Result { + let builder = opentelemetry_otlp::MetricExporter::builder() + .with_http() + .with_http_client(client) + .with_timeout(EXPORT_TIMEOUT) + .with_endpoint(BROKER_REQUEST_ENDPOINT); + build_provider(builder, service_name, attributes) +} + +fn build_provider( + builder: opentelemetry_otlp::MetricExporterBuilder, + service_name: &'static str, + attributes: Vec, +) -> Result { let exporter = builder .build() .map_err(|error| InstallError::Build(error.to_string()))?; diff --git a/crates/capsem-telemetry/src/export/tests.rs b/crates/capsem-telemetry/src/export/tests.rs index 538e0052f..8c9147b94 100644 --- a/crates/capsem-telemetry/src/export/tests.rs +++ b/crates/capsem-telemetry/src/export/tests.rs @@ -165,6 +165,53 @@ fn contains(haystack: &[u8], needle: &str) -> bool { haystack.windows(needle.len()).any(|window| window == needle.as_bytes()) } +#[derive(Clone, Debug, Default)] +struct CapturingHttpClient(Arc>>>); + +#[async_trait::async_trait] +impl opentelemetry_http::HttpClient for CapturingHttpClient { + async fn send_bytes( + &self, + request: http::Request, + ) -> Result, opentelemetry_http::HttpError> { + self.0.lock().unwrap().push(request); + Ok(http::Response::builder() + .status(http::StatusCode::OK) + .body(bytes::Bytes::new())?) + } +} + +#[test] +fn caller_supplied_http_client_receives_otlp_request() { + let client = CapturingHttpClient::default(); + let received = Arc::clone(&client.0); + let provider = provider_with_http_client( + client, + "capsem-broker-test", + vec![KeyValue::new("session.id", "session-7")], + ) + .unwrap(); + let recorder = OtelRecorder::new(provider.meter("capsem")); + metrics::with_local_recorder(&recorder, || { + metrics::counter!(crate::db::DB_WRITE_OPS_TOTAL).increment(1); + }); + provider.force_flush().unwrap(); + + let request = received + .lock() + .unwrap() + .pop() + .expect("the injected client received one export"); + assert_eq!(request.uri(), "http://capsem-metric-broker.invalid/v1/metrics"); + assert_eq!( + request.headers().get(http::header::CONTENT_TYPE).unwrap(), + "application/x-protobuf" + ); + for expected in ["capsem-broker-test", "session-7", crate::db::DB_WRITE_OPS_TOTAL] { + assert!(contains(request.body(), expected), "the OTLP body lacks {expected}"); + } +} + /// The whole export path: the real provider and HTTP client deliver an OTLP /// request to the corp endpoint's `/v1/metrics`, carrying the reporting /// service and what was recorded. Only the pure helpers were tested before, From edff252fde872edd0966b22a2e31010a61a7a572 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:35:32 +0000 Subject: [PATCH 019/464] feat(core): add bounded service byte requests --- crates/capsem-core/src/service_uds.rs | 73 ++++++++++++----- crates/capsem-core/src/service_uds/tests.rs | 89 +++++++++++++++++++++ 2 files changed, 141 insertions(+), 21 deletions(-) create mode 100644 crates/capsem-core/src/service_uds/tests.rs diff --git a/crates/capsem-core/src/service_uds.rs b/crates/capsem-core/src/service_uds.rs index cb4456840..148781091 100644 --- a/crates/capsem-core/src/service_uds.rs +++ b/crates/capsem-core/src/service_uds.rs @@ -1,20 +1,60 @@ //! One HTTP request to the service over its Unix socket, from a process the //! service spawned. //! -//! The VM owner is a client of the service in exactly one place: asking what -//! a private name or member address is, on its VM's behalf. One request per -//! call, no pool, no keep-alive: the answer is small and the socket is local. -use anyhow::{Context, Result}; -use http_body_util::{BodyExt, Full}; +//! Spawned workers use fixed internal service routes for brokered operations. +//! One request uses one connection, with no pool or keep-alive: exchanges are +//! bounded and the socket is local. +use anyhow::{ensure, Context, Result}; +use http_body_util::{BodyExt, Full, Limited}; use hyper::body::Bytes; use hyper_util::rt::TokioIo; use std::path::Path; use std::time::Duration; const REQUEST_DEADLINE: Duration = Duration::from_secs(12); +/// Maximum request or response body carried over the worker-to-service socket. +pub const MAX_BODY_BYTES: usize = 4 * 1024 * 1024; /// POST a JSON document and read back the status and JSON body. pub async fn post_json(socket: &Path, path: &str, body: &serde_json::Value) -> Result<(u16, serde_json::Value)> { + let response = post_bytes( + socket, + path, + http::HeaderValue::from_static("application/json"), + Bytes::from(body.to_string()), + ) + .await?; + let status = response.status().as_u16(); + let bytes = response.into_body(); + let value = if bytes.is_empty() { + serde_json::Value::Null + } else { + serde_json::from_slice(&bytes) + .unwrap_or_else(|_| serde_json::Value::String(String::from_utf8_lossy(&bytes).into_owned())) + }; + Ok((status, value)) +} + +/// POST one bounded byte body and return the bounded response from the service. +/// +/// The path and content type are explicit because callers use fixed internal +/// routes. One connection is used for one request, and the deadline covers +/// connect, write, response headers, and response body collection. +pub async fn post_bytes( + socket: &Path, + path: &str, + content_type: http::HeaderValue, + body: Bytes, +) -> Result> { + ensure!( + body.len() <= MAX_BODY_BYTES, + "service request body exceeds {MAX_BODY_BYTES} bytes" + ); + let request = http::Request::post(path) + .header(http::header::HOST, "capsem") + .header(http::header::CONTENT_TYPE, content_type) + .body(Full::new(body)) + .context("build service request")?; tokio::time::timeout(REQUEST_DEADLINE, async { let stream = tokio::net::UnixStream::connect(socket) .await @@ -27,27 +67,18 @@ pub async fn post_json(socket: &Path, path: &str, body: &serde_json::Value) -> R tracing::debug!(%error, "service connection ended"); } }); - let request = http::Request::post(path) - .header(http::header::HOST, "capsem") - .header(http::header::CONTENT_TYPE, "application/json") - .body(Full::new(Bytes::from(body.to_string()))) - .context("build service request")?; let response = sender.send_request(request).await.context("send service request")?; - let status = response.status().as_u16(); - let bytes = response - .into_body() + let (parts, body) = response.into_parts(); + let bytes = Limited::new(body, MAX_BODY_BYTES) .collect() .await - .context("read service response")? + .map_err(|error| anyhow::anyhow!("service response body exceeds limit or could not be read: {error}"))? .to_bytes(); - let value = if bytes.is_empty() { - serde_json::Value::Null - } else { - serde_json::from_slice(&bytes) - .unwrap_or_else(|_| serde_json::Value::String(String::from_utf8_lossy(&bytes).into_owned())) - }; - Ok((status, value)) + Ok(http::Response::from_parts(parts, bytes)) }) .await .context("service request deadline exceeded")? } + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-core/src/service_uds/tests.rs b/crates/capsem-core/src/service_uds/tests.rs new file mode 100644 index 000000000..143aadd9b --- /dev/null +++ b/crates/capsem-core/src/service_uds/tests.rs @@ -0,0 +1,89 @@ +use axum::body::Bytes; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::post; +use axum::Router; + +use super::*; + +async fn echo(headers: HeaderMap, body: Bytes) -> (StatusCode, [(&'static str, &'static str); 1], Bytes) { + assert_eq!( + headers.get(http::header::CONTENT_TYPE).unwrap(), + "application/x-protobuf" + ); + (StatusCode::ACCEPTED, [("x-capsem-test", "echo")], body) +} + +async fn server(app: Router) -> (tempfile::TempDir, std::path::PathBuf, tokio::task::JoinHandle<()>) { + let directory = tempfile::tempdir().unwrap(); + let socket = directory.path().join("service.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (directory, socket, task) +} + +#[tokio::test] +async fn post_bytes_preserves_status_headers_and_body() { + let (_directory, socket, task) = server(Router::new().route("/bytes", post(echo))).await; + let response = post_bytes( + &socket, + "/bytes", + http::HeaderValue::from_static("application/x-protobuf"), + Bytes::from_static(b"otlp body"), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::ACCEPTED); + assert_eq!(response.headers().get("x-capsem-test").unwrap(), "echo"); + assert_eq!(response.body(), b"otlp body".as_slice()); + task.abort(); +} + +#[tokio::test] +async fn post_bytes_refuses_an_oversized_request_before_connecting() { + let socket = std::path::Path::new("/a/socket/that/must/not/be-opened"); + let error = post_bytes( + socket, + "/bytes", + http::HeaderValue::from_static("application/x-protobuf"), + Bytes::from(vec![0; MAX_BODY_BYTES + 1]), + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("request body exceeds"), "{error:#}"); +} + +#[tokio::test] +async fn post_bytes_refuses_a_malformed_path_before_connecting() { + let socket = std::path::Path::new("/a/socket/that/must/not/be-opened"); + let error = post_bytes( + socket, + "/invalid\npath", + http::HeaderValue::from_static("application/x-protobuf"), + Bytes::new(), + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("build service request"), "{error:#}"); +} + +#[tokio::test] +async fn post_bytes_stops_collecting_an_oversized_response() { + let app = Router::new().route("/large", post(|| async { Bytes::from(vec![0; MAX_BODY_BYTES + 1]) })); + let (_directory, socket, task) = server(app).await; + let error = post_bytes( + &socket, + "/large", + http::HeaderValue::from_static("application/x-protobuf"), + Bytes::new(), + ) + .await + .unwrap_err(); + + assert!(error.to_string().contains("response body exceeds"), "{error:#}"); + task.abort(); +} From 315b1e8a6de76f10474e8bf79a5168bef3036473 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:37:57 +0000 Subject: [PATCH 020/464] test(core): check expired reservation clock --- crates/capsem-core/src/managed_sessions/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/capsem-core/src/managed_sessions/tests.rs b/crates/capsem-core/src/managed_sessions/tests.rs index 49bd75f03..0ee899ef2 100644 --- a/crates/capsem-core/src/managed_sessions/tests.rs +++ b/crates/capsem-core/src/managed_sessions/tests.rs @@ -493,7 +493,7 @@ fn expired_reservation_cannot_admit_a_create_side_effect() { }; let earlier = Instant::now() .checked_sub(Duration::from_secs(2)) - .expect("earlier test instant"); + .expect("the monotonic clock has at least two seconds of history"); store .start_lease( &ticket, From 63e338725a85032091acac5f4bb2a8ee685d7691 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:46:20 +0000 Subject: [PATCH 021/464] feat(process): add brokered metric transport --- Cargo.lock | 4 + crates/capsem-process/Cargo.toml | 4 + crates/capsem-process/src/main.rs | 18 ++- crates/capsem-process/src/metric_export.rs | 118 +++++++++++++++--- .../capsem-process/src/metric_export/tests.rs | 62 +++++++++ crates/capsem-process/src/tests.rs | 24 +++- 6 files changed, 204 insertions(+), 26 deletions(-) create mode 100644 crates/capsem-process/src/metric_export/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 3b955ccbd..e6de4ab5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1145,7 +1145,9 @@ name = "capsem-process" version = "0.7.0" dependencies = [ "anyhow", + "async-trait", "axum", + "bytes", "capsem-assets", "capsem-core", "capsem-foundation", @@ -1156,6 +1158,8 @@ dependencies = [ "clap", "core-foundation-sys", "futures", + "http", + "opentelemetry-http", "serde", "serde_json", "tempfile", diff --git a/crates/capsem-process/Cargo.toml b/crates/capsem-process/Cargo.toml index 0ea8a9b56..2cbd56dcb 100644 --- a/crates/capsem-process/Cargo.toml +++ b/crates/capsem-process/Cargo.toml @@ -25,6 +25,10 @@ serde_json.workspace = true toml.workspace = true clap.workspace = true futures.workspace = true +http.workspace = true +bytes.workspace = true +opentelemetry-http = { version = "0.33", default-features = false } +async-trait = "0.1" [dev-dependencies] axum = { workspace = true, features = ["ws"] } diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index e1d2a9972..d7ad50787 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -127,15 +127,18 @@ struct Args { #[arg(long)] run_dir: Option, /// The service's own socket, where this owner asks on a guest's behalf - /// (private names). Given by the service: it is not always + /// (private names and brokered metrics). Given by the service: it is not always /// `{run_dir}/service.sock`. #[arg(long)] service_socket: Option, + /// Export metrics through the service's generation-authenticated local + /// broker. The service grants this without exposing a collector address. + #[arg(long, conflicts_with = "metric_endpoint")] + metric_broker: bool, #[arg(long)] checkpoint_path: Option, - /// The corp OTLP endpoint this process exports its metrics to, resolved - /// by the service from the corp config. Absent means export is off; the - /// process never reads settings or corp files for it. + /// Transitional direct-export grant, removed after the service switches + /// every launch to `--metric-broker`. #[arg(long)] metric_endpoint: Option, /// Environment variables to inject into guest (repeatable: --env KEY=VALUE) @@ -245,7 +248,12 @@ fn main() -> Result<()> { info!(id = %args.id, "capsem-sandbox-process starting"); // Held until the process exits: dropping it flushes the last measurements. - let _metric_export = metric_export::install(&args.id, args.metric_endpoint.as_deref()); + let _metric_export = metric_export::install( + &args.id, + args.service_socket.as_deref(), + args.metric_broker, + args.metric_endpoint.as_deref(), + ); let guest_dir = prepare_session_layout(&session_dir, args.scratch_disk_size_gb)?; // The image share is attached to every session, read-only at the device: diff --git a/crates/capsem-process/src/metric_export.rs b/crates/capsem-process/src/metric_export.rs index 4357b40dc..dd66da431 100644 --- a/crates/capsem-process/src/metric_export.rs +++ b/crates/capsem-process/src/metric_export.rs @@ -1,34 +1,114 @@ //! Metric export for one VM process. //! //! The process exports its own measurements -- ledger writer, MITM, DNS, -//! security engine, virtio-blk -- tagged with the session it serves, to the -//! corp config's `open_telemetry` endpoint and nowhere else. The service -//! resolves that endpoint and grants it at launch (`--metric-endpoint`): a -//! process takes its runtime config from what the service hands it, never -//! from settings or corp files of its own reading. It never reads -//! `OTEL_EXPORTER_OTLP_*` either: that environment can carry collector -//! credentials, and the service's spawn allowlist keeps it out of this -//! guest-facing process. Authenticated collectors are out of scope here. +//! security engine, virtio-blk -- tagged with the session it serves. Broker +//! mode carries encoded OTLP bytes over the service socket without learning +//! a collector address or credential. The direct endpoint mode remains only +//! while the service launch grant moves to the broker in the next commit. use capsem_telemetry::export::{Destination, Exporter, KeyValue}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; use tracing::{info, warn}; -/// Install export when the service granted an endpoint. Failure is logged -/// and leaves export off; metrics are never a reason a VM does not boot. -pub(crate) fn install(vm_id: &str, endpoint: Option<&str>) -> Option { - let destination = Destination::corp(endpoint)?; - match capsem_telemetry::export::install( - &destination, - "capsem-process", - vec![KeyValue::new("session.id", vm_id.to_string())], - ) { +/// OTLP transport that can reach only this process's fixed service route. +struct ServiceMetricClient { + service_socket: PathBuf, + path: String, + runtime: Mutex, +} + +impl std::fmt::Debug for ServiceMetricClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ServiceMetricClient") + .field("service_socket", &self.service_socket) + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +impl ServiceMetricClient { + fn new(service_socket: PathBuf, vm_id: &str) -> Result { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| format!("build metric broker runtime: {error}"))?; + Ok(Self { + service_socket, + path: format!("/internal/vms/{vm_id}/metrics"), + runtime: Mutex::new(runtime), + }) + } +} + +#[async_trait::async_trait] +impl opentelemetry_http::HttpClient for ServiceMetricClient { + async fn send_bytes( + &self, + request: http::Request, + ) -> Result, opentelemetry_http::HttpError> { + let response = self + .runtime + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .block_on(capsem_core::service_uds::post_bytes( + &self.service_socket, + &self.path, + http::HeaderValue::from_static("application/x-protobuf"), + request.into_body(), + )); + response.map_err(|error| Box::new(std::io::Error::other(format!("metric broker request: {error:#}"))) as _) + } +} + +/// Install export when the service granted broker or transitional direct +/// transport. Failure is logged and leaves export off; metrics never prevent +/// a VM from booting. +pub(crate) fn install( + vm_id: &str, + service_socket: Option<&Path>, + broker: bool, + endpoint: Option<&str>, +) -> Option { + let attributes = vec![KeyValue::new("session.id", vm_id.to_string())]; + let (installation, transport) = if broker { + let socket = match service_socket { + Some(socket) => socket, + None => { + warn!("metric broker granted without a service socket"); + return None; + } + }; + let client = match ServiceMetricClient::new(socket.to_path_buf(), vm_id) { + Ok(client) => client, + Err(error) => { + warn!(%error, "metric export not installed"); + return None; + } + }; + ( + capsem_telemetry::export::install_with_http_client(client, "capsem-process", attributes), + "service-broker", + ) + } else { + let destination = Destination::corp(endpoint)?; + ( + capsem_telemetry::export::install(&destination, "capsem-process", attributes), + "direct-transition", + ) + }; + match installation { Ok(exporter) => { - info!(?destination, "metric export installed"); + info!(transport, "metric export installed"); Some(exporter) } Err(error) => { - warn!(%error, ?destination, "metric export not installed"); + warn!(%error, transport, "metric export not installed"); None } } } + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-process/src/metric_export/tests.rs b/crates/capsem-process/src/metric_export/tests.rs new file mode 100644 index 000000000..d72b66c93 --- /dev/null +++ b/crates/capsem-process/src/metric_export/tests.rs @@ -0,0 +1,62 @@ +use std::sync::{Arc, Mutex}; + +use axum::body::Bytes; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::post; +use axum::Router; +use opentelemetry_http::HttpClient as _; + +use super::*; + +type Received = (String, HeaderMap, Bytes); + +async fn receive( + Path(id): Path, + State(received): State>>>, + headers: HeaderMap, + body: Bytes, +) -> (StatusCode, [(&'static str, &'static str); 1], Bytes) { + received.lock().unwrap().push((id, headers, body)); + ( + StatusCode::ACCEPTED, + [("x-collector", "accepted")], + Bytes::from_static(b"ok"), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn broker_client_uses_only_the_fixed_local_route_and_protobuf_body() { + let directory = tempfile::tempdir().unwrap(); + let socket = directory.path().join("service.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let received = Arc::new(Mutex::new(Vec::new())); + let app = Router::new() + .route("/internal/vms/{id}/metrics", post(receive)) + .with_state(Arc::clone(&received)); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let client = ServiceMetricClient::new(socket, "vm-a").unwrap(); + let request = http::Request::post("https://attacker.example/v1/metrics") + .header(http::header::CONTENT_TYPE, "text/plain") + .header(http::header::AUTHORIZATION, "Bearer must-not-cross") + .body(Bytes::from_static(b"encoded otlp")) + .unwrap(); + + let response = tokio::task::spawn_blocking(move || futures::executor::block_on(client.send_bytes(request))) + .await + .unwrap() + .unwrap(); + + assert_eq!(response.status(), StatusCode::ACCEPTED); + assert_eq!(response.headers().get("x-collector").unwrap(), "accepted"); + assert_eq!(response.body(), b"ok".as_slice()); + let (id, headers, body) = received.lock().unwrap().pop().unwrap(); + assert_eq!(id, "vm-a"); + assert_eq!( + headers.get(http::header::CONTENT_TYPE).unwrap(), + "application/x-protobuf" + ); + assert!(!headers.contains_key(http::header::AUTHORIZATION)); + assert_eq!(body, b"encoded otlp".as_slice()); + server.abort(); +} diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index 82df692f4..1b2f70342 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -61,7 +61,7 @@ fn args_parses_all_required() { /// export is off. The process has no other way to learn it: it may not read /// settings or corp files (`test_process_policy_runtime_contract.py`). #[test] -fn args_metric_endpoint_is_granted_at_launch_and_off_without_it() { +fn args_accept_one_optional_metric_transport_grant() { let required = [ "capsem-process", "--id", @@ -84,8 +84,28 @@ fn args_metric_endpoint_is_granted_at_launch_and_off_without_it() { "/tmp/vm.sock", ]; let off = Args::try_parse_from(required).unwrap(); + assert!(!off.metric_broker); assert_eq!(off.metric_endpoint, None); - assert!(metric_export::install(&off.id, off.metric_endpoint.as_deref()).is_none()); + assert!(metric_export::install(&off.id, None, off.metric_broker, off.metric_endpoint.as_deref()).is_none()); + + let broker = + Args::try_parse_from( + required + .into_iter() + .chain(["--service-socket", "/tmp/service.sock", "--metric-broker"]), + ) + .unwrap(); + assert!(broker.metric_broker); + assert_eq!( + broker.service_socket.as_deref(), + Some(std::path::Path::new("/tmp/service.sock")) + ); + assert!(Args::try_parse_from(required.into_iter().chain([ + "--metric-broker", + "--metric-endpoint", + "https://otel.example" + ]),) + .is_err()); let granted = Args::try_parse_from( required From fd6121a67b46c421ed09443453d8cd258fd496db Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:57:20 +0000 Subject: [PATCH 022/464] security(service): broker worker metric export --- CHANGELOG.md | 4 + crates/capsem-service/src/owner_connection.rs | 18 ++ crates/capsem-service/src/private_routes.rs | 16 +- crates/capsem-service/src/router_runtime.rs | 6 + .../src/service_runtime/telemetry_export.rs | 135 +++++++++- .../src/tests/telemetry_export.rs | 242 +++++++++++++++++- .../src/vm_lifecycle/provision.rs | 2 +- .../src/vm_lifecycle/resume_process.rs | 2 +- 8 files changed, 391 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08bdc6ee8..c10ee640c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Route VM-owner metrics through a generation-authenticated service broker + with a fixed corporate collector destination, bounded protobuf bodies and + no redirects, keeping collector addresses and credentials out of workers. + - Authenticate private-network name lookups with the registered VM owner's kernel Unix identity, eliminating the reusable bearer secret previously written into each session directory and carried in lookup requests. diff --git a/crates/capsem-service/src/owner_connection.rs b/crates/capsem-service/src/owner_connection.rs index fbcdaa50f..880b9828b 100644 --- a/crates/capsem-service/src/owner_connection.rs +++ b/crates/capsem-service/src/owner_connection.rs @@ -11,6 +11,24 @@ pub(crate) struct OwnerConnection { } impl OwnerConnection { + pub(crate) fn current( + state: &ServiceState, + id: &str, + identity: Option, + ) -> Result { + let owner = state + .instances + .lock() + .unwrap() + .get(id) + .map(Self::capture) + .transpose()? + .ok_or_else(|| format!("VM {id} has no running owner"))?; + let identity = identity.ok_or_else(|| "service peer identity unavailable".to_string())?; + owner.authenticate_identity(state, identity)?; + Ok(owner) + } + pub(crate) fn pid(&self) -> u32 { self.identity.pid.get() } diff --git a/crates/capsem-service/src/private_routes.rs b/crates/capsem-service/src/private_routes.rs index bba09b2f2..67effbfcd 100644 --- a/crates/capsem-service/src/private_routes.rs +++ b/crates/capsem-service/src/private_routes.rs @@ -8,20 +8,8 @@ use super::*; /// The asker is the running owner of `vm`, or nobody. fn owner_of(state: &ServiceState, vm: &str, peer: ServicePeer) -> Result<(), AppError> { - let owner = state - .instances - .lock() - .unwrap() - .get(vm) - .map(owner_connection::OwnerConnection::capture) - .transpose() - .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; - let identity = peer - .0 - .ok_or_else(|| AppError(StatusCode::FORBIDDEN, "service peer identity unavailable".into()))?; - owner - .ok_or_else(|| AppError(StatusCode::FORBIDDEN, format!("VM {vm} has no running owner")))? - .authenticate_identity(state, identity) + owner_connection::OwnerConnection::current(state, vm, peer.0) + .map(|_| ()) .map_err(|error| AppError(StatusCode::FORBIDDEN, error)) } diff --git a/crates/capsem-service/src/router_runtime.rs b/crates/capsem-service/src/router_runtime.rs index 283959f9e..ebdc99b24 100644 --- a/crates/capsem-service/src/router_runtime.rs +++ b/crates/capsem-service/src/router_runtime.rs @@ -29,6 +29,12 @@ pub(super) fn build_service_router(state: Arc) -> Router { "/networks/private/resolve", post(private_routes::handle_private_resolve), ) + .route( + "/internal/vms/{id}/metrics", + post(service_runtime::telemetry_export::handle_metric_relay).layer(axum::extract::DefaultBodyLimit::max( + capsem_core::service_uds::MAX_BODY_BYTES, + )), + ) .route( "/networks/{id}/members/{vm_id}", put(network_routes::handle_network_attach).delete(network_routes::handle_network_detach), diff --git a/crates/capsem-service/src/service_runtime/telemetry_export.rs b/crates/capsem-service/src/service_runtime/telemetry_export.rs index afe85284e..ce28f35b2 100644 --- a/crates/capsem-service/src/service_runtime/telemetry_export.rs +++ b/crates/capsem-service/src/service_runtime/telemetry_export.rs @@ -8,7 +8,7 @@ //! read that. Attributes are exactly `session.id` and //! `persistent`. -use std::sync::{Arc, RwLock}; +use std::sync::{Arc, OnceLock, RwLock}; use std::time::Duration; use capsem_core::net::policy_config::SettingsFile; @@ -42,19 +42,134 @@ pub(crate) fn install() -> Option { } } -/// Grant a VM process the endpoint it exports its metrics to. +/// Grant a VM process access to the local metric broker. /// -/// The service resolves it, because a VM process takes its runtime config -/// only from what it is launched with and may not read settings or corp -/// files itself. It is the corp config's endpoint alone: the environment's -/// `OTEL_EXPORTER_OTLP_*` can carry collector credentials, and the spawn -/// allowlist keeps it out of the guest-facing process. -pub(crate) fn grant_metric_endpoint(command: &mut tokio::process::Command, corp: &SettingsFile) { - if let Some(Destination::Corp(endpoint)) = Destination::corp(corp.corp_rule_files.open_telemetry.as_deref()) { - command.arg("--metric-endpoint").arg(endpoint); +/// The presence of a valid corp endpoint decides whether export is on, but +/// neither that endpoint nor environment-carried collector credentials enter +/// the guest-facing process. +pub(crate) fn grant_metric_broker(command: &mut tokio::process::Command, corp: &SettingsFile) { + if Destination::corp(corp.corp_rule_files.open_telemetry.as_deref()).is_some() { + command.arg("--metric-broker"); } } +const COLLECTOR_TIMEOUT: Duration = Duration::from_secs(10); +static RELAY_CLIENT: OnceLock> = OnceLock::new(); + +fn relay_client() -> Result<&'static reqwest::Client, AppError> { + RELAY_CLIENT + .get_or_init(|| { + reqwest::Client::builder() + .timeout(COLLECTOR_TIMEOUT) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| error.to_string()) + }) + .as_ref() + .map_err(|_| AppError(StatusCode::BAD_GATEWAY, "metric collector client unavailable".into())) +} + +fn current_collector_url() -> Result { + let (_, corp) = capsem_core::net::policy_config::load_settings_and_corp_files(); + let Some(Destination::Corp(base)) = Destination::corp(corp.corp_rule_files.open_telemetry.as_deref()) else { + return Err(AppError( + StatusCode::NOT_FOUND, + "metric export is not configured".into(), + )); + }; + let url = reqwest::Url::parse(&format!("{base}/v1/metrics")).map_err(|_| { + AppError( + StatusCode::BAD_GATEWAY, + "configured metric collector endpoint is invalid".into(), + ) + })?; + if !matches!(url.scheme(), "http" | "https") { + return Err(AppError( + StatusCode::BAD_GATEWAY, + "configured metric collector endpoint must use HTTP or HTTPS".into(), + )); + } + Ok(url) +} + +/// Relay one worker's encoded OTLP metrics to the current fixed collector. +pub(crate) async fn handle_metric_relay( + State(state): State>, + axum::extract::ConnectInfo(peer): axum::extract::ConnectInfo, + Path(id): Path, + headers: axum::http::HeaderMap, + body: axum::body::Bytes, +) -> Result<(StatusCode, axum::http::HeaderMap, axum::body::Bytes), AppError> { + let owner = owner_connection::OwnerConnection::current(&state, &id, peer.0) + .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; + if headers.get(axum::http::header::CONTENT_TYPE) + != Some(&axum::http::HeaderValue::from_static("application/x-protobuf")) + { + return Err(AppError( + StatusCode::UNSUPPORTED_MEDIA_TYPE, + "metric broker accepts OTLP protobuf only".into(), + )); + } + if body.len() > capsem_core::service_uds::MAX_BODY_BYTES { + return Err(AppError( + StatusCode::PAYLOAD_TOO_LARGE, + "metric request body is too large".into(), + )); + } + let collector = current_collector_url()?; + owner + .validate(&state, false) + .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; + let response = match relay_client()? + .post(collector) + .header(axum::http::header::CONTENT_TYPE, "application/x-protobuf") + .body(body) + .send() + .await + { + Ok(response) => response, + Err(error) => { + warn!(vm = id, %error, "metric collector request failed"); + let status = if error.is_timeout() { + StatusCode::GATEWAY_TIMEOUT + } else { + StatusCode::BAD_GATEWAY + }; + return Err(AppError(status, "metric collector request failed".into())); + } + }; + let status = response.status(); + let content_type = response.headers().get(axum::http::header::CONTENT_TYPE).cloned(); + let mut stream = response.bytes_stream(); + let mut bytes = Vec::new(); + use futures::StreamExt as _; + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|error| { + warn!(vm = id, %error, "metric collector response failed"); + AppError(StatusCode::BAD_GATEWAY, "metric collector response failed".into()) + })?; + if bytes + .len() + .checked_add(chunk.len()) + .is_none_or(|length| length > capsem_core::service_uds::MAX_BODY_BYTES) + { + return Err(AppError( + StatusCode::BAD_GATEWAY, + "metric collector response is too large".into(), + )); + } + bytes.extend_from_slice(&chunk); + } + owner + .validate(&state, false) + .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; + let mut response_headers = axum::http::HeaderMap::new(); + if let Some(content_type) = content_type { + response_headers.insert(axum::http::header::CONTENT_TYPE, content_type); + } + Ok((status, response_headers, axum::body::Bytes::from(bytes))) +} + /// One session's totals and the attributes its series carry. #[derive(Clone, Debug)] pub(crate) struct SessionTotals { diff --git a/crates/capsem-service/src/tests/telemetry_export.rs b/crates/capsem-service/src/tests/telemetry_export.rs index 554c8a6ca..061ee9470 100644 --- a/crates/capsem-service/src/tests/telemetry_export.rs +++ b/crates/capsem-service/src/tests/telemetry_export.rs @@ -4,7 +4,7 @@ use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; use opentelemetry_sdk::metrics::{InMemoryMetricExporter, SdkMeterProvider}; use super::*; -use crate::service_runtime::telemetry_export::{collect, grant_metric_endpoint, register, SessionTable}; +use crate::service_runtime::telemetry_export::{collect, grant_metric_broker, register, SessionTable}; fn net(decision: capsem_logger::Decision, bytes_sent: u64) -> capsem_logger::WriteOp { capsem_logger::WriteOp::NetEvent( @@ -145,7 +145,7 @@ fn granted(open_telemetry: Option<&str>) -> Vec { let mut corp = capsem_core::net::policy_config::SettingsFile::default(); corp.corp_rule_files.open_telemetry = open_telemetry.map(str::to_string); let mut command = tokio::process::Command::new("capsem-process"); - grant_metric_endpoint(&mut command, &corp); + grant_metric_broker(&mut command, &corp); command .as_std() .get_args() @@ -153,13 +153,239 @@ fn granted(open_telemetry: Option<&str>) -> Vec { .collect() } -/// A VM process learns its export endpoint only from its launch arguments. +/// A VM process learns only whether the service granted brokered export. #[test] -fn a_vm_process_is_granted_the_corp_metric_endpoint_at_launch() { - assert_eq!( - granted(Some("https://otel.example/")), - ["--metric-endpoint", "https://otel.example"] - ); +fn a_vm_process_is_granted_only_the_metric_broker_at_launch() { + assert_eq!(granted(Some("https://otel.example/")), ["--metric-broker"]); assert!(granted(None).is_empty(), "no corp endpoint, no export"); assert!(granted(Some(" ")).is_empty(), "a blank endpoint is no endpoint"); } + +type CollectedMetric = (axum::http::HeaderMap, axum::body::Bytes); + +async fn metric_collector( + State(sender): State>, + headers: axum::http::HeaderMap, + body: axum::body::Bytes, +) -> (StatusCode, [(&'static str, &'static str); 1], axum::body::Bytes) { + sender.send((headers, body)).unwrap(); + ( + StatusCode::CREATED, + [("content-type", "application/x-protobuf")], + axum::body::Bytes::from_static(b"collector response"), + ) +} + +#[test] +fn metric_relay_wrong_process_client() { + let Ok(socket) = std::env::var("CAPSEM_TEST_METRIC_RELAY_SOCKET") else { + return; + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let response = runtime + .block_on(capsem_core::service_uds::post_bytes( + std::path::Path::new(&socket), + "/internal/vms/vm-a/metrics", + axum::http::HeaderValue::from_static("application/x-protobuf"), + axum::body::Bytes::from_static(b"wrong peer"), + )) + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); +} + +#[tokio::test] +async fn metric_relay_uses_only_current_owner_and_configured_collector() { + let _environment = SETTINGS_ENV_LOCK.lock().await; + let settings_dir = tempfile::tempdir().unwrap(); + let (_settings_guard, _, corp_path) = install_empty_settings_env(&settings_dir); + let collector_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let collector_base = format!("http://{}", collector_listener.local_addr().unwrap()); + let (collected_tx, mut collected_rx) = tokio::sync::mpsc::unbounded_channel(); + let collector = tokio::spawn(async move { + axum::serve( + collector_listener, + axum::Router::new() + .route("/v1/metrics", axum::routing::post(metric_collector)) + .with_state(collected_tx), + ) + .await + .unwrap(); + }); + let mut corp = capsem_core::net::policy_config::SettingsFile::default(); + corp.corp_rule_files.open_telemetry = Some(collector_base); + capsem_core::net::policy_config::write_settings_file(&corp_path, &corp).unwrap(); + + let (state, directory) = make_test_state_with_tempdir(); + insert_fake_instance(&state, "vm-a", std::process::id()); + let peer = ServicePeer(Some(capsem_foundation::unix::peer::PeerIdentity { + pid: capsem_foundation::unix::process::ProcessId::try_from(std::process::id()).unwrap(), + uid: capsem_foundation::unix::process::current_uid(), + })); + let socket = directory.path().join("metric-relay.sock"); + let service_listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let service_state = Arc::clone(&state); + let service = tokio::spawn(async move { + axum::serve( + service_listener, + build_service_router(service_state).into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + + let response = capsem_core::service_uds::post_bytes( + &socket, + "/internal/vms/vm-a/metrics", + axum::http::HeaderValue::from_static("application/x-protobuf"), + axum::body::Bytes::from_static(b"encoded metrics"), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + assert_eq!(response.body(), b"collector response".as_slice()); + let (headers, body) = tokio::time::timeout(std::time::Duration::from_secs(2), collected_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!( + headers.get(axum::http::header::CONTENT_TYPE).unwrap(), + "application/x-protobuf" + ); + assert_eq!(body, b"encoded metrics".as_slice()); + + use tower::ServiceExt as _; + let mock_app = build_service_router(Arc::clone(&state)).layer(axum::extract::connect_info::MockConnectInfo(peer)); + let response = mock_app + .clone() + .oneshot( + axum::http::Request::post("/internal/vms/vm-a/metrics") + .header(axum::http::header::CONTENT_TYPE, "application/x-protobuf") + .header(axum::http::header::AUTHORIZATION, "Bearer must-not-cross") + .header("x-otlp-endpoint", "http://attacker.invalid") + .body(axum::body::Body::from("fixed destination")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let (headers, body) = tokio::time::timeout(std::time::Duration::from_secs(2), collected_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(!headers.contains_key(axum::http::header::AUTHORIZATION)); + assert!(!headers.contains_key("x-otlp-endpoint")); + assert_eq!(body, b"fixed destination".as_slice()); + + let response = mock_app + .oneshot( + axum::http::Request::post("/internal/vms/vm-a/metrics") + .header(axum::http::header::CONTENT_TYPE, "application/x-protobuf") + .body(axum::body::Body::from(vec![ + 0; + capsem_core::service_uds::MAX_BODY_BYTES + 1 + ])) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert!( + collected_rx.try_recv().is_err(), + "an oversized body reached the collector" + ); + + let status = tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "tests::telemetry_export::metric_relay_wrong_process_client", + "--nocapture", + ]) + .env("CAPSEM_TEST_METRIC_RELAY_SOCKET", &socket) + .status() + .await + .unwrap(); + assert!(status.success(), "wrong-process probe failed: {status}"); + assert!( + collected_rx.try_recv().is_err(), + "a denied sibling reached the collector" + ); + + state.instances.lock().unwrap().get_mut("vm-a").unwrap().pid = 1; + let response = capsem_core::service_uds::post_bytes( + &socket, + "/internal/vms/vm-a/metrics", + axum::http::HeaderValue::from_static("application/x-protobuf"), + axum::body::Bytes::from_static(b"stale generation"), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert!( + collected_rx.try_recv().is_err(), + "a replaced owner reached the collector" + ); + state.instances.lock().unwrap().get_mut("vm-a").unwrap().pid = std::process::id(); + + let attacker_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let redirect_target = format!("http://{}/stolen", attacker_listener.local_addr().unwrap()); + let redirect_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let redirect_base = format!("http://{}", redirect_listener.local_addr().unwrap()); + let redirect = tokio::spawn(async move { + axum::serve( + redirect_listener, + axum::Router::new() + .route( + "/v1/metrics", + axum::routing::post(|State(target): State| async move { + (StatusCode::TEMPORARY_REDIRECT, [(axum::http::header::LOCATION, target)]) + }), + ) + .with_state(redirect_target), + ) + .await + .unwrap(); + }); + corp.corp_rule_files.open_telemetry = Some(redirect_base); + capsem_core::net::policy_config::write_settings_file(&corp_path, &corp).unwrap(); + let response = capsem_core::service_uds::post_bytes( + &socket, + "/internal/vms/vm-a/metrics", + axum::http::HeaderValue::from_static("application/x-protobuf"), + axum::body::Bytes::from_static(b"do not redirect"), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::TEMPORARY_REDIRECT); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), attacker_listener.accept()) + .await + .is_err(), + "the collector redirect opened an ungranted destination" + ); + + capsem_core::net::policy_config::write_settings_file( + &corp_path, + &capsem_core::net::policy_config::SettingsFile::default(), + ) + .unwrap(); + let response = capsem_core::service_uds::post_bytes( + &socket, + "/internal/vms/vm-a/metrics", + axum::http::HeaderValue::from_static("application/x-protobuf"), + axum::body::Bytes::from_static(b"disabled"), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::NOT_FOUND); + assert!( + collected_rx.try_recv().is_err(), + "disabled export reached the old collector" + ); + + service.abort(); + collector.abort(); + redirect.abort(); +} diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index 07a670dfc..b23a81adc 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -181,7 +181,7 @@ impl ServiceState { child_cmd.arg("--env").arg(format!("CAPSEM_VM_ID={}", id)); child_cmd.arg("--env").arg(format!("CAPSEM_VM_NAME={}", guest_name)); child_cmd.arg("--vm-name").arg(guest_name); - crate::service_runtime::telemetry_export::grant_metric_endpoint( + crate::service_runtime::telemetry_export::grant_metric_broker( &mut child_cmd, &capsem_core::net::policy_config::load_settings_and_corp_files().1, ); diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 82d2c43eb..6d05b057f 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -86,7 +86,7 @@ impl ServiceState { child_cmd.arg("--env").arg(format!("CAPSEM_VM_ID={}", vm_id)); child_cmd.arg("--env").arg(format!("CAPSEM_VM_NAME={}", name)); child_cmd.arg("--vm-name").arg(&name); - crate::service_runtime::telemetry_export::grant_metric_endpoint( + crate::service_runtime::telemetry_export::grant_metric_broker( &mut child_cmd, &capsem_core::net::policy_config::load_settings_and_corp_files().1, ); From 39afd6aa171bfd20f72fcdcf7e66071b7d341c7f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 16:59:59 +0000 Subject: [PATCH 023/464] security(process): remove direct metric export --- crates/capsem-process/src/main.rs | 13 +---- crates/capsem-process/src/metric_export.rs | 63 +++++++++------------- crates/capsem-process/src/tests.rs | 35 ++++++------ 3 files changed, 41 insertions(+), 70 deletions(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index d7ad50787..07fa483fc 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -133,14 +133,10 @@ struct Args { service_socket: Option, /// Export metrics through the service's generation-authenticated local /// broker. The service grants this without exposing a collector address. - #[arg(long, conflicts_with = "metric_endpoint")] + #[arg(long)] metric_broker: bool, #[arg(long)] checkpoint_path: Option, - /// Transitional direct-export grant, removed after the service switches - /// every launch to `--metric-broker`. - #[arg(long)] - metric_endpoint: Option, /// Environment variables to inject into guest (repeatable: --env KEY=VALUE) #[arg(long = "env")] env: Vec, @@ -248,12 +244,7 @@ fn main() -> Result<()> { info!(id = %args.id, "capsem-sandbox-process starting"); // Held until the process exits: dropping it flushes the last measurements. - let _metric_export = metric_export::install( - &args.id, - args.service_socket.as_deref(), - args.metric_broker, - args.metric_endpoint.as_deref(), - ); + let _metric_export = metric_export::install(&args.id, args.service_socket.as_deref(), args.metric_broker); let guest_dir = prepare_session_layout(&session_dir, args.scratch_disk_size_gb)?; // The image share is attached to every session, read-only at the device: diff --git a/crates/capsem-process/src/metric_export.rs b/crates/capsem-process/src/metric_export.rs index dd66da431..387da2d1e 100644 --- a/crates/capsem-process/src/metric_export.rs +++ b/crates/capsem-process/src/metric_export.rs @@ -3,10 +3,9 @@ //! The process exports its own measurements -- ledger writer, MITM, DNS, //! security engine, virtio-blk -- tagged with the session it serves. Broker //! mode carries encoded OTLP bytes over the service socket without learning -//! a collector address or credential. The direct endpoint mode remains only -//! while the service launch grant moves to the broker in the next commit. +//! a collector address or credential. Without a broker grant, export is off. -use capsem_telemetry::export::{Destination, Exporter, KeyValue}; +use capsem_telemetry::export::{Exporter, KeyValue}; use std::path::{Path, PathBuf}; use std::sync::Mutex; use tracing::{info, warn}; @@ -62,49 +61,35 @@ impl opentelemetry_http::HttpClient for ServiceMetricClient { } } -/// Install export when the service granted broker or transitional direct -/// transport. Failure is logged and leaves export off; metrics never prevent -/// a VM from booting. -pub(crate) fn install( - vm_id: &str, - service_socket: Option<&Path>, - broker: bool, - endpoint: Option<&str>, -) -> Option { +/// Install export when the service granted its broker. Failure is logged and +/// leaves export off; metrics never prevent a VM from booting. +pub(crate) fn install(vm_id: &str, service_socket: Option<&Path>, broker: bool) -> Option { + if !broker { + return None; + } let attributes = vec![KeyValue::new("session.id", vm_id.to_string())]; - let (installation, transport) = if broker { - let socket = match service_socket { - Some(socket) => socket, - None => { - warn!("metric broker granted without a service socket"); - return None; - } - }; - let client = match ServiceMetricClient::new(socket.to_path_buf(), vm_id) { - Ok(client) => client, - Err(error) => { - warn!(%error, "metric export not installed"); - return None; - } - }; - ( - capsem_telemetry::export::install_with_http_client(client, "capsem-process", attributes), - "service-broker", - ) - } else { - let destination = Destination::corp(endpoint)?; - ( - capsem_telemetry::export::install(&destination, "capsem-process", attributes), - "direct-transition", - ) + let socket = match service_socket { + Some(socket) => socket, + None => { + warn!("metric broker granted without a service socket"); + return None; + } + }; + let client = match ServiceMetricClient::new(socket.to_path_buf(), vm_id) { + Ok(client) => client, + Err(error) => { + warn!(%error, "metric export not installed"); + return None; + } }; + let installation = capsem_telemetry::export::install_with_http_client(client, "capsem-process", attributes); match installation { Ok(exporter) => { - info!(transport, "metric export installed"); + info!(transport = "service-broker", "metric export installed"); Some(exporter) } Err(error) => { - warn!(%error, transport, "metric export not installed"); + warn!(%error, transport = "service-broker", "metric export not installed"); None } } diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index 1b2f70342..d88effb26 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -21,6 +21,16 @@ fn private_name_broker_carries_no_session_bearer() { } } +#[test] +fn metric_export_has_no_collector_destination_surface() { + for source in [include_str!("main.rs"), include_str!("metric_export.rs")] { + assert!(!source.contains("metric_endpoint")); + assert!(!source.contains("--metric-endpoint")); + assert!(!source.contains("Destination::Corp")); + assert!(!source.contains("OTEL_EXPORTER")); + } +} + // ----------------------------------------------------------------------- // Args parsing // ----------------------------------------------------------------------- @@ -57,11 +67,11 @@ fn args_parses_all_required() { assert_eq!(args.uds_path, PathBuf::from("/tmp/vm.sock")); } -/// The metric endpoint is granted by the service at launch; without the flag -/// export is off. The process has no other way to learn it: it may not read -/// settings or corp files (`test_process_policy_runtime_contract.py`). +/// The metric broker is granted by the service at launch; without the flag +/// export is off. The process has no collector configuration surface and may +/// not read settings or corp files (`test_process_policy_runtime_contract.py`). #[test] -fn args_accept_one_optional_metric_transport_grant() { +fn args_accept_an_optional_metric_broker_grant() { let required = [ "capsem-process", "--id", @@ -85,8 +95,7 @@ fn args_accept_one_optional_metric_transport_grant() { ]; let off = Args::try_parse_from(required).unwrap(); assert!(!off.metric_broker); - assert_eq!(off.metric_endpoint, None); - assert!(metric_export::install(&off.id, None, off.metric_broker, off.metric_endpoint.as_deref()).is_none()); + assert!(metric_export::install(&off.id, None, off.metric_broker).is_none()); let broker = Args::try_parse_from( @@ -100,20 +109,6 @@ fn args_accept_one_optional_metric_transport_grant() { broker.service_socket.as_deref(), Some(std::path::Path::new("/tmp/service.sock")) ); - assert!(Args::try_parse_from(required.into_iter().chain([ - "--metric-broker", - "--metric-endpoint", - "https://otel.example" - ]),) - .is_err()); - - let granted = Args::try_parse_from( - required - .into_iter() - .chain(["--metric-endpoint", "https://otel.example"]), - ) - .unwrap(); - assert_eq!(granted.metric_endpoint.as_deref(), Some("https://otel.example")); } #[test] From 4273c290b264c8125662836333d9c63a6c619b79 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 17:11:16 +0000 Subject: [PATCH 024/464] security(service): cancel revoked broker work --- CHANGELOG.md | 3 +- crates/capsem-service/src/instance.rs | 41 ++++++++- crates/capsem-service/src/instance/tests.rs | 29 +++++++ crates/capsem-service/src/instance_reaper.rs | 7 +- crates/capsem-service/src/main.rs | 1 + crates/capsem-service/src/owner_connection.rs | 6 ++ .../src/service_runtime/telemetry_export.rs | 14 ++- crates/capsem-service/src/tests.rs | 3 +- crates/capsem-service/src/tests/files_api.rs | 3 +- .../src/tests/instance_reaper.rs | 7 +- .../src/tests/settings_files.rs | 6 +- .../src/tests/telemetry_export.rs | 85 +++++++++++++++++++ .../src/vm_lifecycle/provision.rs | 1 + .../src/vm_lifecycle/resume_process.rs | 1 + 14 files changed, 195 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c10ee640c..e461d2a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Route VM-owner metrics through a generation-authenticated service broker with a fixed corporate collector destination, bounded protobuf bodies and - no redirects, keeping collector addresses and credentials out of workers. + no redirects, keeping collector addresses and credentials out of workers; + revoking or replacing a worker generation cancels its in-flight exports. - Authenticate private-network name lookups with the registered VM owner's kernel Unix identity, eliminating the reusable bearer secret previously diff --git a/crates/capsem-service/src/instance.rs b/crates/capsem-service/src/instance.rs index 1b9c5772f..a1cb67957 100644 --- a/crates/capsem-service/src/instance.rs +++ b/crates/capsem-service/src/instance.rs @@ -1,11 +1,47 @@ //! One running VM as the service sees it: the owner process, its sockets, //! the boot assets it booted, and how its records are removed. use super::*; +use tokio_util::sync::CancellationToken; + +/// Cancellation owned by one registered worker generation. +pub(crate) struct WorkerAuthority(CancellationToken); + +/// A non-owning grant canceled when its registered generation is revoked. +pub(crate) struct WorkerGrant(CancellationToken); + +impl Default for WorkerAuthority { + fn default() -> Self { + Self(CancellationToken::new()) + } +} + +impl WorkerAuthority { + pub(crate) fn grant(&self) -> WorkerGrant { + WorkerGrant(self.0.clone()) + } + + pub(crate) fn revoke(&self) { + self.0.cancel(); + } +} + +impl Drop for WorkerAuthority { + fn drop(&mut self) { + self.revoke(); + } +} + +impl WorkerGrant { + pub(crate) async fn revoked(&self) { + self.0.cancelled().await; + } +} pub(crate) struct InstanceInfo { pub(crate) id: String, /// One actual spawn, never reused when an ID is resumed or replaced. pub(crate) generation: uuid::Uuid, + pub(crate) authority: WorkerAuthority, pub(crate) name: String, pub(crate) asset_pins: BootAssetPins, pub(crate) pid: u32, @@ -82,7 +118,10 @@ impl ServiceState { if instances.get(id)?.generation != generation { return None; } - instances.remove(id) + let removed = instances.remove(id)?; + drop(instances); + removed.authority.revoke(); + Some(removed) } /// Unregister a persistent VM. An absent entry is already forgotten, so diff --git a/crates/capsem-service/src/instance/tests.rs b/crates/capsem-service/src/instance/tests.rs index b1a78909f..8d48ddb86 100644 --- a/crates/capsem-service/src/instance/tests.rs +++ b/crates/capsem-service/src/instance/tests.rs @@ -186,3 +186,32 @@ fn teardown_claim_is_bound_to_generation_even_when_id_and_pid_are_reused() { assert!(claim_shutdown_instance(&state, "vm", replacement_generation)); assert!(!claim_shutdown_instance(&state, "vm", replacement_generation)); } + +#[tokio::test] +async fn removing_or_replacing_an_instance_revokes_its_grants() { + let state = make_test_state(); + let original = test_instance(); + let generation = original.generation; + let grant = original.authority.grant(); + let disposable = original.authority.grant(); + drop(disposable); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(1), grant.revoked()) + .await + .is_err(), + "dropping a non-owning grant cannot revoke its worker" + ); + state.instances.lock().unwrap().insert("vm".into(), original); + assert!(state.evict_instance("vm", generation).is_some()); + tokio::time::timeout(std::time::Duration::from_millis(100), grant.revoked()) + .await + .expect("registry removal revokes the generation"); + + let replacement_target = test_instance(); + let replaced = replacement_target.authority.grant(); + state.instances.lock().unwrap().insert("vm".into(), replacement_target); + state.instances.lock().unwrap().insert("vm".into(), test_instance()); + tokio::time::timeout(std::time::Duration::from_millis(100), replaced.revoked()) + .await + .expect("registry replacement revokes the displaced generation"); +} diff --git a/crates/capsem-service/src/instance_reaper.rs b/crates/capsem-service/src/instance_reaper.rs index 01302604f..79c25d4bb 100644 --- a/crates/capsem-service/src/instance_reaper.rs +++ b/crates/capsem-service/src/instance_reaper.rs @@ -224,5 +224,10 @@ pub(super) async fn revoke_exited_generation( { return ExitedGeneration::Replaced; } - ExitedGeneration::Current(instances.remove(id).map(Box::new)) + let removed = instances.remove(id); + drop(instances); + if let Some(instance) = removed.as_ref() { + instance.authority.revoke(); + } + ExitedGeneration::Current(removed.map(Box::new)) } diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 5cd8117cd..2d696ea81 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -553,6 +553,7 @@ impl ServiceState { .extract_if(|_, info| probe.is_gone(info.pid)) .map(|(id, info)| { tracing::warn!(id, "drain_dead_instances removing instance"); + info.authority.revoke(); (id, info) }) .collect(); diff --git a/crates/capsem-service/src/owner_connection.rs b/crates/capsem-service/src/owner_connection.rs index 880b9828b..1c13b04c4 100644 --- a/crates/capsem-service/src/owner_connection.rs +++ b/crates/capsem-service/src/owner_connection.rs @@ -7,6 +7,7 @@ pub(crate) struct OwnerConnection { id: String, generation: uuid::Uuid, identity: peer::PeerIdentity, + authority: instance::WorkerGrant, pub(crate) uds_path: PathBuf, } @@ -65,6 +66,7 @@ impl OwnerConnection { pid: process::ProcessId::try_from(instance.pid).map_err(|e| e.to_string())?, uid: process::current_uid(), }, + authority: instance.authority.grant(), uds_path: instance.uds_path.clone(), }) } @@ -101,6 +103,10 @@ impl OwnerConnection { Ok(()) } + pub(crate) async fn revoked(&self) { + self.authority.revoked().await; + } + pub(crate) async fn open( &self, state: &ServiceState, diff --git a/crates/capsem-service/src/service_runtime/telemetry_export.rs b/crates/capsem-service/src/service_runtime/telemetry_export.rs index ce28f35b2..ce9e0cb89 100644 --- a/crates/capsem-service/src/service_runtime/telemetry_export.rs +++ b/crates/capsem-service/src/service_runtime/telemetry_export.rs @@ -120,13 +120,19 @@ pub(crate) async fn handle_metric_relay( owner .validate(&state, false) .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; - let response = match relay_client()? + let request = relay_client()? .post(collector) .header(axum::http::header::CONTENT_TYPE, "application/x-protobuf") .body(body) - .send() - .await - { + .send(); + tokio::pin!(request); + let response = match tokio::select! { + biased; + () = owner.revoked() => { + return Err(AppError(StatusCode::FORBIDDEN, "VM owner authority was revoked".into())); + } + response = &mut request => response, + } { Ok(response) => response, Err(error) => { warn!(vm = id, %error, "metric collector request failed"); diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 2cc3f6981..d4c8b1e7c 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -138,7 +138,8 @@ pub(super) fn make_asset_state(assets_dir: PathBuf) -> Arc { /// The fields every fake instance shares; a test names only what it varies. pub(crate) fn test_instance() -> InstanceInfo { InstanceInfo { - generation: capsem_crypto::uuid_v4(), + generation: uuid::Uuid::new_v4(), + authority: Default::default(), id: String::new(), name: String::new(), asset_pins: test_asset_pins(), diff --git a/crates/capsem-service/src/tests/files_api.rs b/crates/capsem-service/src/tests/files_api.rs index 09995920d..d530eb599 100644 --- a/crates/capsem-service/src/tests/files_api.rs +++ b/crates/capsem-service/src/tests/files_api.rs @@ -83,7 +83,8 @@ pub(super) fn setup_vm_with_workspace_and_uds( state.instances.lock().unwrap().insert( vm_id.into(), InstanceInfo { - generation: capsem_crypto::uuid_v4(), + generation: uuid::Uuid::new_v4(), + authority: Default::default(), id: vm_id.into(), name: vm_id.into(), asset_pins: test_asset_pins(), diff --git a/crates/capsem-service/src/tests/instance_reaper.rs b/crates/capsem-service/src/tests/instance_reaper.rs index 50a70e259..36585f09a 100644 --- a/crates/capsem-service/src/tests/instance_reaper.rs +++ b/crates/capsem-service/src/tests/instance_reaper.rs @@ -13,15 +13,20 @@ async fn exited_generation_is_revoked_while_its_pid_is_still_reserved() { let pid = child.id().unwrap(); insert_fake_instance_with_session_dir(&state, id, pid, session); let generation = state.instances.lock().unwrap()[id].generation; + let grant = state.instances.lock().unwrap()[id].authority.grant(); - crate::instance_reaper::revoke_exited_generation(&child, id, &state, generation).await; + let revoked = crate::instance_reaper::revoke_exited_generation(&child, id, &state, generation).await; assert!(!state.instances.lock().unwrap().contains_key(id)); + tokio::time::timeout(std::time::Duration::from_millis(100), grant.revoked()) + .await + .expect("exit observation revokes grants before the removed instance is dropped"); let pid = capsem_foundation::unix::process::ProcessId::try_from(pid).unwrap(); assert!( capsem_foundation::unix::process::child_has_exited(pid).unwrap(), "generation revocation must happen before the child is reaped" ); + drop(revoked); assert_eq!(child.wait().await.unwrap().code(), Some(11)); } diff --git a/crates/capsem-service/src/tests/settings_files.rs b/crates/capsem-service/src/tests/settings_files.rs index 79aebfaf3..a109e6e56 100644 --- a/crates/capsem-service/src/tests/settings_files.rs +++ b/crates/capsem-service/src/tests/settings_files.rs @@ -187,7 +187,8 @@ fn resolve_rejects_symlink_escape() { state.instances.lock().unwrap().insert( "test-vm".into(), InstanceInfo { - generation: capsem_crypto::uuid_v4(), + generation: uuid::Uuid::new_v4(), + authority: Default::default(), id: "test-vm".into(), name: "test-vm".into(), asset_pins: test_asset_pins(), @@ -220,7 +221,8 @@ fn resolve_valid_path_inside_workspace() { state.instances.lock().unwrap().insert( "test-vm".into(), InstanceInfo { - generation: capsem_crypto::uuid_v4(), + generation: uuid::Uuid::new_v4(), + authority: Default::default(), id: "test-vm".into(), name: "test-vm".into(), asset_pins: test_asset_pins(), diff --git a/crates/capsem-service/src/tests/telemetry_export.rs b/crates/capsem-service/src/tests/telemetry_export.rs index 061ee9470..b1aba9867 100644 --- a/crates/capsem-service/src/tests/telemetry_export.rs +++ b/crates/capsem-service/src/tests/telemetry_export.rs @@ -389,3 +389,88 @@ async fn metric_relay_uses_only_current_owner_and_configured_collector() { collector.abort(); redirect.abort(); } + +#[tokio::test] +async fn metric_relay_drops_the_upstream_connection_when_generation_is_revoked() { + use tokio::io::{AsyncBufReadExt as _, AsyncReadExt as _, BufReader}; + + let _environment = SETTINGS_ENV_LOCK.lock().await; + let settings_dir = tempfile::tempdir().unwrap(); + let (_settings_guard, _, corp_path) = install_empty_settings_env(&settings_dir); + let collector_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let collector_base = format!("http://{}", collector_listener.local_addr().unwrap()); + let mut corp = capsem_core::net::policy_config::SettingsFile::default(); + corp.corp_rule_files.open_telemetry = Some(collector_base); + capsem_core::net::policy_config::write_settings_file(&corp_path, &corp).unwrap(); + let (accepted_tx, accepted_rx) = tokio::sync::oneshot::channel(); + let (closed_tx, closed_rx) = tokio::sync::oneshot::channel(); + let collector = tokio::spawn(async move { + let (stream, _) = collector_listener.accept().await.unwrap(); + let mut reader = BufReader::new(stream); + let mut content_length = 0; + loop { + let mut line = Vec::new(); + reader.read_until(b'\n', &mut line).await.unwrap(); + if line == b"\r\n" { + break; + } + let text = String::from_utf8_lossy(&line).to_ascii_lowercase(); + if let Some(value) = text.strip_prefix("content-length:") { + content_length = value.trim().parse().unwrap(); + } + } + let mut body = vec![0; content_length]; + reader.read_exact(&mut body).await.unwrap(); + accepted_tx.send(body).unwrap(); + let mut byte = [0]; + closed_tx.send(reader.read(&mut byte).await.unwrap() == 0).unwrap(); + }); + + let (state, directory) = make_test_state_with_tempdir(); + insert_fake_instance(&state, "vm-a", std::process::id()); + let generation = state.instances.lock().unwrap()["vm-a"].generation; + let socket = directory.path().join("metric-revocation.sock"); + let service_listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let service_state = Arc::clone(&state); + let service = tokio::spawn(async move { + axum::serve( + service_listener, + build_service_router(service_state).into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + let request_socket = socket.clone(); + let request = tokio::spawn(async move { + capsem_core::service_uds::post_bytes( + &request_socket, + "/internal/vms/vm-a/metrics", + axum::http::HeaderValue::from_static("application/x-protobuf"), + axum::body::Bytes::from_static(b"cancel me"), + ) + .await + .unwrap() + }); + assert_eq!( + tokio::time::timeout(std::time::Duration::from_secs(2), accepted_rx) + .await + .unwrap() + .unwrap(), + b"cancel me" + ); + assert!(state.evict_instance("vm-a", generation).is_some()); + let response = tokio::time::timeout(std::time::Duration::from_secs(1), request) + .await + .expect("revocation cancels the in-flight collector request") + .unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(1), closed_rx) + .await + .unwrap() + .unwrap(), + "canceling the upstream future closes its collector connection" + ); + service.abort(); + collector.abort(); +} diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index b23a81adc..e87f61b91 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -332,6 +332,7 @@ impl ServiceState { id.to_string(), InstanceInfo { generation, + authority: Default::default(), id: id.to_string(), name: name.to_string(), asset_pins, diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 6d05b057f..dc3648437 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -228,6 +228,7 @@ impl ServiceState { vm_id.clone(), InstanceInfo { generation, + authority: Default::default(), id: vm_id.clone(), name: entry.name.clone(), asset_pins: entry.asset_pins.clone(), From 08a1a9f6422731ed9ec3f78ed2a14be9edd5f193 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 17:16:09 +0000 Subject: [PATCH 025/464] test(service): qualify metric broker end to end --- Cargo.lock | 6 + crates/capsem-service/Cargo.toml | 5 + .../src/tests/telemetry_export.rs | 153 ++++++++++++++++++ 3 files changed, 164 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index e6de4ab5c..aa0c573a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,8 +1225,11 @@ name = "capsem-service" version = "0.7.0" dependencies = [ "anyhow", + "async-trait", "axum", "base64 0.22.1", + "blake3", + "bytes", "capsem-api", "capsem-assets", "capsem-config", @@ -1242,8 +1245,11 @@ dependencies = [ "clap", "filetime", "futures", + "http", "humantime", + "metrics", "opentelemetry", + "opentelemetry-http", "opentelemetry_sdk", "reqwest 0.12.28", "rusqlite", diff --git a/crates/capsem-service/Cargo.toml b/crates/capsem-service/Cargo.toml index 4821131aa..b44a5222f 100644 --- a/crates/capsem-service/Cargo.toml +++ b/crates/capsem-service/Cargo.toml @@ -42,7 +42,12 @@ uuid = { version = "1", features = ["v4"] } workspace = true [dev-dependencies] +async-trait = "0.1" +bytes.workspace = true +http.workspace = true +metrics = "0.24" opentelemetry = { version = "0.33", default-features = false, features = ["metrics"] } +opentelemetry-http = { version = "0.33", default-features = false } opentelemetry_sdk = { version = "0.33", default-features = false, features = ["metrics", "testing"] } tokio-tungstenite = "0.29.0" capsem-config = { path = "../capsem-config" } diff --git a/crates/capsem-service/src/tests/telemetry_export.rs b/crates/capsem-service/src/tests/telemetry_export.rs index b1aba9867..1e8eb7533 100644 --- a/crates/capsem-service/src/tests/telemetry_export.rs +++ b/crates/capsem-service/src/tests/telemetry_export.rs @@ -163,6 +163,76 @@ fn a_vm_process_is_granted_only_the_metric_broker_at_launch() { type CollectedMetric = (axum::http::HeaderMap, axum::body::Bytes); +/// The qualification child uses the worker's production transport shape: the +/// OTLP encoder controls only the body, while this client fixes both the local +/// service socket and the session-specific broker route. +struct QualificationMetricClient { + service_socket: std::path::PathBuf, + path: String, + runtime: std::sync::Mutex, +} + +impl std::fmt::Debug for QualificationMetricClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("QualificationMetricClient") + .field("service_socket", &self.service_socket) + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +#[async_trait::async_trait] +impl opentelemetry_http::HttpClient for QualificationMetricClient { + async fn send_bytes( + &self, + request: http::Request, + ) -> Result, opentelemetry_http::HttpError> { + self.runtime + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .block_on(capsem_core::service_uds::post_bytes( + &self.service_socket, + &self.path, + http::HeaderValue::from_static("application/x-protobuf"), + request.into_body(), + )) + .map_err(|error| Box::new(std::io::Error::other(format!("metric broker request: {error:#}"))) as _) + } +} + +#[test] +fn metric_relay_attribution_child() { + use std::io::Read as _; + + let (Ok(socket), Ok(id)) = ( + std::env::var("CAPSEM_TEST_METRIC_ATTRIBUTION_SOCKET"), + std::env::var("CAPSEM_TEST_METRIC_ATTRIBUTION_ID"), + ) else { + return; + }; + let mut release = [0]; + std::io::stdin().read_exact(&mut release).unwrap(); + let client = QualificationMetricClient { + service_socket: socket.into(), + path: format!("/internal/vms/{id}/metrics"), + runtime: std::sync::Mutex::new( + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(), + ), + }; + let exporter = capsem_telemetry::export::install_with_http_client( + client, + "capsem-process", + vec![capsem_telemetry::export::KeyValue::new("session.id", id)], + ) + .unwrap(); + metrics::counter!(capsem_telemetry::db::DB_WRITE_OPS_TOTAL).increment(1); + exporter.flush().unwrap(); +} + async fn metric_collector( State(sender): State>, headers: axum::http::HeaderMap, @@ -176,6 +246,89 @@ async fn metric_collector( ) } +fn protobuf_contains(body: &[u8], expected: &str) -> bool { + body.windows(expected.len()).any(|window| window == expected.as_bytes()) +} + +#[tokio::test] +async fn worker_metrics_keep_exact_attribution_through_the_service_broker() { + use tokio::io::AsyncWriteExt as _; + + let _environment = SETTINGS_ENV_LOCK.lock().await; + let settings_dir = tempfile::tempdir().unwrap(); + let (_settings_guard, _, corp_path) = install_empty_settings_env(&settings_dir); + let collector_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let collector_base = format!("http://{}", collector_listener.local_addr().unwrap()); + let (collected_tx, mut collected_rx) = tokio::sync::mpsc::unbounded_channel(); + let collector = tokio::spawn(async move { + axum::serve( + collector_listener, + axum::Router::new() + .route("/v1/metrics", axum::routing::post(metric_collector)) + .with_state(collected_tx), + ) + .await + .unwrap(); + }); + let mut corp = capsem_core::net::policy_config::SettingsFile::default(); + corp.corp_rule_files.open_telemetry = Some(collector_base); + capsem_core::net::policy_config::write_settings_file(&corp_path, &corp).unwrap(); + + let (state, directory) = make_test_state_with_tempdir(); + let socket = directory.path().join("metric-attribution.sock"); + let service_listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let service_state = Arc::clone(&state); + let service = tokio::spawn(async move { + axum::serve( + service_listener, + build_service_router(service_state).into_make_service_with_connect_info::(), + ) + .await + .unwrap(); + }); + + let id = "attributed-vm"; + let mut child = tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "tests::telemetry_export::metric_relay_attribution_child", + "--nocapture", + ]) + .env("CAPSEM_TEST_METRIC_ATTRIBUTION_SOCKET", &socket) + .env("CAPSEM_TEST_METRIC_ATTRIBUTION_ID", id) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + insert_fake_instance(&state, id, child.id().unwrap()); + child.stdin.take().unwrap().write_all(b"1").await.unwrap(); + let status = tokio::time::timeout(std::time::Duration::from_secs(5), child.wait()) + .await + .expect("metric-export child completes") + .unwrap(); + assert!(status.success(), "metric-export child failed: {status}"); + + let (headers, body) = tokio::time::timeout(std::time::Duration::from_secs(2), collected_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!( + headers.get(axum::http::header::CONTENT_TYPE).unwrap(), + "application/x-protobuf" + ); + for expected in [ + "service.name", + "capsem-process", + "session.id", + id, + capsem_telemetry::db::DB_WRITE_OPS_TOTAL, + ] { + assert!(protobuf_contains(&body, expected), "OTLP body lacks {expected}"); + } + + service.abort(); + collector.abort(); +} + #[test] fn metric_relay_wrong_process_client() { let Ok(socket) = std::env::var("CAPSEM_TEST_METRIC_RELAY_SOCKET") else { From d3761201e74b5b2d45fe7e4a1949266e39bd04dc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 17:21:38 +0000 Subject: [PATCH 026/464] refactor(foundation): generalize descriptor channel --- .../src/unix/router_channel.rs | 52 +++++++++----- .../src/unix/router_channel/tests.rs | 67 +++++++++++++++++++ 2 files changed, 101 insertions(+), 18 deletions(-) diff --git a/crates/capsem-foundation/src/unix/router_channel.rs b/crates/capsem-foundation/src/unix/router_channel.rs index adad4f79f..3aa52ea30 100644 --- a/crates/capsem-foundation/src/unix/router_channel.rs +++ b/crates/capsem-foundation/src/unix/router_channel.rs @@ -1,4 +1,4 @@ -//! Fixed records and owned SCM_RIGHTS descriptors for the confined router. +//! Fixed records and owned SCM_RIGHTS descriptors for confined workers. //! Cancellation poisons the socket rather than resuming a partial record. //! Senders retain socket owners until the receiver acknowledges adoption; //! Darwin may flush sockets referenced only by queued SCM_RIGHTS messages. @@ -13,11 +13,14 @@ use tokio::sync::Mutex; pub const FRAME_SIZE: usize = 10; pub const MAX_FDS: usize = 2; -pub struct Frame { - pub bytes: [u8; FRAME_SIZE], +pub struct DescriptorFrame { + pub bytes: [u8; RECORD_SIZE], pub fds: Vec, } +/// The existing router wire format. +pub type Frame = DescriptorFrame; + struct Channel { io: AsyncFd, operation: Mutex<()>, @@ -44,17 +47,23 @@ impl Drop for Operation<'_> { } } } -pub struct Sender(Channel); -pub struct Receiver(Channel); -impl Sender { +pub struct DescriptorSender(Channel); +pub struct DescriptorReceiver(Channel); + +/// Sender for the existing router wire format. +pub type Sender = DescriptorSender; +/// Receiver for the existing router wire format. +pub type Receiver = DescriptorReceiver; + +impl DescriptorSender { pub fn new(stream: UnixStream) -> io::Result { Channel::new(stream).map(Self) } pub async fn send(&self, bytes: &[u8], fds: &[RawFd]) -> io::Result { - if bytes.len() != FRAME_SIZE || fds.len() > MAX_FDS { + if RECORD_SIZE == 0 || bytes.len() != RECORD_SIZE || fds.len() > FD_LIMIT { return Err(io::Error::new( ErrorKind::InvalidInput, - "invalid router record size or descriptor count", + "invalid descriptor record size or count", )); } let _lock = self.0.operation.lock().await; @@ -83,25 +92,32 @@ impl Sender { Ok(offset) } } -impl Receiver { +impl DescriptorReceiver { pub fn new(stream: UnixStream) -> io::Result { Channel::new(stream).map(Self) } - pub async fn recv(&self) -> io::Result { + pub async fn recv(&self) -> io::Result> { + if RECORD_SIZE == 0 { + return Err(io::Error::new( + ErrorKind::InvalidInput, + "descriptor record cannot be empty", + )); + } let _lock = self.0.operation.lock().await; let mut operation = Operation { stream: self.0.io.get_ref(), complete: false, }; - let mut frame = Frame { - bytes: [0; FRAME_SIZE], - fds: Vec::with_capacity(MAX_FDS), + let mut frame = DescriptorFrame { + bytes: [0; RECORD_SIZE], + fds: Vec::with_capacity(FD_LIMIT), }; let mut offset = 0; - while offset < FRAME_SIZE { + while offset < RECORD_SIZE { let mut ready = self.0.io.readable().await?; - match ready.try_io(|socket| receive_record(socket.as_raw_fd(), &mut frame.bytes[offset..], &mut frame.fds)) - { + match ready.try_io(|socket| { + receive_record(socket.as_raw_fd(), &mut frame.bytes[offset..], &mut frame.fds, FD_LIMIT) + }) { Ok(Ok(0)) => return Err(ErrorKind::UnexpectedEof.into()), Ok(Ok(count)) => offset += count, Ok(Err(error)) if error.kind() == ErrorKind::Interrupted => continue, @@ -145,7 +161,7 @@ fn send_record(socket: RawFd, bytes: &[u8], fds: &[RawFd]) -> io::Result } } } -fn receive_record(socket: RawFd, bytes: &mut [u8], fds: &mut Vec) -> io::Result { +fn receive_record(socket: RawFd, bytes: &mut [u8], fds: &mut Vec, fd_limit: usize) -> io::Result { // SAFETY: the kernel initializes bounded aligned ancillary storage. Adopt // all delivered descriptors before rejecting truncation to avoid leaks. unsafe { @@ -198,7 +214,7 @@ fn receive_record(socket: RawFd, bytes: &mut [u8], fds: &mut Vec) -> io } header = libc::CMSG_NXTHDR(&message, header); } - if invalid || fds.len() > MAX_FDS { + if invalid || fds.len() > fd_limit { return Err(io::Error::new( ErrorKind::InvalidData, "invalid router ancillary descriptors", diff --git a/crates/capsem-foundation/src/unix/router_channel/tests.rs b/crates/capsem-foundation/src/unix/router_channel/tests.rs index 27982ec55..34933f21e 100644 --- a/crates/capsem-foundation/src/unix/router_channel/tests.rs +++ b/crates/capsem-foundation/src/unix/router_channel/tests.rs @@ -7,6 +7,73 @@ fn async_stream(stream: UnixStream) -> tokio::net::UnixStream { tokio::net::UnixStream::from_std(stream).unwrap() } +#[tokio::test] +async fn larger_single_descriptor_records_reuse_the_hardened_channel() { + type GrantSender = DescriptorSender<272, 1>; + type GrantReceiver = DescriptorReceiver<272, 1>; + + let (parent, child) = UnixStream::pair().unwrap(); + let sender = GrantSender::new(parent).unwrap(); + let receiver = GrantReceiver::new(child).unwrap(); + let (data, peer) = UnixStream::pair().unwrap(); + let mut peer = async_stream(peer); + let mut record = [0; 272]; + record[0] = 7; + record[271] = 9; + sender.send(&record, &[data.as_raw_fd()]).await.unwrap(); + let mut frame = receiver.recv().await.unwrap(); + assert_eq!(frame.bytes, record); + assert_eq!(frame.fds.len(), 1); + let flags = fcntl(frame.fds[0].as_raw_fd(), FcntlArg::F_GETFD).unwrap(); + assert!(FdFlag::from_bits_truncate(flags).contains(FdFlag::FD_CLOEXEC)); + let mut adopted = async_stream(UnixStream::from(frame.fds.pop().unwrap())); + adopted.write_all(b"grant").await.unwrap(); + let mut bytes = [0; 5]; + peer.read_exact(&mut bytes).await.unwrap(); + assert_eq!(&bytes, b"grant"); +} + +#[tokio::test] +async fn generic_limits_reject_wrong_shapes_without_consuming_descriptors() { + type GrantSender = DescriptorSender<272, 1>; + + let (parent, _child) = UnixStream::pair().unwrap(); + let sender = GrantSender::new(parent).unwrap(); + let (data, _peer) = UnixStream::pair().unwrap(); + assert_eq!( + sender.send(&[0; 271], &[data.as_raw_fd()]).await.unwrap_err().kind(), + ErrorKind::InvalidInput + ); + assert_eq!( + sender + .send(&[0; 272], &[data.as_raw_fd(), data.as_raw_fd()]) + .await + .unwrap_err() + .kind(), + ErrorKind::InvalidInput + ); + assert!(data.peer_addr().is_ok()); +} + +#[tokio::test] +async fn generic_receiver_closes_descriptors_above_its_limit() { + type GrantReceiver = DescriptorReceiver<272, 1>; + + let (parent, child) = UnixStream::pair().unwrap(); + let receiver = GrantReceiver::new(child).unwrap(); + let (data, peer) = UnixStream::pair().unwrap(); + send_record(parent.as_raw_fd(), &[0; 272], &[data.as_raw_fd(); 2]).unwrap(); + drop(data); + assert_eq!(receiver.recv().await.err().unwrap().kind(), ErrorKind::InvalidData); + assert_eq!( + timeout(Duration::from_secs(1), async_stream(peer).read(&mut [0])) + .await + .unwrap() + .unwrap(), + 0 + ); +} + #[tokio::test] async fn acknowledged_socket_remains_usable_after_sender_drops_original() { let (parent, child) = UnixStream::pair().unwrap(); From 78031849849e7639659a0e971bb67d879e7efa61 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 17:29:51 +0000 Subject: [PATCH 027/464] feat(proto): define upstream descriptor grants --- crates/capsem-proto/src/lib.rs | 1 + crates/capsem-proto/src/upstream_grant.rs | 509 ++++++++++++++++++ .../capsem-proto/src/upstream_grant/tests.rs | 265 +++++++++ 3 files changed, 775 insertions(+) create mode 100644 crates/capsem-proto/src/upstream_grant.rs create mode 100644 crates/capsem-proto/src/upstream_grant/tests.rs diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 9f1285286..4311da04c 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -26,6 +26,7 @@ pub mod privatelink; pub mod repeated; pub mod router; mod sparse; +pub mod upstream_grant; mod wire_bytes; /// Where the guest mounts the host-visible workspace share: every VM sees its diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs new file mode 100644 index 000000000..b7453369e --- /dev/null +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -0,0 +1,509 @@ +//! Fixed, versioned records for coordinator-minted upstream descriptors. +//! +//! A worker first asks the trusted coordinator to resolve a TCP target. The +//! response carries an opaque selection id, the address policy must judge and +//! the effective HTTP/TLS protocol. Only after policy admits that selection +//! does the worker ask the coordinator to connect it. DNS requests name a +//! configured upstream by index, never by worker-supplied address. + +use std::net::IpAddr; + +use anyhow::{bail, Result}; + +/// Exact record size carried by the bounded SCM_RIGHTS channel. +pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 288; +/// A response grants at most one connected TCP or UDP descriptor. +pub const UPSTREAM_GRANT_MAX_FDS: usize = 1; +/// DNS names are at most 253 wire-text bytes without a root dot. +pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; + +const MAGIC: [u8; 2] = *b"UG"; +const VERSION: u8 = 1; +const MAGIC_RANGE: std::ops::Range = 0..2; +const VERSION_OFFSET: usize = 2; +const KIND_OFFSET: usize = 3; +const REQUEST_ID_RANGE: std::ops::Range = 4..12; +const RESOURCE_ID_RANGE: std::ops::Range = 12..20; +const PORT_RANGE: std::ops::Range = 20..22; +const DETAIL_RANGE: std::ops::Range = 22..24; +const NAME_LENGTH_OFFSET: usize = 24; +const NAME_RANGE: std::ops::Range = 25..25 + MAX_UPSTREAM_HOST_BYTES; +const RESERVED_RANGE: std::ops::Range = NAME_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; + +const RESOLVE_TCP: u8 = 1; +const CONNECT_TCP: u8 = 2; +const OPEN_DNS: u8 = 3; +const ADOPTED: u8 = 4; +const RELEASE: u8 = 5; +const TCP_RESOLVED: u8 = 101; +const DESCRIPTOR_GRANTED: u8 = 102; +const DENIED: u8 = 103; + +/// Application protocol spoken over a granted TCP stream. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum UpstreamProtocol { + Http, + Tls, +} + +impl UpstreamProtocol { + const fn code(self) -> u16 { + match self { + Self::Http => 1, + Self::Tls => 2, + } + } + + fn decode(code: u16) -> Result { + match code { + 1 => Ok(Self::Http), + 2 => Ok(Self::Tls), + _ => bail!("invalid upstream protocol {code}"), + } + } +} + +/// Kind of connected socket carried by a descriptor response. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum UpstreamDescriptorKind { + Tcp, + DnsUdp, +} + +impl UpstreamDescriptorKind { + const fn code(self) -> u16 { + match self { + Self::Tcp => 1, + Self::DnsUdp => 2, + } + } + + fn decode(code: u16) -> Result { + match code { + 1 => Ok(Self::Tcp), + 2 => Ok(Self::DnsUdp), + _ => bail!("invalid upstream descriptor kind {code}"), + } + } +} + +/// Bounded denial reasons. Details stay in trusted coordinator logs. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum UpstreamGrantDenial { + NotConfigured, + NotAllowed, + Capacity, + ResolveFailed, + ConnectFailed, + Revoked, + InvalidResource, +} + +impl UpstreamGrantDenial { + const fn code(self) -> u16 { + match self { + Self::NotConfigured => 1, + Self::NotAllowed => 2, + Self::Capacity => 3, + Self::ResolveFailed => 4, + Self::ConnectFailed => 5, + Self::Revoked => 6, + Self::InvalidResource => 7, + } + } + + fn decode(code: u16) -> Result { + match code { + 1 => Ok(Self::NotConfigured), + 2 => Ok(Self::NotAllowed), + 3 => Ok(Self::Capacity), + 4 => Ok(Self::ResolveFailed), + 5 => Ok(Self::ConnectFailed), + 6 => Ok(Self::Revoked), + 7 => Ok(Self::InvalidResource), + _ => bail!("invalid upstream denial reason {code}"), + } + } +} + +/// Worker-to-coordinator record. Request ids correlate replies; resource ids +/// are opaque coordinator-minted selections or live descriptor grants. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum UpstreamGrantRequest { + ResolveTcp { + request_id: u64, + protocol: UpstreamProtocol, + host: String, + port: u16, + }, + ConnectTcp { + request_id: u64, + selection_id: u64, + }, + OpenDns { + request_id: u64, + upstream_index: u16, + }, + Adopted { + grant_id: u64, + }, + Release { + resource_id: u64, + }, +} + +/// Coordinator-to-worker record. Only `DescriptorGranted` carries one fd. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum UpstreamGrantResponse { + TcpResolved { + request_id: u64, + selection_id: u64, + protocol: UpstreamProtocol, + judged_ip: Option, + }, + DescriptorGranted { + request_id: u64, + grant_id: u64, + kind: UpstreamDescriptorKind, + }, + Denied { + request_id: u64, + reason: UpstreamGrantDenial, + }, +} + +impl UpstreamGrantRequest { + /// Requests never carry descriptors from the untrusted worker. + pub const fn expected_descriptor_count(&self) -> usize { + 0 + } +} + +impl UpstreamGrantResponse { + /// Only a successful connected-socket grant carries a descriptor. + pub const fn expected_descriptor_count(&self) -> usize { + match self { + Self::DescriptorGranted { .. } => 1, + Self::TcpResolved { .. } | Self::Denied { .. } => 0, + } + } +} + +/// Encode one worker request into its exact fixed-size record. +pub fn encode_upstream_grant_request(request: &UpstreamGrantRequest) -> Result<[u8; UPSTREAM_GRANT_FRAME_SIZE]> { + let mut frame = empty_frame(); + match request { + UpstreamGrantRequest::ResolveTcp { + request_id, + protocol, + host, + port, + } => { + require_nonzero("request id", *request_id)?; + if *port == 0 { + bail!("upstream port cannot be zero"); + } + validate_normalized_host(host)?; + frame[KIND_OFFSET] = RESOLVE_TCP; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u16(&mut frame, PORT_RANGE, *port); + put_u16(&mut frame, DETAIL_RANGE, protocol.code()); + put_name(&mut frame, host)?; + } + UpstreamGrantRequest::ConnectTcp { + request_id, + selection_id, + } => { + require_nonzero("request id", *request_id)?; + require_nonzero("selection id", *selection_id)?; + frame[KIND_OFFSET] = CONNECT_TCP; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u64(&mut frame, RESOURCE_ID_RANGE, *selection_id); + } + UpstreamGrantRequest::OpenDns { + request_id, + upstream_index, + } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = OPEN_DNS; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u16(&mut frame, DETAIL_RANGE, *upstream_index); + } + UpstreamGrantRequest::Adopted { grant_id } => { + require_nonzero("grant id", *grant_id)?; + frame[KIND_OFFSET] = ADOPTED; + put_u64(&mut frame, RESOURCE_ID_RANGE, *grant_id); + } + UpstreamGrantRequest::Release { resource_id } => { + require_nonzero("resource id", *resource_id)?; + frame[KIND_OFFSET] = RELEASE; + put_u64(&mut frame, RESOURCE_ID_RANGE, *resource_id); + } + } + Ok(frame) +} + +/// Decode and fully validate one worker request. +pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result { + validate_envelope(frame)?; + let request_id = get_u64(frame, REQUEST_ID_RANGE); + let resource_id = get_u64(frame, RESOURCE_ID_RANGE); + let port = get_u16(frame, PORT_RANGE); + let detail = get_u16(frame, DETAIL_RANGE); + let name = get_name(frame)?; + match frame[KIND_OFFSET] { + RESOLVE_TCP => { + require_nonzero("request id", request_id)?; + if resource_id != 0 || port == 0 { + bail!("invalid TCP resolution fields"); + } + let protocol = UpstreamProtocol::decode(detail)?; + validate_normalized_host(&name)?; + Ok(UpstreamGrantRequest::ResolveTcp { + request_id, + protocol, + host: name, + port, + }) + } + CONNECT_TCP => { + require_nonzero("request id", request_id)?; + require_nonzero("selection id", resource_id)?; + require_empty_fields(port, detail, &name)?; + Ok(UpstreamGrantRequest::ConnectTcp { + request_id, + selection_id: resource_id, + }) + } + OPEN_DNS => { + require_nonzero("request id", request_id)?; + if resource_id != 0 || port != 0 || !name.is_empty() { + bail!("invalid DNS grant fields"); + } + Ok(UpstreamGrantRequest::OpenDns { + request_id, + upstream_index: detail, + }) + } + ADOPTED => { + require_nonzero("grant id", resource_id)?; + if request_id != 0 { + bail!("adoption cannot carry a request id"); + } + require_empty_fields(port, detail, &name)?; + Ok(UpstreamGrantRequest::Adopted { grant_id: resource_id }) + } + RELEASE => { + require_nonzero("resource id", resource_id)?; + if request_id != 0 { + bail!("release cannot carry a request id"); + } + require_empty_fields(port, detail, &name)?; + Ok(UpstreamGrantRequest::Release { resource_id }) + } + kind => bail!("invalid upstream request kind {kind}"), + } +} + +/// Encode one coordinator response into its exact fixed-size record. +pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Result<[u8; UPSTREAM_GRANT_FRAME_SIZE]> { + let mut frame = empty_frame(); + match response { + UpstreamGrantResponse::TcpResolved { + request_id, + selection_id, + protocol, + judged_ip, + } => { + require_nonzero("request id", *request_id)?; + require_nonzero("selection id", *selection_id)?; + frame[KIND_OFFSET] = TCP_RESOLVED; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u64(&mut frame, RESOURCE_ID_RANGE, *selection_id); + put_u16(&mut frame, DETAIL_RANGE, protocol.code()); + if let Some(address) = judged_ip { + put_name(&mut frame, &address.to_string())?; + } + } + UpstreamGrantResponse::DescriptorGranted { + request_id, + grant_id, + kind, + } => { + require_nonzero("request id", *request_id)?; + require_nonzero("grant id", *grant_id)?; + frame[KIND_OFFSET] = DESCRIPTOR_GRANTED; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u64(&mut frame, RESOURCE_ID_RANGE, *grant_id); + put_u16(&mut frame, DETAIL_RANGE, kind.code()); + } + UpstreamGrantResponse::Denied { request_id, reason } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = DENIED; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u16(&mut frame, DETAIL_RANGE, reason.code()); + } + } + Ok(frame) +} + +/// Decode and fully validate one coordinator response. +pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result { + validate_envelope(frame)?; + let request_id = get_u64(frame, REQUEST_ID_RANGE); + let resource_id = get_u64(frame, RESOURCE_ID_RANGE); + let port = get_u16(frame, PORT_RANGE); + let detail = get_u16(frame, DETAIL_RANGE); + let name = get_name(frame)?; + require_nonzero("request id", request_id)?; + if port != 0 { + bail!("upstream response cannot carry a port"); + } + match frame[KIND_OFFSET] { + TCP_RESOLVED => { + require_nonzero("selection id", resource_id)?; + let protocol = UpstreamProtocol::decode(detail)?; + let judged_ip = if name.is_empty() { + None + } else { + let address: IpAddr = name.parse().map_err(|_| anyhow::anyhow!("invalid judged IP address"))?; + if address.to_string() != name { + bail!("judged IP address is not canonical"); + } + Some(address) + }; + Ok(UpstreamGrantResponse::TcpResolved { + request_id, + selection_id: resource_id, + protocol, + judged_ip, + }) + } + DESCRIPTOR_GRANTED => { + require_nonzero("grant id", resource_id)?; + if !name.is_empty() { + bail!("descriptor grant cannot carry a name"); + } + Ok(UpstreamGrantResponse::DescriptorGranted { + request_id, + grant_id: resource_id, + kind: UpstreamDescriptorKind::decode(detail)?, + }) + } + DENIED => { + if resource_id != 0 || !name.is_empty() { + bail!("denial cannot carry a resource"); + } + Ok(UpstreamGrantResponse::Denied { + request_id, + reason: UpstreamGrantDenial::decode(detail)?, + }) + } + kind => bail!("invalid upstream response kind {kind}"), + } +} + +fn empty_frame() -> [u8; UPSTREAM_GRANT_FRAME_SIZE] { + let mut frame = [0; UPSTREAM_GRANT_FRAME_SIZE]; + frame[MAGIC_RANGE].copy_from_slice(&MAGIC); + frame[VERSION_OFFSET] = VERSION; + frame +} + +fn validate_envelope(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result<()> { + if frame[MAGIC_RANGE] != MAGIC { + bail!("invalid upstream grant magic"); + } + if frame[VERSION_OFFSET] != VERSION { + bail!("unsupported upstream grant version {}", frame[VERSION_OFFSET]); + } + if frame[RESERVED_RANGE].iter().any(|byte| *byte != 0) { + bail!("upstream grant reserved bytes are nonzero"); + } + Ok(()) +} + +fn validate_normalized_host(host: &str) -> Result<()> { + let bytes = host.as_bytes(); + if bytes.is_empty() || bytes.len() > MAX_UPSTREAM_HOST_BYTES { + bail!("invalid upstream host length {}", bytes.len()); + } + if !host.is_ascii() || host != host.to_ascii_lowercase() || host.ends_with('.') { + bail!("upstream host is not normalized"); + } + if host.contains(':') { + let address: IpAddr = host + .parse() + .map_err(|_| anyhow::anyhow!("invalid upstream IP literal"))?; + if address.to_string() != host { + bail!("upstream IP literal is not canonical"); + } + return Ok(()); + } + for label in host.split('.') { + if label.is_empty() + || label.len() > 63 + || !label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'-' | b'_')) + { + bail!("invalid normalized upstream host"); + } + } + Ok(()) +} + +fn put_name(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], name: &str) -> Result<()> { + let length = u8::try_from(name.len()).map_err(|_| anyhow::anyhow!("upstream name is too long"))?; + if usize::from(length) > MAX_UPSTREAM_HOST_BYTES { + bail!("upstream name is too long"); + } + frame[NAME_LENGTH_OFFSET] = length; + frame[NAME_RANGE.start..NAME_RANGE.start + usize::from(length)].copy_from_slice(name.as_bytes()); + Ok(()) +} + +fn get_name(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result { + let length = usize::from(frame[NAME_LENGTH_OFFSET]); + if length > NAME_RANGE.len() { + bail!("upstream name length exceeds its field"); + } + let (used, padding) = frame[NAME_RANGE].split_at(length); + if padding.iter().any(|byte| *byte != 0) { + bail!("upstream name padding is nonzero"); + } + String::from_utf8(used.to_vec()).map_err(|_| anyhow::anyhow!("upstream name is not UTF-8")) +} + +fn require_nonzero(label: &str, value: u64) -> Result<()> { + if value == 0 { + bail!("{label} cannot be zero"); + } + Ok(()) +} + +fn require_empty_fields(port: u16, detail: u16, name: &str) -> Result<()> { + if port != 0 || detail != 0 || !name.is_empty() { + bail!("ownership record carries unrelated fields"); + } + Ok(()) +} + +fn put_u16(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range, value: u16) { + frame[range].copy_from_slice(&value.to_be_bytes()); +} + +fn get_u16(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range) -> u16 { + u16::from_be_bytes(frame[range].try_into().unwrap()) +} + +fn put_u64(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range, value: u64) { + frame[range].copy_from_slice(&value.to_be_bytes()); +} + +fn get_u64(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range) -> u64 { + u64::from_be_bytes(frame[range].try_into().unwrap()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs new file mode 100644 index 000000000..c5905d976 --- /dev/null +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -0,0 +1,265 @@ +use super::*; + +fn request_roundtrip(request: UpstreamGrantRequest) { + let encoded = encode_upstream_grant_request(&request).unwrap(); + assert_eq!(decode_upstream_grant_request(&encoded).unwrap(), request); +} + +fn response_roundtrip(response: UpstreamGrantResponse) { + let encoded = encode_upstream_grant_response(&response).unwrap(); + assert_eq!(decode_upstream_grant_response(&encoded).unwrap(), response); +} + +#[test] +fn every_request_round_trips_exactly() { + for request in [ + UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Tls, + host: "api.example.com".into(), + port: 443, + }, + UpstreamGrantRequest::ResolveTcp { + request_id: 2, + protocol: UpstreamProtocol::Http, + host: "2001:db8::1".into(), + port: 8080, + }, + UpstreamGrantRequest::ConnectTcp { + request_id: 3, + selection_id: 41, + }, + UpstreamGrantRequest::OpenDns { + request_id: 4, + upstream_index: 0, + }, + UpstreamGrantRequest::Adopted { grant_id: 51 }, + UpstreamGrantRequest::Release { resource_id: 61 }, + ] { + request_roundtrip(request); + } +} + +#[test] +fn every_response_round_trips_exactly() { + for response in [ + UpstreamGrantResponse::TcpResolved { + request_id: 1, + selection_id: 11, + protocol: UpstreamProtocol::Tls, + judged_ip: Some("127.0.0.1".parse().unwrap()), + }, + UpstreamGrantResponse::TcpResolved { + request_id: 2, + selection_id: 12, + protocol: UpstreamProtocol::Http, + judged_ip: Some("2001:db8::1".parse().unwrap()), + }, + UpstreamGrantResponse::TcpResolved { + request_id: 3, + selection_id: 13, + protocol: UpstreamProtocol::Http, + judged_ip: None, + }, + UpstreamGrantResponse::DescriptorGranted { + request_id: 4, + grant_id: 14, + kind: UpstreamDescriptorKind::Tcp, + }, + UpstreamGrantResponse::DescriptorGranted { + request_id: 5, + grant_id: 15, + kind: UpstreamDescriptorKind::DnsUdp, + }, + UpstreamGrantResponse::Denied { + request_id: 6, + reason: UpstreamGrantDenial::Revoked, + }, + ] { + response_roundtrip(response); + } +} + +#[test] +fn descriptor_expectations_are_explicit() { + let request = UpstreamGrantRequest::OpenDns { + request_id: 1, + upstream_index: 0, + }; + assert_eq!(request.expected_descriptor_count(), 0); + assert_eq!( + UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 2, + kind: UpstreamDescriptorKind::DnsUdp, + } + .expected_descriptor_count(), + 1 + ); + assert_eq!( + UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::NotAllowed, + } + .expected_descriptor_count(), + 0 + ); +} + +#[test] +fn request_encoding_rejects_noncanonical_or_oversized_hosts() { + for host in [ + "Example.com".to_string(), + "example.com.".to_string(), + "example..com".to_string(), + "[::1]".to_string(), + "0:0:0:0:0:0:0:1".to_string(), + "a".repeat(MAX_UPSTREAM_HOST_BYTES + 1), + ] { + assert!(encode_upstream_grant_request(&UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Tls, + host, + port: 443, + }) + .is_err()); + } +} + +#[test] +fn maximum_host_fits_and_hostile_length_bytes_do_not_panic() { + let host = ["a".repeat(63), "b".repeat(63), "c".repeat(63), "d".repeat(61)].join("."); + assert_eq!(host.len(), MAX_UPSTREAM_HOST_BYTES); + request_roundtrip(UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Tls, + host, + port: 443, + }); + + let valid = encode_upstream_grant_request(&UpstreamGrantRequest::ConnectTcp { + request_id: 2, + selection_id: 3, + }) + .unwrap(); + for hostile_length in [254, 255] { + let mut malformed = valid; + malformed[NAME_LENGTH_OFFSET] = hostile_length; + assert!(decode_upstream_grant_request(&malformed).is_err()); + } +} + +#[test] +fn zero_identifiers_and_port_are_rejected() { + for request in [ + UpstreamGrantRequest::ResolveTcp { + request_id: 0, + protocol: UpstreamProtocol::Tls, + host: "example.com".into(), + port: 443, + }, + UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Tls, + host: "example.com".into(), + port: 0, + }, + UpstreamGrantRequest::ConnectTcp { + request_id: 1, + selection_id: 0, + }, + UpstreamGrantRequest::Adopted { grant_id: 0 }, + UpstreamGrantRequest::Release { resource_id: 0 }, + ] { + assert!(encode_upstream_grant_request(&request).is_err(), "accepted {request:?}"); + } +} + +#[test] +fn envelope_and_trailing_bytes_are_strict() { + let valid = encode_upstream_grant_request(&UpstreamGrantRequest::ConnectTcp { + request_id: 1, + selection_id: 2, + }) + .unwrap(); + for mutate in [ + |frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE]| frame[0] = b'X', + |frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE]| frame[VERSION_OFFSET] = VERSION + 1, + |frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE]| frame[RESERVED_RANGE.start] = 1, + |frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE]| frame[NAME_RANGE.start + 1] = 1, + ] { + let mut malformed = valid; + mutate(&mut malformed); + assert!(decode_upstream_grant_request(&malformed).is_err()); + } +} + +#[test] +fn wrong_direction_and_unknown_kinds_are_rejected() { + let request = encode_upstream_grant_request(&UpstreamGrantRequest::OpenDns { + request_id: 1, + upstream_index: 0, + }) + .unwrap(); + assert!(decode_upstream_grant_response(&request).is_err()); + let response = encode_upstream_grant_response(&UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::NotConfigured, + }) + .unwrap(); + assert!(decode_upstream_grant_request(&response).is_err()); + let mut unknown = request; + unknown[KIND_OFFSET] = 99; + assert!(decode_upstream_grant_request(&unknown).is_err()); +} + +#[test] +fn malformed_protocol_reason_and_ownership_fields_are_rejected() { + let mut resolve = encode_upstream_grant_request(&UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Tls, + host: "example.com".into(), + port: 443, + }) + .unwrap(); + put_u16(&mut resolve, DETAIL_RANGE, 99); + assert!(decode_upstream_grant_request(&resolve).is_err()); + + let mut denial = encode_upstream_grant_response(&UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::Capacity, + }) + .unwrap(); + put_u16(&mut denial, DETAIL_RANGE, 99); + assert!(decode_upstream_grant_response(&denial).is_err()); + + let mut adopted = encode_upstream_grant_request(&UpstreamGrantRequest::Adopted { grant_id: 7 }).unwrap(); + put_u64(&mut adopted, REQUEST_ID_RANGE, 8); + assert!(decode_upstream_grant_request(&adopted).is_err()); + + let mut released = encode_upstream_grant_request(&UpstreamGrantRequest::Release { resource_id: 9 }).unwrap(); + put_u16(&mut released, PORT_RANGE, 80); + assert!(decode_upstream_grant_request(&released).is_err()); +} + +#[test] +fn response_requires_canonical_ip_and_resource_shape() { + let mut resolved = encode_upstream_grant_response(&UpstreamGrantResponse::TcpResolved { + request_id: 1, + selection_id: 2, + protocol: UpstreamProtocol::Tls, + judged_ip: Some("::1".parse().unwrap()), + }) + .unwrap(); + put_name(&mut resolved, "0:0:0:0:0:0:0:1").unwrap(); + assert!(decode_upstream_grant_response(&resolved).is_err()); + + let mut granted = encode_upstream_grant_response(&UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 3, + kind: UpstreamDescriptorKind::Tcp, + }) + .unwrap(); + put_name(&mut granted, "unexpected").unwrap(); + assert!(decode_upstream_grant_response(&granted).is_err()); +} From 6a458e0e1296405191d21a9e5aeaf08747f7090a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 17:34:20 +0000 Subject: [PATCH 028/464] refactor(core): compile active policy once --- .../src/net/policy_config/active_policy.rs | 42 +++++++++++- .../net/policy_config/active_policy/tests.rs | 65 +++++++++++++++++++ .../capsem-core/src/net/policy_config/mod.rs | 2 +- 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/crates/capsem-core/src/net/policy_config/active_policy.rs b/crates/capsem-core/src/net/policy_config/active_policy.rs index 6155c49f0..8e6bc5ed6 100644 --- a/crates/capsem-core/src/net/policy_config/active_policy.rs +++ b/crates/capsem-core/src/net/policy_config/active_policy.rs @@ -11,10 +11,11 @@ //! is refused, not adapted. use std::collections::BTreeMap; +use std::net::SocketAddr; use serde::{Deserialize, Serialize}; -use super::builder::{merge_plugin_policy, network_config_from_policy_and_dns, MergedPolicies}; +use super::builder::{apply_network_config, merge_plugin_policy, network_config_from_policy_and_dns, MergedPolicies}; use super::provider_profile::{ModelEndpointRegistry, ProviderRuleProfile}; use super::security_rule_profile::{SecurityPluginConfig, SecurityRuleProfile, SecurityRuleSet}; use super::types::{NetworkConfig, SettingsFile}; @@ -39,6 +40,18 @@ pub struct ActivePolicyFile { pub mcp: Option, } +/// Every runtime concern compiled from one validated active-policy value. +/// Callers attach the exact-byte digest and source path they read; this value +/// contains no fallback to ambient settings or built-in defaults on failure. +pub struct CompiledActivePolicy { + pub network: crate::net::policy::NetworkMechanics, + pub dns_upstreams: Vec, + pub security_rules: SecurityRuleSet, + pub plugins: BTreeMap, + pub model_endpoints: ModelEndpointRegistry, + pub mcp: McpConfig, +} + /// Digest of an active policy's serialized bytes, as `blake3:`. /// /// The service writes a session's active policy and capsem-process loads it @@ -103,6 +116,33 @@ impl ActivePolicyFile { Ok(()) } + /// Compile one complete runtime snapshot from this file alone. + pub fn compile_runtime(&self) -> Result { + self.validate()?; + let (user, corp) = self.merged_policy_inputs(); + let mut merged = MergedPolicies::from_files(&user, &corp)?; + apply_network_config(&self.network, &mut merged.network); + let dns_upstreams = self + .network + .dns + .upstreams + .iter() + .map(|upstream| { + upstream + .parse::() + .map_err(|error| format!("parse DNS upstream {upstream:?}: {error}")) + }) + .collect::, _>>()?; + Ok(CompiledActivePolicy { + network: merged.network, + dns_upstreams, + security_rules: merged.security_rules, + plugins: self.plugins.clone(), + model_endpoints: merged.model_endpoints, + mcp: self.mcp.clone().unwrap_or_default(), + }) + } + /// The (user, corp) settings pair `MergedPolicies` builds a session from. pub fn merged_policy_inputs(&self) -> (SettingsFile, SettingsFile) { let user = SettingsFile { diff --git a/crates/capsem-core/src/net/policy_config/active_policy/tests.rs b/crates/capsem-core/src/net/policy_config/active_policy/tests.rs index d9457a061..2c152ea91 100644 --- a/crates/capsem-core/src/net/policy_config/active_policy/tests.rs +++ b/crates/capsem-core/src/net/policy_config/active_policy/tests.rs @@ -127,6 +127,71 @@ upstreams = ["127.0.0.1:5353"] assert_eq!(active.network.dns.upstreams, vec!["127.0.0.1:5353".to_string()]); } +#[test] +fn one_runtime_snapshot_compiles_every_active_policy_concern() { + let active: ActivePolicyFile = toml::from_str( + r#" +[user_rules.profiles.rules.runtime_http] +name = "runtime_http" +action = "allow" +priority = 10 +match = 'http.host == "policy.example"' + +[corp_rules] + +[plugins.credential_broker] +mode = "rewrite" + +[network] +log_bodies = true +http_upstream_ports = [80, 3713] + +[network.dns] +upstreams = ["127.0.0.1:5353", "[::1]:5354"] + +[network.upstream_overrides."policy.example:443"] +dial = "127.0.0.1:3713" +protocol = "http" + +[mcp.server_enabled] +local = false +"#, + ) + .unwrap(); + + let expected_endpoints = active.model_endpoint_registry().unwrap(); + let runtime = active.compile_runtime().unwrap(); + + assert!(runtime + .security_rules + .rules() + .iter() + .any(|rule| rule.rule_id == "profiles.rules.runtime_http")); + assert_eq!(runtime.plugins["credential_broker"].mode, SecurityPluginMode::Rewrite); + assert!(!runtime.mcp.server_enabled["local"]); + assert_eq!(runtime.model_endpoints, expected_endpoints); + assert!(runtime.network.log_bodies); + assert_eq!(runtime.network.http_upstream_ports, vec![80, 3713]); + assert_eq!( + runtime.network.upstream_overrides["policy.example:443"].dial, + "127.0.0.1:3713" + ); + assert_eq!( + runtime.dns_upstreams, + vec!["127.0.0.1:5353".parse().unwrap(), "[::1]:5354".parse().unwrap()] + ); +} + +#[test] +fn runtime_snapshot_refuses_an_invalid_dns_upstream_without_fallback() { + let mut active = + ActivePolicyFile::from_settings_and_corp(&SettingsFile::default(), &SettingsFile::default()).unwrap(); + active.network.dns.upstreams = vec!["resolver.example:domain".into()]; + + let error = active.compile_runtime().err().expect("invalid DNS must fail"); + assert!(error.contains("resolver.example:domain"), "{error}"); +} + #[test] fn mcp_servers_merge_settings_under_corp() { let settings = parse( diff --git a/crates/capsem-core/src/net/policy_config/mod.rs b/crates/capsem-core/src/net/policy_config/mod.rs index dbc7ca4e1..201fdbf52 100644 --- a/crates/capsem-core/src/net/policy_config/mod.rs +++ b/crates/capsem-core/src/net/policy_config/mod.rs @@ -22,7 +22,7 @@ mod settings_policy; mod tree; mod types; -pub use active_policy::{active_policy_digest, ActivePolicyFile}; +pub use active_policy::{active_policy_digest, ActivePolicyFile, CompiledActivePolicy}; pub use builder::*; pub use capsem_config::*; pub use lint::load_merged_lint; From 25479480f5dd320abee0919250e048a335cd8fad Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 17:36:36 +0000 Subject: [PATCH 029/464] refactor(process): use compiled policy snapshot --- crates/capsem-process/src/runtime_config.rs | 47 ++++--------------- .../src/runtime_config/tests.rs | 23 +++++++++ 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/crates/capsem-process/src/runtime_config.rs b/crates/capsem-process/src/runtime_config.rs index 02e6ddaf6..00597425e 100644 --- a/crates/capsem-process/src/runtime_config.rs +++ b/crates/capsem-process/src/runtime_config.rs @@ -1,8 +1,6 @@ use anyhow::{Context, Result}; use capsem_core::net::policy::NetworkMechanics; -use capsem_core::net::policy_config::{ - ActivePolicyFile, MergedPolicies, ModelEndpointRegistry, SecurityPluginConfig, SecurityRuleSet, -}; +use capsem_core::net::policy_config::{ActivePolicyFile, ModelEndpointRegistry, SecurityPluginConfig, SecurityRuleSet}; use capsem_proto::mcp_contracts::McpServerDef; use std::collections::{BTreeMap, HashMap}; use std::net::SocketAddr; @@ -55,45 +53,20 @@ impl RuntimePolicyConfig { active_policy_digest: String, ) -> Result { let path = active_policy_path.display().to_string(); - active - .validate() + let compiled = active + .compile_runtime() .map_err(anyhow::Error::msg) - .with_context(|| format!("validate {path}"))?; - let (user_settings, corp_settings) = active.merged_policy_inputs(); - let merged = MergedPolicies::from_files(&user_settings, &corp_settings) - .map_err(anyhow::Error::msg) - .with_context(|| format!("merge active policy {path}"))?; - let mut network = merged.network; - capsem_core::net::policy_config::apply_network_config(&active.network, &mut network); - let security_rules = active - .compile_security_rule_set() - .map_err(anyhow::Error::msg) - .with_context(|| format!("compile active policy rules from {path}"))?; - let model_endpoints = active - .model_endpoint_registry() - .map_err(anyhow::Error::msg) - .with_context(|| format!("compile active policy model endpoints from {path}"))?; - let dns_upstreams = active - .network - .dns - .upstreams - .iter() - .map(|upstream| { - upstream - .parse::() - .with_context(|| format!("parse DNS upstream {upstream:?} from {path}")) - }) - .collect::>>()?; + .with_context(|| format!("compile active policy {path}"))?; Ok(Self { active_policy_path, active_policy_digest, - network, - dns_upstreams, - security_rules, - plugins: active.plugins.clone(), - model_endpoints, - mcp: active.mcp.clone().unwrap_or_default(), + network: compiled.network, + dns_upstreams: compiled.dns_upstreams, + security_rules: compiled.security_rules, + plugins: compiled.plugins, + model_endpoints: compiled.model_endpoints, + mcp: compiled.mcp, }) } diff --git a/crates/capsem-process/src/runtime_config/tests.rs b/crates/capsem-process/src/runtime_config/tests.rs index df7e3e369..0e8f26177 100644 --- a/crates/capsem-process/src/runtime_config/tests.rs +++ b/crates/capsem-process/src/runtime_config/tests.rs @@ -148,3 +148,26 @@ protocol = "http" .is_none()); assert!(runtime.network.find_upstream_override("evil.example", 443).is_none()); } + +#[test] +fn invalid_runtime_snapshot_names_its_active_policy_without_fallback() { + let dir = tempfile::tempdir().unwrap(); + let active_path = dir.path().join("vm/active_policy.toml"); + std::fs::create_dir_all(active_path.parent().unwrap()).unwrap(); + std::fs::write( + &active_path, + r#" +[user_rules] +[corp_rules] +[network] +[network.dns] +upstreams = ["resolver.example:domain"] +"#, + ) + .unwrap(); + + let error = RuntimePolicySource::new(&active_path).load().unwrap_err(); + let chain = format!("{error:#}"); + assert!(chain.contains(&active_path.display().to_string()), "{chain}"); + assert!(chain.contains("resolver.example:domain"), "{chain}"); +} From 97d30225070116d6efec784c082ae9cf1065d42a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 17:41:02 +0000 Subject: [PATCH 030/464] refactor(core): share upstream target selection --- crates/capsem-core/src/net/mitm_proxy/mod.rs | 3 +- .../src/net/mitm_proxy/upstream.rs | 44 +++++++++++++------ .../src/net/mitm_proxy/upstream/tests.rs | 40 +++++++++++++++++ 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/crates/capsem-core/src/net/mitm_proxy/mod.rs b/crates/capsem-core/src/net/mitm_proxy/mod.rs index 340c8a6b5..26353feaa 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mod.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mod.rs @@ -63,7 +63,7 @@ use fd_stream::{AsyncFdStream, ReplayReader}; use mcp_observe::{observed_mcp_http_request_for_body, should_sniff_mcp_http_body, ObservedMcpHttpRequest}; use protocol::Protocol; use telemetry_hook::TelemetryRequestContext; -use upstream::{CachedUpstream, UpstreamCache, UpstreamTarget}; +use upstream::{CachedUpstream, UpstreamCache}; use util::{ current_unix_ms, format_headers, format_headers_for_domain, http_upstream_port_allowed, is_anthropic_model_name, is_google_model_name, is_llm_api_path, is_openai_model_name, materialize_collected_response_headers, @@ -72,6 +72,7 @@ use util::{ pub use mcp_endpoint::{McpEndpointState, McpTimeouts, ScopedMcpTools}; pub use mcp_frame::{dispatch_logged_mcp_request, McpTransport}; +pub use upstream::UpstreamTarget; /// Re-exported so capsem-app can reference the type without depending on rustls. pub type UpstreamTlsConfig = rustls::ClientConfig; diff --git a/crates/capsem-core/src/net/mitm_proxy/upstream.rs b/crates/capsem-core/src/net/mitm_proxy/upstream.rs index 27c83f5c0..b98d5fea6 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upstream.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upstream.rs @@ -24,7 +24,7 @@ use crate::net::upstream_address::{judged_address, UpstreamResolver}; /// The upstream connection one request will use. #[derive(Clone, Debug, PartialEq, Eq)] -pub(super) enum UpstreamTarget { +pub enum UpstreamTarget { /// An administrator's `upstream_overrides` route, dialed as configured. Override { dial: String, protocol: Protocol }, /// The addresses the guest's name resolved to, in the order to try. @@ -34,6 +34,19 @@ pub(super) enum UpstreamTarget { } impl UpstreamTarget { + /// Resolve one fresh target from trusted policy. The returned address set + /// is what policy judges and what [`Self::connect`] later dials; the name + /// is never resolved a second time. + pub async fn resolve(resolver: &UpstreamResolver, policy: &NetworkMechanics, domain: &str, port: u16) -> Self { + if let Some(target) = Self::override_target(policy, domain, port) { + return target; + } + match resolver.resolve(domain, port).await { + Ok(addresses) => Self::Resolved(addresses), + Err(error) => Self::Unresolved(error), + } + } + /// Fix the target for `domain:port` before the rules run. A keep-alive /// request for the same `domain:port` stays on the address its /// connection was judged and opened with instead of resolving again. @@ -44,15 +57,8 @@ impl UpstreamTarget { port: u16, cache: &UpstreamCache, ) -> Self { - if let Some(route) = policy.find_upstream_override(domain, port) { - let protocol = match route.protocol { - UpstreamOverrideProtocol::Http => Protocol::Http, - UpstreamOverrideProtocol::Tls => Protocol::Tls, - }; - return Self::Override { - dial: route.dial.clone(), - protocol, - }; + if let Some(target) = Self::override_target(policy, domain, port) { + return target; } if let Some(pinned) = cache .lock() @@ -68,10 +74,22 @@ impl UpstreamTarget { } } + fn override_target(policy: &NetworkMechanics, domain: &str, port: u16) -> Option { + let route = policy.find_upstream_override(domain, port)?; + let protocol = match route.protocol { + UpstreamOverrideProtocol::Http => Protocol::Http, + UpstreamOverrideProtocol::Tls => Protocol::Tls, + }; + Some(Self::Override { + dial: route.dial.clone(), + protocol, + }) + } + /// The address the rules see as `ip.value`. A resolved name fails /// closed: any local or private answer speaks for the whole set. An /// override reports only what the guest's literal host said, as before. - pub(super) fn judged_ip(&self, domain: &str) -> Option { + pub fn judged_ip(&self, domain: &str) -> Option { match self { Self::Resolved(addresses) => judged_address(addresses), Self::Override { .. } | Self::Unresolved(_) => domain.parse().ok(), @@ -79,7 +97,7 @@ impl UpstreamTarget { } /// The protocol spoken to the upstream: the override's, else the guest's. - pub(super) fn protocol(&self, guest: Protocol) -> Protocol { + pub fn protocol(&self, guest: Protocol) -> Protocol { match self { Self::Override { protocol, .. } => *protocol, Self::Resolved(_) | Self::Unresolved(_) => guest, @@ -89,7 +107,7 @@ impl UpstreamTarget { /// Connect to exactly this target. Returns the stream and the target /// pinned to the peer it reached, which is what a reused connection is /// keyed by. - pub(super) async fn connect(&self) -> std::io::Result<(TcpStream, Self)> { + pub async fn connect(&self) -> std::io::Result<(TcpStream, Self)> { let stream = match self { Self::Override { dial, .. } => TcpStream::connect(dial.as_str()).await?, Self::Resolved(addresses) => TcpStream::connect(addresses.as_slice()).await?, diff --git a/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs b/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs index f4117c6b5..af0f5b194 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs @@ -40,6 +40,22 @@ async fn select(resolver: &UpstreamResolver, policy: &NetworkMechanics, domain: UpstreamTarget::select(resolver, policy, domain, port, &UpstreamCache::default()).await } +#[tokio::test] +async fn fresh_selection_is_the_same_target_the_mitm_path_judges() { + let resolver = UpstreamResolver::system().with_fixed_answer( + "same.example", + vec!["93.184.216.34".parse().unwrap(), "10.1.2.3".parse().unwrap()], + ); + let policy = NetworkMechanics::new(); + + let trusted = UpstreamTarget::resolve(&resolver, &policy, "same.example", 443).await; + let mitm = select(&resolver, &policy, "same.example", 443).await; + + assert_eq!(trusted, mitm); + assert_eq!(trusted.judged_ip("same.example"), Some("10.1.2.3".parse().unwrap())); + assert_eq!(trusted.protocol(Protocol::Tls), Protocol::Tls); +} + #[tokio::test] async fn a_public_looking_name_that_resolves_locally_is_asked_by_the_default_guard() { for (name, address) in [ @@ -189,3 +205,27 @@ async fn keep_alive_requests_reuse_only_their_own_pinned_upstream() { "the host is part of the key" ); } + +#[tokio::test] +async fn a_trusted_override_preempts_an_existing_resolved_connection() { + let pinned = UpstreamTarget::Resolved(vec!["93.184.216.34:443".parse().unwrap()]); + let cache = UpstreamCache::new(Some(CachedUpstream::new("site.example", 443, pinned, sender().await))); + let mut policy = NetworkMechanics::new(); + policy.upstream_overrides = BTreeMap::from([( + "site.example:443".to_string(), + UpstreamOverride { + dial: "127.0.0.1:3713".to_string(), + protocol: UpstreamOverrideProtocol::Http, + }, + )]); + + let selected = UpstreamTarget::select(&UpstreamResolver::system(), &policy, "site.example", 443, &cache).await; + + assert_eq!( + selected, + UpstreamTarget::Override { + dial: "127.0.0.1:3713".to_string(), + protocol: Protocol::Http, + } + ); +} From e3d64c18755aca5ff9595f45ad8492ab29be72a4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:03:50 +0000 Subject: [PATCH 031/464] test(citadel): record waitid observation shape --- tests/citadel/unix_boundary_debt.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/citadel/unix_boundary_debt.toml b/tests/citadel/unix_boundary_debt.toml index be0d68fc4..d7f77d130 100644 --- a/tests/citadel/unix_boundary_debt.toml +++ b/tests/citadel/unix_boundary_debt.toml @@ -88,7 +88,7 @@ fingerprint = "c1be64a4c316e2674c3627d8193db0d380ccf0d8cf6c4c7bf701ee9e525b51d8" reason = "macOS libproc process-group listing; no nix wrapper" [foundation.raw_calls."crates/capsem-foundation/src/unix/process.rs::waitid"] -fingerprint = "28b3eead89af899046069d614987786cfce11ef7131806a89f7b6da5df2d3363" +fingerprint = "ad944aed3e75f6a36c63b0db8815e8d6bdfd845e5607db8b1f9475ce192d7c91" reason = "observe a child's exit without reaping it (WNOWAIT) on macOS and Linux alike" [foundation.raw_calls."crates/capsem-foundation/src/unix/router_channel.rs::CMSG_DATA"] From 0319c8d28d571345c9dde16e4171cbe775f59df3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:04:12 +0000 Subject: [PATCH 032/464] security(service): broker upstream socket grants --- CHANGELOG.md | 5 + crates/capsem-service/src/active_policy.rs | 53 +- crates/capsem-service/src/instance.rs | 1 + crates/capsem-service/src/main.rs | 1 + crates/capsem-service/src/tests.rs | 1 + crates/capsem-service/src/tests/files_api.rs | 1 + .../capsem-service/src/tests/policy_push.rs | 37 ++ .../src/tests/settings_files.rs | 2 + crates/capsem-service/src/upstream_broker.rs | 521 ++++++++++++++++++ .../src/upstream_broker/tests.rs | 444 +++++++++++++++ .../src/vm_lifecycle/provision.rs | 13 +- .../src/vm_lifecycle/resume_process.rs | 13 +- 12 files changed, 1075 insertions(+), 17 deletions(-) create mode 100644 crates/capsem-service/src/upstream_broker.rs create mode 100644 crates/capsem-service/src/upstream_broker/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index e461d2a6b..6f52545ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Broker VM DNS and TCP upstream access through generation-bound connected + descriptors selected from the service's trusted active-policy snapshot. + Policy reload, worker replacement, release and malformed ownership attempts + revoke live sockets; workers cannot choose DNS servers or dial addresses. + - Route VM-owner metrics through a generation-authenticated service broker with a fixed corporate collector destination, bounded protobuf bodies and no redirects, keeping collector addresses and credentials out of workers; diff --git a/crates/capsem-service/src/active_policy.rs b/crates/capsem-service/src/active_policy.rs index 41be1a9d7..d13145aa6 100644 --- a/crates/capsem-service/src/active_policy.rs +++ b/crates/capsem-service/src/active_policy.rs @@ -12,6 +12,13 @@ use capsem_core::net::policy_config::ActivePolicyFile; pub(crate) struct PublishedActivePolicy { pub(crate) path: PathBuf, pub(crate) digest: String, + runtime: Arc, +} + +impl PublishedActivePolicy { + pub(crate) fn broker_policy(&self) -> Arc { + crate::upstream_broker::BrokerPolicy::new(self.digest.clone(), Arc::clone(&self.runtime)) + } } impl ServiceState { @@ -28,19 +35,27 @@ impl ServiceState { .with_context(|| format!("create {}", active_policy_dir.display()))?; let active_policy_path = active_policy_dir.join(ACTIVE_POLICY_FILE); let serialized = toml::to_string_pretty(&active_policy).context("serialize active policy")?; + let exact: ActivePolicyFile = toml::from_str(&serialized).context("parse serialized active policy")?; + let runtime = Arc::new( + exact + .compile_runtime() + .map_err(anyhow::Error::msg) + .context("compile serialized active policy")?, + ); // capsem-process reads this on every reload: publish it whole. capsem_foundation::unix::fs::atomic_write_private(&active_policy_path, serialized.as_bytes()) .with_context(|| format!("write {}", active_policy_path.display()))?; Ok(PublishedActivePolicy { path: active_policy_path, digest: capsem_core::net::policy_config::active_policy_digest(serialized.as_bytes()), + runtime, }) } /// Re-materialize the active policy of every running VM. Returns each VM's /// id with the digest it was given, or why it could not be; one VM that /// cannot take the policy does not keep the others on the old one. - pub(crate) fn refresh_active_policies(&self) -> Vec<(String, std::result::Result)> { + pub(crate) fn refresh_active_policies(&self) -> Vec<(String, std::result::Result)> { // Copied out first: the instances lock is not held across file writes. let mut targets = Vec::new(); for (id, info) in self.instances.lock().unwrap().iter() { @@ -51,7 +66,6 @@ impl ServiceState { .map(|(id, session_dir)| { let published = self .materialize_active_policy(&session_dir) - .map(|active| active.digest) .map_err(|error| format!("{error:#}")); (id, published) }) @@ -109,17 +123,16 @@ pub(crate) async fn push_policy_to_running_instances( None } // A VM that stopped since it was listed has nothing to reload. - Ok(digest) => Some(( - id.clone(), - instances.get(&id)?.uds_path.clone(), - instances.get(&id)?.generation, - digest, - )), + Ok(published) => { + let instance = instances.get(&id)?; + Some((id.clone(), instance.generation, instance.uds_path.clone(), published)) + } }) .collect::>() }; - let results = futures::future::join_all(targets.iter().map(|(id, uds_path, generation, expected)| async move { + let results = futures::future::join_all(targets.iter().map(|(id, generation, uds_path, published)| async move { + let expected = &published.digest; let request = ServiceToProcess::ReloadConfig { id: state.next_job_id(), }; @@ -128,10 +141,24 @@ pub(crate) async fn push_policy_to_running_instances( active_policy_digest: Some(applied), error: None, .. - }) if &applied == expected => connection_routes::policy::acknowledge(state, id, *generation, expected) - .await - .err() - .map(|_| format!("{id}: applied credential policy ownership unavailable")), + }) if &applied == expected => { + let publisher = { + let instances = state.instances.lock().unwrap(); + match instances.get(id) { + Some(instance) if instance.generation == *generation => Some(instance.upstream_policy.clone()), + Some(_) => return Some(format!("{id}: owner changed before broker policy publication")), + None => None, + } + }; + match publisher { + Some(publisher) => publisher + .publish(published.broker_policy()) + .await + .err() + .map(|error| format!("{id}: {error}")), + None => None, + } + } Ok(ProcessToService::ConfigReloadResult { active_policy_digest: Some(applied), error: None, diff --git a/crates/capsem-service/src/instance.rs b/crates/capsem-service/src/instance.rs index a1cb67957..829700844 100644 --- a/crates/capsem-service/src/instance.rs +++ b/crates/capsem-service/src/instance.rs @@ -42,6 +42,7 @@ pub(crate) struct InstanceInfo { /// One actual spawn, never reused when an ID is resumed or replaced. pub(crate) generation: uuid::Uuid, pub(crate) authority: WorkerAuthority, + pub(crate) upstream_policy: crate::upstream_broker::PolicyPublisher, pub(crate) name: String, pub(crate) asset_pins: BootAssetPins, pub(crate) pid: u32, diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 2d696ea81..cfc5f193a 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -65,6 +65,7 @@ mod startup; mod suspend_confirmation; mod update_command; mod update_status; +mod upstream_broker; mod vm_files; mod vm_lifecycle; use asset_routes::*; diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index d4c8b1e7c..343374aa2 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -140,6 +140,7 @@ pub(crate) fn test_instance() -> InstanceInfo { InstanceInfo { generation: uuid::Uuid::new_v4(), authority: Default::default(), + upstream_policy: upstream_broker::test_policy_publisher(), id: String::new(), name: String::new(), asset_pins: test_asset_pins(), diff --git a/crates/capsem-service/src/tests/files_api.rs b/crates/capsem-service/src/tests/files_api.rs index d530eb599..c2855757f 100644 --- a/crates/capsem-service/src/tests/files_api.rs +++ b/crates/capsem-service/src/tests/files_api.rs @@ -85,6 +85,7 @@ pub(super) fn setup_vm_with_workspace_and_uds( InstanceInfo { generation: uuid::Uuid::new_v4(), authority: Default::default(), + upstream_policy: upstream_broker::test_policy_publisher(), id: vm_id.into(), name: vm_id.into(), asset_pins: test_asset_pins(), diff --git a/crates/capsem-service/src/tests/policy_push.rs b/crates/capsem-service/src/tests/policy_push.rs index 545ee25d6..4776223e2 100644 --- a/crates/capsem-service/src/tests/policy_push.rs +++ b/crates/capsem-service/src/tests/policy_push.rs @@ -219,6 +219,43 @@ async fn a_reload_acknowledging_another_active_policy_fails_the_push() { process.await.unwrap(); } +#[tokio::test] +async fn a_reload_ack_from_a_replaced_generation_cannot_update_its_broker() { + let _env_lock = SETTINGS_ENV_LOCK.lock().await; + let dir = tempfile::tempdir().unwrap(); + let (state, sessions, _env) = state_with_running_vms(&dir, &["replaced-vm"]); + let session_dir = sessions[0].clone(); + let active_path = session_dir.join("vm/active_policy.toml"); + let process = spawn_fake_process(&uds_path(&state, "replaced-vm"), 1, { + let state = Arc::clone(&state); + move |message| { + let ServiceToProcess::ReloadConfig { id } = *message else { + return Box::pin(async { None }); + }; + insert_fake_instance_with_session_dir(&state, "replaced-vm", std::process::id(), session_dir.clone()); + let digest = capsem_core::net::policy_config::active_policy_digest(&std::fs::read(&active_path).unwrap()); + Box::pin(async move { + Some(ProcessToService::ConfigReloadResult { + id, + active_policy_digest: Some(digest), + error: None, + }) + }) + } + }); + + let error = handle_mcp_default_edit(State(Arc::clone(&state)), permission(SecurityRuleAction::Block)) + .await + .expect_err("a stale generation cannot publish into its replacement's broker"); + assert_eq!(error.0, StatusCode::INTERNAL_SERVER_ERROR); + assert!( + error.1.contains("replaced-vm: VM owner changed during IPC admission"), + "{}", + error.1 + ); + process.await.unwrap(); +} + /// Start edit A, hold its VM acknowledgement, then start edit B. Release A /// once B is either waiting for A or has already rewritten A's session. The /// fake VM answers each reload with the digest of the active policy it finds diff --git a/crates/capsem-service/src/tests/settings_files.rs b/crates/capsem-service/src/tests/settings_files.rs index a109e6e56..1bbef41d7 100644 --- a/crates/capsem-service/src/tests/settings_files.rs +++ b/crates/capsem-service/src/tests/settings_files.rs @@ -189,6 +189,7 @@ fn resolve_rejects_symlink_escape() { InstanceInfo { generation: uuid::Uuid::new_v4(), authority: Default::default(), + upstream_policy: upstream_broker::test_policy_publisher(), id: "test-vm".into(), name: "test-vm".into(), asset_pins: test_asset_pins(), @@ -223,6 +224,7 @@ fn resolve_valid_path_inside_workspace() { InstanceInfo { generation: uuid::Uuid::new_v4(), authority: Default::default(), + upstream_policy: upstream_broker::test_policy_publisher(), id: "test-vm".into(), name: "test-vm".into(), asset_pins: test_asset_pins(), diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs new file mode 100644 index 000000000..f9476d50a --- /dev/null +++ b/crates/capsem-service/src/upstream_broker.rs @@ -0,0 +1,521 @@ +//! Generation-bound connected-socket grants for one VM owner. +//! +//! The coordinator selects and opens upstream sockets from its trusted active +//! policy snapshot, then passes the connected descriptor over the inherited +//! socketpair. Payload bytes never transit the coordinator. The worker cannot +//! name a DNS server or connect an address directly; TCP resolution yields an +//! opaque selection that must be connected on this same generation channel. + +use std::collections::HashMap; +use std::io; +use std::os::fd::{AsFd as _, AsRawFd as _, OwnedFd}; +use std::os::unix::net::UnixStream; +use std::sync::Arc; +use std::time::Duration; + +use capsem_core::net::mitm_proxy::protocol::Protocol; +use capsem_core::net::mitm_proxy::UpstreamTarget; +use capsem_core::net::policy_config::CompiledActivePolicy; +use capsem_core::net::upstream_address::UpstreamResolver; +use capsem_foundation::unix::fd::{self, SocketShutdown}; +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::upstream_grant::{ + decode_upstream_grant_request, encode_upstream_grant_response, UpstreamDescriptorKind, UpstreamGrantDenial, + UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, +}; +use tokio::sync::{mpsc, oneshot}; +use tracing::{debug, warn}; + +use crate::instance::WorkerGrant; + +const MAX_SELECTIONS: usize = 64; +const MAX_ACTIVE_GRANTS: usize = 256; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const RESPONSE_TIMEOUT: Duration = Duration::from_secs(1); + +type WireSender = DescriptorSender; +type WireReceiver = DescriptorReceiver; + +pub(crate) struct BrokerPolicy { + digest: String, + runtime: Arc, +} + +#[derive(Clone)] +pub(crate) struct PolicyPublisher { + updates: Option>, +} +pub(crate) struct PendingBroker { + coordinator: UnixStream, + worker: UnixStream, + initial_policy: Arc, + updates: mpsc::Receiver, +} + +struct PolicyUpdate { + policy: Arc, + applied: oneshot::Sender>, +} + +struct Selection { + policy_digest: String, + target: UpstreamTarget, +} + +struct ActiveGrant { + descriptor: OwnedFd, + kind: UpstreamDescriptorKind, + adopted: bool, +} + +impl BrokerPolicy { + pub(crate) fn new(digest: String, runtime: Arc) -> Arc { + Arc::new(Self { digest, runtime }) + } +} + +impl PolicyPublisher { + pub(crate) async fn publish(&self, policy: Arc) -> Result<(), String> { + let Some(updates) = &self.updates else { + return Ok(()); + }; + let (applied, confirmation) = oneshot::channel(); + updates + .send(PolicyUpdate { policy, applied }) + .await + .map_err(|_| "upstream policy broker stopped".to_string())?; + confirmation + .await + .map_err(|_| "upstream policy broker stopped before applying the policy".to_string())? + } +} + +impl PendingBroker { + pub(crate) fn pair(initial_policy: Arc) -> io::Result<(Self, PolicyPublisher)> { + let (coordinator, worker) = UnixStream::pair()?; + let (updates_tx, updates) = mpsc::channel(1); + Ok(( + Self { + coordinator, + worker, + initial_policy, + updates, + }, + PolicyPublisher { + updates: Some(updates_tx), + }, + )) + } + + pub(crate) fn worker_stdio(&self) -> io::Result { + let descriptor: OwnedFd = self.worker.try_clone()?.into(); + Ok(std::process::Stdio::from(descriptor)) + } + + pub(crate) fn start(self, authority: WorkerGrant) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + if let Err(error) = run(self.coordinator, self.initial_policy, self.updates, authority).await { + debug!(%error, "upstream broker stopped"); + } + }) + } +} + +async fn run( + socket: UnixStream, + mut policy: Arc, + mut updates: mpsc::Receiver, + authority: WorkerGrant, +) -> Result<(), String> { + let requests = + WireReceiver::new(socket.try_clone().map_err(|error| error.to_string())?).map_err(|error| error.to_string())?; + let responses = WireSender::new(socket).map_err(|error| error.to_string())?; + let (request_tx, mut request_rx) = mpsc::channel(1); + let request_pump = tokio::spawn(async move { + loop { + let frame = requests.recv().await; + let complete = frame.is_err(); + if request_tx.send(frame).await.is_err() || complete { + return; + } + } + }); + let mut selections = HashMap::new(); + let mut active = HashMap::new(); + let mut next_resource_id = 1_u64; + let mut awaiting_adoption = None; + let result = async { + 'requests: loop { + let frame = tokio::select! { + biased; + () = authority.revoked() => break Err("worker authority revoked".to_string()), + update = updates.recv() => { + apply_policy_update( + update.ok_or_else(|| "upstream policy publisher closed".to_string())?, + &mut policy, + &mut selections, + &mut active, + &mut awaiting_adoption, + )?; + continue; + } + frame = request_rx.recv() => frame + .ok_or_else(|| "upstream grant request pump stopped".to_string())? + .map_err(|error| format!("receive upstream grant request: {error}"))?, + }; + if !frame.fds.is_empty() { + break Err("worker sent an upstream descriptor".into()); + } + let request = decode_upstream_grant_request(&frame.bytes) + .map_err(|error| format!("decode upstream grant request: {error:#}"))?; + if let Some(expected) = awaiting_adoption { + if request != (UpstreamGrantRequest::Adopted { grant_id: expected }) { + break Err(format!( + "descriptor grant {expected} was not adopted before the next request" + )); + } + } + match request { + UpstreamGrantRequest::ResolveTcp { + request_id, + protocol, + host, + port, + } => { + let current = Arc::clone(&policy); + selections.retain(|_, selection: &mut Selection| selection.policy_digest == current.digest); + if selections.len() >= MAX_SELECTIONS { + send_denied(&responses, request_id, UpstreamGrantDenial::Capacity).await?; + continue; + } + if protocol == UpstreamProtocol::Http + && !current.runtime.network.http_upstream_ports.is_empty() + && !current.runtime.network.http_upstream_ports.contains(&port) + { + send_denied(&responses, request_id, UpstreamGrantDenial::NotAllowed).await?; + continue; + } + let guest_protocol = core_protocol(protocol); + let resolver = UpstreamResolver::system(); + let target = tokio::select! { + biased; + () = authority.revoked() => break Err("worker authority revoked".to_string()), + update = updates.recv() => { + apply_policy_update( + update.ok_or_else(|| "upstream policy publisher closed".to_string())?, + &mut policy, + &mut selections, + &mut active, + &mut awaiting_adoption, + )?; + continue 'requests; + } + target = UpstreamTarget::resolve( + &resolver, + ¤t.runtime.network, + &host, + port, + ) => target, + }; + if matches!(target, UpstreamTarget::Unresolved(_)) { + send_denied(&responses, request_id, UpstreamGrantDenial::ResolveFailed).await?; + continue; + } + let selection_id = allocate_id(&mut next_resource_id)?; + let judged_ip = target.judged_ip(&host); + let effective_protocol = wire_protocol(target.protocol(guest_protocol))?; + selections.insert( + selection_id, + Selection { + policy_digest: current.digest.clone(), + target, + }, + ); + send_response( + &responses, + &UpstreamGrantResponse::TcpResolved { + request_id, + selection_id, + protocol: effective_protocol, + judged_ip, + }, + None, + ) + .await?; + } + UpstreamGrantRequest::ConnectTcp { + request_id, + selection_id, + } => { + if active.len() >= MAX_ACTIVE_GRANTS { + send_denied(&responses, request_id, UpstreamGrantDenial::Capacity).await?; + continue; + } + let current_digest = policy.digest.clone(); + let Some(selection) = selections.remove(&selection_id) else { + send_denied(&responses, request_id, UpstreamGrantDenial::InvalidResource).await?; + continue; + }; + if selection.policy_digest != current_digest { + send_denied(&responses, request_id, UpstreamGrantDenial::InvalidResource).await?; + continue; + } + let connected = tokio::select! { + biased; + () = authority.revoked() => break Err("worker authority revoked".to_string()), + update = updates.recv() => { + apply_policy_update( + update.ok_or_else(|| "upstream policy publisher closed".to_string())?, + &mut policy, + &mut selections, + &mut active, + &mut awaiting_adoption, + )?; + continue 'requests; + } + result = tokio::time::timeout(CONNECT_TIMEOUT, selection.target.connect()) => result, + }; + let stream = match connected { + Ok(Ok((stream, _pinned))) => stream, + Ok(Err(error)) => { + warn!(selection_id, %error, "upstream broker TCP connect failed"); + send_denied(&responses, request_id, UpstreamGrantDenial::ConnectFailed).await?; + continue; + } + Err(_) => { + warn!(selection_id, "upstream broker TCP connect timed out"); + send_denied(&responses, request_id, UpstreamGrantDenial::ConnectFailed).await?; + continue; + } + }; + let descriptor: OwnedFd = stream + .into_std() + .map_err(|error| format!("adopt connected TCP stream: {error}"))? + .into(); + let grant_id = allocate_id(&mut next_resource_id)?; + active.insert( + grant_id, + ActiveGrant { + descriptor, + kind: UpstreamDescriptorKind::Tcp, + adopted: false, + }, + ); + send_active_grant(&responses, &active, request_id, grant_id, UpstreamDescriptorKind::Tcp).await?; + awaiting_adoption = Some(grant_id); + } + UpstreamGrantRequest::OpenDns { + request_id, + upstream_index, + } => { + if active.len() >= MAX_ACTIVE_GRANTS { + send_denied(&responses, request_id, UpstreamGrantDenial::Capacity).await?; + continue; + } + let current = Arc::clone(&policy); + let Some(upstream) = current.runtime.dns_upstreams.get(usize::from(upstream_index)).copied() else { + send_denied(&responses, request_id, UpstreamGrantDenial::NotConfigured).await?; + continue; + }; + let bind = if upstream.is_ipv6() { "[::]:0" } else { "0.0.0.0:0" }; + let socket = tokio::select! { + biased; + () = authority.revoked() => break Err("worker authority revoked".to_string()), + update = updates.recv() => { + apply_policy_update( + update.ok_or_else(|| "upstream policy publisher closed".to_string())?, + &mut policy, + &mut selections, + &mut active, + &mut awaiting_adoption, + )?; + continue 'requests; + } + result = tokio::net::UdpSocket::bind(bind) => result, + } + .map_err(|error| format!("bind broker DNS socket: {error}"))?; + let connected = tokio::select! { + biased; + () = authority.revoked() => break Err("worker authority revoked".to_string()), + update = updates.recv() => { + apply_policy_update( + update.ok_or_else(|| "upstream policy publisher closed".to_string())?, + &mut policy, + &mut selections, + &mut active, + &mut awaiting_adoption, + )?; + continue 'requests; + } + result = socket.connect(upstream) => result, + }; + if let Err(error) = connected { + warn!(%upstream, %error, "upstream broker DNS connect failed"); + send_denied(&responses, request_id, UpstreamGrantDenial::ConnectFailed).await?; + continue; + } + let descriptor: OwnedFd = socket + .into_std() + .map_err(|error| format!("adopt connected DNS socket: {error}"))? + .into(); + let grant_id = allocate_id(&mut next_resource_id)?; + active.insert( + grant_id, + ActiveGrant { + descriptor, + kind: UpstreamDescriptorKind::DnsUdp, + adopted: false, + }, + ); + send_active_grant( + &responses, + &active, + request_id, + grant_id, + UpstreamDescriptorKind::DnsUdp, + ) + .await?; + awaiting_adoption = Some(grant_id); + } + UpstreamGrantRequest::Adopted { grant_id } => { + let Some(grant) = active.get_mut(&grant_id) else { + break Err(format!("adopted unknown descriptor grant {grant_id}")); + }; + if grant.adopted || awaiting_adoption != Some(grant_id) { + break Err(format!("invalid descriptor adoption {grant_id}")); + } + grant.adopted = true; + awaiting_adoption = None; + } + UpstreamGrantRequest::Release { resource_id } => { + if selections.remove(&resource_id).is_none() { + let Some(grant) = active.remove(&resource_id) else { + break Err(format!("released unknown upstream resource {resource_id}")); + }; + if !grant.adopted { + break Err(format!("released unadopted descriptor grant {resource_id}")); + } + revoke_grant(resource_id, grant)?; + } + } + } + } + } + .await; + request_pump.abort(); + let _ = request_pump.await; + let cleanup = revoke_active(active); + match (result, cleanup) { + (Ok(()), cleanup) => cleanup, + (Err(error), Ok(())) => Err(error), + (Err(error), Err(cleanup)) => Err(format!("{error}; {cleanup}")), + } +} + +async fn send_active_grant( + sender: &WireSender, + active: &HashMap, + request_id: u64, + grant_id: u64, + kind: UpstreamDescriptorKind, +) -> Result<(), String> { + let grant = active.get(&grant_id).ok_or("new descriptor grant disappeared")?; + send_response( + sender, + &UpstreamGrantResponse::DescriptorGranted { + request_id, + grant_id, + kind, + }, + Some(&grant.descriptor), + ) + .await +} + +async fn send_denied(sender: &WireSender, request_id: u64, reason: UpstreamGrantDenial) -> Result<(), String> { + send_response(sender, &UpstreamGrantResponse::Denied { request_id, reason }, None).await +} + +async fn send_response( + sender: &WireSender, + response: &UpstreamGrantResponse, + descriptor: Option<&OwnedFd>, +) -> Result<(), String> { + let bytes = + encode_upstream_grant_response(response).map_err(|error| format!("encode grant response: {error:#}"))?; + let descriptors = descriptor.map_or_else(Vec::new, |fd| vec![fd.as_raw_fd()]); + tokio::time::timeout(RESPONSE_TIMEOUT, sender.send(&bytes, &descriptors)) + .await + .map_err(|_| "send upstream grant response timed out".to_string())? + .map_err(|error| format!("send upstream grant response: {error}"))?; + Ok(()) +} + +fn allocate_id(next: &mut u64) -> Result { + let id = *next; + if id == 0 { + return Err("upstream broker exhausted resource ids".into()); + } + *next = next.checked_add(1).unwrap_or(0); + Ok(id) +} + +fn apply_policy_update( + update: PolicyUpdate, + current: &mut Arc, + selections: &mut HashMap, + active: &mut HashMap, + awaiting_adoption: &mut Option, +) -> Result<(), String> { + *current = update.policy; + selections.clear(); + let result = revoke_active(std::mem::take(active)); + *awaiting_adoption = None; + let _ = update.applied.send(result.clone()); + result +} + +fn core_protocol(protocol: UpstreamProtocol) -> Protocol { + match protocol { + UpstreamProtocol::Http => Protocol::Http, + UpstreamProtocol::Tls => Protocol::Tls, + } +} + +fn wire_protocol(protocol: Protocol) -> Result { + match protocol { + Protocol::Http => Ok(UpstreamProtocol::Http), + Protocol::Tls => Ok(UpstreamProtocol::Tls), + Protocol::McpFrame | Protocol::Unknown => Err("invalid upstream broker protocol".into()), + } +} + +fn revoke_active(active: HashMap) -> Result<(), String> { + let mut failures = Vec::new(); + for (grant_id, grant) in active { + if let Err(error) = revoke_grant(grant_id, grant) { + failures.push(error); + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("; ")) + } +} + +fn revoke_grant(grant_id: u64, grant: ActiveGrant) -> Result<(), String> { + let result = match grant.kind { + UpstreamDescriptorKind::Tcp => fd::reset_tcp(grant.descriptor.as_fd()).map(|_| ()), + UpstreamDescriptorKind::DnsUdp => fd::shutdown(grant.descriptor.as_fd(), SocketShutdown::Both), + }; + result.map_err(|error| format!("revoke upstream descriptor {grant_id}: {error}")) +} + +#[cfg(test)] +pub(crate) fn test_policy_publisher() -> PolicyPublisher { + PolicyPublisher { updates: None } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs new file mode 100644 index 000000000..46eae894c --- /dev/null +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -0,0 +1,444 @@ +use std::collections::BTreeMap; +use std::fs::File; +use std::net::SocketAddr; +use std::os::fd::{AsRawFd as _, OwnedFd}; + +use capsem_core::net::policy::{UpstreamOverride, UpstreamOverrideProtocol}; +use capsem_core::net::policy_config::{ActivePolicyFile, SettingsFile}; +use capsem_proto::upstream_grant::{ + decode_upstream_grant_response, encode_upstream_grant_request, UpstreamGrantResponse, +}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + +use super::*; +use crate::instance::WorkerAuthority; + +struct TestClient { + requests: WireSender, + responses: WireReceiver, +} + +impl TestClient { + fn new(socket: UnixStream) -> Self { + Self { + requests: WireSender::new(socket.try_clone().unwrap()).unwrap(), + responses: WireReceiver::new(socket).unwrap(), + } + } + + async fn send(&self, request: &UpstreamGrantRequest) { + let bytes = encode_upstream_grant_request(request).unwrap(); + self.requests.send(&bytes, &[]).await.unwrap(); + } + + async fn request(&self, request: &UpstreamGrantRequest) -> (UpstreamGrantResponse, Vec) { + self.send(request).await; + let frame = self.responses.recv().await.unwrap(); + let response = decode_upstream_grant_response(&frame.bytes).unwrap(); + assert_eq!(frame.fds.len(), response.expected_descriptor_count()); + (response, frame.fds) + } +} + +fn test_policy(digest: &str, tcp_override: Option, dns_upstreams: Vec) -> Arc { + let active = ActivePolicyFile::from_settings_and_corp(&SettingsFile::default(), &SettingsFile::default()).unwrap(); + let mut runtime = active.compile_runtime().unwrap(); + runtime.dns_upstreams = dns_upstreams; + runtime.network.upstream_overrides = tcp_override.map_or_else(BTreeMap::new, |dial| { + BTreeMap::from([( + "policy.example:443".into(), + UpstreamOverride { + dial: dial.to_string(), + protocol: UpstreamOverrideProtocol::Http, + }, + )]) + }); + BrokerPolicy::new(digest.into(), Arc::new(runtime)) +} + +fn start_broker( + policy: Arc, +) -> ( + TestClient, + PolicyPublisher, + WorkerAuthority, + tokio::task::JoinHandle>, +) { + let (pending, publisher) = PendingBroker::pair(policy).unwrap(); + let client = TestClient::new(pending.worker); + let authority = WorkerAuthority::default(); + let task = tokio::spawn(run( + pending.coordinator, + pending.initial_policy, + pending.updates, + authority.grant(), + )); + (client, publisher, authority, task) +} + +async fn resolve_override(client: &TestClient, request_id: u64) -> u64 { + let (response, fds) = client + .request(&UpstreamGrantRequest::ResolveTcp { + request_id, + protocol: UpstreamProtocol::Tls, + host: "policy.example".into(), + port: 443, + }) + .await; + assert!(fds.is_empty()); + let UpstreamGrantResponse::TcpResolved { + request_id: response_id, + selection_id, + protocol, + judged_ip, + } = response + else { + panic!("expected a resolved target, got {response:?}"); + }; + assert_eq!(response_id, request_id); + assert_eq!(protocol, UpstreamProtocol::Http); + assert_eq!(judged_ip, None); + selection_id +} + +async fn connect_selection(client: &TestClient, request_id: u64, selection_id: u64) -> (u64, OwnedFd) { + let (response, mut fds) = client + .request(&UpstreamGrantRequest::ConnectTcp { + request_id, + selection_id, + }) + .await; + let UpstreamGrantResponse::DescriptorGranted { + request_id: response_id, + grant_id, + kind, + } = response + else { + panic!("expected a descriptor grant, got {response:?}"); + }; + assert_eq!(response_id, request_id); + assert_eq!(kind, UpstreamDescriptorKind::Tcp); + (grant_id, fds.pop().unwrap()) +} + +async fn stop_broker(authority: WorkerAuthority, task: tokio::task::JoinHandle>) { + authority.revoke(); + let error = tokio::time::timeout(Duration::from_secs(1), task) + .await + .expect("broker stops when authority is revoked") + .unwrap() + .unwrap_err(); + assert_eq!(error, "worker authority revoked"); +} + +fn tcp_stream(descriptor: OwnedFd) -> tokio::net::TcpStream { + let stream = std::net::TcpStream::from(descriptor); + stream.set_nonblocking(true).unwrap(); + tokio::net::TcpStream::from_std(stream).unwrap() +} + +#[tokio::test] +async fn trusted_override_grants_only_its_connected_tcp_stream() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (client, _publisher, authority, task) = + start_broker(test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![])); + + let selection_id = resolve_override(&client, 1).await; + let (grant_id, descriptor) = connect_selection(&client, 2, selection_id).await; + let (mut upstream, peer) = listener.accept().await.unwrap(); + assert_eq!(peer.ip().to_string(), "127.0.0.1"); + client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; + + let mut granted = tcp_stream(descriptor); + granted.write_all(b"brokered").await.unwrap(); + let mut received = [0_u8; 8]; + upstream.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"brokered"); + + client + .send(&UpstreamGrantRequest::Release { resource_id: grant_id }) + .await; + let mut byte = [0_u8; 1]; + let revoked = tokio::time::timeout(Duration::from_secs(1), upstream.read(&mut byte)) + .await + .expect("release revokes the peer connection"); + assert!( + matches!(revoked, Err(_) | Ok(0)), + "peer stayed usable after release: {revoked:?}" + ); + stop_broker(authority, task).await; +} + +#[tokio::test] +async fn tcp_selection_is_single_use() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (client, _publisher, authority, task) = + start_broker(test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![])); + let selection_id = resolve_override(&client, 1).await; + let (grant_id, descriptor) = connect_selection(&client, 2, selection_id).await; + let (_upstream, _) = listener.accept().await.unwrap(); + client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; + + let (response, fds) = client + .request(&UpstreamGrantRequest::ConnectTcp { + request_id: 3, + selection_id, + }) + .await; + assert!(fds.is_empty()); + assert_eq!( + response, + UpstreamGrantResponse::Denied { + request_id: 3, + reason: UpstreamGrantDenial::InvalidResource, + } + ); + assert!(tokio::time::timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err()); + client + .send(&UpstreamGrantRequest::Release { resource_id: grant_id }) + .await; + drop(descriptor); + stop_broker(authority, task).await; +} + +#[tokio::test] +async fn unresolved_selection_count_is_bounded() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (client, _publisher, authority, task) = + start_broker(test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![])); + let mut selections = Vec::new(); + for request_id in 1..=MAX_SELECTIONS as u64 { + selections.push(resolve_override(&client, request_id).await); + } + let request_id = MAX_SELECTIONS as u64 + 1; + let (response, fds) = client + .request(&UpstreamGrantRequest::ResolveTcp { + request_id, + protocol: UpstreamProtocol::Tls, + host: "policy.example".into(), + port: 443, + }) + .await; + assert!(fds.is_empty()); + assert_eq!( + response, + UpstreamGrantResponse::Denied { + request_id, + reason: UpstreamGrantDenial::Capacity, + } + ); + client + .send(&UpstreamGrantRequest::Release { + resource_id: selections[0], + }) + .await; + assert_ne!(resolve_override(&client, request_id + 1).await, 0); + stop_broker(authority, task).await; +} + +#[tokio::test] +async fn dns_grant_uses_only_the_configured_index() { + let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let (client, _publisher, authority, task) = + start_broker(test_policy("policy-a", None, vec![upstream.local_addr().unwrap()])); + + let (response, mut fds) = client + .request(&UpstreamGrantRequest::OpenDns { + request_id: 1, + upstream_index: 0, + }) + .await; + let UpstreamGrantResponse::DescriptorGranted { grant_id, kind, .. } = response else { + panic!("expected a DNS descriptor grant, got {response:?}"); + }; + assert_eq!(kind, UpstreamDescriptorKind::DnsUdp); + client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; + let socket = std::net::UdpSocket::from(fds.pop().unwrap()); + socket.set_nonblocking(true).unwrap(); + let granted = tokio::net::UdpSocket::from_std(socket).unwrap(); + granted.send(b"query").await.unwrap(); + let mut received = [0_u8; 5]; + let (count, _peer) = upstream.recv_from(&mut received).await.unwrap(); + assert_eq!(&received[..count], b"query"); + + let (denial, fds) = client + .request(&UpstreamGrantRequest::OpenDns { + request_id: 2, + upstream_index: 1, + }) + .await; + assert!(fds.is_empty()); + assert_eq!( + denial, + UpstreamGrantResponse::Denied { + request_id: 2, + reason: UpstreamGrantDenial::NotConfigured, + } + ); + client + .send(&UpstreamGrantRequest::Release { resource_id: grant_id }) + .await; + let (barrier, barrier_fds) = client + .request(&UpstreamGrantRequest::OpenDns { + request_id: 3, + upstream_index: 1, + }) + .await; + assert!(barrier_fds.is_empty()); + assert!(matches!( + barrier, + UpstreamGrantResponse::Denied { + request_id: 3, + reason: UpstreamGrantDenial::NotConfigured + } + )); + assert!(granted.send(b"after-release").await.is_err()); + stop_broker(authority, task).await; +} + +#[tokio::test] +async fn policy_publication_revokes_grants_and_invalidates_selections() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let (client, publisher, authority, task) = start_broker(test_policy("policy-a", Some(address), vec![])); + let live_selection = resolve_override(&client, 1).await; + let (grant_id, descriptor) = connect_selection(&client, 2, live_selection).await; + let (mut upstream, _) = listener.accept().await.unwrap(); + client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; + let stale_selection = resolve_override(&client, 3).await; + + publisher + .publish(test_policy("policy-b", Some(address), vec![])) + .await + .unwrap(); + let (denial, fds) = client + .request(&UpstreamGrantRequest::ConnectTcp { + request_id: 4, + selection_id: stale_selection, + }) + .await; + assert!(fds.is_empty()); + assert_eq!( + denial, + UpstreamGrantResponse::Denied { + request_id: 4, + reason: UpstreamGrantDenial::InvalidResource, + } + ); + let mut byte = [0_u8; 1]; + let revoked = tokio::time::timeout(Duration::from_secs(1), upstream.read(&mut byte)) + .await + .expect("policy publication revokes an old grant"); + assert!( + matches!(revoked, Err(_) | Ok(0)), + "old policy grant stayed usable: {revoked:?}" + ); + drop(descriptor); + stop_broker(authority, task).await; +} + +#[tokio::test] +async fn generation_revocation_resets_every_live_grant() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (client, _publisher, authority, task) = + start_broker(test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![])); + let selection_id = resolve_override(&client, 1).await; + let (grant_id, descriptor) = connect_selection(&client, 2, selection_id).await; + let (mut upstream, _) = listener.accept().await.unwrap(); + client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; + + authority.revoke(); + let error = tokio::time::timeout(Duration::from_secs(1), task) + .await + .expect("revoked generation stops its broker") + .unwrap() + .unwrap_err(); + assert_eq!(error, "worker authority revoked"); + let mut byte = [0_u8; 1]; + let revoked = tokio::time::timeout(Duration::from_secs(1), upstream.read(&mut byte)) + .await + .expect("generation revocation resets its granted connection"); + assert!(matches!(revoked, Err(_) | Ok(0))); + drop(descriptor); +} + +#[tokio::test] +async fn next_request_before_adoption_terminates_and_revokes_the_channel() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (client, _publisher, _authority, task) = + start_broker(test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![])); + let selection_id = resolve_override(&client, 1).await; + let (_grant_id, descriptor) = connect_selection(&client, 2, selection_id).await; + let (mut upstream, _) = listener.accept().await.unwrap(); + + client + .send(&UpstreamGrantRequest::Release { + resource_id: selection_id, + }) + .await; + let error = tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!(error.contains("was not adopted"), "{error}"); + let mut byte = [0_u8; 1]; + let revoked = tokio::time::timeout(Duration::from_secs(1), upstream.read(&mut byte)) + .await + .expect("protocol violation revokes the unadopted grant"); + assert!(matches!(revoked, Err(_) | Ok(0))); + drop(descriptor); +} + +#[tokio::test] +async fn worker_supplied_descriptor_terminates_before_resolution() { + let (client, _publisher, _authority, task) = start_broker(test_policy("policy-a", None, vec![])); + let request = UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Tls, + host: "example.com".into(), + port: 443, + }; + let bytes = encode_upstream_grant_request(&request).unwrap(); + let descriptor = File::open("/dev/null").unwrap(); + client.requests.send(&bytes, &[descriptor.as_raw_fd()]).await.unwrap(); + + let error = tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(error, "worker sent an upstream descriptor"); + assert!(client.responses.recv().await.is_err()); +} + +#[tokio::test] +async fn disallowed_plain_http_port_is_denied_before_dial() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + assert!(![80, 3128, 3713, 8080, 11434].contains(&port)); + let (client, _publisher, authority, task) = start_broker(test_policy("policy-a", None, vec![])); + + let (response, fds) = client + .request(&UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Http, + host: "127.0.0.1".into(), + port, + }) + .await; + assert!(fds.is_empty()); + assert_eq!( + response, + UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::NotAllowed, + } + ); + assert!(tokio::time::timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err()); + stop_broker(authority, task).await; +} diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index e87f61b91..b8205d2b4 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -154,7 +154,11 @@ impl ServiceState { } } - let active_policy_path = self.materialize_active_policy(&session_dir)?.path; + let active_policy = self.materialize_active_policy(&session_dir)?; + let active_policy_path = active_policy.path.clone(); + let (upstream_broker, upstream_policy) = + crate::upstream_broker::PendingBroker::pair(active_policy.broker_policy())?; + let upstream_stdio = upstream_broker.worker_stdio()?; info!(process_binary = %self.process_binary.display(), exists = self.process_binary.exists(), "checking process_binary"); @@ -262,6 +266,7 @@ impl ServiceState { .arg(&self.run_dir) .arg("--service-socket") .arg(&self.service_socket) + .stdin(upstream_stdio) .stdout(std::process::Stdio::from(process_log_file.try_clone()?)) .stderr(std::process::Stdio::from(process_log_file)); self.retirements.spawn(id, generation, &mut child_cmd) @@ -327,12 +332,15 @@ impl ServiceState { } } + let authority = crate::instance::WorkerAuthority::default(); + let upstream_grant = authority.grant(); let mut instances = self.instances.lock().unwrap(); instances.insert( id.to_string(), InstanceInfo { generation, - authority: Default::default(), + authority, + upstream_policy, id: id.to_string(), name: name.to_string(), asset_pins, @@ -349,6 +357,7 @@ impl ServiceState { }, ); drop(instances); + let _upstream_broker = upstream_broker.start(upstream_grant); let _reaper = instance_reaper::spawn_exit_reaper( child, id.to_string(), diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index dc3648437..8be58b454 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -65,7 +65,11 @@ impl ServiceState { service_runtime::remove_instance_sentinels(&uds_path); self.validate_persistent_entry(&entry)?; - let active_policy_path = self.materialize_active_policy(&entry.session_dir)?.path; + let active_policy = self.materialize_active_policy(&entry.session_dir)?; + let active_policy_path = active_policy.path.clone(); + let (upstream_broker, upstream_policy) = + crate::upstream_broker::PendingBroker::pair(active_policy.broker_policy())?; + let upstream_stdio = upstream_broker.worker_stdio()?; let scratch_disk_size_gb = session_rootfs_size_gb(&entry)?; let resolved = self.resolve_pinned_asset_paths(&entry.asset_pins)?; self.validate_pinned_asset_files(&resolved, &entry.asset_pins)?; @@ -191,6 +195,7 @@ impl ServiceState { .arg(&self.run_dir) .arg("--service-socket") .arg(&self.service_socket) + .stdin(upstream_stdio) .stdout(std::process::Stdio::from(process_log_file.try_clone()?)) .stderr(std::process::Stdio::from(process_log_file)); self.retirements.spawn(&vm_id, generation, &mut child_cmd) @@ -222,13 +227,16 @@ impl ServiceState { ); } + let authority = crate::instance::WorkerAuthority::default(); + let upstream_grant = authority.grant(); let session_dir = entry.session_dir.clone(); let mut instances = self.instances.lock().unwrap(); instances.insert( vm_id.clone(), InstanceInfo { generation, - authority: Default::default(), + authority, + upstream_policy, id: vm_id.clone(), name: entry.name.clone(), asset_pins: entry.asset_pins.clone(), @@ -245,6 +253,7 @@ impl ServiceState { }, ); drop(instances); + let _upstream_broker = upstream_broker.start(upstream_grant); let _reaper = instance_reaper::spawn_exit_reaper( child, vm_id.clone(), From 442074da71e45d0e3e96f258e5431772f74fbbb5 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:07:10 +0000 Subject: [PATCH 033/464] security(foundation): seal adopted descriptors on exec --- crates/capsem-foundation/src/unix/fd.rs | 61 +++---------------- crates/capsem-foundation/src/unix/fd/tests.rs | 12 +++- 2 files changed, 21 insertions(+), 52 deletions(-) diff --git a/crates/capsem-foundation/src/unix/fd.rs b/crates/capsem-foundation/src/unix/fd.rs index 62e0dcae1..192e22d3f 100644 --- a/crates/capsem-foundation/src/unix/fd.rs +++ b/crates/capsem-foundation/src/unix/fd.rs @@ -5,7 +5,7 @@ use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, OwnedFd, RawFd}; use std::time::Duration; use nix::errno::Errno; -use nix::fcntl::{fcntl, FcntlArg, OFlag}; +use nix::fcntl::{fcntl, FcntlArg, FdFlag, OFlag}; use nix::sys::socket::{self, Shutdown}; use super::errno; @@ -85,57 +85,16 @@ pub fn duplicate(fd: BorrowedFd<'_>) -> io::Result { Ok(unsafe { OwnedFd::from_raw_fd(raw) }) } -/// Read at a checked file offset without changing the descriptor's cursor. -/// Interrupted operations are retried; EOF and short reads remain visible. -pub fn read_at(fd: BorrowedFd<'_>, bytes: &mut [u8], offset: u64) -> io::Result { - let offset = i64::try_from(offset).map_err(|_| io::Error::from_raw_os_error(Errno::EINVAL as i32))?; - retry_eintr(|| nix::sys::uio::pread(fd, bytes, offset)).map_err(errno::io) -} - -/// Write at a checked file offset without changing the descriptor's cursor. -/// Callers own any short-write loop so their protocol can account for progress. -pub fn write_at(fd: BorrowedFd<'_>, bytes: &[u8], offset: u64) -> io::Result { - let offset = i64::try_from(offset).map_err(|_| io::Error::from_raw_os_error(Errno::EINVAL as i32))?; - retry_eintr(|| nix::sys::uio::pwrite(fd, bytes, offset)).map_err(errno::io) -} - -/// Exact logical size of an already-open regular file, retaining descriptor identity. -pub fn regular_file_size(fd: BorrowedFd<'_>) -> io::Result { - regular_file_metadata(fd).map(|metadata| metadata.len) -} - -/// Descriptor-owned logical and allocated sizes for a regular file. -#[derive(Debug, Clone, Copy)] -pub struct RegularFileMetadata { - pub len: u64, - pub allocated_bytes: u64, -} - -pub fn regular_file_metadata(fd: BorrowedFd<'_>) -> io::Result { - use nix::sys::stat::{fstat, SFlag}; - let stat = retry_eintr(|| fstat(fd.as_raw_fd())).map_err(errno::io)?; - if SFlag::from_bits_truncate(stat.st_mode) & SFlag::S_IFMT != SFlag::S_IFREG { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "descriptor is not a regular file", - )); - } - let len = u64::try_from(stat.st_size) - .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "file length is negative"))?; - let allocated_bytes = u64::try_from(stat.st_blocks) - .ok() - .and_then(|blocks| blocks.checked_mul(512)) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "file allocation is invalid"))?; - Ok(RegularFileMetadata { len, allocated_bytes }) -} - -/// Signal one eventfd completion without borrowing raw Unix mechanics at callers. -pub fn signal_counter(fd: BorrowedFd<'_>) -> io::Result<()> { - let count = retry_eintr(|| nix::unistd::write(fd, &1u64.to_ne_bytes())).map_err(errno::io)?; - if count != std::mem::size_of::() { - return Err(io::Error::new(io::ErrorKind::WriteZero, "short eventfd counter write")); +/// Prevent an existing descriptor from crossing a later exec boundary. +/// Returns whether `FD_CLOEXEC` was already set and preserves other flags. +pub fn set_close_on_exec(fd: BorrowedFd<'_>) -> io::Result { + let raw_flags = retry_eintr(|| fcntl(fd.as_raw_fd(), FcntlArg::F_GETFD)).map_err(errno::io)?; + let flags = FdFlag::from_bits_truncate(raw_flags); + let was_enabled = flags.contains(FdFlag::FD_CLOEXEC); + if !was_enabled { + retry_eintr(|| fcntl(fd.as_raw_fd(), FcntlArg::F_SETFD(flags | FdFlag::FD_CLOEXEC))).map_err(errno::io)?; } - Ok(()) + Ok(was_enabled) } /// Set or clear `O_NONBLOCK`, returning its previous state. diff --git a/crates/capsem-foundation/src/unix/fd/tests.rs b/crates/capsem-foundation/src/unix/fd/tests.rs index 70110866f..5facbe187 100644 --- a/crates/capsem-foundation/src/unix/fd/tests.rs +++ b/crates/capsem-foundation/src/unix/fd/tests.rs @@ -4,7 +4,7 @@ use std::os::unix::net::UnixStream; use nix::fcntl::{fcntl, FcntlArg, FdFlag, OFlag}; -use super::{duplicate, retry_eintr, set_nonblocking, shutdown, wait_readable, SocketShutdown}; +use super::{duplicate, retry_eintr, set_close_on_exec, set_nonblocking, shutdown, wait_readable, SocketShutdown}; use nix::errno::Errno; #[test] @@ -204,6 +204,16 @@ fn duplicate_owns_an_independent_cloexec_descriptor() { assert_eq!(&bytes, b"owned"); } +#[test] +fn close_on_exec_change_reports_previous_state_and_preserves_descriptor_flags() { + let (stream, _peer) = UnixStream::pair().unwrap(); + fcntl(stream.as_raw_fd(), FcntlArg::F_SETFD(FdFlag::empty())).unwrap(); + assert!(!set_close_on_exec(stream.as_fd()).unwrap()); + assert!(set_close_on_exec(stream.as_fd()).unwrap()); + let flags = FdFlag::from_bits_truncate(fcntl(stream.as_raw_fd(), FcntlArg::F_GETFD).unwrap()); + assert!(flags.contains(FdFlag::FD_CLOEXEC)); +} + #[test] fn nonblocking_change_reports_previous_state_and_preserves_other_flags() { let (stream, _peer) = UnixStream::pair().unwrap(); From 1a9b5a783d27031ca78176f42e1fb06445881605 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:11:10 +0000 Subject: [PATCH 034/464] refactor(core): inject DNS datagram grants --- crates/capsem-core/src/net/dns/mod.rs | 2 +- crates/capsem-core/src/net/dns/resolver.rs | 103 +++++++++++++++--- .../capsem-core/src/net/dns/resolver/tests.rs | 73 +++++++++++++ .../src/net/policy_config/active_policy.rs | 6 + .../net/policy_config/active_policy/tests.rs | 19 ++++ 5 files changed, 187 insertions(+), 16 deletions(-) diff --git a/crates/capsem-core/src/net/dns/mod.rs b/crates/capsem-core/src/net/dns/mod.rs index 8b1530d0e..9a4286bcc 100644 --- a/crates/capsem-core/src/net/dns/mod.rs +++ b/crates/capsem-core/src/net/dns/mod.rs @@ -45,6 +45,6 @@ pub mod server; pub mod telemetry; pub use cache::{DnsAnswerCache, DEFAULT_CAPACITY, DEFAULT_MAX_TTL_SECS}; -pub use resolver::{DnsResolver, DEFAULT_UPSTREAMS}; +pub use resolver::{DnsDatagram, DnsGrantFuture, DnsResolver, DnsUpstreamGrants, DEFAULT_UPSTREAMS}; pub use server::{DnsHandler, DnsHandlerResult, SharedPolicy}; pub use telemetry::{build_dns_event, security_event_from_dns_event}; diff --git a/crates/capsem-core/src/net/dns/resolver.rs b/crates/capsem-core/src/net/dns/resolver.rs index da4b299f9..daafddcb1 100644 --- a/crates/capsem-core/src/net/dns/resolver.rs +++ b/crates/capsem-core/src/net/dns/resolver.rs @@ -17,8 +17,12 @@ //! scale (<200ms typical) and timing out the whole query rather than //! the per-attempt is fine for an interactive sandbox. +use std::fmt; +use std::future::Future; use std::io; use std::net::SocketAddr; +use std::pin::Pin; +use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{anyhow, Result}; @@ -37,30 +41,103 @@ pub const DEFAULT_UPSTREAMS: &[&str] = &["1.1.1.1:53", "8.8.8.8:53"]; /// into the backlog while we are still waiting on the first. const DEFAULT_TIMEOUT: Duration = Duration::from_millis(2000); +/// One connected upstream datagram plus the action that releases its grant. +pub struct DnsDatagram { + socket: UdpSocket, + release: Option>, +} + +impl DnsDatagram { + pub fn new(socket: UdpSocket, release: impl FnOnce() + Send + 'static) -> Self { + Self { + socket, + release: Some(Box::new(release)), + } + } + + fn socket(&self) -> &UdpSocket { + &self.socket + } +} + +impl Drop for DnsDatagram { + fn drop(&mut self) { + if let Some(release) = self.release.take() { + release(); + } + } +} + +/// Future returned by a trusted connected-datagram grant provider. +pub type DnsGrantFuture<'a> = Pin> + Send + 'a>>; + +/// Supplies a connected UDP socket for one configured upstream index. +pub trait DnsUpstreamGrants: Send + Sync { + fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_>; +} + +struct DirectDnsUpstreams(Vec); + +impl DnsUpstreamGrants for DirectDnsUpstreams { + fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_> { + Box::pin(async move { + let upstream = self + .0 + .get(usize::from(upstream_index)) + .copied() + .ok_or_else(|| anyhow!("DNS upstream index {upstream_index} is not configured"))?; + let bind_addr: SocketAddr = if upstream.is_ipv6() { + "[::]:0".parse().unwrap() + } else { + "0.0.0.0:0".parse().unwrap() + }; + let socket = UdpSocket::bind(bind_addr).await?; + socket.connect(upstream).await?; + Ok(DnsDatagram::new(socket, || {})) + }) + } +} + /// UDP DNS forwarder. Iterates the upstream list per query; first /// successful response wins. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct DnsResolver { upstreams: Vec, + grants: Arc, per_attempt_timeout: Duration, } +impl fmt::Debug for DnsResolver { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DnsResolver") + .field("upstreams", &self.upstreams) + .field("per_attempt_timeout", &self.per_attempt_timeout) + .finish_non_exhaustive() + } +} + impl DnsResolver { /// Build a resolver targeting the [`DEFAULT_UPSTREAMS`] nameservers /// with the default per-attempt timeout. pub fn new() -> Self { let upstreams = DEFAULT_UPSTREAMS.iter().filter_map(|s| s.parse().ok()).collect(); - Self { - upstreams, - per_attempt_timeout: DEFAULT_TIMEOUT, - } + Self::with_upstreams(upstreams) } /// Build a resolver targeting an explicit list of upstreams. /// Used by tests + future operator config (capsem.toml). pub fn with_upstreams(upstreams: Vec) -> Self { + let grants = Arc::new(DirectDnsUpstreams(upstreams.clone())); + Self::with_grants(upstreams, grants) + } + + /// Build a resolver whose connected sockets come from a trusted grant + /// provider. The addresses are labels for ordering and telemetry only. + pub fn with_grants(upstreams: Vec, grants: Arc) -> Self { Self { upstreams, + grants, per_attempt_timeout: DEFAULT_TIMEOUT, } } @@ -90,9 +167,10 @@ impl DnsResolver { return Err(anyhow!("no upstream nameservers configured")); } let mut last_err: Option = None; - for upstream in &self.upstreams { + for (index, upstream) in self.upstreams.iter().enumerate() { let t0 = Instant::now(); - match self.try_one(*upstream, query_bytes).await { + let upstream_index = u16::try_from(index).map_err(|_| anyhow!("too many DNS upstreams configured"))?; + match self.try_one(upstream_index, *upstream, query_bytes).await { Ok(resp) => { let elapsed = t0.elapsed(); debug!( @@ -112,15 +190,10 @@ impl DnsResolver { Err(last_err.unwrap_or_else(|| anyhow!("all DNS upstreams failed"))) } - async fn try_one(&self, upstream: SocketAddr, query_bytes: &[u8]) -> Result> { + async fn try_one(&self, upstream_index: u16, upstream: SocketAddr, query_bytes: &[u8]) -> Result> { let expected = ExpectedAnswer::for_query(query_bytes)?; - let bind_addr: SocketAddr = if upstream.is_ipv6() { - "[::]:0".parse().unwrap() - } else { - "0.0.0.0:0".parse().unwrap() - }; - let sock = UdpSocket::bind(bind_addr).await?; - sock.connect(upstream).await?; + let grant = self.grants.open(upstream_index).await?; + let sock = grant.socket(); sock.send(query_bytes).await?; let deadline = Instant::now() + self.per_attempt_timeout; let mut buf = vec![0u8; 4096]; diff --git a/crates/capsem-core/src/net/dns/resolver/tests.rs b/crates/capsem-core/src/net/dns/resolver/tests.rs index a864123e1..0c09cbebb 100644 --- a/crates/capsem-core/src/net/dns/resolver/tests.rs +++ b/crates/capsem-core/src/net/dns/resolver/tests.rs @@ -10,6 +10,7 @@ use super::*; use hickory_proto::op::{Message, MessageType, OpCode, Query}; use hickory_proto::rr::{Name, RecordType}; use std::str::FromStr; +use std::sync::{Arc, Mutex}; fn query_bytes(id: u16, name: &str) -> Vec { let mut message = Message::new(id, MessageType::Query, OpCode::Query); @@ -190,3 +191,75 @@ async fn a_hundred_concurrent_resolves_complete_against_one_upstream() { task.await.unwrap(); } } + +#[derive(Clone)] +struct IndexedGrants { + upstreams: Arc>, + requested: Arc>>, + refuse: bool, +} + +impl DnsUpstreamGrants for IndexedGrants { + fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_> { + Box::pin(async move { + self.requested.lock().unwrap().push(upstream_index); + if self.refuse { + return Err(anyhow!("grant refused")); + } + let upstream = self.upstreams[usize::from(upstream_index)]; + let socket = UdpSocket::bind("127.0.0.1:0").await?; + socket.connect(upstream).await?; + Ok(DnsDatagram::new(socket, || {})) + }) + } +} + +#[tokio::test] +async fn injected_grants_preserve_configured_failover_order() { + let sink = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let primary = sink.local_addr().unwrap(); + tokio::spawn(async move { + let mut bytes = [0_u8; 512]; + while sink.recv_from(&mut bytes).await.is_ok() {} + }); + let failover = fake_upstream(|query| { + let id = u16::from_be_bytes([query[0], query[1]]); + vec![answer_for(query, id, "ordered.example.")] + }) + .await; + let requested = Arc::new(Mutex::new(Vec::new())); + let grants = IndexedGrants { + upstreams: Arc::new(vec![primary, failover]), + requested: Arc::clone(&requested), + refuse: false, + }; + let resolver = + DnsResolver::with_grants(vec![primary, failover], Arc::new(grants)).with_timeout(Duration::from_millis(50)); + + resolver + .resolve(&query_bytes(7, "ordered.example.")) + .await + .expect("the second configured grant answers"); + assert_eq!(*requested.lock().unwrap(), vec![0, 1]); +} + +#[tokio::test] +async fn refused_grant_never_falls_back_to_direct_udp() { + let upstream = fake_upstream(|query| { + let id = u16::from_be_bytes([query[0], query[1]]); + vec![answer_for(query, id, "closed.example.")] + }) + .await; + let requested = Arc::new(Mutex::new(Vec::new())); + let grants = IndexedGrants { + upstreams: Arc::new(vec![upstream]), + requested: Arc::clone(&requested), + refuse: true, + }; + let error = DnsResolver::with_grants(vec![upstream], Arc::new(grants)) + .resolve(&query_bytes(9, "closed.example.")) + .await + .expect_err("a missing descriptor grant fails closed"); + assert!(error.to_string().contains("grant refused"), "{error:#}"); + assert_eq!(*requested.lock().unwrap(), vec![0]); +} diff --git a/crates/capsem-core/src/net/policy_config/active_policy.rs b/crates/capsem-core/src/net/policy_config/active_policy.rs index 8e6bc5ed6..47f1f32f5 100644 --- a/crates/capsem-core/src/net/policy_config/active_policy.rs +++ b/crates/capsem-core/src/net/policy_config/active_policy.rs @@ -90,6 +90,12 @@ impl ActivePolicyFile { } let merged = MergedPolicies::from_files(settings, corp)?; let mut network = network_config_from_policy_and_dns(&merged.network, corp.network.dns.clone()); + if network.dns.upstreams.is_empty() { + network.dns.upstreams = crate::net::dns::DEFAULT_UPSTREAMS + .iter() + .map(|upstream| (*upstream).to_string()) + .collect(); + } network.upstream_overrides = corp.network.upstream_overrides.clone(); let active = Self { diff --git a/crates/capsem-core/src/net/policy_config/active_policy/tests.rs b/crates/capsem-core/src/net/policy_config/active_policy/tests.rs index 2c152ea91..209c0a749 100644 --- a/crates/capsem-core/src/net/policy_config/active_policy/tests.rs +++ b/crates/capsem-core/src/net/policy_config/active_policy/tests.rs @@ -192,6 +192,25 @@ fn runtime_snapshot_refuses_an_invalid_dns_upstream_without_fallback() { assert!(error.contains("resolver.example:domain"), "{error}"); } +#[test] +fn default_dns_upstreams_are_explicit_in_the_active_policy_snapshot() { + let active = ActivePolicyFile::from_settings_and_corp(&SettingsFile::default(), &SettingsFile::default()).unwrap(); + assert_eq!( + active.network.dns.upstreams, + crate::net::dns::DEFAULT_UPSTREAMS + .iter() + .map(|upstream| (*upstream).to_string()) + .collect::>() + ); + assert_eq!( + active.compile_runtime().unwrap().dns_upstreams, + crate::net::dns::DEFAULT_UPSTREAMS + .iter() + .map(|upstream| upstream.parse().unwrap()) + .collect::>() + ); +} + #[test] fn mcp_servers_merge_settings_under_corp() { let settings = parse( From 75e348b1ddf5aef160f5696852fb06dec3d0a6e9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:14:23 +0000 Subject: [PATCH 035/464] refactor(core): reload DNS grant order --- crates/capsem-core/src/net/dns/resolver.rs | 37 +++++++++++++------ .../capsem-core/src/net/dns/resolver/tests.rs | 27 ++++++++++++++ 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/crates/capsem-core/src/net/dns/resolver.rs b/crates/capsem-core/src/net/dns/resolver.rs index daafddcb1..14f23edb6 100644 --- a/crates/capsem-core/src/net/dns/resolver.rs +++ b/crates/capsem-core/src/net/dns/resolver.rs @@ -22,7 +22,7 @@ use std::future::Future; use std::io; use std::net::SocketAddr; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; use anyhow::{anyhow, Result}; @@ -76,13 +76,15 @@ pub trait DnsUpstreamGrants: Send + Sync { fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_>; } -struct DirectDnsUpstreams(Vec); +struct DirectDnsUpstreams(Arc>>); impl DnsUpstreamGrants for DirectDnsUpstreams { fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_> { Box::pin(async move { let upstream = self .0 + .read() + .unwrap() .get(usize::from(upstream_index)) .copied() .ok_or_else(|| anyhow!("DNS upstream index {upstream_index} is not configured"))?; @@ -102,7 +104,7 @@ impl DnsUpstreamGrants for DirectDnsUpstreams { /// successful response wins. #[derive(Clone)] pub struct DnsResolver { - upstreams: Vec, + upstreams: Arc>>, grants: Arc, per_attempt_timeout: Duration, } @@ -111,7 +113,7 @@ impl fmt::Debug for DnsResolver { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter .debug_struct("DnsResolver") - .field("upstreams", &self.upstreams) + .field("upstreams", &*self.upstreams.read().unwrap()) .field("per_attempt_timeout", &self.per_attempt_timeout) .finish_non_exhaustive() } @@ -128,15 +130,19 @@ impl DnsResolver { /// Build a resolver targeting an explicit list of upstreams. /// Used by tests + future operator config (capsem.toml). pub fn with_upstreams(upstreams: Vec) -> Self { - let grants = Arc::new(DirectDnsUpstreams(upstreams.clone())); - Self::with_grants(upstreams, grants) + let upstreams = Arc::new(RwLock::new(upstreams)); + Self { + grants: Arc::new(DirectDnsUpstreams(Arc::clone(&upstreams))), + upstreams, + per_attempt_timeout: DEFAULT_TIMEOUT, + } } /// Build a resolver whose connected sockets come from a trusted grant /// provider. The addresses are labels for ordering and telemetry only. pub fn with_grants(upstreams: Vec, grants: Arc) -> Self { Self { - upstreams, + upstreams: Arc::new(RwLock::new(upstreams)), grants, per_attempt_timeout: DEFAULT_TIMEOUT, } @@ -150,9 +156,15 @@ impl DnsResolver { self } - /// Borrow the configured upstream list (debugging / metrics only). - pub fn upstreams(&self) -> &[SocketAddr] { - &self.upstreams + /// Snapshot the configured upstream list (debugging / metrics only). + pub fn upstreams(&self) -> Vec { + self.upstreams.read().unwrap().clone() + } + + /// Replace the configured order after the owning policy reload succeeds. + /// An in-flight query keeps the one snapshot it began with. + pub fn replace_upstreams(&self, upstreams: Vec) { + *self.upstreams.write().unwrap() = upstreams; } /// Forward `query_bytes` upstream. Returns the raw response bytes @@ -163,11 +175,12 @@ impl DnsResolver { /// On total failure (every upstream timed out or errored) returns /// the cumulative error so the caller can synthesize a SERVFAIL. pub async fn resolve(&self, query_bytes: &[u8]) -> Result<(Vec, Duration)> { - if self.upstreams.is_empty() { + let upstreams = self.upstreams(); + if upstreams.is_empty() { return Err(anyhow!("no upstream nameservers configured")); } let mut last_err: Option = None; - for (index, upstream) in self.upstreams.iter().enumerate() { + for (index, upstream) in upstreams.iter().enumerate() { let t0 = Instant::now(); let upstream_index = u16::try_from(index).map_err(|_| anyhow!("too many DNS upstreams configured"))?; match self.try_one(upstream_index, *upstream, query_bytes).await { diff --git a/crates/capsem-core/src/net/dns/resolver/tests.rs b/crates/capsem-core/src/net/dns/resolver/tests.rs index 0c09cbebb..bcf7a8fae 100644 --- a/crates/capsem-core/src/net/dns/resolver/tests.rs +++ b/crates/capsem-core/src/net/dns/resolver/tests.rs @@ -263,3 +263,30 @@ async fn refused_grant_never_falls_back_to_direct_udp() { assert!(error.to_string().contains("grant refused"), "{error:#}"); assert_eq!(*requested.lock().unwrap(), vec![0]); } + +#[tokio::test] +async fn a_policy_reload_replaces_the_configured_grant_order() { + let upstream = fake_upstream(|query| { + let id = u16::from_be_bytes([query[0], query[1]]); + vec![answer_for(query, id, "reloaded.example.")] + }) + .await; + let requested = Arc::new(Mutex::new(Vec::new())); + let grants = IndexedGrants { + upstreams: Arc::new(vec![upstream]), + requested: Arc::clone(&requested), + refuse: false, + }; + let resolver = DnsResolver::with_grants(Vec::new(), Arc::new(grants)); + resolver + .resolve(&query_bytes(10, "reloaded.example.")) + .await + .expect_err("empty policy has no implicit direct fallback"); + + resolver.replace_upstreams(vec![upstream]); + resolver + .resolve(&query_bytes(11, "reloaded.example.")) + .await + .expect("the reloaded upstream order is used"); + assert_eq!(*requested.lock().unwrap(), vec![0]); +} From 137f479d72a7bf8d23afe0584727f2d15f839b5d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:17:51 +0000 Subject: [PATCH 036/464] security(proto): bind grants to active policy --- crates/capsem-proto/src/upstream_grant.rs | 60 ++++++++++++++++++- .../capsem-proto/src/upstream_grant/tests.rs | 48 +++++++++++++++ 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index b7453369e..83a5611f9 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -11,7 +11,7 @@ use std::net::IpAddr; use anyhow::{bail, Result}; /// Exact record size carried by the bounded SCM_RIGHTS channel. -pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 288; +pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 360; /// A response grants at most one connected TCP or UDP descriptor. pub const UPSTREAM_GRANT_MAX_FDS: usize = 1; /// DNS names are at most 253 wire-text bytes without a root dot. @@ -28,7 +28,9 @@ const PORT_RANGE: std::ops::Range = 20..22; const DETAIL_RANGE: std::ops::Range = 22..24; const NAME_LENGTH_OFFSET: usize = 24; const NAME_RANGE: std::ops::Range = 25..25 + MAX_UPSTREAM_HOST_BYTES; -const RESERVED_RANGE: std::ops::Range = NAME_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; +const POLICY_DIGEST_BYTES: usize = 71; +const POLICY_DIGEST_RANGE: std::ops::Range = NAME_RANGE.end..NAME_RANGE.end + POLICY_DIGEST_BYTES; +const RESERVED_RANGE: std::ops::Range = POLICY_DIGEST_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; const RESOLVE_TCP: u8 = 1; const CONNECT_TCP: u8 = 2; @@ -160,11 +162,13 @@ pub enum UpstreamGrantResponse { selection_id: u64, protocol: UpstreamProtocol, judged_ip: Option, + policy_digest: String, }, DescriptorGranted { request_id: u64, grant_id: u64, kind: UpstreamDescriptorKind, + policy_digest: String, }, Denied { request_id: u64, @@ -246,6 +250,7 @@ pub fn encode_upstream_grant_request(request: &UpstreamGrantRequest) -> Result<[ /// Decode and fully validate one worker request. pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result { validate_envelope(frame)?; + require_empty_policy_digest(frame)?; let request_id = get_u64(frame, REQUEST_ID_RANGE); let resource_id = get_u64(frame, RESOURCE_ID_RANGE); let port = get_u16(frame, PORT_RANGE); @@ -314,6 +319,7 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul selection_id, protocol, judged_ip, + policy_digest, } => { require_nonzero("request id", *request_id)?; require_nonzero("selection id", *selection_id)?; @@ -324,11 +330,13 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul if let Some(address) = judged_ip { put_name(&mut frame, &address.to_string())?; } + put_policy_digest(&mut frame, policy_digest)?; } UpstreamGrantResponse::DescriptorGranted { request_id, grant_id, kind, + policy_digest, } => { require_nonzero("request id", *request_id)?; require_nonzero("grant id", *grant_id)?; @@ -336,6 +344,7 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); put_u64(&mut frame, RESOURCE_ID_RANGE, *grant_id); put_u16(&mut frame, DETAIL_RANGE, kind.code()); + put_policy_digest(&mut frame, policy_digest)?; } UpstreamGrantResponse::Denied { request_id, reason } => { require_nonzero("request id", *request_id)?; @@ -355,6 +364,7 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - let port = get_u16(frame, PORT_RANGE); let detail = get_u16(frame, DETAIL_RANGE); let name = get_name(frame)?; + let policy_digest = get_policy_digest(frame)?; require_nonzero("request id", request_id)?; if port != 0 { bail!("upstream response cannot carry a port"); @@ -377,6 +387,7 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - selection_id: resource_id, protocol, judged_ip, + policy_digest: require_policy_digest(policy_digest)?, }) } DESCRIPTOR_GRANTED => { @@ -388,12 +399,16 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - request_id, grant_id: resource_id, kind: UpstreamDescriptorKind::decode(detail)?, + policy_digest: require_policy_digest(policy_digest)?, }) } DENIED => { if resource_id != 0 || !name.is_empty() { bail!("denial cannot carry a resource"); } + if policy_digest.is_some() { + bail!("denial cannot carry a policy digest"); + } Ok(UpstreamGrantResponse::Denied { request_id, reason: UpstreamGrantDenial::decode(detail)?, @@ -475,6 +490,47 @@ fn get_name(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result { String::from_utf8(used.to_vec()).map_err(|_| anyhow::anyhow!("upstream name is not UTF-8")) } +fn put_policy_digest(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], digest: &str) -> Result<()> { + validate_policy_digest(digest)?; + frame[POLICY_DIGEST_RANGE].copy_from_slice(digest.as_bytes()); + Ok(()) +} + +fn get_policy_digest(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result> { + let bytes = &frame[POLICY_DIGEST_RANGE]; + if bytes.iter().all(|byte| *byte == 0) { + return Ok(None); + } + let digest = std::str::from_utf8(bytes).map_err(|_| anyhow::anyhow!("policy digest is not UTF-8"))?; + validate_policy_digest(digest)?; + Ok(Some(digest.to_owned())) +} + +fn require_policy_digest(digest: Option) -> Result { + digest.ok_or_else(|| anyhow::anyhow!("successful upstream response lacks a policy digest")) +} + +fn require_empty_policy_digest(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result<()> { + if frame[POLICY_DIGEST_RANGE].iter().any(|byte| *byte != 0) { + bail!("upstream request cannot carry a policy digest"); + } + Ok(()) +} + +fn validate_policy_digest(digest: &str) -> Result<()> { + let Some(hash) = digest.strip_prefix("blake3:") else { + bail!("policy digest must use blake3"); + }; + if hash.len() != 64 + || !hash + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + bail!("policy digest must contain 64 lowercase hexadecimal bytes"); + } + Ok(()) +} + fn require_nonzero(label: &str, value: u64) -> Result<()> { if value == 0 { bail!("{label} cannot be zero"); diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index c5905d976..b4d4a4ed0 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -1,5 +1,7 @@ use super::*; +const POLICY_DIGEST: &str = "blake3:0000000000000000000000000000000000000000000000000000000000000001"; + fn request_roundtrip(request: UpstreamGrantRequest) { let encoded = encode_upstream_grant_request(&request).unwrap(); assert_eq!(decode_upstream_grant_request(&encoded).unwrap(), request); @@ -48,28 +50,33 @@ fn every_response_round_trips_exactly() { selection_id: 11, protocol: UpstreamProtocol::Tls, judged_ip: Some("127.0.0.1".parse().unwrap()), + policy_digest: POLICY_DIGEST.into(), }, UpstreamGrantResponse::TcpResolved { request_id: 2, selection_id: 12, protocol: UpstreamProtocol::Http, judged_ip: Some("2001:db8::1".parse().unwrap()), + policy_digest: POLICY_DIGEST.into(), }, UpstreamGrantResponse::TcpResolved { request_id: 3, selection_id: 13, protocol: UpstreamProtocol::Http, judged_ip: None, + policy_digest: POLICY_DIGEST.into(), }, UpstreamGrantResponse::DescriptorGranted { request_id: 4, grant_id: 14, kind: UpstreamDescriptorKind::Tcp, + policy_digest: POLICY_DIGEST.into(), }, UpstreamGrantResponse::DescriptorGranted { request_id: 5, grant_id: 15, kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_DIGEST.into(), }, UpstreamGrantResponse::Denied { request_id: 6, @@ -92,6 +99,7 @@ fn descriptor_expectations_are_explicit() { request_id: 1, grant_id: 2, kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_DIGEST.into(), } .expected_descriptor_count(), 1 @@ -249,6 +257,7 @@ fn response_requires_canonical_ip_and_resource_shape() { selection_id: 2, protocol: UpstreamProtocol::Tls, judged_ip: Some("::1".parse().unwrap()), + policy_digest: POLICY_DIGEST.into(), }) .unwrap(); put_name(&mut resolved, "0:0:0:0:0:0:0:1").unwrap(); @@ -258,8 +267,47 @@ fn response_requires_canonical_ip_and_resource_shape() { request_id: 1, grant_id: 3, kind: UpstreamDescriptorKind::Tcp, + policy_digest: POLICY_DIGEST.into(), }) .unwrap(); put_name(&mut granted, "unexpected").unwrap(); assert!(decode_upstream_grant_response(&granted).is_err()); } + +#[test] +fn successful_responses_require_an_exact_policy_digest() { + for policy_digest in [ + "policy-a", + "blake3:000000000000000000000000000000000000000000000000000000000000000A", + "blake3:000000000000000000000000000000000000000000000000000000000000001", + ] { + assert!( + encode_upstream_grant_response(&UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 2, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: policy_digest.into(), + }) + .is_err() + ); + } + + let valid = encode_upstream_grant_response(&UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 2, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_DIGEST.into(), + }) + .unwrap(); + let mut missing = valid; + missing[POLICY_DIGEST_RANGE].fill(0); + assert!(decode_upstream_grant_response(&missing).is_err()); + + let mut denied = encode_upstream_grant_response(&UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::Revoked, + }) + .unwrap(); + denied[POLICY_DIGEST_RANGE].copy_from_slice(&valid[POLICY_DIGEST_RANGE]); + assert!(decode_upstream_grant_response(&denied).is_err()); +} From d26c1a4fb8b2d2393e26d0229b7df9c51c855f08 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:20:45 +0000 Subject: [PATCH 037/464] security(service): stamp upstream policy grants --- crates/capsem-service/src/upstream_broker.rs | 14 +++++++++++++- .../src/upstream_broker/tests.rs | 19 +++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index f9476d50a..8df546a9e 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -238,6 +238,7 @@ async fn run( selection_id, protocol: effective_protocol, judged_ip, + policy_digest: current.digest.clone(), }, None, ) @@ -301,7 +302,15 @@ async fn run( adopted: false, }, ); - send_active_grant(&responses, &active, request_id, grant_id, UpstreamDescriptorKind::Tcp).await?; + send_active_grant( + &responses, + &active, + request_id, + grant_id, + UpstreamDescriptorKind::Tcp, + ¤t_digest, + ) + .await?; awaiting_adoption = Some(grant_id); } UpstreamGrantRequest::OpenDns { @@ -373,6 +382,7 @@ async fn run( request_id, grant_id, UpstreamDescriptorKind::DnsUdp, + ¤t.digest, ) .await?; awaiting_adoption = Some(grant_id); @@ -418,6 +428,7 @@ async fn send_active_grant( request_id: u64, grant_id: u64, kind: UpstreamDescriptorKind, + policy_digest: &str, ) -> Result<(), String> { let grant = active.get(&grant_id).ok_or("new descriptor grant disappeared")?; send_response( @@ -426,6 +437,7 @@ async fn send_active_grant( request_id, grant_id, kind, + policy_digest: policy_digest.to_owned(), }, Some(&grant.descriptor), ) diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index 46eae894c..fad7780d5 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -53,7 +53,11 @@ fn test_policy(digest: &str, tcp_override: Option, dns_upstreams: Ve }, )]) }); - BrokerPolicy::new(digest.into(), Arc::new(runtime)) + BrokerPolicy::new(policy_digest(digest), Arc::new(runtime)) +} + +fn policy_digest(label: &str) -> String { + format!("blake3:{}", blake3::hash(label.as_bytes()).to_hex()) } fn start_broker( @@ -91,6 +95,7 @@ async fn resolve_override(client: &TestClient, request_id: u64) -> u64 { selection_id, protocol, judged_ip, + policy_digest: response_digest, } = response else { panic!("expected a resolved target, got {response:?}"); @@ -98,6 +103,7 @@ async fn resolve_override(client: &TestClient, request_id: u64) -> u64 { assert_eq!(response_id, request_id); assert_eq!(protocol, UpstreamProtocol::Http); assert_eq!(judged_ip, None); + assert_eq!(response_digest, policy_digest("policy-a")); selection_id } @@ -112,12 +118,14 @@ async fn connect_selection(client: &TestClient, request_id: u64, selection_id: u request_id: response_id, grant_id, kind, + policy_digest: response_digest, } = response else { panic!("expected a descriptor grant, got {response:?}"); }; assert_eq!(response_id, request_id); assert_eq!(kind, UpstreamDescriptorKind::Tcp); + assert_eq!(response_digest, policy_digest("policy-a")); (grant_id, fds.pop().unwrap()) } @@ -250,10 +258,17 @@ async fn dns_grant_uses_only_the_configured_index() { upstream_index: 0, }) .await; - let UpstreamGrantResponse::DescriptorGranted { grant_id, kind, .. } = response else { + let UpstreamGrantResponse::DescriptorGranted { + grant_id, + kind, + policy_digest: response_digest, + .. + } = response + else { panic!("expected a DNS descriptor grant, got {response:?}"); }; assert_eq!(kind, UpstreamDescriptorKind::DnsUdp); + assert_eq!(response_digest, policy_digest("policy-a")); client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; let socket = std::net::UdpSocket::from(fds.pop().unwrap()); socket.set_nonblocking(true).unwrap(); From 7ffb3f8f094a216d6a6267f81ca8ea968a4113e7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:26:48 +0000 Subject: [PATCH 038/464] security(process): broker upstream DNS sockets --- crates/capsem-process/src/ipc.rs | 4 + crates/capsem-process/src/ipc/tests.rs | 20 ++ crates/capsem-process/src/main.rs | 26 +- crates/capsem-process/src/upstream_grant.rs | 227 ++++++++++++++++++ .../src/upstream_grant/tests.rs | 202 ++++++++++++++++ 5 files changed, 473 insertions(+), 6 deletions(-) create mode 100644 crates/capsem-process/src/upstream_grant.rs create mode 100644 crates/capsem-process/src/upstream_grant/tests.rs diff --git a/crates/capsem-process/src/ipc.rs b/crates/capsem-process/src/ipc.rs index cee6442e8..46ef9af6f 100644 --- a/crates/capsem-process/src/ipc.rs +++ b/crates/capsem-process/src/ipc.rs @@ -142,6 +142,8 @@ pub(crate) async fn handle_ipc_connection( mcp_builtin_binary: Option, mcp_builtin_env: HashMap, vm_ready: Arc, + dns_resolver: Arc, + upstream_grants: Arc, ) -> Result<()> { // First frame on every IPC connection is a Hello -- detect cross-version // mixes (capsem-service built before X, capsem-process built after) in @@ -684,6 +686,8 @@ pub(crate) async fn handle_ipc_connection( let reply = match runtime_source.load() { Ok(runtime_config) => { let digest = runtime_config.active_policy_digest.clone(); + upstream_grants.replace_policy_digest(digest.clone()); + dns_resolver.replace_upstreams(runtime_config.dns_upstreams.clone()); runtime_config.apply(&net_state, &mcp_runtime); ProcessToService::ConfigReloadResult { id, diff --git a/crates/capsem-process/src/ipc/tests.rs b/crates/capsem-process/src/ipc/tests.rs index 8b2300a31..62e70def1 100644 --- a/crates/capsem-process/src/ipc/tests.rs +++ b/crates/capsem-process/src/ipc/tests.rs @@ -100,6 +100,8 @@ pub(in crate::ipc) struct Dispatcher { pub(in crate::ipc) mcp_runtime: Arc, runtime_source: RuntimePolicySource, pub(in crate::ipc) ready: Arc, + dns_resolver: Arc, + upstream_grants: Arc, } impl Dispatcher { @@ -191,6 +193,10 @@ impl Dispatcher { let job_store = Arc::new(JobStore::new()); let (ctrl_tx, ctrl_rx) = mpsc::channel(16); let (events_tx, _) = broadcast::channel(16); + let initial_digest = + capsem_core::net::policy_config::active_policy_digest(&std::fs::read(&active_policy).unwrap()); + let dns_resolver = Arc::new(capsem_core::net::dns::DnsResolver::with_upstreams(Vec::new())); + let upstream_grants = Arc::new(crate::upstream_grant::UpstreamGrantClient::test_handle(initial_digest)); ( Self { job_store, @@ -201,6 +207,8 @@ impl Dispatcher { mcp_runtime, runtime_source: RuntimePolicySource::new(active_policy), ready: Arc::new(AtomicBool::new(true)), + dns_resolver, + upstream_grants, }, ctrl_rx, ) @@ -221,6 +229,8 @@ impl Dispatcher { None, HashMap::new(), Arc::clone(&self.ready), + Arc::clone(&self.dns_resolver), + Arc::clone(&self.upstream_grants), )) } } @@ -559,6 +569,11 @@ async fn negotiated_dispatcher_covers_stream_jobs_queries_and_lifecycle() { } if error == "read fixture failed" )); + std::fs::write( + temp.path().join("active_policy.toml"), + "[user_rules]\n[corp_rules]\n[network]\n[network.dns]\nupstreams = [\"127.0.0.1:5353\"]\n", + ) + .unwrap(); service_tx .send(ServiceToProcess::ReloadConfig { id: 30 }) .await @@ -571,6 +586,11 @@ async fn negotiated_dispatcher_covers_stream_jobs_queries_and_lifecycle() { ProcessToService::ConfigReloadResult { id: 30, active_policy_digest: Some(digest), error: None } if digest == applied )); + assert_eq!(dispatcher.upstream_grants.policy_digest(), applied); + assert_eq!( + dispatcher.dns_resolver.upstreams(), + vec!["127.0.0.1:5353".parse().unwrap()] + ); service_tx .send(ServiceToProcess::McpListServers { id: 15 }) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 07fa483fc..de06d64fb 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -10,6 +10,7 @@ mod private_names; mod private_seats; mod runtime_config; mod terminal; +mod upstream_grant; mod vsock; use anyhow::{Context, Result}; @@ -19,6 +20,7 @@ use capsem_core::{boot_vm, BootOptions, VirtioFsShare, VsockConnection}; use capsem_logger::DbWriter; use capsem_proto::ipc::{ProcessToService, ServiceToProcess}; use clap::Parser; +use std::os::fd::AsFd as _; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::net::UnixListener; @@ -211,6 +213,8 @@ fn main() -> Result<()> { // SAFETY: process entry precedes argument parsing, telemetry, descriptor // owners and runtime threads. Broker grants will be named here explicitly. unsafe { capsem_foundation::unix::fd::close_inherited_descriptors()? }; + let upstream_socket = + upstream_grant::adopt_inherited(std::io::stdin().as_fd()).context("adopt inherited upstream grant channel")?; let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { service: "capsem-process", sink: capsem_foundation::telemetry::LogSink::Stderr, @@ -338,6 +342,7 @@ fn main() -> Result<()> { trace_id_for_loop, session_dir_for_loop, shutdown_for_loop, + upstream_socket, ) .await { @@ -400,6 +405,7 @@ fn main() -> Result<()> { Ok(()) } +#[allow(clippy::too_many_arguments)] async fn run_async_main_loop( args: Args, controller: capsem_foundation::unix::peer::PeerIdentity, @@ -408,9 +414,14 @@ async fn run_async_main_loop( trace_id: String, session_dir: std::path::PathBuf, shutdown: Arc>, + upstream_socket: std::os::unix::net::UnixStream, ) -> Result<()> { let runtime_source = runtime_config::RuntimePolicySource::new(args.active_policy.clone()); let runtime_config = runtime_source.load()?; + let upstream_grants = Arc::new(upstream_grant::UpstreamGrantClient::start( + upstream_socket, + runtime_config.active_policy_digest.clone(), + )?); let terminal_output = Arc::new(capsem_core::TerminalOutputQueue::new()); // 1024 queued events: a guest resolving and fetching in parallel enqueues @@ -607,11 +618,10 @@ async fn run_async_main_loop( // DNS handler shares the same security rule/plugin handles as MITM // so admin enforcement edits take effect across protocols at once. - let dns_resolver = if runtime_config.dns_upstreams.is_empty() { - DnsResolver::new() - } else { - DnsResolver::with_upstreams(runtime_config.dns_upstreams.clone()) - }; + let dns_resolver = Arc::new(DnsResolver::with_grants( + runtime_config.dns_upstreams.clone(), + Arc::clone(&upstream_grants) as Arc, + )); // The private zone is the service's to answer, for this VM's networks. let private_names = Arc::new(private_names::ServicePrivateNames::new( seats.service_socket, @@ -622,7 +632,7 @@ async fn run_async_main_loop( Arc::clone(&net_state.policy), Arc::clone(&security_rules), Arc::clone(&plugin_policy), - Arc::new(dns_resolver), + Arc::clone(&dns_resolver), Arc::new(DnsAnswerCache::default()), ) .with_private_names(private_names), @@ -751,6 +761,8 @@ async fn run_async_main_loop( let builtin_bin_c = builtin_bin.clone(); let builtin_env_c = builtin_env.clone(); let ready_c = Arc::clone(&vm_ready); + let dns_resolver_c = Arc::clone(&dns_resolver); + let upstream_grants_c = Arc::clone(&upstream_grants); tokio::spawn(async move { if let Err(e) = ipc::handle_ipc_connection( @@ -766,6 +778,8 @@ async fn run_async_main_loop( builtin_bin_c, builtin_env_c, ready_c, + dns_resolver_c, + upstream_grants_c, ) .await { diff --git a/crates/capsem-process/src/upstream_grant.rs b/crates/capsem-process/src/upstream_grant.rs new file mode 100644 index 000000000..129be63da --- /dev/null +++ b/crates/capsem-process/src/upstream_grant.rs @@ -0,0 +1,227 @@ +//! Process-side client for coordinator-minted upstream descriptors. + +use std::io; +use std::os::fd::{AsFd as _, BorrowedFd, OwnedFd}; +use std::os::unix::net::UnixStream; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use anyhow::{anyhow, Result}; +use capsem_core::net::dns::{DnsDatagram, DnsGrantFuture, DnsUpstreamGrants}; +use capsem_foundation::unix::fd::{self, SocketShutdown}; +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::upstream_grant::{ + decode_upstream_grant_response, encode_upstream_grant_request, UpstreamDescriptorKind, UpstreamGrantRequest, + UpstreamGrantResponse, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, +}; +use tokio::sync::{mpsc, oneshot}; + +const COMMAND_CAPACITY: usize = 64; +const WIRE_TIMEOUT: Duration = Duration::from_secs(2); + +type WireSender = DescriptorSender; +type WireReceiver = DescriptorReceiver; + +struct OpenDns { + upstream_index: u16, + policy_digest: String, + reply: oneshot::Sender>, +} + +/// Serializes grant protocol traffic over the inherited generation channel. +#[derive(Clone)] +pub(crate) struct UpstreamGrantClient { + opens: mpsc::Sender, + policy_digest: Arc>, +} + +impl UpstreamGrantClient { + pub(crate) fn start(socket: UnixStream, policy_digest: String) -> io::Result { + let sender = WireSender::new(socket.try_clone()?)?; + let receiver = WireReceiver::new(socket.try_clone()?)?; + let (opens_tx, opens) = mpsc::channel(COMMAND_CAPACITY); + let (actor_releases, releases) = mpsc::unbounded_channel(); + tokio::spawn(async move { + let result = run_actor(sender, receiver, opens, releases, actor_releases).await; + if let Err(error) = result { + tracing::warn!(%error, "upstream grant channel stopped"); + } + if let Err(error) = fd::shutdown(socket.as_fd(), SocketShutdown::Both) { + tracing::debug!(%error, "failed to shut down upstream grant channel"); + } + }); + Ok(Self { + opens: opens_tx, + policy_digest: Arc::new(RwLock::new(policy_digest)), + }) + } + + pub(crate) fn replace_policy_digest(&self, policy_digest: String) { + *self.policy_digest.write().unwrap() = policy_digest; + } + + #[cfg(test)] + pub(crate) fn policy_digest(&self) -> String { + self.policy_digest.read().unwrap().clone() + } + + #[cfg(test)] + pub(crate) fn test_handle(policy_digest: String) -> Self { + let (opens, _commands) = mpsc::channel(COMMAND_CAPACITY); + Self { + opens, + policy_digest: Arc::new(RwLock::new(policy_digest)), + } + } +} + +impl DnsUpstreamGrants for UpstreamGrantClient { + fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_> { + Box::pin(async move { + let policy_digest = self.policy_digest.read().unwrap().clone(); + let (reply, result) = oneshot::channel(); + self.opens + .send(OpenDns { + upstream_index, + policy_digest, + reply, + }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + }) + } +} + +async fn run_actor( + sender: WireSender, + receiver: WireReceiver, + mut opens: mpsc::Receiver, + mut releases: mpsc::UnboundedReceiver, + release_sender: mpsc::UnboundedSender, +) -> Result<(), String> { + let mut next_request_id = 1_u64; + loop { + tokio::select! { + biased; + release = releases.recv() => { + let Some(resource_id) = release else { + if opens.is_closed() { + return Ok(()); + } + continue; + }; + send_request(&sender, &UpstreamGrantRequest::Release { resource_id }).await?; + } + open = opens.recv() => { + let Some(open) = open else { + return Ok(()); + }; + let request_id = next_request_id; + next_request_id = next_request_id.checked_add(1).ok_or("upstream request id exhausted")?; + match open_dns(&sender, &receiver, &release_sender, request_id, &open).await { + Ok(result) => { + let _ = open.reply.send(result); + } + Err(error) => { + let _ = open.reply.send(Err(error.clone())); + return Err(error); + } + } + } + } + } +} + +async fn open_dns( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + open: &OpenDns, +) -> Result, String> { + send_request( + sender, + &UpstreamGrantRequest::OpenDns { + request_id, + upstream_index: open.upstream_index, + }, + ) + .await?; + let frame = tokio::time::timeout(WIRE_TIMEOUT, receiver.recv()) + .await + .map_err(|_| "receive upstream grant response timed out".to_string())? + .map_err(|error| format!("receive upstream grant response: {error}"))?; + let response = decode_upstream_grant_response(&frame.bytes) + .map_err(|error| format!("decode upstream grant response: {error:#}"))?; + if frame.fds.len() != response.expected_descriptor_count() { + return Err(format!( + "upstream grant response carried {} descriptors, expected {}", + frame.fds.len(), + response.expected_descriptor_count() + )); + } + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("upstream DNS grant denied: {reason:?}"))), + UpstreamGrantResponse::DescriptorGranted { + request_id: response_id, + grant_id, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest, + } if response_id == request_id => { + let descriptor = frame.fds.into_iter().next().ok_or("DNS grant omitted its descriptor")?; + send_request(sender, &UpstreamGrantRequest::Adopted { grant_id }).await?; + if policy_digest != open.policy_digest { + send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; + return Ok(Err(format!( + "upstream DNS grant policy mismatch: expected {}, received {policy_digest}", + open.policy_digest + ))); + } + let socket = datagram_from_descriptor(descriptor)?; + let releases = releases.clone(); + Ok(Ok(DnsDatagram::new(socket, move || { + let _ = releases.send(grant_id); + }))) + } + response => Err(format!( + "unexpected upstream DNS grant response for request {request_id}: {response:?}" + )), + } +} + +async fn send_request(sender: &WireSender, request: &UpstreamGrantRequest) -> Result<(), String> { + let bytes = + encode_upstream_grant_request(request).map_err(|error| format!("encode upstream grant request: {error:#}"))?; + tokio::time::timeout(WIRE_TIMEOUT, sender.send(&bytes, &[])) + .await + .map_err(|_| "send upstream grant request timed out".to_string())? + .map_err(|error| format!("send upstream grant request: {error}"))?; + Ok(()) +} + +fn datagram_from_descriptor(descriptor: OwnedFd) -> Result { + let socket = std::net::UdpSocket::from(descriptor); + socket + .set_nonblocking(true) + .map_err(|error| format!("make granted DNS socket nonblocking: {error}"))?; + tokio::net::UdpSocket::from_std(socket).map_err(|error| format!("adopt granted DNS socket: {error}")) +} + +/// Duplicate the inherited generation channel before runtime threads exist. +pub(crate) fn adopt_inherited(descriptor: BorrowedFd<'_>) -> io::Result { + fd::set_close_on_exec(descriptor)?; + let owned = fd::duplicate(descriptor)?; + let socket = UnixStream::from(owned); + socket.peer_addr()?; + Ok(socket) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-process/src/upstream_grant/tests.rs b/crates/capsem-process/src/upstream_grant/tests.rs new file mode 100644 index 000000000..797919bf0 --- /dev/null +++ b/crates/capsem-process/src/upstream_grant/tests.rs @@ -0,0 +1,202 @@ +use std::io::{Read as _, Write as _}; +use std::os::fd::{AsFd as _, AsRawFd as _}; + +use capsem_core::net::dns::{DnsResolver, DnsUpstreamGrants}; +use capsem_proto::upstream_grant::{ + decode_upstream_grant_request, encode_upstream_grant_response, UpstreamDescriptorKind, UpstreamGrantDenial, + UpstreamGrantRequest, UpstreamGrantResponse, +}; + +use super::*; + +const POLICY_A: &str = "blake3:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const POLICY_B: &str = "blake3:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + +async fn open_error(client: &UpstreamGrantClient) -> String { + match client.open(0).await { + Ok(_) => panic!("unexpected DNS grant"), + Err(error) => error.to_string(), + } +} + +fn channels(socket: UnixStream) -> (WireReceiver, WireSender) { + let receiver = WireReceiver::new(socket.try_clone().unwrap()).unwrap(); + let sender = WireSender::new(socket).unwrap(); + (receiver, sender) +} + +async fn receive_request(receiver: &WireReceiver) -> UpstreamGrantRequest { + let frame = receiver.recv().await.unwrap(); + assert!(frame.fds.is_empty()); + decode_upstream_grant_request(&frame.bytes).unwrap() +} + +async fn send_response( + sender: &WireSender, + response: &UpstreamGrantResponse, + descriptor: Option<&std::net::UdpSocket>, +) { + let bytes = encode_upstream_grant_response(response).unwrap(); + let fds = descriptor.map_or_else(Vec::new, |socket| vec![socket.as_raw_fd()]); + sender.send(&bytes, &fds).await.unwrap(); +} + +fn query() -> Vec { + let mut query = vec![0x12, 0x34, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + query.extend_from_slice(&[7, b'e', b'x', b'a', b'm', b'p', b'l', b'e', 3, b'c', b'o', b'm', 0]); + query.extend_from_slice(&[0, 1, 0, 1]); + query +} + +#[tokio::test] +async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { + let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let upstream_address = upstream.local_addr().unwrap(); + let (broker, worker) = UnixStream::pair().unwrap(); + let client = Arc::new(UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap()); + let (requests, responses) = channels(broker); + let expected_query = query(); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenDns { + request_id: 1, + upstream_index: 0, + } + ); + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.connect(upstream_address).unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 41, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_A.into(), + }, + Some(&socket), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { grant_id: 41 } + ); + let mut received = [0_u8; 512]; + let (count, peer) = upstream.recv_from(&mut received).await.unwrap(); + assert_eq!(&received[..count], expected_query); + let mut answer = received[..count].to_vec(); + answer[2] = 0x81; + answer[3] = 0x80; + upstream.send_to(&answer, peer).await.unwrap(); + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { resource_id: 41 } + ); + }); + + let resolver = DnsResolver::with_grants(vec![upstream_address], client); + let (answer, _) = resolver.resolve(&query()).await.unwrap(); + assert_eq!(&answer[..2], &[0x12, 0x34]); + assert_eq!(&answer[2..4], &[0x81, 0x80]); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn stale_policy_grant_is_adopted_released_and_channel_remains_usable() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenDns { request_id: 1, .. } + )); + let peer = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.connect(peer.local_addr().unwrap()).unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 51, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_B.into(), + }, + Some(&socket), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { grant_id: 51 } + ); + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { resource_id: 51 } + ); + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenDns { request_id: 2, .. } + )); + send_response( + &responses, + &UpstreamGrantResponse::Denied { + request_id: 2, + reason: UpstreamGrantDenial::NotConfigured, + }, + None, + ) + .await; + }); + + let error = open_error(&client).await; + assert!(error.contains("policy mismatch"), "{error}"); + let error = open_error(&client).await; + assert!(error.contains("NotConfigured"), "{error}"); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn malformed_or_revoked_grant_channel_fails_closed() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenDns { request_id: 1, .. } + )); + send_response( + &responses, + &UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 61, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_A.into(), + }, + None, + ) + .await; + }); + let error = open_error(&client).await; + assert!(error.contains("carried 0 descriptors"), "{error}"); + broker_task.await.unwrap(); + let error = open_error(&client).await; + assert!(error.contains("closed") || error.contains("stopped"), "{error}"); + + let (broker, worker) = UnixStream::pair().unwrap(); + let revoked = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + drop(broker); + assert!(!open_error(&revoked).await.is_empty()); +} + +#[test] +fn inherited_channel_is_duplicated_cloexec_and_connected() { + let (inherited, peer) = UnixStream::pair().unwrap(); + let mut adopted = adopt_inherited(inherited.as_fd()).unwrap(); + let mut peer = peer; + assert!(fd::set_close_on_exec(inherited.as_fd()).unwrap()); + adopted.write_all(b"x").unwrap(); + let mut byte = [0_u8; 1]; + peer.read_exact(&mut byte).unwrap(); + assert_eq!(byte, [b'x']); +} From de3152f97e5440ef0dae34ec1aef3022dc2cd606 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:34:39 +0000 Subject: [PATCH 039/464] refactor(core): inject upstream TCP grants --- crates/capsem-core/benches/mitm_request.rs | 1 + crates/capsem-core/src/net/mitm_proxy/mod.rs | 30 ++- .../capsem-core/src/net/mitm_proxy/upgrade.rs | 6 +- .../src/net/mitm_proxy/upstream.rs | 240 +++++++++++++++++- .../src/net/mitm_proxy/upstream/tests.rs | 134 +++++++++- crates/capsem-core/tests/mitm_integration.rs | 1 + 6 files changed, 399 insertions(+), 13 deletions(-) diff --git a/crates/capsem-core/benches/mitm_request.rs b/crates/capsem-core/benches/mitm_request.rs index 48f801f38..809efa5c8 100644 --- a/crates/capsem-core/benches/mitm_request.rs +++ b/crates/capsem-core/benches/mitm_request.rs @@ -99,6 +99,7 @@ fn proxy_config(upstream_port: u16) -> Arc { pipeline, mcp_endpoint: None, upstream_resolver: Default::default(), + upstream_grants: None, }) } diff --git a/crates/capsem-core/src/net/mitm_proxy/mod.rs b/crates/capsem-core/src/net/mitm_proxy/mod.rs index 26353feaa..826a7245f 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mod.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mod.rs @@ -72,7 +72,10 @@ use util::{ pub use mcp_endpoint::{McpEndpointState, McpTimeouts, ScopedMcpTools}; pub use mcp_frame::{dispatch_logged_mcp_request, McpTransport}; -pub use upstream::UpstreamTarget; +pub use upstream::{ + GrantedTcpStream, TcpConnectGrantFuture, TcpGrantSelection, TcpResolveGrantFuture, TcpUpstreamGrants, + UpstreamTarget, +}; /// Re-exported so capsem-app can reference the type without depending on rustls. pub type UpstreamTlsConfig = rustls::ClientConfig; @@ -124,6 +127,8 @@ pub struct MitmProxyConfig { pub mcp_endpoint: Option>, /// Resolves guest-named upstreams before policy; the dial goes only to what it judged. pub upstream_resolver: crate::net::upstream_address::UpstreamResolver, + /// Trusted coordinator connection grants for confined workers. + pub upstream_grants: Option>, } /// Build an empty pipeline for callers that do not install production hooks. @@ -874,7 +879,16 @@ async fn handle_request( let policy: Arc = config.policy.read().unwrap().clone(); // Resolve before the rules run, so they judge the address the dial reaches. let resolver = &config.upstream_resolver; - let target = UpstreamTarget::select(resolver, &policy, domain, upstream_port, cached_upstream).await; + let target = UpstreamTarget::select( + resolver, + &policy, + protocol, + domain, + upstream_port, + cached_upstream, + config.upstream_grants.as_deref(), + ) + .await; let upstream_ip = target.judged_ip(domain); let log_bodies = policy.log_bodies; let max_body = policy.max_body_capture; @@ -1739,7 +1753,11 @@ async fn handle_request( } else { let dial_start = Instant::now(); let tcp_start = Instant::now(); - let upstream_tcp = match target.connect().instrument(upstream_prepare_span.clone()).await { + let upstream_tcp = match target + .connect_with_grants(config.upstream_grants.as_deref()) + .instrument(upstream_prepare_span.clone()) + .await + { Ok((tcp, pinned)) => { connected = pinned; tcp @@ -1983,7 +2001,11 @@ async fn handle_request( error = %e, "cached upstream sender failed on send; reconnecting replayable request" ); - let upstream_tcp = match target.connect().instrument(upstream_send_span.clone()).await { + let upstream_tcp = match target + .connect_with_grants(config.upstream_grants.as_deref()) + .instrument(upstream_send_span.clone()) + .await + { Ok((tcp, pinned)) => { connected = pinned; tcp diff --git a/crates/capsem-core/src/net/mitm_proxy/upgrade.rs b/crates/capsem-core/src/net/mitm_proxy/upgrade.rs index b32cdc127..377fd1129 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upgrade.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upgrade.rs @@ -212,7 +212,11 @@ pub(super) async fn handle_upgrade( .clone() .unwrap_or_else(|| matched_rule.clone()); - let upstream_tcp = match target.connect().instrument(ws_span.clone()).await { + let upstream_tcp = match target + .connect_with_grants(config.upstream_grants.as_deref()) + .instrument(ws_span.clone()) + .await + { Ok((stream, _pinned)) => stream, Err(error) => { ws_span.record("decision", "error"); diff --git a/crates/capsem-core/src/net/mitm_proxy/upstream.rs b/crates/capsem-core/src/net/mitm_proxy/upstream.rs index b98d5fea6..5e3f7f861 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upstream.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upstream.rs @@ -12,9 +12,16 @@ //! target exactly as written, and the rules see what they always saw. use std::fmt; +use std::future::Future; +use std::io; use std::net::{IpAddr, SocketAddr}; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; use hyper::client::conn::http1::SendRequest; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::net::TcpStream; use super::body::ProxyBoxBody; @@ -22,6 +29,130 @@ use super::protocol::Protocol; use crate::net::policy::{NetworkMechanics, UpstreamOverrideProtocol}; use crate::net::upstream_address::{judged_address, UpstreamResolver}; +/// Future returned by a trusted TCP selection provider. +pub type TcpResolveGrantFuture<'a> = Pin> + Send + 'a>>; +/// Future returned by a trusted connected-stream provider. +pub type TcpConnectGrantFuture<'a> = Pin> + Send + 'a>>; + +/// Selects and connects upstreams without exposing address choice to a worker. +pub trait TcpUpstreamGrants: Send + Sync { + fn resolve(&self, protocol: Protocol, host: &str, port: u16) -> TcpResolveGrantFuture<'_>; + fn connect(&self, selection_id: u64) -> TcpConnectGrantFuture<'_>; +} + +struct SelectionLease { + id: u64, + claimed: AtomicBool, + release: Mutex>>, +} + +impl Drop for SelectionLease { + fn drop(&mut self) { + if !self.claimed.load(Ordering::Acquire) { + if let Some(release) = self.release.get_mut().unwrap().take() { + release(); + } + } + } +} + +/// One coordinator-selected target, released if policy rejects it before use. +#[derive(Clone)] +pub struct TcpGrantSelection { + lease: Arc, + pub protocol: Protocol, + pub judged_ip: Option, +} + +impl TcpGrantSelection { + pub fn new( + selection_id: u64, + protocol: Protocol, + judged_ip: Option, + release: impl FnOnce() + Send + 'static, + ) -> Self { + Self { + lease: Arc::new(SelectionLease { + id: selection_id, + claimed: AtomicBool::new(false), + release: Mutex::new(Some(Box::new(release))), + }), + protocol, + judged_ip, + } + } + + fn claim(&self) -> io::Result { + self.lease + .claimed + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map(|_| self.lease.id) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "upstream selection was already consumed")) + } +} + +impl fmt::Debug for TcpGrantSelection { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("TcpGrantSelection") + .field("id", &self.lease.id) + .field("protocol", &self.protocol) + .field("judged_ip", &self.judged_ip) + .finish() + } +} + +impl PartialEq for TcpGrantSelection { + fn eq(&self, other: &Self) -> bool { + self.lease.id == other.lease.id && self.protocol == other.protocol && self.judged_ip == other.judged_ip + } +} + +impl Eq for TcpGrantSelection {} + +/// A connected upstream descriptor whose coordinator grant is released on drop. +pub struct GrantedTcpStream { + stream: TcpStream, + release: Option>, +} + +impl GrantedTcpStream { + pub fn new(stream: TcpStream, release: impl FnOnce() + Send + 'static) -> Self { + Self { + stream, + release: Some(Box::new(release)), + } + } +} + +impl Drop for GrantedTcpStream { + fn drop(&mut self) { + if let Some(release) = self.release.take() { + release(); + } + } +} + +impl AsyncRead for GrantedTcpStream { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buffer: &mut ReadBuf<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_read(cx, buffer) + } +} + +impl AsyncWrite for GrantedTcpStream { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, bytes: &[u8]) -> Poll> { + Pin::new(&mut self.stream).poll_write(cx, bytes) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.stream).poll_shutdown(cx) + } +} + /// The upstream connection one request will use. #[derive(Clone, Debug, PartialEq, Eq)] pub enum UpstreamTarget { @@ -29,6 +160,16 @@ pub enum UpstreamTarget { Override { dial: String, protocol: Protocol }, /// The addresses the guest's name resolved to, in the order to try. Resolved(Vec), + /// A trusted coordinator's opaque selection. Connected targets retain + /// their judged identity while a retry obtains a fresh one-shot token. + Granted { + host: String, + port: u16, + guest_protocol: Protocol, + selection: Option, + protocol: Protocol, + judged_ip: Option, + }, /// The name did not resolve. Nothing is dialed; the reason is the error. Unresolved(String), } @@ -53,12 +194,16 @@ impl UpstreamTarget { pub(super) async fn select( resolver: &UpstreamResolver, policy: &NetworkMechanics, + guest_protocol: Protocol, domain: &str, port: u16, cache: &UpstreamCache, + grants: Option<&dyn TcpUpstreamGrants>, ) -> Self { - if let Some(target) = Self::override_target(policy, domain, port) { - return target; + if grants.is_none() { + if let Some(target) = Self::override_target(policy, domain, port) { + return target; + } } if let Some(pinned) = cache .lock() @@ -68,12 +213,29 @@ impl UpstreamTarget { { return pinned; } + if let Some(grants) = grants { + return Self::resolve_granted(grants, guest_protocol, domain, port).await; + } match resolver.resolve(domain, port).await { Ok(addresses) => Self::Resolved(addresses), Err(error) => Self::Unresolved(error), } } + async fn resolve_granted(grants: &dyn TcpUpstreamGrants, protocol: Protocol, host: &str, port: u16) -> Self { + match grants.resolve(protocol, host, port).await { + Ok(selection) => Self::Granted { + host: host.to_owned(), + port, + guest_protocol: protocol, + protocol: selection.protocol, + judged_ip: selection.judged_ip, + selection: Some(selection), + }, + Err(error) => Self::Unresolved(error.to_string()), + } + } + fn override_target(policy: &NetworkMechanics, domain: &str, port: u16) -> Option { let route = policy.find_upstream_override(domain, port)?; let protocol = match route.protocol { @@ -92,6 +254,7 @@ impl UpstreamTarget { pub fn judged_ip(&self, domain: &str) -> Option { match self { Self::Resolved(addresses) => judged_address(addresses), + Self::Granted { judged_ip, .. } => *judged_ip, Self::Override { .. } | Self::Unresolved(_) => domain.parse().ok(), } } @@ -100,6 +263,7 @@ impl UpstreamTarget { pub fn protocol(&self, guest: Protocol) -> Protocol { match self { Self::Override { protocol, .. } => *protocol, + Self::Granted { protocol, .. } => *protocol, Self::Resolved(_) | Self::Unresolved(_) => guest, } } @@ -111,6 +275,12 @@ impl UpstreamTarget { let stream = match self { Self::Override { dial, .. } => TcpStream::connect(dial.as_str()).await?, Self::Resolved(addresses) => TcpStream::connect(addresses.as_slice()).await?, + Self::Granted { .. } => { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "descriptor grant required for brokered upstream target", + )); + } Self::Unresolved(error) => return Err(std::io::Error::new(std::io::ErrorKind::NotFound, error.clone())), }; let _ = stream.set_nodelay(true); @@ -120,6 +290,58 @@ impl UpstreamTarget { }; Ok((stream, pinned)) } + + /// Connect through the trusted grant provider. Direct targets are refused + /// whenever a provider is installed, preventing an accidental fallback. + pub async fn connect_with_grants( + &self, + grants: Option<&dyn TcpUpstreamGrants>, + ) -> io::Result<(GrantedTcpStream, Self)> { + let Some(grants) = grants else { + let (stream, pinned) = self.connect().await?; + return Ok((GrantedTcpStream::new(stream, || {}), pinned)); + }; + let Self::Granted { + host, + port, + guest_protocol, + selection, + protocol, + judged_ip, + } = self + else { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "direct upstream target refused while descriptor grants are required", + )); + }; + let refreshed; + let selection = match selection { + Some(selection) => selection, + None => { + refreshed = grants.resolve(*guest_protocol, host, *port).await?; + if refreshed.protocol != *protocol || refreshed.judged_ip != *judged_ip { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "refreshed upstream selection changed the judged target", + )); + } + &refreshed + } + }; + let stream = grants.connect(selection.claim()?).await?; + Ok(( + stream, + Self::Granted { + host: host.clone(), + port: *port, + guest_protocol: *guest_protocol, + selection: None, + protocol: *protocol, + judged_ip: *judged_ip, + }, + )) + } } impl fmt::Display for UpstreamTarget { @@ -127,6 +349,13 @@ impl fmt::Display for UpstreamTarget { match self { Self::Override { dial, .. } => write!(f, "override {dial}"), Self::Resolved(addresses) => write!(f, "{addresses:?}"), + Self::Granted { + host, + port, + protocol, + judged_ip, + .. + } => write!(f, "granted {host}:{port} ({protocol:?}, {judged_ip:?})"), Self::Unresolved(error) => write!(f, "unresolved ({error})"), } } @@ -161,7 +390,12 @@ impl CachedUpstream { } fn pinned_for(&self, domain: &str, port: u16) -> Option { - let pinned = self.domain == domain && self.port == port && matches!(self.target, UpstreamTarget::Resolved(_)); + let pinned = self.domain == domain + && self.port == port + && matches!( + self.target, + UpstreamTarget::Resolved(_) | UpstreamTarget::Granted { selection: None, .. } + ); pinned.then(|| self.target.clone()) } } diff --git a/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs b/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs index af0f5b194..cba198c59 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use super::super::{http_request_security_event, HttpRequestSecurityEventInput}; use super::*; @@ -11,6 +12,41 @@ const DEFAULT_RULES: &str = include_str!(concat!( )); const LOCAL_NETWORK: &str = "profiles.rules.default_000_local_network"; +struct TestGrants { + address: SocketAddr, + next_id: AtomicU64, + selection_releases: Arc, + stream_releases: Arc, +} + +impl TcpUpstreamGrants for TestGrants { + fn resolve(&self, _protocol: Protocol, _host: &str, _port: u16) -> TcpResolveGrantFuture<'_> { + let id = self.next_id.fetch_add(1, Ordering::Relaxed); + let releases = Arc::clone(&self.selection_releases); + Box::pin(async move { + Ok(TcpGrantSelection::new( + id, + Protocol::Http, + Some("127.0.0.1".parse().unwrap()), + move || { + releases.fetch_add(1, Ordering::Relaxed); + }, + )) + }) + } + + fn connect(&self, _selection_id: u64) -> TcpConnectGrantFuture<'_> { + let address = self.address; + let releases = Arc::clone(&self.stream_releases); + Box::pin(async move { + let stream = TcpStream::connect(address).await?; + Ok(GrantedTcpStream::new(stream, move || { + releases.fetch_add(1, Ordering::Relaxed); + })) + }) + } +} + fn default_rules() -> SecurityRuleSet { let profile = SecurityRuleProfile::parse_toml(DEFAULT_RULES).expect("defaults parse"); SecurityRuleSet::compile_profile(&profile, SecurityRuleSource::BuiltinDefault).expect("defaults compile") @@ -37,7 +73,16 @@ fn decision(target: &UpstreamTarget, domain: &str, port: u16) -> (String, Securi } async fn select(resolver: &UpstreamResolver, policy: &NetworkMechanics, domain: &str, port: u16) -> UpstreamTarget { - UpstreamTarget::select(resolver, policy, domain, port, &UpstreamCache::default()).await + UpstreamTarget::select( + resolver, + policy, + Protocol::Tls, + domain, + port, + &UpstreamCache::default(), + None, + ) + .await } #[tokio::test] @@ -186,11 +231,12 @@ async fn keep_alive_requests_reuse_only_their_own_pinned_upstream() { .with_fixed_answer("other.example", vec!["127.0.0.1".parse().unwrap()]); let policy = NetworkMechanics::new(); - let same = UpstreamTarget::select(&rebound, &policy, "site.example", 80, &cache).await; + let same = UpstreamTarget::select(&rebound, &policy, Protocol::Http, "site.example", 80, &cache, None).await; assert_eq!(same, pinned, "the same host stays on the address it was judged with"); - let other_port = UpstreamTarget::select(&rebound, &policy, "site.example", 8080, &cache).await; - let other_host = UpstreamTarget::select(&rebound, &policy, "other.example", 80, &cache).await; + let other_port = + UpstreamTarget::select(&rebound, &policy, Protocol::Http, "site.example", 8080, &cache, None).await; + let other_host = UpstreamTarget::select(&rebound, &policy, Protocol::Http, "other.example", 80, &cache, None).await; assert_eq!( other_host.judged_ip("other.example"), Some("127.0.0.1".parse().unwrap()) @@ -219,7 +265,16 @@ async fn a_trusted_override_preempts_an_existing_resolved_connection() { }, )]); - let selected = UpstreamTarget::select(&UpstreamResolver::system(), &policy, "site.example", 443, &cache).await; + let selected = UpstreamTarget::select( + &UpstreamResolver::system(), + &policy, + Protocol::Tls, + "site.example", + 443, + &cache, + None, + ) + .await; assert_eq!( selected, @@ -229,3 +284,72 @@ async fn a_trusted_override_preempts_an_existing_resolved_connection() { } ); } + +#[tokio::test] +async fn brokered_selection_is_judged_connected_pinned_and_released() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let selection_releases = Arc::new(AtomicUsize::new(0)); + let stream_releases = Arc::new(AtomicUsize::new(0)); + let grants = TestGrants { + address: listener.local_addr().unwrap(), + next_id: AtomicU64::new(1), + selection_releases: Arc::clone(&selection_releases), + stream_releases: Arc::clone(&stream_releases), + }; + let cache = UpstreamCache::default(); + let target = UpstreamTarget::select( + &UpstreamResolver::system(), + &NetworkMechanics::new(), + Protocol::Tls, + "brokered.example", + 443, + &cache, + Some(&grants), + ) + .await; + assert_eq!(target.judged_ip("brokered.example"), Some("127.0.0.1".parse().unwrap())); + assert_eq!(target.protocol(Protocol::Tls), Protocol::Http); + + let (connected, accepted) = tokio::join!(target.connect_with_grants(Some(&grants)), listener.accept()); + let (stream, pinned) = connected.unwrap(); + let (_peer, _) = accepted.unwrap(); + assert!(matches!(pinned, UpstreamTarget::Granted { selection: None, .. })); + assert_eq!(selection_releases.load(Ordering::Relaxed), 0); + drop(stream); + assert_eq!(stream_releases.load(Ordering::Relaxed), 1); + + let unused = UpstreamTarget::select( + &UpstreamResolver::system(), + &NetworkMechanics::new(), + Protocol::Tls, + "blocked.example", + 443, + &UpstreamCache::default(), + Some(&grants), + ) + .await; + drop(unused); + assert_eq!(selection_releases.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn installed_grants_never_fall_back_to_a_direct_target() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let grants = TestGrants { + address: listener.local_addr().unwrap(), + next_id: AtomicU64::new(1), + selection_releases: Arc::new(AtomicUsize::new(0)), + stream_releases: Arc::new(AtomicUsize::new(0)), + }; + let direct = UpstreamTarget::Resolved(vec![listener.local_addr().unwrap()]); + let error = match direct.connect_with_grants(Some(&grants)).await { + Ok(_) => panic!("direct connection bypassed installed grants"), + Err(error) => error, + }; + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), listener.accept()) + .await + .is_err() + ); +} diff --git a/crates/capsem-core/tests/mitm_integration.rs b/crates/capsem-core/tests/mitm_integration.rs index e97e60fbc..2338fc54f 100644 --- a/crates/capsem-core/tests/mitm_integration.rs +++ b/crates/capsem-core/tests/mitm_integration.rs @@ -195,6 +195,7 @@ fn make_proxy_config_with_mechanics( pipeline, mcp_endpoint: None, upstream_resolver: Default::default(), + upstream_grants: None, }); (config, db) } From b061ccd35784f1e895365d6297061a5a86919665 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:40:04 +0000 Subject: [PATCH 040/464] test(core): exercise MITM through TCP grants --- crates/capsem-core/tests/mitm_integration.rs | 67 +++++++++++++++++-- .../mitm_integration/resolved_upstream.rs | 4 ++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/crates/capsem-core/tests/mitm_integration.rs b/crates/capsem-core/tests/mitm_integration.rs index 2338fc54f..765220c89 100644 --- a/crates/capsem-core/tests/mitm_integration.rs +++ b/crates/capsem-core/tests/mitm_integration.rs @@ -6,12 +6,18 @@ /// - Denied domains are rejected before TLS handshake completes /// - Telemetry records correct decisions, methods, and status codes /// -use std::collections::BTreeMap; -use std::sync::Arc; +use std::collections::{BTreeMap, HashMap}; +use std::io; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; use capsem_core::net::cert_authority::CertAuthority; -use capsem_core::net::mitm_proxy::{self, MitmProxyConfig}; +use capsem_core::net::mitm_proxy::{ + self, protocol::Protocol, GrantedTcpStream, MitmProxyConfig, TcpConnectGrantFuture, TcpGrantSelection, + TcpResolveGrantFuture, TcpUpstreamGrants, UpstreamTarget, +}; use capsem_core::net::policy::{NetworkMechanics, UpstreamOverride, UpstreamOverrideProtocol}; +use capsem_core::net::upstream_address::UpstreamResolver; use capsem_logger::{DbWriter, Decision}; use http_body_util::{BodyExt, Full}; use hyper::body::Bytes; @@ -28,6 +34,58 @@ const CA_KEY: &str = include_str!("../resources/ca/capsem-ca.key"); const CA_CERT: &str = include_str!("../resources/ca/capsem-ca.crt"); const HERMETIC_UPSTREAM_DOMAIN: &str = "fixture.capsem.test"; +struct IntegrationGrants { + policy: Arc>>, + resolver: UpstreamResolver, + selections: Arc>>, + next_id: AtomicU64, +} + +impl IntegrationGrants { + fn new(policy: Arc>>, resolver: UpstreamResolver) -> Self { + Self { + policy, + resolver, + selections: Arc::new(Mutex::new(HashMap::new())), + next_id: AtomicU64::new(1), + } + } +} + +impl TcpUpstreamGrants for IntegrationGrants { + fn resolve(&self, protocol: Protocol, host: &str, port: u16) -> TcpResolveGrantFuture<'_> { + let host = host.to_owned(); + Box::pin(async move { + let policy = self.policy.read().unwrap().clone(); + let target = UpstreamTarget::resolve(&self.resolver, &policy, &host, port).await; + if let UpstreamTarget::Unresolved(error) = &target { + return Err(io::Error::new(io::ErrorKind::NotFound, error.clone())); + } + let judged_ip = target.judged_ip(&host); + let protocol = target.protocol(protocol); + let selection_id = self.next_id.fetch_add(1, Ordering::Relaxed); + self.selections.lock().unwrap().insert(selection_id, target); + let selections = Arc::clone(&self.selections); + Ok(TcpGrantSelection::new(selection_id, protocol, judged_ip, move || { + selections.lock().unwrap().remove(&selection_id); + })) + }) + } + + fn connect(&self, selection_id: u64) -> TcpConnectGrantFuture<'_> { + Box::pin(async move { + let target = self + .selections + .lock() + .unwrap() + .remove(&selection_id) + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "unknown integration selection"))?; + let (stream, _pinned) = target.connect().await?; + Ok(GrantedTcpStream::new(stream, || {})) + }) + } +} + /// Build a proxy config from allow/block lists for integration tests. /// /// Enforcement intent is compiled into `SecurityRuleSet` so tests exercise the @@ -180,6 +238,7 @@ fn make_proxy_config_with_mechanics( plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); let pipeline = mitm_proxy::make_production_pipeline(Arc::clone(&policy), Arc::clone(&telemetry)); + let upstream_grants = Arc::new(IntegrationGrants::new(Arc::clone(&policy), UpstreamResolver::system())); let config = Arc::new(MitmProxyConfig { server_tls: mitm_proxy::make_server_tls_config(&ca), ca, @@ -195,7 +254,7 @@ fn make_proxy_config_with_mechanics( pipeline, mcp_endpoint: None, upstream_resolver: Default::default(), - upstream_grants: None, + upstream_grants: Some(upstream_grants), }); (config, db) } diff --git a/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs b/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs index 4455351b5..716fbe1f1 100644 --- a/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs +++ b/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs @@ -177,6 +177,10 @@ fn default_rules() -> capsem_core::net::policy_config::SecurityRuleSet { fn resolving(config: Arc, name: &str, address: &str) -> Arc { let mut config = Arc::try_unwrap(config).ok().expect("config not yet shared"); config.upstream_resolver = UpstreamResolver::system().with_fixed_answer(name, vec![address.parse().unwrap()]); + config.upstream_grants = Some(Arc::new(IntegrationGrants::new( + Arc::clone(&config.policy), + config.upstream_resolver.clone(), + ))); Arc::new(config) } From c902a8b7ade645810f20e9f4c0a6ade2c663e14f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:40:59 +0000 Subject: [PATCH 041/464] security(process): broker MITM upstream TCP --- crates/capsem-process/src/main.rs | 1 + crates/capsem-process/src/upstream_grant.rs | 359 +++++++++++++++--- .../src/upstream_grant/tests.rs | 146 ++++++- 3 files changed, 450 insertions(+), 56 deletions(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index de06d64fb..91664bb18 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -614,6 +614,7 @@ async fn run_async_main_loop( pipeline: mitm_pipeline, mcp_endpoint: Some(mcp_endpoint), upstream_resolver: capsem_core::net::upstream_address::UpstreamResolver::system(), + upstream_grants: Some(Arc::clone(&upstream_grants) as Arc), }); // DNS handler shares the same security rule/plugin handles as MITM diff --git a/crates/capsem-process/src/upstream_grant.rs b/crates/capsem-process/src/upstream_grant.rs index 129be63da..d037b9f30 100644 --- a/crates/capsem-process/src/upstream_grant.rs +++ b/crates/capsem-process/src/upstream_grant.rs @@ -8,30 +8,50 @@ use std::time::Duration; use anyhow::{anyhow, Result}; use capsem_core::net::dns::{DnsDatagram, DnsGrantFuture, DnsUpstreamGrants}; +use capsem_core::net::mitm_proxy::{ + protocol::Protocol, GrantedTcpStream, TcpConnectGrantFuture, TcpGrantSelection, TcpResolveGrantFuture, + TcpUpstreamGrants, +}; use capsem_foundation::unix::fd::{self, SocketShutdown}; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; use capsem_proto::upstream_grant::{ decode_upstream_grant_response, encode_upstream_grant_request, UpstreamDescriptorKind, UpstreamGrantRequest, - UpstreamGrantResponse, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, + UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, }; use tokio::sync::{mpsc, oneshot}; const COMMAND_CAPACITY: usize = 64; -const WIRE_TIMEOUT: Duration = Duration::from_secs(2); +// The coordinator permits a TCP dial to take ten seconds. Leave room for its +// response framing while still bounding a wedged generation channel. +const WIRE_TIMEOUT: Duration = Duration::from_secs(12); type WireSender = DescriptorSender; type WireReceiver = DescriptorReceiver; -struct OpenDns { - upstream_index: u16, - policy_digest: String, - reply: oneshot::Sender>, +enum Command { + OpenDns { + upstream_index: u16, + policy_digest: String, + reply: oneshot::Sender>, + }, + ResolveTcp { + protocol: Protocol, + host: String, + port: u16, + policy_digest: String, + reply: oneshot::Sender>, + }, + ConnectTcp { + selection_id: u64, + policy_digest: String, + reply: oneshot::Sender>, + }, } /// Serializes grant protocol traffic over the inherited generation channel. #[derive(Clone)] pub(crate) struct UpstreamGrantClient { - opens: mpsc::Sender, + commands: mpsc::Sender, policy_digest: Arc>, } @@ -39,10 +59,10 @@ impl UpstreamGrantClient { pub(crate) fn start(socket: UnixStream, policy_digest: String) -> io::Result { let sender = WireSender::new(socket.try_clone()?)?; let receiver = WireReceiver::new(socket.try_clone()?)?; - let (opens_tx, opens) = mpsc::channel(COMMAND_CAPACITY); + let (commands_tx, commands) = mpsc::channel(COMMAND_CAPACITY); let (actor_releases, releases) = mpsc::unbounded_channel(); tokio::spawn(async move { - let result = run_actor(sender, receiver, opens, releases, actor_releases).await; + let result = run_actor(sender, receiver, commands, releases, actor_releases).await; if let Err(error) = result { tracing::warn!(%error, "upstream grant channel stopped"); } @@ -51,7 +71,7 @@ impl UpstreamGrantClient { } }); Ok(Self { - opens: opens_tx, + commands: commands_tx, policy_digest: Arc::new(RwLock::new(policy_digest)), }) } @@ -67,9 +87,9 @@ impl UpstreamGrantClient { #[cfg(test)] pub(crate) fn test_handle(policy_digest: String) -> Self { - let (opens, _commands) = mpsc::channel(COMMAND_CAPACITY); + let (commands, _requests) = mpsc::channel(COMMAND_CAPACITY); Self { - opens, + commands, policy_digest: Arc::new(RwLock::new(policy_digest)), } } @@ -80,8 +100,8 @@ impl DnsUpstreamGrants for UpstreamGrantClient { Box::pin(async move { let policy_digest = self.policy_digest.read().unwrap().clone(); let (reply, result) = oneshot::channel(); - self.opens - .send(OpenDns { + self.commands + .send(Command::OpenDns { upstream_index, policy_digest, reply, @@ -96,10 +116,53 @@ impl DnsUpstreamGrants for UpstreamGrantClient { } } +impl TcpUpstreamGrants for UpstreamGrantClient { + fn resolve(&self, protocol: Protocol, host: &str, port: u16) -> TcpResolveGrantFuture<'_> { + let host = host.to_owned(); + Box::pin(async move { + let policy_digest = self.policy_digest.read().unwrap().clone(); + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::ResolveTcp { + protocol, + host, + port, + policy_digest, + reply, + }) + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; + result + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? + .map_err(io::Error::other) + }) + } + + fn connect(&self, selection_id: u64) -> TcpConnectGrantFuture<'_> { + Box::pin(async move { + let policy_digest = self.policy_digest.read().unwrap().clone(); + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::ConnectTcp { + selection_id, + policy_digest, + reply, + }) + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; + result + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? + .map_err(io::Error::other) + }) + } +} + async fn run_actor( sender: WireSender, receiver: WireReceiver, - mut opens: mpsc::Receiver, + mut commands: mpsc::Receiver, mut releases: mpsc::UnboundedReceiver, release_sender: mpsc::UnboundedSender, ) -> Result<(), String> { @@ -109,61 +172,105 @@ async fn run_actor( biased; release = releases.recv() => { let Some(resource_id) = release else { - if opens.is_closed() { + if commands.is_closed() { return Ok(()); } continue; }; send_request(&sender, &UpstreamGrantRequest::Release { resource_id }).await?; } - open = opens.recv() => { - let Some(open) = open else { + command = commands.recv() => { + let Some(command) = command else { return Ok(()); }; let request_id = next_request_id; next_request_id = next_request_id.checked_add(1).ok_or("upstream request id exhausted")?; - match open_dns(&sender, &receiver, &release_sender, request_id, &open).await { - Ok(result) => { - let _ = open.reply.send(result); - } - Err(error) => { - let _ = open.reply.send(Err(error.clone())); - return Err(error); - } - } + dispatch(&sender, &receiver, &release_sender, request_id, command).await?; } } } } +async fn dispatch( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + command: Command, +) -> Result<(), String> { + match command { + Command::OpenDns { + upstream_index, + policy_digest, + reply, + } => finish( + reply, + open_dns(sender, receiver, releases, request_id, upstream_index, &policy_digest).await, + ), + Command::ResolveTcp { + protocol, + host, + port, + policy_digest, + reply, + } => finish( + reply, + resolve_tcp( + sender, + receiver, + releases, + request_id, + protocol, + &host, + port, + &policy_digest, + ) + .await, + ), + Command::ConnectTcp { + selection_id, + policy_digest, + reply, + } => finish( + reply, + connect_tcp(sender, receiver, releases, request_id, selection_id, &policy_digest).await, + ), + } +} + +fn finish( + reply: oneshot::Sender>, + result: Result, String>, +) -> Result<(), String> { + match result { + Ok(result) => { + let _ = reply.send(result); + Ok(()) + } + Err(error) => { + let _ = reply.send(Err(error.clone())); + Err(error) + } + } +} + async fn open_dns( sender: &WireSender, receiver: &WireReceiver, releases: &mpsc::UnboundedSender, request_id: u64, - open: &OpenDns, + upstream_index: u16, + expected_policy_digest: &str, ) -> Result, String> { send_request( sender, &UpstreamGrantRequest::OpenDns { request_id, - upstream_index: open.upstream_index, + upstream_index, }, ) .await?; - let frame = tokio::time::timeout(WIRE_TIMEOUT, receiver.recv()) - .await - .map_err(|_| "receive upstream grant response timed out".to_string())? - .map_err(|error| format!("receive upstream grant response: {error}"))?; - let response = decode_upstream_grant_response(&frame.bytes) - .map_err(|error| format!("decode upstream grant response: {error:#}"))?; - if frame.fds.len() != response.expected_descriptor_count() { - return Err(format!( - "upstream grant response carried {} descriptors, expected {}", - frame.fds.len(), - response.expected_descriptor_count() - )); - } + let (response, fds) = receive_response(receiver).await?; match response { UpstreamGrantResponse::Denied { request_id: response_id, @@ -173,15 +280,14 @@ async fn open_dns( request_id: response_id, grant_id, kind: UpstreamDescriptorKind::DnsUdp, - policy_digest, + policy_digest: response_digest, } if response_id == request_id => { - let descriptor = frame.fds.into_iter().next().ok_or("DNS grant omitted its descriptor")?; + let descriptor = fds.into_iter().next().ok_or("DNS grant omitted its descriptor")?; send_request(sender, &UpstreamGrantRequest::Adopted { grant_id }).await?; - if policy_digest != open.policy_digest { + if response_digest != expected_policy_digest { send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; return Ok(Err(format!( - "upstream DNS grant policy mismatch: expected {}, received {policy_digest}", - open.policy_digest + "upstream DNS grant policy mismatch: expected {expected_policy_digest}, received {response_digest}" ))); } let socket = datagram_from_descriptor(descriptor)?; @@ -196,6 +302,154 @@ async fn open_dns( } } +#[allow(clippy::too_many_arguments)] +async fn resolve_tcp( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + protocol: Protocol, + host: &str, + port: u16, + expected_policy_digest: &str, +) -> Result, String> { + let protocol = match wire_protocol(protocol) { + Ok(protocol) => protocol, + Err(error) => return Ok(Err(error)), + }; + send_request( + sender, + &UpstreamGrantRequest::ResolveTcp { + request_id, + protocol, + host: host.to_owned(), + port, + }, + ) + .await?; + let (response, _fds) = receive_response(receiver).await?; + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("upstream TCP selection denied: {reason:?}"))), + UpstreamGrantResponse::TcpResolved { + request_id: response_id, + selection_id, + protocol, + judged_ip, + policy_digest, + } if response_id == request_id => { + if policy_digest != expected_policy_digest { + send_request( + sender, + &UpstreamGrantRequest::Release { + resource_id: selection_id, + }, + ) + .await?; + return Ok(Err(format!( + "upstream TCP selection policy mismatch: expected {expected_policy_digest}, received {policy_digest}" + ))); + } + let releases = releases.clone(); + Ok(Ok(TcpGrantSelection::new( + selection_id, + core_protocol(protocol), + judged_ip, + move || { + let _ = releases.send(selection_id); + }, + ))) + } + response => Err(format!( + "unexpected upstream TCP selection response for request {request_id}: {response:?}" + )), + } +} + +async fn connect_tcp( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + selection_id: u64, + expected_policy_digest: &str, +) -> Result, String> { + send_request( + sender, + &UpstreamGrantRequest::ConnectTcp { + request_id, + selection_id, + }, + ) + .await?; + let (response, fds) = receive_response(receiver).await?; + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("upstream TCP connection denied: {reason:?}"))), + UpstreamGrantResponse::DescriptorGranted { + request_id: response_id, + grant_id, + kind: UpstreamDescriptorKind::Tcp, + policy_digest, + } if response_id == request_id => { + let descriptor = fds.into_iter().next().ok_or("TCP grant omitted its descriptor")?; + send_request(sender, &UpstreamGrantRequest::Adopted { grant_id }).await?; + if policy_digest != expected_policy_digest { + send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; + return Ok(Err(format!( + "upstream TCP grant policy mismatch: expected {expected_policy_digest}, received {policy_digest}" + ))); + } + let stream = stream_from_descriptor(descriptor)?; + let releases = releases.clone(); + Ok(Ok(GrantedTcpStream::new(stream, move || { + let _ = releases.send(grant_id); + }))) + } + response => Err(format!( + "unexpected upstream TCP grant response for request {request_id}: {response:?}" + )), + } +} + +async fn receive_response(receiver: &WireReceiver) -> Result<(UpstreamGrantResponse, Vec), String> { + let frame = tokio::time::timeout(WIRE_TIMEOUT, receiver.recv()) + .await + .map_err(|_| "receive upstream grant response timed out".to_string())? + .map_err(|error| format!("receive upstream grant response: {error}"))?; + let response = decode_upstream_grant_response(&frame.bytes) + .map_err(|error| format!("decode upstream grant response: {error:#}"))?; + if frame.fds.len() != response.expected_descriptor_count() { + return Err(format!( + "upstream grant response carried {} descriptors, expected {}", + frame.fds.len(), + response.expected_descriptor_count() + )); + } + Ok((response, frame.fds)) +} + +fn wire_protocol(protocol: Protocol) -> Result { + match protocol { + Protocol::Http => Ok(UpstreamProtocol::Http), + Protocol::Tls => Ok(UpstreamProtocol::Tls), + Protocol::McpFrame | Protocol::Unknown => { + Err(format!("protocol {} cannot select a TCP upstream", protocol.label())) + } + } +} + +fn core_protocol(protocol: UpstreamProtocol) -> Protocol { + match protocol { + UpstreamProtocol::Http => Protocol::Http, + UpstreamProtocol::Tls => Protocol::Tls, + } +} + async fn send_request(sender: &WireSender, request: &UpstreamGrantRequest) -> Result<(), String> { let bytes = encode_upstream_grant_request(request).map_err(|error| format!("encode upstream grant request: {error:#}"))?; @@ -214,6 +468,17 @@ fn datagram_from_descriptor(descriptor: OwnedFd) -> Result Result { + let stream = std::net::TcpStream::from(descriptor); + stream + .set_nonblocking(true) + .map_err(|error| format!("make granted TCP stream nonblocking: {error}"))?; + stream + .set_nodelay(true) + .map_err(|error| format!("set granted TCP stream nodelay: {error}"))?; + tokio::net::TcpStream::from_std(stream).map_err(|error| format!("adopt granted TCP stream: {error}")) +} + /// Duplicate the inherited generation channel before runtime threads exist. pub(crate) fn adopt_inherited(descriptor: BorrowedFd<'_>) -> io::Result { fd::set_close_on_exec(descriptor)?; diff --git a/crates/capsem-process/src/upstream_grant/tests.rs b/crates/capsem-process/src/upstream_grant/tests.rs index 797919bf0..992b1165d 100644 --- a/crates/capsem-process/src/upstream_grant/tests.rs +++ b/crates/capsem-process/src/upstream_grant/tests.rs @@ -1,13 +1,15 @@ use std::io::{Read as _, Write as _}; -use std::os::fd::{AsFd as _, AsRawFd as _}; +use std::os::fd::{AsFd as _, AsRawFd as _, RawFd}; use capsem_core::net::dns::{DnsResolver, DnsUpstreamGrants}; +use capsem_core::net::mitm_proxy::{TcpUpstreamGrants, UpstreamTarget}; use capsem_proto::upstream_grant::{ decode_upstream_grant_request, encode_upstream_grant_response, UpstreamDescriptorKind, UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, }; use super::*; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; const POLICY_A: &str = "blake3:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const POLICY_B: &str = "blake3:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; @@ -31,13 +33,9 @@ async fn receive_request(receiver: &WireReceiver) -> UpstreamGrantRequest { decode_upstream_grant_request(&frame.bytes).unwrap() } -async fn send_response( - sender: &WireSender, - response: &UpstreamGrantResponse, - descriptor: Option<&std::net::UdpSocket>, -) { +async fn send_response(sender: &WireSender, response: &UpstreamGrantResponse, descriptor: Option) { let bytes = encode_upstream_grant_response(response).unwrap(); - let fds = descriptor.map_or_else(Vec::new, |socket| vec![socket.as_raw_fd()]); + let fds = descriptor.map_or_else(Vec::new, |descriptor| vec![descriptor]); sender.send(&bytes, &fds).await.unwrap(); } @@ -74,7 +72,7 @@ async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { kind: UpstreamDescriptorKind::DnsUdp, policy_digest: POLICY_A.into(), }, - Some(&socket), + Some(socket.as_raw_fd()), ) .await; assert_eq!( @@ -122,7 +120,7 @@ async fn stale_policy_grant_is_adopted_released_and_channel_remains_usable() { kind: UpstreamDescriptorKind::DnsUdp, policy_digest: POLICY_B.into(), }, - Some(&socket), + Some(socket.as_raw_fd()), ) .await; assert_eq!( @@ -189,6 +187,136 @@ async fn malformed_or_revoked_grant_channel_fails_closed() { assert!(!open_error(&revoked).await.is_empty()); } +#[tokio::test] +async fn tcp_selection_and_connected_stream_are_brokered_and_released() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream_address = listener.local_addr().unwrap(); + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Tls, + host: "api.example".into(), + port: 443, + } + ); + send_response( + &responses, + &UpstreamGrantResponse::TcpResolved { + request_id: 1, + selection_id: 71, + protocol: UpstreamProtocol::Http, + judged_ip: Some("127.0.0.1".parse().unwrap()), + policy_digest: POLICY_A.into(), + }, + None, + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::ConnectTcp { + request_id: 2, + selection_id: 71, + } + ); + let socket = tokio::net::TcpStream::connect(upstream_address) + .await + .unwrap() + .into_std() + .unwrap(); + let (mut upstream, _) = listener.accept().await.unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::DescriptorGranted { + request_id: 2, + grant_id: 72, + kind: UpstreamDescriptorKind::Tcp, + policy_digest: POLICY_A.into(), + }, + Some(socket.as_raw_fd()), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { grant_id: 72 } + ); + let mut request = [0_u8; 4]; + upstream.read_exact(&mut request).await.unwrap(); + assert_eq!(&request, b"ping"); + upstream.write_all(b"pong").await.unwrap(); + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { resource_id: 72 } + ); + }); + + let selection = client.resolve(Protocol::Tls, "api.example", 443).await.unwrap(); + assert_eq!(selection.protocol, Protocol::Http); + assert_eq!(selection.judged_ip, Some("127.0.0.1".parse().unwrap())); + let target = UpstreamTarget::Granted { + host: "api.example".into(), + port: 443, + guest_protocol: Protocol::Tls, + protocol: selection.protocol, + judged_ip: selection.judged_ip, + selection: Some(selection), + }; + let (mut stream, pinned) = target.connect_with_grants(Some(&client)).await.unwrap(); + assert!(matches!(pinned, UpstreamTarget::Granted { selection: None, .. })); + stream.write_all(b"ping").await.unwrap(); + let mut response = [0_u8; 4]; + stream.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"pong"); + drop(stream); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn unused_or_stale_tcp_selections_are_released_without_connecting() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + for (request_id, selection_id, digest) in [(1, 81, POLICY_A), (2, 82, POLICY_B)] { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::ResolveTcp { + request_id: actual, + .. + } if actual == request_id + )); + send_response( + &responses, + &UpstreamGrantResponse::TcpResolved { + request_id, + selection_id, + protocol: UpstreamProtocol::Tls, + judged_ip: Some("93.184.216.34".parse().unwrap()), + policy_digest: digest.into(), + }, + None, + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { + resource_id: selection_id, + } + ); + } + }); + + let unused = client.resolve(Protocol::Tls, "unused.example", 443).await.unwrap(); + drop(unused); + let error = client.resolve(Protocol::Tls, "stale.example", 443).await.unwrap_err(); + assert!(error.to_string().contains("policy mismatch"), "{error}"); + broker_task.await.unwrap(); +} + #[test] fn inherited_channel_is_duplicated_cloexec_and_connected() { let (inherited, peer) = UnixStream::pair().unwrap(); From bffe04aa69a09cde2496c7950a24d980b408340b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:44:37 +0000 Subject: [PATCH 042/464] test(service): stress upstream grant isolation --- .../src/upstream_broker/tests.rs | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index fad7780d5..5a3945aeb 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -457,3 +457,114 @@ async fn disallowed_plain_http_port_is_denied_before_dial() { .is_err()); stop_broker(authority, task).await; } + +#[tokio::test] +async fn opaque_selections_cannot_cross_worker_generations() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let policy = || test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![]); + let (first, _first_publisher, first_authority, first_task) = start_broker(policy()); + let (second, _second_publisher, second_authority, second_task) = start_broker(policy()); + let selection_id = resolve_override(&first, 1).await; + + let (response, fds) = second + .request(&UpstreamGrantRequest::ConnectTcp { + request_id: 1, + selection_id, + }) + .await; + assert!(fds.is_empty()); + assert_eq!( + response, + UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::InvalidResource, + } + ); + assert!(tokio::time::timeout(Duration::from_millis(100), listener.accept()) + .await + .is_err()); + first + .send(&UpstreamGrantRequest::Release { + resource_id: selection_id, + }) + .await; + stop_broker(first_authority, first_task).await; + stop_broker(second_authority, second_task).await; +} + +#[tokio::test] +async fn active_descriptor_capacity_is_bounded_and_recovers_after_release() { + let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let (client, _publisher, authority, task) = + start_broker(test_policy("policy-a", None, vec![upstream.local_addr().unwrap()])); + let mut descriptors = Vec::with_capacity(MAX_ACTIVE_GRANTS); + let mut grant_ids = Vec::with_capacity(MAX_ACTIVE_GRANTS); + for request_id in 1..=MAX_ACTIVE_GRANTS as u64 { + let (response, mut fds) = client + .request(&UpstreamGrantRequest::OpenDns { + request_id, + upstream_index: 0, + }) + .await; + let UpstreamGrantResponse::DescriptorGranted { grant_id, .. } = response else { + panic!("expected descriptor grant, got {response:?}"); + }; + client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; + grant_ids.push(grant_id); + descriptors.push(fds.pop().unwrap()); + } + + let denied_id = MAX_ACTIVE_GRANTS as u64 + 1; + let (response, fds) = client + .request(&UpstreamGrantRequest::OpenDns { + request_id: denied_id, + upstream_index: 0, + }) + .await; + assert!(fds.is_empty()); + assert_eq!( + response, + UpstreamGrantResponse::Denied { + request_id: denied_id, + reason: UpstreamGrantDenial::Capacity, + } + ); + + for grant_id in grant_ids { + client + .send(&UpstreamGrantRequest::Release { resource_id: grant_id }) + .await; + } + drop(descriptors); + let (response, mut fds) = client + .request(&UpstreamGrantRequest::OpenDns { + request_id: denied_id + 1, + upstream_index: 0, + }) + .await; + let UpstreamGrantResponse::DescriptorGranted { grant_id, .. } = response else { + panic!("capacity did not recover: {response:?}"); + }; + client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; + client + .send(&UpstreamGrantRequest::Release { resource_id: grant_id }) + .await; + drop(fds.pop().unwrap()); + stop_broker(authority, task).await; +} + +#[tokio::test] +async fn malformed_frame_closes_the_generation_before_any_response() { + let (client, _publisher, _authority, task) = start_broker(test_policy("policy-a", None, vec![])); + let mut malformed = [0_u8; UPSTREAM_GRANT_FRAME_SIZE]; + malformed[..2].copy_from_slice(b"XX"); + client.requests.send(&malformed, &[]).await.unwrap(); + + let error = tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert!(error.contains("invalid upstream grant magic"), "{error}"); + assert!(client.responses.recv().await.is_err()); +} From ceed697ea8f8d46d6ced427a2c9d9ff08fa35816 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:48:21 +0000 Subject: [PATCH 043/464] security(core): add resolver without host authority --- .../capsem-core/src/net/upstream_address.rs | 27 ++++++++++++++++--- .../src/net/upstream_address/tests.rs | 11 ++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/crates/capsem-core/src/net/upstream_address.rs b/crates/capsem-core/src/net/upstream_address.rs index 577f4ba67..b1f3b5652 100644 --- a/crates/capsem-core/src/net/upstream_address.rs +++ b/crates/capsem-core/src/net/upstream_address.rs @@ -83,15 +83,35 @@ pub async fn resolve_upstream(host: &str, port: u16) -> Result, /// Where guest-named upstreams resolve. The system resolver unless a name /// carries a fixed answer -- a deterministic stand-in for a DNS record, so a /// test can point a public-looking name at loopback without touching DNS. -#[derive(Clone, Debug, Default)] +#[derive(Clone, Debug)] pub struct UpstreamResolver { fixed: BTreeMap>, + system: bool, +} + +impl Default for UpstreamResolver { + fn default() -> Self { + Self::system() + } } impl UpstreamResolver { /// Resolve every name through the host's system resolver. pub fn system() -> Self { - Self::default() + Self { + fixed: BTreeMap::new(), + system: true, + } + } + + /// Carry no host resolution authority. Confined workers use connected + /// descriptor grants instead, so even an accidental direct selection + /// must fail closed before consulting DNS. + pub fn disabled() -> Self { + Self { + fixed: BTreeMap::new(), + system: false, + } } /// Answer `name` with `addresses` instead of asking the system resolver. @@ -108,7 +128,8 @@ impl UpstreamResolver { .iter() .map(|address| SocketAddr::new(address.to_canonical(), port)) .collect()), - None => resolve_upstream(host, port).await, + None if self.system => resolve_upstream(host, port).await, + None => Err("upstream resolution is disabled in this process".to_owned()), } } } diff --git a/crates/capsem-core/src/net/upstream_address/tests.rs b/crates/capsem-core/src/net/upstream_address/tests.rs index 4453b9c19..e9cba707f 100644 --- a/crates/capsem-core/src/net/upstream_address/tests.rs +++ b/crates/capsem-core/src/net/upstream_address/tests.rs @@ -110,6 +110,17 @@ async fn a_fixed_answer_replaces_the_system_resolver_for_that_name_only() { ); } +#[tokio::test] +async fn a_disabled_resolver_has_no_system_or_literal_authority() { + let resolver = UpstreamResolver::disabled(); + for host in ["localhost", "127.0.0.1", "::1"] { + assert_eq!( + resolver.resolve(host, 443).await.unwrap_err(), + "upstream resolution is disabled in this process" + ); + } +} + #[test] fn the_judged_address_fails_closed_on_any_local_answer() { assert_eq!(judged_address(&[]), None); From 02b1b2e2c263eb16473a3dd44c5ff76109dea197 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:48:58 +0000 Subject: [PATCH 044/464] security(process): remove host resolver authority --- crates/capsem-process/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 91664bb18..7b11054dc 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -613,7 +613,7 @@ async fn run_async_main_loop( telemetry: telemetry_deps, pipeline: mitm_pipeline, mcp_endpoint: Some(mcp_endpoint), - upstream_resolver: capsem_core::net::upstream_address::UpstreamResolver::system(), + upstream_resolver: capsem_core::net::upstream_address::UpstreamResolver::disabled(), upstream_grants: Some(Arc::clone(&upstream_grants) as Arc), }); From 6701fe688ef0f8d5863a91529f8f414e5adfea5a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:49:04 +0000 Subject: [PATCH 045/464] test(citadel): guard upstream grant boundary --- tests/citadel/test_upstream_grant_boundary.py | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 tests/citadel/test_upstream_grant_boundary.py diff --git a/tests/citadel/test_upstream_grant_boundary.py b/tests/citadel/test_upstream_grant_boundary.py new file mode 100644 index 000000000..17af05add --- /dev/null +++ b/tests/citadel/test_upstream_grant_boundary.py @@ -0,0 +1,130 @@ +"""Citadel guard for generation-bound upstream descriptor grants. + +The VM owner may name a host, protocol and configured DNS index. Only the +trusted service resolves or connects those names. The service hands the owner +a connected descriptor and never relays workload bytes, keeping policy and +telemetry on the trusted side without putting traffic on the control channel. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] + +RATIONALE = """\ +Confined VM owners obtain upstream access only through generation-bound grants. + +capsem-process may adopt connected TCP and DNS descriptors, but it must not +resolve hosts, bind an outbound socket, or dial an address. The trusted service +selects the peer from its active policy, opens the socket, and passes the file +descriptor. MITM request, retry and WebSocket paths must consume those grants. +Payload bytes stay on the connected descriptor and never cross the coordinator +control channel. This prevents arbitrary egress, stale-generation reuse and a +high-bandwidth workload relay through the trusted service. +""" + +DIRECT_UPSTREAM = re.compile( + r"\b(?:TcpStream|UdpSocket|TcpSocket|lookup_host|ToSocketAddrs)\b" + r"|\bUpstreamResolver::system\b|\bUpstreamTarget::resolve\b" +) + +# These are descriptor adoption only. The exact source lines make a new +# constructor, alias or socket operation a reviewed change to this boundary. +ALLOWED_PROCESS_SOCKET_LINES = { + "crates/capsem-process/src/cables.rs": { + "let source = capsem_core::container::publish::Source(std::net::TcpStream::from(" + }, + "crates/capsem-process/src/upstream_grant.rs": { + "fn datagram_from_descriptor(descriptor: OwnedFd) -> Result {", + "let socket = std::net::UdpSocket::from(descriptor);", + 'tokio::net::UdpSocket::from_std(socket).map_err(|error| format!("adopt granted DNS socket: {error}"))', + "fn stream_from_descriptor(descriptor: OwnedFd) -> Result {", + "let stream = std::net::TcpStream::from(descriptor);", + 'tokio::net::TcpStream::from_std(stream).map_err(|error| format!("adopt granted TCP stream: {error}"))', + }, +} + + +def _production_rust(root: Path) -> list[Path]: + source = root / "crates" / "capsem-process" / "src" + return [ + path + for path in sorted(source.rglob("*.rs")) + if "tests" not in path.relative_to(source).parts and path.name != "tests.rs" + ] + + +def _code_lines(path: Path) -> list[tuple[int, str]]: + lines = [] + for number, source in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + code = source.split("//", 1)[0].strip() + if code: + lines.append((number, code)) + return lines + + +def process_direct_upstream_references(root: Path = PROJECT_ROOT) -> list[str]: + found = [] + for path in _production_rust(root): + relative = path.relative_to(root).as_posix() + allowed = ALLOWED_PROCESS_SOCKET_LINES.get(relative, set()) + for number, code in _code_lines(path): + if DIRECT_UPSTREAM.search(code) and code not in allowed: + found.append(f"{relative}:{number}: {code}") + return found + + +def test_process_can_only_adopt_upstream_descriptors() -> None: + found = process_direct_upstream_references() + assert not found, RATIONALE + "\n" + "\n".join(found) + + +def test_process_installs_grants_for_dns_and_mitm() -> None: + main = (PROJECT_ROOT / "crates/capsem-process/src/main.rs").read_text(encoding="utf-8") + assert "DnsResolver::with_grants(" in main, RATIONALE + assert "upstream_grants: Some(" in main, RATIONALE + assert "Arc" in main, RATIONALE + assert "Arc" in main, RATIONALE + + +def test_every_mitm_dial_path_consumes_a_grant() -> None: + mitm = PROJECT_ROOT / "crates/capsem-core/src/net/mitm_proxy" + paths = [mitm / "mod.rs", mitm / "upgrade.rs"] + source = "\n".join(path.read_text(encoding="utf-8") for path in paths) + assert source.count(".connect_with_grants(config.upstream_grants.as_deref())") == 3, RATIONALE + assert not re.search(r"\btarget\s*\.\s*connect\s*\(", source), RATIONALE + + +def test_service_broker_opens_sockets_without_relaying_bodies() -> None: + source = (PROJECT_ROOT / "crates/capsem-service/src/upstream_broker.rs").read_text(encoding="utf-8") + for operation in ( + "UpstreamTarget::resolve(", + "selection.target.connect()", + "tokio::net::UdpSocket::bind(", + "socket.connect(upstream)", + ): + assert operation in source, RATIONALE + forbidden_relay = re.compile( + r"\bcopy_bidirectional\b|\bAsyncReadExt\b|\bAsyncWriteExt\b" + r"|\.read_exact\s*\(|\.write_all\s*\(" + ) + assert not forbidden_relay.search(source), RATIONALE + + +def test_process_socket_evasions_are_reported(tmp_path: Path) -> None: + source = tmp_path / "crates/capsem-process/src" + source.mkdir(parents=True) + (source / "main.rs").write_text( + "use tokio::net::TcpStream as Dialer;\n" + 'let tcp = Dialer::connect("203.0.113.7:443").await?;\n' + 'let dns = std::net::UdpSocket::bind("0.0.0.0:0")?;\n' + "let target = UpstreamTarget::resolve(&UpstreamResolver::system(), policy, host, port).await;\n", + encoding="utf-8", + ) + assert process_direct_upstream_references(tmp_path) == [ + "crates/capsem-process/src/main.rs:1: use tokio::net::TcpStream as Dialer;", + 'crates/capsem-process/src/main.rs:3: let dns = std::net::UdpSocket::bind("0.0.0.0:0")?;', + "crates/capsem-process/src/main.rs:4: let target = UpstreamTarget::resolve(&UpstreamResolver::system(), policy, host, port).await;", + ] From f0ce7e037ff44df6085b905ae4f4be15bff3f2ee Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:51:57 +0000 Subject: [PATCH 046/464] fix(citadel): scope sparse fields to serde types --- .../test_sparse_messagepack_boundary.py | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/citadel/test_sparse_messagepack_boundary.py b/tests/citadel/test_sparse_messagepack_boundary.py index c46b6cdd8..78009c440 100644 --- a/tests/citadel/test_sparse_messagepack_boundary.py +++ b/tests/citadel/test_sparse_messagepack_boundary.py @@ -47,13 +47,25 @@ def default_field_violations(source: str) -> list[str]: """Return defaultable wire fields whose omission/decoding pair is incomplete.""" violations: list[str] = [] depth = 0 + inspect_type = False + type_attributes: list[str] = [] + type_attribute_open = False attributes: list[str] = [] attribute_open = False for number, original in enumerate(source.splitlines(), 1): line = original.split("//", 1)[0].strip() if depth == 0: + if line.startswith("#[") or type_attribute_open: + type_attributes.append(line) + type_attribute_open = not line.endswith(")]" ) + continue if TYPE_START.search(line): depth = line.count("{") - line.count("}") + derives = " ".join(type_attributes) + inspect_type = "Serialize" in derives or "Deserialize" in derives + type_attributes.clear() + elif line: + type_attributes.clear() continue if line.startswith("#[serde(") or attribute_open: attributes.append(line) @@ -63,7 +75,9 @@ def default_field_violations(source: str) -> list[str]: else: attrs = " ".join(attributes) missing_pair = "default" not in attrs or "skip_serializing_if" not in attrs - if (FIELD.search(line) and missing_pair) or ("default" in attrs and "skip_serializing_if" not in attrs): + if inspect_type and ( + (FIELD.search(line) and missing_pair) or ("default" in attrs and "skip_serializing_if" not in attrs) + ): violations.append(f"line {number}: {line}") attributes.clear() depth += line.count("{") - line.count("}") @@ -80,22 +94,29 @@ def encoder_violations(relative: Path, source: str) -> list[str]: def test_guard_rejects_missing_omission_and_wild_encoding() -> None: - missing = "pub struct Event {\n pub value: Option,\n}" - assert default_field_violations(missing) == ["line 2: pub value: Option,"] - default_scalar = "pub struct Event {\n #[serde(default)]\n pub count: u64,\n}" - assert default_field_violations(default_scalar) == ["line 3: pub count: u64,"] - multiline = "pub struct Event {\n #[serde(\n default,\n )]\n pub count: u64,\n}" - assert default_field_violations(multiline) == ["line 5: pub count: u64,"] + missing = "#[derive(Serialize)]\npub struct Event {\n pub value: Option,\n}" + assert default_field_violations(missing) == ["line 3: pub value: Option,"] + default_scalar = "#[derive(Deserialize)]\npub struct Event {\n #[serde(default)]\n pub count: u64,\n}" + assert default_field_violations(default_scalar) == ["line 4: pub count: u64,"] + multiline = ( + "#[derive(\n Serialize,\n)]\n" + "pub struct Event {\n #[serde(\n default,\n )]\n pub count: u64,\n}" + ) + assert default_field_violations(multiline) == ["line 8: pub count: u64,"] + fixed_record = "pub enum FixedRecord {\n Answer { value: Option },\n}" + assert default_field_violations(fixed_record) == [] assert encoder_violations(Path("crates/rogue/src/lib.rs"), "rmp_serde::to_vec_named(&event)") assert encoder_violations(Path("crates/rogue/src/lib.rs"), "use rmp_serde as codec;") assert encoder_violations(Path("crates/capsem-proto/src/lib.rs"), "rmp_serde::to_vec(&event)") def test_new_protocol_modules_are_scanned(tmp_path: Path) -> None: - (tmp_path / "new_wire.rs").write_text("pub struct NewWire {\n pub value: Option,\n}\n") + (tmp_path / "new_wire.rs").write_text( + "#[derive(serde::Serialize)]\npub struct NewWire {\n pub value: Option,\n}\n" + ) sources = wire_type_sources(tmp_path) assert [path.name for path in sources] == ["new_wire.rs"] - assert default_field_violations(sources[0].read_text()) == ["line 2: pub value: Option,"] + assert default_field_violations(sources[0].read_text()) == ["line 3: pub value: Option,"] def test_capsem_messagepack_is_sparse_and_owned() -> None: From 905316be4255cac700d10035e89b91c522be0155 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:52:14 +0000 Subject: [PATCH 047/464] style(python): sort standard library imports --- sdk/python/tests/image_package_acceptance.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/python/tests/image_package_acceptance.py b/sdk/python/tests/image_package_acceptance.py index a36350d80..949095778 100644 --- a/sdk/python/tests/image_package_acceptance.py +++ b/sdk/python/tests/image_package_acceptance.py @@ -10,7 +10,6 @@ import sys import tarfile import threading -import time import tomllib import zipfile from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer From d9ea374a7fc639fdfd6dada081a48a3e22d728dc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:52:34 +0000 Subject: [PATCH 048/464] test(citadel): record isolation dependency lock --- tests/citadel/cargo_lock_identity.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/citadel/cargo_lock_identity.toml b/tests/citadel/cargo_lock_identity.toml index 127f5e43d..0142520bb 100644 --- a/tests/citadel/cargo_lock_identity.toml +++ b/tests/citadel/cargo_lock_identity.toml @@ -7,5 +7,5 @@ # each moved the lock. `test_cargo_lock_changes_are_deliberate` fails until # this record matches the lock again, so the cost is chosen, reviewed and # batched rather than discovered twenty minutes into a gate. -sha256 = "4e1e9e68d7778900a31eb802b2268630772da326be45945c7cf9e49b91255843" -reason = "Approved broker contract integration deliberately reuses locked time 0.3.47 with its parsing feature to normalize RFC3339 offsets and precise instants; the prior UTC-only parser refused valid offset timestamps. Only the credential crate's existing-package dependency edge changes, with no registry upgrades. Previous deliberate capsem-api, humantime and UUID parsing edges, approved safe-Rust Sigma migration, crypto ownership and normal BLAKE2 comparisons remain intact. Native vendor identities and owned-client product setup are not activated. This intentionally advances the guest-builder identity for the correct shared timestamp parser." +sha256 = "a4a1a88cdc170cdc693203262e857e56c20d6ec0843e3328d1c6793ec41c08ab" +reason = "The 0.7 worker-isolation line adds existing capsem-guard and HTTP/OpenTelemetry packages to capsem-process and service tests for parent admission and brokered metrics; workspace versions stay 0.7.0 and registry versions remain unchanged" From d16c45cd639a26743a8e9d9c2233ea4d4624f154 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:52:58 +0000 Subject: [PATCH 049/464] test(gate): classify internal metric ingestion --- config/gate.toml | 2 +- tests/citadel/test_bench_route_coverage.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index a3798aeca..90ef3616f 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3009,7 +3009,7 @@ slowest_steps = 10 # `internal` is a reasoned, ratcheted inventory for service-UDS control routes # that are deliberately neither public gateway routes nor benchmark subjects. [benchmark.routes] -internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS" } +internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/vms/{id}/metrics" = "An authenticated VM owner submits its OTLP metrics to the service over the local UDS for host export; the gateway must never expose this ingestion route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS" } measured = [ "/connections", "/vms/{id}/credential-grants", diff --git a/tests/citadel/test_bench_route_coverage.py b/tests/citadel/test_bench_route_coverage.py index d2e1d6332..81faee5f9 100644 --- a/tests/citadel/test_bench_route_coverage.py +++ b/tests/citadel/test_bench_route_coverage.py @@ -126,10 +126,10 @@ def test_each_route_has_exactly_one_benchmark_classification() -> None: def test_internal_routes_carry_a_reason_and_are_a_tight_inventory() -> None: internal = _config()["internal"] assert all(reason.strip() for reason in internal.values()) - # Five real exceptions: doctor failure preservation, private-network - # resolution, and the three preview admission exchanges between the - # gateway and service over the local UDS. - assert len(internal) <= 5, ( + # Six real exceptions: doctor failure preservation, private-network + # resolution, owner metric ingestion, and the three preview admission + # exchanges between the gateway and service over the local UDS. + assert len(internal) <= 6, ( "the internal service-only route inventory grew; prefer a public, " "measured gateway route unless the control-plane exception is real" ) From 632324ee0c0b149249dadb8b9fcf5b75625d11fe Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:54:47 +0000 Subject: [PATCH 050/464] test(service): split private name peer checks --- .../src/tests/network_routes.rs | 54 +------------------ .../src/tests/network_routes/private_names.rs | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 53 deletions(-) create mode 100644 crates/capsem-service/src/tests/network_routes/private_names.rs diff --git a/crates/capsem-service/src/tests/network_routes.rs b/crates/capsem-service/src/tests/network_routes.rs index 21c3e91ae..0e343946b 100644 --- a/crates/capsem-service/src/tests/network_routes.rs +++ b/crates/capsem-service/src/tests/network_routes.rs @@ -5,6 +5,7 @@ use axum::http::Method; use std::time::Duration; mod handoff; +mod private_names; fn app(state: &Arc) -> axum::Router { build_service_router(Arc::clone(state)) @@ -850,59 +851,6 @@ async fn private_resolve_from( route_request(app, Method::POST, "/networks/private/resolve", Some(request)).await } -#[test] -fn private_name_wrong_process_client() { - let Ok(socket) = std::env::var("CAPSEM_TEST_PRIVATE_NAME_SOCKET") else { - return; - }; - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let (status, _) = runtime - .block_on(capsem_core::service_uds::post_json( - std::path::Path::new(&socket), - "/networks/private/resolve", - &json!({"source_vm": "vm-a", "name": "vm-a"}), - )) - .unwrap(); - assert_eq!(status, StatusCode::FORBIDDEN.as_u16()); -} - -#[tokio::test] -async fn private_names_reject_a_real_same_uid_wrong_process() { - let (state, dir) = make_test_state_with_tempdir(); - insert_fake_instance(&state, "vm-a", std::process::id()); - let socket = dir.path().join("private-name-peer.sock"); - let listener = tokio::net::UnixListener::bind(&socket).unwrap(); - let router = app(&state).into_make_service_with_connect_info::(); - let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); - let (owner_status, _) = capsem_core::service_uds::post_json( - &socket, - "/networks/private/resolve", - &json!({"source_vm": "vm-a", "name": "nobody"}), - ) - .await - .unwrap(); - assert_eq!( - owner_status, - StatusCode::NOT_FOUND.as_u16(), - "the registered owner passes authentication before name lookup" - ); - let status = tokio::process::Command::new(std::env::current_exe().unwrap()) - .args([ - "--exact", - "tests::network_routes::private_name_wrong_process_client", - "--nocapture", - ]) - .env("CAPSEM_TEST_PRIVATE_NAME_SOCKET", &socket) - .status() - .await - .unwrap(); - server.abort(); - assert!(status.success(), "wrong-process probe failed: {status}"); -} - #[tokio::test] async fn private_names_resolve_only_to_members_the_asker_shares_a_network_with() { let (state, _dir) = make_test_state_with_tempdir(); diff --git a/crates/capsem-service/src/tests/network_routes/private_names.rs b/crates/capsem-service/src/tests/network_routes/private_names.rs new file mode 100644 index 000000000..082d1a1a8 --- /dev/null +++ b/crates/capsem-service/src/tests/network_routes/private_names.rs @@ -0,0 +1,54 @@ +use super::*; + +#[test] +fn private_name_wrong_process_client() { + let Ok(socket) = std::env::var("CAPSEM_TEST_PRIVATE_NAME_SOCKET") else { + return; + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (status, _) = runtime + .block_on(capsem_core::service_uds::post_json( + std::path::Path::new(&socket), + "/networks/private/resolve", + &json!({"source_vm": "vm-a", "name": "vm-a"}), + )) + .unwrap(); + assert_eq!(status, StatusCode::FORBIDDEN.as_u16()); +} + +#[tokio::test] +async fn private_names_reject_a_real_same_uid_wrong_process() { + let (state, dir) = make_test_state_with_tempdir(); + insert_fake_instance(&state, "vm-a", std::process::id()); + let socket = dir.path().join("private-name-peer.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let router = app(&state).into_make_service_with_connect_info::(); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let (owner_status, _) = capsem_core::service_uds::post_json( + &socket, + "/networks/private/resolve", + &json!({"source_vm": "vm-a", "name": "nobody"}), + ) + .await + .unwrap(); + assert_eq!( + owner_status, + StatusCode::NOT_FOUND.as_u16(), + "the registered owner passes authentication before name lookup" + ); + let status = tokio::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "tests::network_routes::private_names::private_name_wrong_process_client", + "--nocapture", + ]) + .env("CAPSEM_TEST_PRIVATE_NAME_SOCKET", &socket) + .status() + .await + .unwrap(); + server.abort(); + assert!(status.success(), "wrong-process probe failed: {status}"); +} From 0b0e6bc6c7dc44fec0539a9fa7f9140bd94aaee0 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 18:58:12 +0000 Subject: [PATCH 051/464] refactor(core): split MITM configuration fixtures --- config/gate.toml | 8 +- .../capsem-core/src/net/mitm_proxy/config.rs | 52 +++++++++++ crates/capsem-core/src/net/mitm_proxy/mod.rs | 52 ++--------- crates/capsem-core/tests/mitm_integration.rs | 91 +++---------------- .../tests/mitm_integration/grants.rs | 63 +++++++++++++ .../capsem-core/tests/mitm_integration/tls.rs | 24 +++++ 6 files changed, 161 insertions(+), 129 deletions(-) create mode 100644 crates/capsem-core/src/net/mitm_proxy/config.rs create mode 100644 crates/capsem-core/tests/mitm_integration/grants.rs create mode 100644 crates/capsem-core/tests/mitm_integration/tls.rs diff --git a/config/gate.toml b/config/gate.toml index 90ef3616f..4126884f8 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1099,11 +1099,11 @@ must_stay_below_lines = 3000 "crates/capsem-core/src/hypervisor/kvm/sys.rs" = 1895 "crates/capsem-core/src/hypervisor/kvm/virtio_blk.rs" = 1768 "crates/capsem-core/src/hypervisor/kvm/virtio_blk/tests.rs" = 1059 -"crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs" = 1987 -"crates/capsem-core/src/net/mitm_proxy/mod.rs" = 2426 +"crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs" = 1997 +"crates/capsem-core/src/net/mitm_proxy/mod.rs" = 2410 "crates/capsem-core/src/net/policy_config/security_rule_profile/tests.rs" = 1549 -"crates/capsem-core/src/security_engine/mod.rs" = 2071 -"crates/capsem-core/tests/mitm_integration.rs" = 2023 +"crates/capsem-core/src/security_engine/mod.rs" = 2073 +"crates/capsem-core/tests/mitm_integration.rs" = 2014 "crates/capsem-logger/src/writer/tests.rs" = 1395 "crates/capsem-mock-server/src/main.rs" = 1134 "crates/capsem-process/src/vsock.rs" = 1227 diff --git a/crates/capsem-core/src/net/mitm_proxy/config.rs b/crates/capsem-core/src/net/mitm_proxy/config.rs new file mode 100644 index 000000000..8ea1f6c68 --- /dev/null +++ b/crates/capsem-core/src/net/mitm_proxy/config.rs @@ -0,0 +1,52 @@ +use std::sync::Arc; + +use capsem_logger::DbWriter; + +use super::mcp_endpoint::McpEndpointState; +use super::pipeline; +use super::telemetry_hook; +use super::upstream::TcpUpstreamGrants; +use crate::net::cert_authority::CertAuthority; +use crate::net::policy::NetworkMechanics; + +/// Configuration for the MITM proxy. +pub struct MitmProxyConfig { + pub ca: Arc, + /// Guest-facing TLS config, built once (`make_server_tls_config`): one session cache for all. + pub server_tls: Arc, + /// Live policy, swappable via RwLock so settings changes take effect + /// without restarting the VM. Each HTTP request snapshots the Arc so + /// that disabling a provider blocks the next request even on an + /// existing keep-alive connection. + pub policy: Arc>>, + /// Live model endpoint registry from settings and corp provider blocks. + /// MITM resolves host -> model protocol once per request and then passes + /// that typed metadata to enforcement, hooks, broker substitution, and + /// telemetry. Provider hooks must not infer protocol from domains. + pub model_endpoints: Arc>>, + pub db: Arc, + /// Cached upstream TLS config (shared across all connections). + pub upstream_tls: Arc, + /// Telemetry deps shared with the `TelemetryHook` registered in + /// `pipeline`. Held here as the same `Arc` so the hook and any + /// remaining direct callers (rare; should fold into the hook) read + /// the same `pricing` table + `trace_state` mutex. The Arc breaks + /// the would-be cycle (config → pipeline → hook → config); the + /// hook only points at this `TelemetryDeps`, not the surrounding + /// `MitmProxyConfig`. + pub telemetry: Arc, + /// Hook pipeline. `make_production_pipeline` registers the sync + /// ChunkHook chain (decompression → SSE parse → + /// provider interpreters → telemetry). `handle_request` dispatches + /// L1 events through this pipeline and seeds per-request context + /// into the `ChunkDispatchBody`'s `HookState` before serving. + pub pipeline: Arc, + /// T3 framed MCP endpoint on the MITM listener. Dispatch state lives + /// here so the low-privilege aggregator remains DB-free while MITM + /// owns policy, timeouts, protocol telemetry, and MCP-origin `tool_calls`. + pub mcp_endpoint: Option>, + /// Resolves guest-named upstreams before policy; the dial goes only to what it judged. + pub upstream_resolver: crate::net::upstream_address::UpstreamResolver, + /// Trusted coordinator connection grants for confined workers. + pub upstream_grants: Option>, +} diff --git a/crates/capsem-core/src/net/mitm_proxy/mod.rs b/crates/capsem-core/src/net/mitm_proxy/mod.rs index 826a7245f..f98e00947 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mod.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mod.rs @@ -12,6 +12,7 @@ /// 7. Forward request, stream response back /// 8. Emit per-request telemetry (one NetEvent per HTTP request, not per connection) pub mod body; +mod config; pub mod decompression_hook; pub mod events; mod fd_stream; @@ -39,7 +40,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; use std::time::{Instant, SystemTime}; -use capsem_logger::{DbWriter, Decision, McpCall, NetEvent, WriteOp}; +use capsem_logger::{Decision, McpCall, NetEvent, WriteOp}; use capsem_telemetry::mitm as m; use http_body_util::Full; use hyper::body::Bytes; @@ -54,7 +55,6 @@ trait TokioReadWrite: AsyncRead + AsyncWrite {} impl TokioReadWrite for T where T: AsyncRead + AsyncWrite {} -use super::cert_authority::CertAuthority; use super::policy::NetworkMechanics; use crate::net::ai_traffic::provider::{route_provider, ModelProtocol, ProviderKind}; use crate::security_engine::{HttpSecurityEvent, IpSecurityEvent, ModelSecurityEvent, SecurityEvent, TcpSecurityEvent}; @@ -70,6 +70,7 @@ use util::{ parse_http_host_target, provider_label, request_can_replay_empty_body, split_path_query, }; +pub use config::MitmProxyConfig; pub use mcp_endpoint::{McpEndpointState, McpTimeouts, ScopedMcpTools}; pub use mcp_frame::{dispatch_logged_mcp_request, McpTransport}; pub use upstream::{ @@ -89,49 +90,10 @@ const CREDENTIAL_BODY_CAPTURE_LIMIT: usize = HTTP_BODY_CAPTURE_LIMIT; static FIRST_NETWORK_READY_EMITTED: AtomicBool = AtomicBool::new(false); -/// Configuration for the MITM proxy. -pub struct MitmProxyConfig { - pub ca: Arc, - /// Guest-facing TLS config, built once (`make_server_tls_config`): one session cache for all. - pub server_tls: Arc, - /// Live policy, swappable via RwLock so settings changes take effect - /// without restarting the VM. Each HTTP request snapshots the Arc so - /// that disabling a provider blocks the next request even on an - /// existing keep-alive connection. - pub policy: Arc>>, - /// Live model endpoint registry from settings and corp provider blocks. - /// MITM resolves host -> model protocol once per request and then passes - /// that typed metadata to enforcement, hooks, broker substitution, and - /// telemetry. Provider hooks must not infer protocol from domains. - pub model_endpoints: Arc>>, - pub db: Arc, - /// Cached upstream TLS config (shared across all connections). - pub upstream_tls: Arc, - /// Telemetry deps shared with the `TelemetryHook` registered in - /// `pipeline`. Held here as the same `Arc` so the hook and any - /// remaining direct callers (rare; should fold into the hook) read - /// the same `pricing` table + `trace_state` mutex. The Arc breaks - /// the would-be cycle (config → pipeline → hook → config); the - /// hook only points at this `TelemetryDeps`, not the surrounding - /// `MitmProxyConfig`. - pub telemetry: Arc, - /// Hook pipeline. `make_production_pipeline` registers the sync - /// ChunkHook chain (decompression → SSE parse → - /// provider interpreters → telemetry). `handle_request` dispatches - /// L1 events through this pipeline and seeds per-request context - /// into the `ChunkDispatchBody`'s `HookState` before serving. - pub pipeline: Arc, - /// T3 framed MCP endpoint on the MITM listener. Dispatch state lives - /// here so the low-privilege aggregator remains DB-free while MITM - /// owns policy, timeouts, protocol telemetry, and MCP-origin `tool_calls`. - pub mcp_endpoint: Option>, - /// Resolves guest-named upstreams before policy; the dial goes only to what it judged. - pub upstream_resolver: crate::net::upstream_address::UpstreamResolver, - /// Trusted coordinator connection grants for confined workers. - pub upstream_grants: Option>, -} - -/// Build an empty pipeline for callers that do not install production hooks. +/// Build the default (empty) hook pipeline. T1 slices 2 + 3 will +/// extend this to register the production hook set; until then the +/// pipeline is wired through `MitmProxyConfig` but no dispatch +/// happens from `handle_request`. pub fn make_default_pipeline() -> Arc { Arc::new(pipeline::Pipeline::builder().build()) } diff --git a/crates/capsem-core/tests/mitm_integration.rs b/crates/capsem-core/tests/mitm_integration.rs index 765220c89..0b6544946 100644 --- a/crates/capsem-core/tests/mitm_integration.rs +++ b/crates/capsem-core/tests/mitm_integration.rs @@ -6,86 +6,36 @@ /// - Denied domains are rejected before TLS handshake completes /// - Telemetry records correct decisions, methods, and status codes /// -use std::collections::{BTreeMap, HashMap}; -use std::io; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex}; +use std::collections::BTreeMap; +use std::sync::Arc; use capsem_core::net::cert_authority::CertAuthority; -use capsem_core::net::mitm_proxy::{ - self, protocol::Protocol, GrantedTcpStream, MitmProxyConfig, TcpConnectGrantFuture, TcpGrantSelection, - TcpResolveGrantFuture, TcpUpstreamGrants, UpstreamTarget, -}; +use capsem_core::net::mitm_proxy::{self, MitmProxyConfig}; use capsem_core::net::policy::{NetworkMechanics, UpstreamOverride, UpstreamOverrideProtocol}; use capsem_core::net::upstream_address::UpstreamResolver; use capsem_logger::{DbWriter, Decision}; use http_body_util::{BodyExt, Full}; use hyper::body::Bytes; use hyper_util::rt::TokioIo; -use rustls::pki_types::{pem::PemObject, CertificateDer, ServerName}; +use rustls::pki_types::ServerName; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_rustls::TlsConnector; // Case modules live beside this file; the ratcheted line count stays flat. #[path = "mitm_integration/mod.rs"] mod cases; +#[path = "mitm_integration/grants.rs"] +mod grants; +#[path = "mitm_integration/tls.rs"] +mod tls; + +use grants::IntegrationGrants; +use tls::make_tls_client_config; const CA_KEY: &str = include_str!("../resources/ca/capsem-ca.key"); const CA_CERT: &str = include_str!("../resources/ca/capsem-ca.crt"); const HERMETIC_UPSTREAM_DOMAIN: &str = "fixture.capsem.test"; -struct IntegrationGrants { - policy: Arc>>, - resolver: UpstreamResolver, - selections: Arc>>, - next_id: AtomicU64, -} - -impl IntegrationGrants { - fn new(policy: Arc>>, resolver: UpstreamResolver) -> Self { - Self { - policy, - resolver, - selections: Arc::new(Mutex::new(HashMap::new())), - next_id: AtomicU64::new(1), - } - } -} - -impl TcpUpstreamGrants for IntegrationGrants { - fn resolve(&self, protocol: Protocol, host: &str, port: u16) -> TcpResolveGrantFuture<'_> { - let host = host.to_owned(); - Box::pin(async move { - let policy = self.policy.read().unwrap().clone(); - let target = UpstreamTarget::resolve(&self.resolver, &policy, &host, port).await; - if let UpstreamTarget::Unresolved(error) = &target { - return Err(io::Error::new(io::ErrorKind::NotFound, error.clone())); - } - let judged_ip = target.judged_ip(&host); - let protocol = target.protocol(protocol); - let selection_id = self.next_id.fetch_add(1, Ordering::Relaxed); - self.selections.lock().unwrap().insert(selection_id, target); - let selections = Arc::clone(&self.selections); - Ok(TcpGrantSelection::new(selection_id, protocol, judged_ip, move || { - selections.lock().unwrap().remove(&selection_id); - })) - }) - } - - fn connect(&self, selection_id: u64) -> TcpConnectGrantFuture<'_> { - Box::pin(async move { - let target = self - .selections - .lock() - .unwrap() - .remove(&selection_id) - .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "unknown integration selection"))?; - let (stream, _pinned) = target.connect().await?; - Ok(GrantedTcpStream::new(stream, || {})) - }) - } -} - /// Build a proxy config from allow/block lists for integration tests. /// /// Enforcement intent is compiled into `SecurityRuleSet` so tests exercise the @@ -269,25 +219,6 @@ fn security_rules_from_toml(toml: &str) -> capsem_core::net::policy_config::Secu .expect("test security rules") } -/// Build a rustls ClientConfig that trusts the Capsem MITM CA. -fn make_tls_client_config() -> rustls::ClientConfig { - let mut root_store = rustls::RootCertStore::empty(); - let certs: Vec<_> = CertificateDer::pem_slice_iter(CA_CERT.as_bytes()) - .collect::>() - .unwrap(); - for cert in certs { - root_store.add(cert).unwrap(); - } - let provider = Arc::new(capsem_crypto::tls::provider()); - let mut config = rustls::ClientConfig::builder_with_provider(provider) - .with_safe_default_protocol_versions() - .unwrap() - .with_root_certificates(root_store) - .with_no_client_auth(); - config.alpn_protocols = vec![b"http/1.1".to_vec()]; - config -} - /// Spawn the MITM proxy on a TCP listener and return the address. /// The proxy handles exactly one connection then exits. async fn spawn_proxy(config: Arc) -> (tokio::task::JoinHandle<()>, std::net::SocketAddr) { diff --git a/crates/capsem-core/tests/mitm_integration/grants.rs b/crates/capsem-core/tests/mitm_integration/grants.rs new file mode 100644 index 000000000..24200c190 --- /dev/null +++ b/crates/capsem-core/tests/mitm_integration/grants.rs @@ -0,0 +1,63 @@ +use std::collections::HashMap; +use std::io; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; + +use capsem_core::net::mitm_proxy::{ + protocol::Protocol, GrantedTcpStream, TcpConnectGrantFuture, TcpGrantSelection, TcpResolveGrantFuture, + TcpUpstreamGrants, UpstreamTarget, +}; +use capsem_core::net::policy::NetworkMechanics; +use capsem_core::net::upstream_address::UpstreamResolver; + +pub(super) struct IntegrationGrants { + policy: Arc>>, + resolver: UpstreamResolver, + selections: Arc>>, + next_id: AtomicU64, +} + +impl IntegrationGrants { + pub(super) fn new(policy: Arc>>, resolver: UpstreamResolver) -> Self { + Self { + policy, + resolver, + selections: Arc::new(Mutex::new(HashMap::new())), + next_id: AtomicU64::new(1), + } + } +} + +impl TcpUpstreamGrants for IntegrationGrants { + fn resolve(&self, protocol: Protocol, host: &str, port: u16) -> TcpResolveGrantFuture<'_> { + let host = host.to_owned(); + Box::pin(async move { + let policy = self.policy.read().unwrap().clone(); + let target = UpstreamTarget::resolve(&self.resolver, &policy, &host, port).await; + if let UpstreamTarget::Unresolved(error) = &target { + return Err(io::Error::new(io::ErrorKind::NotFound, error.clone())); + } + let judged_ip = target.judged_ip(&host); + let protocol = target.protocol(protocol); + let selection_id = self.next_id.fetch_add(1, Ordering::Relaxed); + self.selections.lock().unwrap().insert(selection_id, target); + let selections = Arc::clone(&self.selections); + Ok(TcpGrantSelection::new(selection_id, protocol, judged_ip, move || { + selections.lock().unwrap().remove(&selection_id); + })) + }) + } + + fn connect(&self, selection_id: u64) -> TcpConnectGrantFuture<'_> { + Box::pin(async move { + let target = self + .selections + .lock() + .unwrap() + .remove(&selection_id) + .ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "unknown integration selection"))?; + let (stream, _pinned) = target.connect().await?; + Ok(GrantedTcpStream::new(stream, || {})) + }) + } +} diff --git a/crates/capsem-core/tests/mitm_integration/tls.rs b/crates/capsem-core/tests/mitm_integration/tls.rs new file mode 100644 index 000000000..2bce74a43 --- /dev/null +++ b/crates/capsem-core/tests/mitm_integration/tls.rs @@ -0,0 +1,24 @@ +use std::sync::Arc; + +use rustls::pki_types::{pem::PemObject, CertificateDer}; + +use super::CA_CERT; + +/// Build a rustls client config that trusts the Capsem MITM CA. +pub(super) fn make_tls_client_config() -> rustls::ClientConfig { + let mut root_store = rustls::RootCertStore::empty(); + let certs: Vec<_> = CertificateDer::pem_slice_iter(CA_CERT.as_bytes()) + .collect::>() + .unwrap(); + for cert in certs { + root_store.add(cert).unwrap(); + } + let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let mut config = rustls::ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .unwrap() + .with_root_certificates(root_store) + .with_no_client_auth(); + config.alpn_protocols = vec![b"http/1.1".to_vec()]; + config +} From 9b58393cc3a0bfb5a01b379ad1858deefb996ab8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 19:28:18 +0000 Subject: [PATCH 052/464] fix(gate): refresh qualified debug image pin --- config/gate.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/gate.toml b/config/gate.toml index 4126884f8..9a8680c5b 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2342,7 +2342,7 @@ context = "images/capsem-debug" script = "tests/fixtures/oci/pinned_image.py" repository = "ghcr.io/google/capsem/capsem-debug" cache_stage = "test-images" -digests = { "linux/arm64" = "sha256:eb349ca15ee89715842252af6b253314d656ae41e9a685645393d8ec7a5666b6", "linux/amd64" = "sha256:7e5df4cf8375e28939d710c221ac95ac823c961799638b18dfe64f8242763d53" } +digests = { "linux/arm64" = "sha256:eb349ca15ee89715842252af6b253314d656ae41e9a685645393d8ec7a5666b6", "linux/amd64" = "sha256:3b0fd69477cfe0dccbad777608244664e72f828ba938db4a51ff7a2ac4635d95" } # The reference image: a pinned digest of the official `dev` image. Every # runtime test that boots a workload boots this one, so main development From b0bdb7dd252917e20e49a3b0dea1780bf1ca0c80 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 19:48:12 +0000 Subject: [PATCH 053/464] test(api): approve internal metrics ingestion --- config/public-surface.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/public-surface.toml b/config/public-surface.toml index 322307db2..9830cb5db 100644 --- a/config/public-surface.toml +++ b/config/public-surface.toml @@ -83,7 +83,7 @@ approved = [ ] [http] -count = 106 +count = 95 approved = [ # All nine connection/grant routes approved by the user on 2026-10-14. "GET /connections", @@ -160,6 +160,7 @@ approved = [ "POST /internal/vms/{id}/exposures/{exposure_id}/preview-admission", "POST /internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap", "POST /internal/vms/{id}/exposures/{exposure_id}/preview-session", + "POST /internal/vms/{id}/metrics", "POST /update/apply", "POST /update/check", "POST /images/pull", From 56371cd5ced55136d11e1189018a018474d2b290 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 19:48:50 +0000 Subject: [PATCH 054/464] test(frontend): follow shared session errors --- tests/frontend/test_ui_contract.py | 4 ++-- .../src/lib/__tests__/session-language-contract.test.ts | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/frontend/test_ui_contract.py b/tests/frontend/test_ui_contract.py index 3c1f2b47d..e5bb7f83a 100644 --- a/tests/frontend/test_ui_contract.py +++ b/tests/frontend/test_ui_contract.py @@ -12,14 +12,14 @@ def read(relative: str) -> str: def test_frontend_uses_current_route_vocabulary_not_retired_policy_vm_terms() -> None: dashboard = read("lib/components/shell/NewTabPage.svelte") + session_errors = read("lib/models/session-action-error.ts") settings = read("lib/components/shell/SettingsPage.svelte") stats = read("lib/components/views/StatsView.svelte") toolbar = read("lib/components/shell/Toolbar.svelte") assert "Sessions" in dashboard - assert "import { sessionActionError }" in dashboard assert "sessionActionError(error)" in dashboard - assert "Failed to create session" in read("lib/models/session-action-error.ts") + assert "Failed to create session" in session_errors assert "Session {vmId} ledger" in stats assert "Session Logs" in toolbar diff --git a/web/app/src/lib/__tests__/session-language-contract.test.ts b/web/app/src/lib/__tests__/session-language-contract.test.ts index b1df8cee9..e9e4c2135 100644 --- a/web/app/src/lib/__tests__/session-language-contract.test.ts +++ b/web/app/src/lib/__tests__/session-language-contract.test.ts @@ -6,6 +6,10 @@ const dashboard = readFileSync( new URL('../components/shell/NewTabPage.svelte', import.meta.url), 'utf8', ); +const sessionActionErrors = readFileSync( + new URL('../models/session-action-error.ts', import.meta.url), + 'utf8', +); const appShell = readFileSync( new URL('../components/shell/App.svelte', import.meta.url), 'utf8', @@ -46,8 +50,8 @@ describe('user-facing session language contract', () => { expect(dashboard).toContain('Sessions'); expect(dashboard).toContain('Loading sessions'); expect(dashboard).toContain('No sessions'); - expect(dashboard).toContain('actionError = sessionActionError(error)'); - expect(sessionActionError(null)).toBe('Failed to create session'); + expect(dashboard).toContain('sessionActionError(error)'); + expect(sessionActionErrors).toContain('Failed to create session'); expect(dashboard).not.toContain('>' + legacyVmPlural + '<'); expect(dashboard).not.toContain('Customize ' + 'VM'); expect(dashboard).not.toContain('Loading ' + legacyVmPlural); From 7ad48160350ab8d427d266bea72eb48e5d71d417 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 19:49:11 +0000 Subject: [PATCH 055/464] test(ironbank): follow split Anthropic fixtures --- tests/ironbank/test_model_client_scripts.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/ironbank/test_model_client_scripts.py b/tests/ironbank/test_model_client_scripts.py index 2ba1f80a0..df311dcbc 100644 --- a/tests/ironbank/test_model_client_scripts.py +++ b/tests/ironbank/test_model_client_scripts.py @@ -77,14 +77,21 @@ def test_gemini_replay_uses_release_target_model() -> None: def test_anthropic_replay_uses_release_target_model() -> None: sdk_test = PROJECT_ROOT / "tests" / "ironbank" / "test_model_sdk_ledger.py" - mock_server = PROJECT_ROOT / "crates" / "capsem-mock-server" / "src" / "anthropic.rs" + mock_sources = [ + PROJECT_ROOT / "crates" / "capsem-mock-server" / "src" / name + for name in ("anthropic.rs", "agent_mcp.rs") + ] assert HERMETIC_ANTHROPIC_MODEL == "claude-sonnet-4-6" sdk_text = sdk_test.read_text(encoding="utf-8") - mock_text = mock_server.read_text(encoding="utf-8") assert "HERMETIC_ANTHROPIC_MODEL" in sdk_text - assert HERMETIC_ANTHROPIC_MODEL in mock_text - for path, text in ((sdk_test, sdk_text), (mock_server, mock_text)): + sources = [ + (sdk_test, sdk_text), + *[(path, path.read_text(encoding="utf-8")) for path in mock_sources], + ] + for path, text in sources: + if path in mock_sources: + assert HERMETIC_ANTHROPIC_MODEL in text, path assert "claude-sonnet-4-20250514" not in text, path @@ -94,7 +101,10 @@ def test_agy_noninteractive_script_selects_model_explicitly() -> None: assert '"agy",' in script assert '"--model",' in script assert f'HERMETIC_AGY_MODEL_DISPLAY = "{HERMETIC_AGY_MODEL_DISPLAY}"' in script - assert 'emit_result("google", "daily-cloudcode-pa.googleapis.com", "/v1internal:streamGenerateContent"' in script + assert ( + 'emit_result("google", "daily-cloudcode-pa.googleapis.com", "/v1internal:streamGenerateContent"' + in script + ) assert '"run_command"' in script assert '"CommandLine": "printf' in script assert '"/api/chat"' not in script From dac5a4c7a7bc9aae89d4d1b8b977d753994c4d04 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 19:50:04 +0000 Subject: [PATCH 056/464] fix(container): budget image unpack separately --- guest/artifacts/container/launch.py | 9 +++++++-- tests/fixtures/oci/test_launch_config.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/guest/artifacts/container/launch.py b/guest/artifacts/container/launch.py index 307248de9..5b61e67da 100644 --- a/guest/artifacts/container/launch.py +++ b/guest/artifacts/container/launch.py @@ -86,6 +86,10 @@ TERMINAL_SHELL = ["/bin/sh", "-c", "command -v bash >/dev/null 2>&1 && exec bash -l; exec sh -l"] TERMINAL_TYPE = "TERM=xterm-256color" ATTACH_INTERVAL = 1.0 +COMMAND_TIMEOUT = 30 +# Large test-tooling images can take longer to materialize when several VMs +# unpack in parallel. Keep this below the service's 110-second create wait. +IMAGE_UNPACK_TIMEOUT = 90 # A detached workload's terminal: runc runs in the foreground on a PTY this # launcher holds (runc gives the container its own and relays the two; it # allows a console socket only when it detaches, which would lose the exit @@ -487,8 +491,8 @@ def idmap_workspace(id_map, target): os.waitpid(child, 0) -def command(*args, check=True, **kwargs): - return subprocess.run(args, check=check, timeout=30, **kwargs) +def command(*args, check=True, timeout=COMMAND_TIMEOUT, **kwargs): + return subprocess.run(args, check=check, timeout=timeout, **kwargs) def hosts_file(): @@ -866,6 +870,7 @@ def unpacked_root(digest, id_map, share=image_share): command( "umoci", "unpack", "--uid-map", mapping, "--gid-map", mapping, "--image", f"{layout}:image", str(root / "bundle"), + timeout=IMAGE_UNPACK_TIMEOUT, ) (root / "bundle").chmod(0o711) shutil.copyfile(root / "bundle" / "config.json", root / "runtime.json") diff --git a/tests/fixtures/oci/test_launch_config.py b/tests/fixtures/oci/test_launch_config.py index f636c227a..53cfebc1f 100644 --- a/tests/fixtures/oci/test_launch_config.py +++ b/tests/fixtures/oci/test_launch_config.py @@ -1085,6 +1085,23 @@ def test_a_named_session_unpacks_its_image_once(launcher, tmp_path, monkeypatch) assert not (launcher.RUNTIME / "image").exists(), "the assembled layout is not kept" +def test_image_unpack_has_headroom_under_parallel_vm_load(launcher, tmp_path, monkeypatch): + _, _, digest = _fake_unpack(launcher, tmp_path, monkeypatch) + delegated = launcher.command + calls = [] + + def command(*argv, **kwargs): + calls.append((argv, kwargs.copy())) + return delegated(*argv, **kwargs) + + monkeypatch.setattr(launcher, "command", command) + launcher.unpacked_root(digest, SECURITY["id_map"], launcher.image_share) + + assert calls[0][0][0:2] == ("umoci", "unpack") + assert calls[0][1]["timeout"] == launcher.IMAGE_UNPACK_TIMEOUT + assert launcher.IMAGE_UNPACK_TIMEOUT > launcher.COMMAND_TIMEOUT + + def test_only_the_current_digest_keeps_a_root(launcher, tmp_path, monkeypatch): _, _, digest = _fake_unpack(launcher, tmp_path, monkeypatch) stale = tmp_path / "roots" / ("d" * 64) From ecc2f111a8c09527ced1bcc3e50c7c2c8006d178 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 20:26:24 +0000 Subject: [PATCH 057/464] fix(test): qualify mock helper imports --- build_system/tests/helpers/mock_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_system/tests/helpers/mock_server.py b/build_system/tests/helpers/mock_server.py index f4de68215..3f0ba1d22 100644 --- a/build_system/tests/helpers/mock_server.py +++ b/build_system/tests/helpers/mock_server.py @@ -16,7 +16,7 @@ from capsem_builder.cache.config import load_paths from build_system.scripts.release.release_test_binary import ensure_host_test_binary -from helpers.constants import host_bin_root +from tests.helpers.constants import host_bin_root PROJECT_ROOT = Path(__file__).resolve().parents[3] MOCK_SERVER_BINARY = host_bin_root() / "capsem-mock-server" From 62ce41d7a036351243ec9d854b3a5cda30b5ec0e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 20:26:30 +0000 Subject: [PATCH 058/464] test(service): preserve startup failure coverage --- .../test_svc_startup_failure_logged.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/capsem-service/test_svc_startup_failure_logged.py b/tests/capsem-service/test_svc_startup_failure_logged.py index cf1c3f2eb..c951584f2 100644 --- a/tests/capsem-service/test_svc_startup_failure_logged.py +++ b/tests/capsem-service/test_svc_startup_failure_logged.py @@ -1,8 +1,10 @@ """A service that fails to start says why in its own log. -The gate and installer point to the service log when startup fails before -listening. A corrupt host ledger must record its cause there, rather than -leaving it only on the detached process's stderr. +The gate and the installer launch capsem-service detached and, when it dies +before listening, point at its log. A startup error returned from +`run_service` went only to stderr, which nobody reads: a dev build refusing +a malformed installed ledger left a service log that ended at "sqlite mmap +telemetry recorded" with no cause anywhere. """ from __future__ import annotations @@ -21,9 +23,11 @@ def test_a_startup_error_is_written_to_the_service_log() -> None: home_dir, run_dir = make_service_home_run_dirs() sessions = home_dir / "sessions" sessions.mkdir() - # Pre-v4 ledgers are archived and replaced. A corrupt SQLite file still - # fails startup, so it exercises the error-log contract directly. - (sessions / "host.db").write_bytes(b"corrupt host ledger") + # A malformed current ledger must fail loudly. A pre-v4 ledger is kept + # aside and replaced, so it cannot exercise the startup failure path. + with closing(sqlite3.connect(sessions / "host.db")) as broken: + broken.execute("CREATE TABLE archive_state (id INTEGER PRIMARY KEY)") + broken.commit() sign_binary(PROCESS_BINARY) sign_binary(SERVICE_BINARY) env = { @@ -55,6 +59,6 @@ def test_a_startup_error_is_written_to_the_service_log() -> None: logs = sorted(run_dir.glob("service*.log")) assert logs, f"no service log in {run_dir}" text = "".join(path.read_text() for path in logs) - assert "file is not a database" in text, ( + assert "archive_state must contain exactly one row" in text, ( "the cause of a failed start must reach the service log, not only stderr" ) From f70d130f67b58fe6a061c6bdfc6f6ab137b40986 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 20:26:33 +0000 Subject: [PATCH 059/464] test(ironbank): flush plugin import ledger --- tests/ironbank/test_doctor_ledger.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/ironbank/test_doctor_ledger.py b/tests/ironbank/test_doctor_ledger.py index 3600254bb..803032cdd 100644 --- a/tests/ironbank/test_doctor_ledger.py +++ b/tests/ironbank/test_doctor_ledger.py @@ -764,18 +764,11 @@ def test_runtime_plugin_action_matrix_pays_file_import_ledger_debt(): assert read_status == 200 assert read_body.decode() == EICAR_TEXT - plugins = client.get("/plugins/list", timeout=30) - by_id = {plugin["id"]: plugin for plugin in plugins["plugins"]} - assert by_id["dummy_pre_eicar"]["runtime"]["enabled"] is False - assert by_id["dummy_post_allow"]["runtime"]["enabled"] is True - assert by_id["dummy_post_allow"]["runtime"]["execution_count"] == 0 - # The info route aggregates the plugin's runtime across sessions. - dummy_post_detail = client.get("/plugins/dummy_post_allow/info", timeout=30) - assert dummy_post_detail["runtime"]["enabled"] is True - assert dummy_post_detail["runtime"]["execution_count"] >= 1 - # Stop awaits the owner and its DB flush before inspecting disk rows. - stopped = client.post(f"/vms/{vm_id}/stop", timeout=60) - assert stopped is not None + # DbWriter.write().await accepts into the process-owned buffer. Stop + # the persistent VM to drain and checkpoint that writer before opening + # session.db directly; polling the live file is not a flush barrier. + stopped = client.post(f"/vms/{vm_id}/stop", {}, timeout=60) + assert stopped == {"success": True, "persistent": True}, stopped conn = ledgers.enter_context(contextlib.closing(_connect_session_db(service, client, vm_id))) security_rows = conn.execute( From 991ee5e3f5945a3c0cedcae9727e91d984298b01 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 21:28:10 +0000 Subject: [PATCH 060/464] fix(gate): track shared worktree processes --- .../gate/tools/ci/check_orphan_processes.py | 10 ++++++++-- .../tests/gate/test_orphan_process_gate.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/build_system/builder/gate/tools/ci/check_orphan_processes.py b/build_system/builder/gate/tools/ci/check_orphan_processes.py index ec13d2910..81853bc91 100644 --- a/build_system/builder/gate/tools/ci/check_orphan_processes.py +++ b/build_system/builder/gate/tools/ci/check_orphan_processes.py @@ -91,7 +91,7 @@ def _process_facts(proc: psutil.Process) -> dict | None: def _owned_roots(root: Path) -> list[Path]: - """`root`, plus wherever its own `cache/target/` profile links actually point. + """`root`, plus wherever its own shared Cargo links actually point. Compiler output is shared between runs at one absolute path, so a prefix reaches it through `cache/target/cargo/debug` and `cache/target/release` symlinks. Resolving @@ -106,7 +106,13 @@ def _owned_roots(root: Path) -> list[Path]: reason the sharing is safe at all: one gate runs per machine under `flock`. """ roots = [root.resolve()] - for link in sorted((root / "cache" / "target" / "cargo").glob("*")): + cargo = root / "cache" / "target" / "cargo" + try: + if cargo.is_symlink(): + roots.append(cargo.resolve()) + except OSError: + pass + for link in sorted(cargo.glob("*")): try: if link.is_symlink(): roots.append(link.resolve()) diff --git a/build_system/tests/gate/test_orphan_process_gate.py b/build_system/tests/gate/test_orphan_process_gate.py index dbbb49921..86f24f318 100644 --- a/build_system/tests/gate/test_orphan_process_gate.py +++ b/build_system/tests/gate/test_orphan_process_gate.py @@ -388,6 +388,20 @@ def test_a_process_from_this_checkout_is_ours() -> None: assert ORPHANS._from_this_tree(ours, ROOT) +def test_a_process_from_a_worktrees_shared_cargo_root_is_ours(tmp_path: Path) -> None: + checkout = tmp_path / "checkout" + shared = tmp_path / "shared-cargo" + (checkout / "cache" / "target").mkdir(parents=True) + (shared / "debug").mkdir(parents=True) + (checkout / "cache" / "target" / "cargo").symlink_to(shared, target_is_directory=True) + ours = _facts(14, 100.0, exe=str(shared / "debug" / "capsem-service")) + + assert ORPHANS._from_this_tree(ours, checkout), ( + "isolated gate worktrees link the whole shared Cargo root, so their " + "process accounting must follow that ownership edge" + ) + + # --------------------------------------------------------------------------- # Functional: the check finds and reaps a real survivor # --------------------------------------------------------------------------- From 355da894b6eb0a41e98d9e148d37cc26aae94fc7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 22:11:20 +0000 Subject: [PATCH 061/464] fix(core): expire caller-owned workspace attributes --- CHANGELOG.md | 171 +----------------- crates/capsem-core/src/hypervisor/fuse/mod.rs | 3 + .../src/hypervisor/kvm/virtio_fs/ops_dir.rs | 8 +- .../src/hypervisor/kvm/virtio_fs/ops_file.rs | 2 +- .../src/hypervisor/kvm/virtio_fs/ops_meta.rs | 4 +- .../kvm/virtio_fs/tests/ownership.rs | 28 +++ 6 files changed, 40 insertions(+), 176 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f52545ac..b83db65bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,175 +48,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- Staged release verification includes the shared crypto owner and can verify - SHA-256 without optional BLAKE3 packages. Missing BLAKE3 reports a setup hint. - -- Decision ledger flushes use the full snapshot index when recording - occurrences, avoiding repeated scans as distinct decisions accumulate. - -- Idle ledger writers honor the original flush deadline when a late event - arrives, and failed idle flushes retain their normal retry interval. - -- Connection inventory supports bounded `limit` and `cursor` pages through the - service and SDKs, with malformed queries refused. - -- Interrupted credential disconnects resume bounded provider revocation after - service restart; maintenance retries unresolved retirement while grants stay revoked. - -- Archive index flushes batch SQLite inserts within the existing atomic - archive/ledger transaction, reducing repeated statement execution. - -- Stopped named sessions offer a read-only Credentials view with explicit - revocation; inspecting approvals does not wait for the connection catalog. - -- Named stopped sessions expose retained approval inventory and revocation - through canonical session IDs; grant creation still requires a live runtime. - -- Default session grant expiry fits the trusted remaining owner lifetime, - including time spent checking policy; explicit expiry overruns remain refused. - -- Retained credential approvals can be inspected and durably revoked using - their original session owner without creating or rebinding runtime authority. - -- Credential settings show account sign-in, cancellation and provider retirement; - running sessions can explicitly grant and revoke selected account capabilities. - Unsupported sign-in profiles remain disabled with an explanation. - -- Frontend connection and session-grant requests use authenticated SDK calls, - validate returned ownership, and never replay consent or grant mutations. - -- Grant polling retires authority when its consent is unavailable and refuses - exhausted revisions instead of publishing numbers clients cannot represent. - -- Credential grant retries normalize capability sets and exact RFC3339 expiry - instants, including offsets, and share Google's documented scope aliases. - Broker input budgets and timeouts come from one checked-in policy; completed - retry records retire after its 24-hour window without evicting active work. - -- Sigma YAML imports use a maintained safe Rust parser, reject duplicate - selection keys, and bound document, nesting and alias expansion. - -- Sigma selection lists and negated selections retain their logical grouping, - so an alternative cannot bypass another required security rule field. - Compiling grouped alternatives rejects excessive expansion before allocation. - -- Test-image cache retention keeps the OCI and prepared EROFS working set for - concurrent checkouts instead of reclaiming every fixture during enforcement. - -- Development builds optimize SQLite, its adapter and the ledger writer, and - archive index flushes reuse their prepared statement to reduce HTTP completion - stalls under load. - -- Protocol throughput benchmarks explicitly identify guest execution and reject - host/guest comparisons with incorrect execution labels. - -- The shared gateway OpenAPI contract correctly declares network creation's - existing HTTP 201 response, with matching generated SDK contracts. - -- Managed close retains original exit and grant authority when network cleanup - fails. Already-retired switches are awaited even if another departure fails; - a retry completes networking before session deletion and ownership closure. - -- Last-member network departure and retirement commit atomically. Failed - retirement preserves the membership and address for retry instead of - leaving an empty active network that cleanup can no longer find. - -- Logger durability barriers report accepted operations rejected by SQLite - during batch salvage, preventing failed registry writes from returning a - successful flush while preserving valid rows in the same producer batch. - -- Managed policy reload narrows active leases from their original renewal time - and updates expiry workers without extending ownership. Stale admission or - claim completions cannot overwrite a tighter timer; responses report the - enforced lease and valid renewal timing. Invalid policy refuses admission - and renewal while original cleanup authority remains available. - -- Managed close retains its confirmed original native exit outside ephemeral - session files, so restart can finish cleanup after directory deletion and - before the ownership close commit. Receipts survive unresolved cleanup and - retire with ownership tombstones. - -- Normal VM exit persists its native retirement receipt before moving a named - session, preserving confirmed stop authority for resume after host reboot. - -- Registered VM launches release shutdown admission before ongoing exec or - image setup, so workload duration does not hold the launch drain. - -- Failed VM launches retain original child retirement authority. Observed - native spawn failures prove no child exists; failures after spawn kill and - reap that child before publishing retirement to managed cleanup. - -- Managed startup recovery closes durable reservations that never prepared a - VM target, preserving capability authentication without replaying creation. - -- Service shutdown seals launch admission and joins already admitted workers - before selecting children for teardown, including cancelled HTTP callers. - -- Managed ownership stops renewing after ordinary stop/delete or loss of its - original VM spawn. Authenticated claims join the existing cleanup barrier; - changed spawns and failed cleanup cannot regain active ownership. - -- Managed VM spawns cannot transfer to persistence through ordinary save. - Admission uses original spawn ownership and refuses uncertain storage before - any persistence side effect. - -- TypeScript managed scopes cancel the owned callback and join cleanup at - validated lease expiry, including while renewal is stalled. - -- Python managed scopes stop at the validated lease expiry, including while - a renewal request is stalled, and join original-generation cleanup. - -- The Rust SDK provides managed ephemeral scopes with finite renewal, - cancellation-surviving cleanup, original-generation validation and explicit - retry authority when the service cannot confirm closure. - -- Preview and app-surface routes validate VM identifiers using the same rule - as the service and streaming gateway, rejecting malformed leading characters. - -- Linux warm resume restores allocated checkpoint extents into lazy zeroed RAM - and immediately resets stale guest vsock connections. Checkpoints preserve - page-sized sparse holes and existing file and directory handles. - -- The pinned guest kernel honors negotiated FUSE I/O sizes for large explicit - reads while retaining the existing speculative read-ahead limits. - -- Linux VM RAM stays aligned for transparent huge pages, accelerating boot and - cold filesystem reads without eagerly allocating guest memory. VirtioFS - supports full one-MiB requests, event-index wakeups on fresh and resumed VMs, - and bounded reusable I/O buffers through the shared Unix owner. - -- Prepared EROFS images use a catalog-bound BLAKE3 integrity annotation for - accelerated host and guest verification. The version 2 artifact contract - retains OCI transport SHA-256 and exact size, subject and descriptor checks. - -- OCI cache and prepared filesystem verification use the existing accelerated - crypto backend on CPUs without dedicated SHA instructions, retaining SHA-256 - identities and full integrity checks. - -- Explicitly admitted image references select their catalog-bound EROFS - filesystem instead of unnecessarily extracting OCI layers at startup. - -- CLI history and MCP tool tables truncate Unicode text at character - boundaries instead of panicking on a split UTF-8 character. - -- The authenticated gateway forwards host credential injection, so desktop - settings and TCP SDK clients can use file and memory storage. - -- Settings error text and negative badges remain readable on light and dark - surfaces without changing solid-button foregrounds. - -- Credential file-lock waits stop after five seconds instead of indefinitely - delaying startup or an explicit injection behind another writer. - -- Model tool records label observed WebSocket exchanges with their actual - transport while preserving HTTP and SSE labels. - -- Codex code-mode calls retain their tool namespace, input and linked results - in model telemetry. Text-array outputs preserve request metadata, and - Responses tool turns report tool-use completion. - -- WebSocket upgrade handshakes retain network telemetry without inflating - model call counts with an empty inference record. +- KVM image workloads retain write access to their workspace after VM-root + access by expiring caller-owned VirtioFS attributes across user namespaces. - Rust SDK start and resume cover workload readiness with the same deadline budget as creation, retaining larger client defaults and cancellation. diff --git a/crates/capsem-core/src/hypervisor/fuse/mod.rs b/crates/capsem-core/src/hypervisor/fuse/mod.rs index 11ebe4dc9..22c46179f 100644 --- a/crates/capsem-core/src/hypervisor/fuse/mod.rs +++ b/crates/capsem-core/src/hypervisor/fuse/mod.rs @@ -15,6 +15,9 @@ pub use protocol::*; use std::os::unix::fs::MetadataExt; +/// Caller-owned attributes cannot be cached across namespaces sharing an inode. +pub const ATTR_CACHE_SECONDS: u64 = 0; + // --------------------------------------------------------------------------- // Struct serialization helpers // --------------------------------------------------------------------------- diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs index c301ede10..afd51489c 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs @@ -145,7 +145,7 @@ impl FuseProcessor { nodeid: ino, generation: 0, entry_valid: 1, - attr_valid: 1, + attr_valid: fuse::ATTR_CACHE_SECONDS, entry_valid_nsec: 0, attr_valid_nsec: 0, attr: fuse::metadata_to_fuse_attr(ino, &meta, header), @@ -268,7 +268,7 @@ impl FuseProcessor { nodeid: ino, generation: 0, entry_valid: 1, - attr_valid: 1, + attr_valid: fuse::ATTR_CACHE_SECONDS, entry_valid_nsec: 0, attr_valid_nsec: 0, attr: fuse::metadata_to_fuse_attr(ino, &meta, header), @@ -310,7 +310,7 @@ impl FuseProcessor { nodeid: ino, generation: 0, entry_valid: 1, - attr_valid: 1, + attr_valid: fuse::ATTR_CACHE_SECONDS, entry_valid_nsec: 0, attr_valid_nsec: 0, attr: fuse::metadata_to_fuse_attr(ino, &meta, header), @@ -370,7 +370,7 @@ impl FuseProcessor { nodeid: ino, generation: 0, entry_valid: 1, - attr_valid: 1, + attr_valid: fuse::ATTR_CACHE_SECONDS, entry_valid_nsec: 0, attr_valid_nsec: 0, attr: fuse::metadata_to_fuse_attr(ino, &meta, header), diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs index 2ac1f65ef..0b3da08f0 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs @@ -272,7 +272,7 @@ impl FuseProcessor { nodeid: ino, generation: 0, entry_valid: 1, - attr_valid: 1, + attr_valid: fuse::ATTR_CACHE_SECONDS, entry_valid_nsec: 0, attr_valid_nsec: 0, attr: fuse::metadata_to_fuse_attr(ino, meta, header), diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs index 16dfde078..69f1ae0cb 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs @@ -75,7 +75,7 @@ impl FuseProcessor { nodeid: ino, generation: 0, entry_valid: 1, - attr_valid: 1, + attr_valid: fuse::ATTR_CACHE_SECONDS, entry_valid_nsec: 0, attr_valid_nsec: 0, attr: fuse::metadata_to_fuse_attr(ino, &meta, header), @@ -94,7 +94,7 @@ impl FuseProcessor { }; let attr_out = FuseAttrOut { - attr_valid: 1, + attr_valid: fuse::ATTR_CACHE_SECONDS, attr_valid_nsec: 0, dummy: 0, attr: fuse::metadata_to_fuse_attr(header.nodeid, &meta, header), diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/ownership.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/ownership.rs index aa7a608e5..a153ce36c 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/ownership.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/ownership.rs @@ -161,3 +161,31 @@ fn chown_alongside_a_mode_change_still_applies_the_mode() { let mode = std::fs::metadata(dir.join("f")).unwrap().permissions().mode(); assert_eq!(mode & 0o777, 0o640); } + +/// Ownership is derived from each request's caller, so the kernel must ask +/// again when a different namespace reaches an inode that VM root touched. +#[test] +fn caller_owned_attributes_are_never_cached() { + let dir = temp_share("caller-owned-attributes"); + std::fs::write(dir.join("f"), b"x").unwrap(); + let mut proc = test_processor(&dir); + + let mut lookup_header = make_header(FUSE_LOOKUP, 1, 11); + lookup_header.uid = 1000; + lookup_header.gid = 1000; + let lookup_response = proc.handle_request(&build_request(&lookup_header, b"f\0")); + let payload = std::mem::size_of::(); + let entry: FuseEntryOut = fuse::read_struct(&lookup_response[payload..]).unwrap(); + assert_eq!((entry.attr.uid, entry.attr.gid), (1000, 1000)); + assert_eq!(entry.attr_valid, 0); + assert_eq!(entry.attr_valid_nsec, 0); + + let mut getattr_header = make_header(FUSE_GETATTR, entry.nodeid, 12); + getattr_header.uid = 2000; + getattr_header.gid = 2000; + let getattr_response = proc.handle_request(&build_request(&getattr_header, &[])); + let attr: FuseAttrOut = fuse::read_struct(&getattr_response[payload..]).unwrap(); + assert_eq!((attr.attr.uid, attr.attr.gid), (2000, 2000)); + assert_eq!(attr.attr_valid, 0); + assert_eq!(attr.attr_valid_nsec, 0); +} From 4a585f106bbc7e0a75fac4b5ef57a7ad2e2acab1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Thu, 8 Oct 2026 23:18:27 +0000 Subject: [PATCH 062/464] perf(service): cache effective plugin policy --- crates/capsem-service/src/main.rs | 4 ++ crates/capsem-service/src/plugin_routes.rs | 56 ++++++++++++++++++- crates/capsem-service/src/policy_mutation.rs | 1 + crates/capsem-service/src/service_runtime.rs | 1 + crates/capsem-service/src/settings_routes.rs | 4 ++ crates/capsem-service/src/tests.rs | 1 + .../capsem-service/src/tests/plugin_routes.rs | 12 ++++ 7 files changed, 76 insertions(+), 3 deletions(-) diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index cfc5f193a..2fb7505f7 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -231,6 +231,10 @@ pub struct ServiceState { /// by explicit MCP discovery routes. Hot MCP list routes must not read the /// tool cache JSON from disk. mcp_tool_cache: Mutex>, + /// Effective plugin settings loaded on the first plugin route read and + /// invalidated by every service-owned policy edit or explicit reload. + /// Hot plugin list polls must not parse settings and corp TOML per request. + plugin_policy_cache: Mutex, /// Logger-owned DB handle for the host ledger (`sessions/host.db`): host /// events and policy mutations. Routes call `write`; they must never open /// SQLite directly or hold a side `DbWriter`. diff --git a/crates/capsem-service/src/plugin_routes.rs b/crates/capsem-service/src/plugin_routes.rs index 83d08f742..45424dbd9 100644 --- a/crates/capsem-service/src/plugin_routes.rs +++ b/crates/capsem-service/src/plugin_routes.rs @@ -190,7 +190,13 @@ pub(super) fn plugin_catalog() -> &'static BTreeMap /// Each catalogued plugin's effective configuration, and the plugins settings /// or corp set. -type EffectivePluginPolicy = (BTreeMap, BTreeSet); +pub(super) type EffectivePluginPolicy = (BTreeMap, BTreeSet); + +#[derive(Default)] +pub(super) struct PluginPolicyCache { + generation: u64, + policy: Option>, +} /// The effective plugin policy from the current files. fn effective_plugin_policy() -> Result { @@ -205,12 +211,56 @@ fn effective_plugin_policy() -> Result { Ok((policy, overridden)) } +impl ServiceState { + pub(crate) fn invalidate_plugin_policy_cache(&self) { + let mut cache = self.plugin_policy_cache.lock().unwrap(); + cache.generation = cache.generation.wrapping_add(1); + cache.policy = None; + } + + fn cached_plugin_policy(&self) -> (u64, Option>) { + let cache = self.plugin_policy_cache.lock().unwrap(); + (cache.generation, cache.policy.clone()) + } + + fn publish_plugin_policy( + &self, + generation: u64, + policy: EffectivePluginPolicy, + ) -> Option> { + let mut cache = self.plugin_policy_cache.lock().unwrap(); + if cache.generation != generation { + return None; + } + if let Some(policy) = &cache.policy { + return Some(Arc::clone(policy)); + } + let policy = Arc::new(policy); + cache.policy = Some(Arc::clone(&policy)); + drop(cache); + Some(policy) + } +} + +async fn effective_plugin_policy_for(state: &Arc) -> Result, AppError> { + loop { + let (generation, cached) = state.cached_plugin_policy(); + if let Some(policy) = cached { + return Ok(policy); + } + let policy = state.off_worker(|_| effective_plugin_policy()).await??; + if let Some(policy) = state.publish_plugin_policy(generation, policy) { + return Ok(policy); + } + } +} + async fn plugin_info_for( state: &Arc, plugin_id: &str, include_runtime: bool, ) -> Result { - let policy = state.off_worker(|_| effective_plugin_policy()).await??; + let policy = effective_plugin_policy_for(state).await?; plugin_info_from(state, &policy, plugin_id, include_runtime).await } @@ -428,7 +478,7 @@ fn hydrate_credential_broker_runtime( pub(super) async fn handle_plugins( State(state): State>, ) -> Result, AppError> { - let policy = state.off_worker(|_| effective_plugin_policy()).await??; + let policy = effective_plugin_policy_for(&state).await?; let mut plugins = Vec::new(); for plugin_id in plugin_catalog().keys() { plugins.push(plugin_info_from(&state, &policy, plugin_id, false).await?); diff --git a/crates/capsem-service/src/policy_mutation.rs b/crates/capsem-service/src/policy_mutation.rs index 8e24fb857..53871d8ab 100644 --- a/crates/capsem-service/src/policy_mutation.rs +++ b/crates/capsem-service/src/policy_mutation.rs @@ -102,6 +102,7 @@ pub(crate) async fn apply_policy_mutation( .await? .map_err(bad_request) .inspect_err(rejected)?; + state.invalidate_plugin_policy_cache(); let event = write_policy_mutation_event(state, &mutation, summary).await?; info!( target: "capsem.policy_mutation", diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index baf31304e..a2632a4cb 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -272,6 +272,7 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { asset_reconcile_inflight: AtomicBool::new(false), asset_status_path, mcp_tool_cache: Mutex::new(capsem_core::mcp::load_tool_cache()), + plugin_policy_cache: Mutex::new(Default::default()), host_ledger, host_stats: Mutex::new(Default::default()), last_defunct_reconcile_ms: AtomicU64::new(0), diff --git a/crates/capsem-service/src/settings_routes.rs b/crates/capsem-service/src/settings_routes.rs index 99761d6e0..f2ea55330 100644 --- a/crates/capsem-service/src/settings_routes.rs +++ b/crates/capsem-service/src/settings_routes.rs @@ -8,6 +8,7 @@ pub(super) async fn handle_reload_config( ) -> Result, AppError> { let mutation = state.policy_mutation.begin().await; let reloaded = push_policy_to_running_instances(&state, &mutation).await?; + state.invalidate_plugin_policy_cache(); drop(mutation); Ok(Json(serde_json::json!({ "success": true, "reloaded": reloaded }))) } @@ -26,6 +27,7 @@ pub(super) async fn handle_save_settings( let _mutation = state.policy_mutation.begin().await; capsem_core::net::policy_config::batch_update_settings_json(&raw) .map_err(|e| AppError(StatusCode::BAD_REQUEST, e))?; + state.invalidate_plugin_policy_cache(); let resp = capsem_core::net::policy_config::load_settings_response(); Ok(Json(serde_json::to_value(resp).unwrap_or_default())) } @@ -56,6 +58,8 @@ pub(super) async fn handle_corp_config( )); } + state.invalidate_plugin_policy_cache(); + Ok(Json(json!({ "success": true }))) } diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 343374aa2..04920236e 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -74,6 +74,7 @@ pub(crate) fn test_state( asset_reconcile_inflight: AtomicBool::new(false), asset_status_path: asset_status_path_for_run_dir(&run_dir), mcp_tool_cache: Mutex::new(capsem_core::mcp::load_tool_cache()), + plugin_policy_cache: Mutex::new(Default::default()), host_ledger: test_host_ledger(&run_dir), host_stats: Mutex::new(Default::default()), last_defunct_reconcile_ms: AtomicU64::new(0), diff --git a/crates/capsem-service/src/tests/plugin_routes.rs b/crates/capsem-service/src/tests/plugin_routes.rs index 502df2813..9c611f769 100644 --- a/crates/capsem-service/src/tests/plugin_routes.rs +++ b/crates/capsem-service/src/tests/plugin_routes.rs @@ -383,6 +383,18 @@ async fn plugin_edits_through_the_router_change_the_listed_config() { assert_eq!(edited["config"]["detection_level"], "critical"); assert_eq!(edited["overridden"], true); + let (status, list) = route_request(app.clone(), axum::http::Method::GET, "/plugins/list", None).await; + assert_eq!(status, StatusCode::OK, "{list}"); + let listed = list["plugins"] + .as_array() + .unwrap() + .iter() + .find(|plugin| plugin["id"] == "dummy_pre_eicar") + .expect("edited plugin remains listed"); + assert_eq!(listed["config"]["mode"], "block"); + assert_eq!(listed["config"]["detection_level"], "critical"); + assert_eq!(listed["overridden"], true); + let (status, info) = route_request( app.clone(), axum::http::Method::GET, From 5d2645f817588b65c71c69f2ac8326a9c80b9498 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 00:32:38 +0000 Subject: [PATCH 063/464] security(foundation): confine Linux workers --- CHANGELOG.md | 4 + crates/capsem-foundation/src/unix/mod.rs | 1 + .../src/unix/worker_sandbox.rs | 91 ++++++ .../src/unix/worker_sandbox/linux.rs | 291 ++++++++++++++++++ .../src/unix/worker_sandbox/tests.rs | 107 +++++++ tests/citadel/unix_boundary_debt.toml | 18 +- 6 files changed, 497 insertions(+), 15 deletions(-) create mode 100644 crates/capsem-foundation/src/unix/worker_sandbox.rs create mode 100644 crates/capsem-foundation/src/unix/worker_sandbox/linux.rs create mode 100644 crates/capsem-foundation/src/unix/worker_sandbox/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b83db65bf..d61f38700 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Add the Linux worker confinement primitive that restricts every runtime + thread to explicit filesystem grants, inherited network descriptors and its + signal domain while denying ambient control sockets and process execution. + - Broker VM DNS and TCP upstream access through generation-bound connected descriptors selected from the service's trusted active-policy snapshot. Policy reload, worker replacement, release and malformed ownership attempts diff --git a/crates/capsem-foundation/src/unix/mod.rs b/crates/capsem-foundation/src/unix/mod.rs index bc630fd86..98f4b61f2 100644 --- a/crates/capsem-foundation/src/unix/mod.rs +++ b/crates/capsem-foundation/src/unix/mod.rs @@ -15,3 +15,4 @@ pub mod router_sandbox; pub mod router_stream; pub mod tcp; pub mod tree_clone; +pub mod worker_sandbox; diff --git a/crates/capsem-foundation/src/unix/worker_sandbox.rs b/crates/capsem-foundation/src/unix/worker_sandbox.rs new file mode 100644 index 000000000..35a025946 --- /dev/null +++ b/crates/capsem-foundation/src/unix/worker_sandbox.rs @@ -0,0 +1,91 @@ +//! OS confinement shared by host workers that retain bounded file authority. + +use std::io; +use std::path::{Path, PathBuf}; + +/// The worker's post-startup process authority. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Role { + Gateway, + VmOwner, +} + +/// Filesystem access granted after confinement. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Access { + ReadOnly, + ReadWrite, + Executable, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PathRule { + path: PathBuf, + access: Access, +} + +impl PathRule { + pub fn new(path: impl Into, access: Access) -> Self { + Self { + path: path.into(), + access, + } + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn access(&self) -> Access { + self.access + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Policy { + role: Role, + paths: Vec, +} + +impl Policy { + pub fn new(role: Role) -> Self { + Self { + role, + paths: Vec::new(), + } + } + + pub fn allow(mut self, path: impl Into, access: Access) -> Self { + self.paths.push(PathRule::new(path, access)); + self + } + + pub fn role(&self) -> Role { + self.role + } + + pub fn paths(&self) -> &[PathRule] { + &self.paths + } +} + +/// Restrict all existing and future threads before the worker publishes readiness. +pub fn confine(policy: &Policy) -> io::Result<()> { + platform::confine(policy) +} + +#[cfg(target_os = "linux")] +#[path = "worker_sandbox/linux.rs"] +mod platform; + +#[cfg(not(target_os = "linux"))] +mod platform { + use super::*; + + pub(super) fn confine(_policy: &Policy) -> io::Result<()> { + Err(io::Error::new(io::ErrorKind::Unsupported, "worker sandbox unavailable")) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs new file mode 100644 index 000000000..858fc978d --- /dev/null +++ b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs @@ -0,0 +1,291 @@ +use super::{Access, Policy, Role}; +use nix::fcntl::{open, OFlag}; +use nix::sys::stat::{fstat, Mode, SFlag}; +use std::io; +use std::os::fd::{AsRawFd, OwnedFd}; + +const CREATE_RULESET_VERSION: u32 = 1; +const RESTRICT_SELF_TSYNC: u32 = 1 << 3; +const RULE_PATH_BENEATH: u32 = 1; +const MINIMUM_ABI: i32 = 6; + +const FS_EXECUTE: u64 = 1 << 0; +const FS_WRITE_FILE: u64 = 1 << 1; +const FS_READ_FILE: u64 = 1 << 2; +const FS_READ_DIR: u64 = 1 << 3; +const FS_REMOVE_DIR: u64 = 1 << 4; +const FS_REMOVE_FILE: u64 = 1 << 5; +const FS_MAKE_DIR: u64 = 1 << 7; +const FS_MAKE_REG: u64 = 1 << 8; +const FS_MAKE_SOCK: u64 = 1 << 9; +const FS_MAKE_FIFO: u64 = 1 << 10; +const FS_MAKE_SYM: u64 = 1 << 12; +const FS_REFER: u64 = 1 << 13; +const FS_TRUNCATE: u64 = 1 << 14; +const FS_IOCTL_DEV: u64 = 1 << 15; +const FS_HANDLED: u64 = FS_EXECUTE + | FS_WRITE_FILE + | FS_READ_FILE + | FS_READ_DIR + | FS_REMOVE_DIR + | FS_REMOVE_FILE + | FS_MAKE_DIR + | FS_MAKE_REG + | FS_MAKE_SOCK + | FS_MAKE_FIFO + | FS_MAKE_SYM + | FS_REFER + | FS_TRUNCATE + | FS_IOCTL_DEV; +const FS_READ: u64 = FS_READ_FILE | FS_READ_DIR; +const FS_WRITE: u64 = FS_READ + | FS_WRITE_FILE + | FS_REMOVE_DIR + | FS_REMOVE_FILE + | FS_MAKE_DIR + | FS_MAKE_REG + | FS_MAKE_SOCK + | FS_MAKE_FIFO + | FS_MAKE_SYM + | FS_REFER + | FS_TRUNCATE; + +const NET_BIND_TCP: u64 = 1 << 0; +const NET_CONNECT_TCP: u64 = 1 << 1; +const NET_HANDLED: u64 = NET_BIND_TCP | NET_CONNECT_TCP; +const SCOPE_ABSTRACT_UNIX_SOCKET: u64 = 1 << 0; +const SCOPE_SIGNAL: u64 = 1 << 1; + +#[repr(C)] +struct RulesetAttr { + handled_access_fs: u64, + handled_access_net: u64, + scoped: u64, +} + +#[repr(C, packed)] +struct PathBeneathAttr { + allowed_access: u64, + parent_fd: i32, +} + +pub(super) fn confine(policy: &Policy) -> io::Result<()> { + let ruleset = create_ruleset()?; + for rule in policy.paths() { + add_path_rule(&ruleset, rule.path(), rule.access())?; + } + nix::sys::prctl::set_no_new_privs().map_err(super::super::errno::io)?; + restrict_all_threads(&ruleset)?; + install_seccomp(policy.role()) +} + +fn create_ruleset() -> io::Result { + // nix has no Landlock API. The kernel reads no attribute when VERSION is + // set, and returns the supported ABI directly. + let abi = unsafe { + libc::syscall( + libc::SYS_landlock_create_ruleset, + std::ptr::null::(), + 0usize, + CREATE_RULESET_VERSION, + ) + }; + if abi < 0 { + return Err(io::Error::last_os_error()); + } + if abi < i64::from(MINIMUM_ABI) { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("Landlock ABI {MINIMUM_ABI} required; kernel provides {abi}"), + )); + } + let attr = RulesetAttr { + handled_access_fs: FS_HANDLED, + handled_access_net: NET_HANDLED, + scoped: SCOPE_ABSTRACT_UNIX_SOCKET | SCOPE_SIGNAL, + }; + // nix has no Landlock API. The initialized C-layout attribute is copied + // synchronously by the kernel. + let descriptor = unsafe { + libc::syscall( + libc::SYS_landlock_create_ruleset, + &attr, + std::mem::size_of::(), + 0u32, + ) + }; + if descriptor < 0 { + return Err(io::Error::last_os_error()); + } + // SAFETY: a successful create_ruleset result is a new owned descriptor. + Ok(unsafe { OwnedFd::from_raw_fd(descriptor as i32) }) +} + +fn add_path_rule(ruleset: &OwnedFd, path: &std::path::Path, access: Access) -> io::Result<()> { + let raw = open( + path, + OFlag::O_PATH | OFlag::O_CLOEXEC | OFlag::O_NOFOLLOW, + Mode::empty(), + ) + .map_err(super::super::errno::io)?; + // SAFETY: nix::fcntl::open returned a new owned descriptor. + let target = unsafe { OwnedFd::from_raw_fd(raw) }; + let metadata = fstat(target.as_raw_fd()).map_err(super::super::errno::io)?; + let kind = SFlag::from_bits_truncate(metadata.st_mode); + if kind.contains(SFlag::S_IFLNK) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("sandbox grant is a symlink: {}", path.display()), + )); + } + let directory = kind.contains(SFlag::S_IFDIR); + let allowed_access = match (access, directory) { + (Access::ReadOnly, true) => FS_READ, + (Access::ReadOnly, false) => FS_READ_FILE, + (Access::ReadWrite, true) => FS_WRITE, + (Access::ReadWrite, false) => FS_READ_FILE | FS_WRITE_FILE | FS_TRUNCATE, + (Access::Executable, false) => FS_READ_FILE | FS_EXECUTE, + (Access::Executable, true) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("sandbox executable grant is a directory: {}", path.display()), + )); + } + }; + let attr = PathBeneathAttr { + allowed_access, + parent_fd: target.as_raw_fd(), + }; + // nix has no Landlock API. Both descriptors remain live while the kernel + // synchronously copies the packed rule. + let result = unsafe { + libc::syscall( + libc::SYS_landlock_add_rule, + ruleset.as_raw_fd(), + RULE_PATH_BENEATH, + &attr, + 0u32, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +fn restrict_all_threads(ruleset: &OwnedFd) -> io::Result<()> { + // nix has no Landlock API. TSYNC is mandatory: workers create their Tokio + // runtime before confinement, and a single ambient sibling thread would + // invalidate the process boundary. + let result = unsafe { + libc::syscall( + libc::SYS_landlock_restrict_self, + ruleset.as_raw_fd(), + RESTRICT_SELF_TSYNC, + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +fn install_seccomp(role: Role) -> io::Result<()> { + #[cfg(target_arch = "aarch64")] + let architecture = 0xc00000b7; + #[cfg(target_arch = "x86_64")] + let architecture = 0xc000003e; + let instruction = |code, jt, jf, k| libc::sock_filter { code, jt, jf, k }; + let deny = 0x0005_0000 | libc::EPERM as u32; + let allow = 0x7fff_0000; + let mut filter = vec![ + instruction(0x20, 0, 0, 4), // LD W ABS seccomp_data.arch + instruction(0x15, 1, 0, architecture), // JEQ expected arch + instruction(0x06, 0, 0, 0x8000_0000), // RET KILL_PROCESS + instruction(0x20, 0, 0, 0), // LD W ABS seccomp_data.nr + ]; + let mut denied = vec![ + libc::SYS_connect, + libc::SYS_bind, + libc::SYS_listen, + libc::SYS_ptrace, + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + libc::SYS_kcmp, + libc::SYS_pidfd_getfd, + libc::SYS_open_by_handle_at, + libc::SYS_name_to_handle_at, + libc::SYS_mount, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_chroot, + libc::SYS_setns, + libc::SYS_unshare, + libc::SYS_add_key, + libc::SYS_request_key, + libc::SYS_keyctl, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_memfd_create, + libc::SYS_io_uring_setup, + libc::SYS_io_uring_register, + libc::SYS_reboot, + libc::SYS_kexec_load, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_chmod, + libc::SYS_fchmod, + libc::SYS_fchmodat, + libc::SYS_chown, + libc::SYS_fchown, + libc::SYS_lchown, + libc::SYS_fchownat, + libc::SYS_setxattr, + libc::SYS_lsetxattr, + libc::SYS_fsetxattr, + libc::SYS_removexattr, + libc::SYS_lremovexattr, + libc::SYS_fremovexattr, + libc::SYS_utime, + libc::SYS_utimes, + libc::SYS_futimesat, + libc::SYS_utimensat, + ]; + if role == Role::Gateway { + denied.extend([libc::SYS_execve, libc::SYS_execveat]); + } + for syscall in denied { + filter.push(instruction(0x15, 0, 1, syscall as u32)); + filter.push(instruction(0x06, 0, 0, deny)); + } + // New AF_UNIX sockets support descriptor channels and child router + // socketpairs. Internet, packet, netlink and VSOCK sockets must arrive as + // already-connected grants from the trusted coordinator. + filter.extend([ + instruction(0x15, 1, 0, libc::SYS_socket as u32), + instruction(0x06, 0, 0, allow), + instruction(0x20, 0, 0, 16), // LD W ABS seccomp_data.args[0] + instruction(0x15, 0, 1, libc::AF_UNIX as u32), + instruction(0x06, 0, 0, allow), + instruction(0x06, 0, 0, deny), + ]); + let program = libc::sock_fprog { + len: filter.len() as u16, + filter: filter.as_mut_ptr(), + }; + // SAFETY: the kernel copies the initialized filter synchronously. TSYNC + // refuses installation if any existing thread cannot receive the policy. + let result = unsafe { libc::syscall(libc::SYS_seccomp, 1u32, 1u32, &program) }; + if result != 0 { + return Err(if result < 0 { + io::Error::last_os_error() + } else { + io::Error::other(format!("worker seccomp could not confine thread {result}")) + }); + } + Ok(()) +} + +use std::os::fd::FromRawFd; diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs new file mode 100644 index 000000000..bb9ebd8c1 --- /dev/null +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -0,0 +1,107 @@ +use super::*; +use std::io::{Read, Write}; +use std::net::{Ipv4Addr, TcpListener, TcpStream}; +use std::os::fd::AsFd; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +#[cfg(target_os = "linux")] +#[test] +fn linux_policy_preserves_grants_and_denies_ambient_authority() { + let directory = tempfile::tempdir().unwrap(); + let allowed = directory.path().join("allowed"); + let readonly = directory.path().join("readonly"); + let denied = directory.path().join("denied"); + std::fs::create_dir_all(&allowed).unwrap(); + std::fs::create_dir_all(&readonly).unwrap(); + std::fs::create_dir_all(&denied).unwrap(); + std::fs::write(readonly.join("value"), b"readable").unwrap(); + std::fs::write(denied.join("granted"), b"one file").unwrap(); + std::fs::write(denied.join("secret"), b"secret").unwrap(); + let control = denied.join("control.sock"); + let _control = UnixListener::bind(&control).unwrap(); + + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let mut client = TcpStream::connect((Ipv4Addr::LOCALHOST, port)).unwrap(); + let (socket, _) = listener.accept().unwrap(); + + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "unix::worker_sandbox::tests::linux_sandbox_child", + "--nocapture", + ]) + .env_clear() + .env("WORKER_SANDBOX_ALLOWED", &allowed) + .env("WORKER_SANDBOX_READONLY", &readonly) + .env("WORKER_SANDBOX_DENIED", &denied) + .env("WORKER_SANDBOX_EXACT", denied.join("granted")) + .env("WORKER_SANDBOX_CONTROL", &control) + .env("WORKER_SANDBOX_PORT", port.to_string()) + .stdin(Stdio::from(std::os::fd::OwnedFd::from(socket))) + .spawn() + .unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + client.write_all(b"ping").unwrap(); + let mut reply = [0; 4]; + let read = client.read_exact(&mut reply); + let status = child.wait().unwrap(); + assert!(status.success(), "sandbox child failed: {status}"); + read.unwrap(); + assert_eq!(&reply, b"pong"); +} + +#[cfg(target_os = "linux")] +#[test] +fn linux_sandbox_child() { + let Some(allowed) = std::env::var_os("WORKER_SANDBOX_ALLOWED") else { + return; + }; + let readonly = std::path::PathBuf::from(std::env::var_os("WORKER_SANDBOX_READONLY").unwrap()); + let denied = std::path::PathBuf::from(std::env::var_os("WORKER_SANDBOX_DENIED").unwrap()); + let exact = std::path::PathBuf::from(std::env::var_os("WORKER_SANDBOX_EXACT").unwrap()); + let control = std::env::var_os("WORKER_SANDBOX_CONTROL").unwrap(); + let port: u16 = std::env::var("WORKER_SANDBOX_PORT").unwrap().parse().unwrap(); + let allowed = std::path::PathBuf::from(allowed); + let stdin = std::io::stdin(); + let mut inherited = TcpStream::from(super::super::fd::duplicate(stdin.as_fd()).unwrap()); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + + confine( + &Policy::new(Role::Gateway) + .allow(&allowed, Access::ReadWrite) + .allow(&readonly, Access::ReadOnly) + .allow(&exact, Access::ReadOnly), + ) + .unwrap(); + + runtime.block_on(async { + tokio::task::spawn_blocking(move || { + assert_eq!(std::fs::read(readonly.join("value")).unwrap(), b"readable"); + assert_eq!(std::fs::read(exact).unwrap(), b"one file"); + assert!(std::fs::write(readonly.join("value"), b"changed").is_err()); + std::fs::write(allowed.join("created"), b"ok").unwrap(); + assert!(std::fs::read(denied.join("secret")).is_err()); + assert!(UnixStream::connect(control).is_err()); + assert!(TcpStream::connect((Ipv4Addr::LOCALHOST, port)).is_err()); + assert!(Command::new("/usr/bin/true").status().is_err()); + let parent = unsafe { libc::getppid() }; + assert_eq!(unsafe { libc::kill(parent, 0) }, -1, "worker signalled its parent"); + }) + .await + .unwrap(); + tokio::task::spawn_blocking(|| 7usize).await.unwrap() + }); + + let mut request = [0; 4]; + inherited.read_exact(&mut request).unwrap(); + assert_eq!(&request, b"ping"); + inherited.write_all(b"pong").unwrap(); + std::process::exit(0); +} diff --git a/tests/citadel/unix_boundary_debt.toml b/tests/citadel/unix_boundary_debt.toml index d7f77d130..8efe94081 100644 --- a/tests/citadel/unix_boundary_debt.toml +++ b/tests/citadel/unix_boundary_debt.toml @@ -133,18 +133,6 @@ reason = "macOS Seatbelt confinement of the router; no nix wrapper" fingerprint = "d14267a19410254a6633ba264e8046ea3dbd9c9046b72006155d76ee76db008f" reason = "seccomp(2) filter install with TSYNC; nix has no seccomp" -[foundation.raw_calls."crates/capsem-foundation/src/unix/process/pinned.rs::syscall"] -fingerprint = "e7b0fa754e405a0d00ed0808bf38b8c6bedae2df72a82154e0bbd61d42990c93" -reason = "Linux pidfd_open and pidfd_send_signal retain original process identity across exit and PID reuse; pinned nix 0.29 provides no wrapper for either syscall" - -[foundation.raw_calls."crates/capsem-foundation/src/unix/process/pinned_macos.rs::proc_pidinfo"] -fingerprint = "33ed8bd6a7cea2e0bef25a259110c71112d2e9e0bcebaef0297bf64e85b8af63" -reason = "Darwin combined native BSD status and immutable process identity including zombies; nix has no libproc query for this fixed ABI" - -[foundation.raw_calls."crates/capsem-foundation/src/unix/process/pinned_macos.rs::dlsym"] -fingerprint = "806c2d3062055d3c1bf7a97779715cbadc6d136ffae2fa2c59141886d0222b3e" -reason = "Resolve Darwin's PID-version guarded signal API only when present; macOS 14 lacks it and must retain exit observation without raw PID fallback; nix has no libproc symbol wrapper" - -[foundation.raw_calls."crates/capsem-foundation/src/unix/process/scope.rs::sysctlbyname"] -fingerprint = "017ea7709880405f8f0ef6262209e68cb7250c7c308852bdd237dbe936d36771" -reason = "Read Darwin's bounded kernel boot-session UUID so durable process witnesses cannot cross reboot; nix has no Darwin named sysctl wrapper" +[foundation.raw_calls."crates/capsem-foundation/src/unix/worker_sandbox/linux.rs::syscall"] +fingerprint = "47970efa29a92b2183616ae27524b3453bc9c3abdd320c180f24a4423dd7488d" +reason = "Landlock rules and thread-synchronized seccomp installation have no nix API" From 95d629516ef716881982b638d2ce850b0d55fb58 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 00:38:42 +0000 Subject: [PATCH 064/464] security(service): scope VM owner routes --- CHANGELOG.md | 4 ++ crates/capsem-service/src/router_runtime.rs | 4 ++ crates/capsem-service/src/service_peer.rs | 43 +++++++++++++++++++ .../capsem-service/src/service_peer/tests.rs | 28 ++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 crates/capsem-service/src/service_peer/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d61f38700..a3cc227d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Restrict registered VM-owner service connections to private-name lookup and + that owner's metrics route, preventing a compromised worker from invoking + the coordinator's public API over its same-user control socket. + - Add the Linux worker confinement primitive that restricts every runtime thread to explicit filesystem grants, inherited network descriptors and its signal domain while denying ambient control sockets and process execution. diff --git a/crates/capsem-service/src/router_runtime.rs b/crates/capsem-service/src/router_runtime.rs index ebdc99b24..07043e5de 100644 --- a/crates/capsem-service/src/router_runtime.rs +++ b/crates/capsem-service/src/router_runtime.rs @@ -180,6 +180,10 @@ pub(super) fn build_service_router(state: Arc) -> Router { // refused file uploads the API documents. .layer(axum::extract::DefaultBodyLimit::max(capsem_api::MAX_REQUEST_BODY_BYTES)) .layer(TraceLayer::new_for_http().on_request(()).on_response(())) + .layer(axum::middleware::from_fn_with_state( + Arc::clone(&state), + service_peer::restrict_owner_routes, + )) .with_state(state) } diff --git a/crates/capsem-service/src/service_peer.rs b/crates/capsem-service/src/service_peer.rs index 6d87bfec5..5a08b9dc7 100644 --- a/crates/capsem-service/src/service_peer.rs +++ b/crates/capsem-service/src/service_peer.rs @@ -13,3 +13,46 @@ impl Connected> for ServicePeer { Self(capsem_foundation::unix::peer::identity(stream.io().as_fd()).ok()) } } + +/// A registered VM owner may reach only the two service routes it needs after +/// startup. Route handlers still validate the VM ID and generation; this +/// outer boundary prevents a compromised owner from using the public API as a +/// same-user coordinator client. +pub(crate) async fn restrict_owner_routes( + axum::extract::State(state): axum::extract::State>, + request: axum::extract::Request, + next: axum::middleware::Next, +) -> axum::response::Response { + let Some(peer) = request + .extensions() + .get::>() + .and_then(|info| info.0 .0) + else { + return next.run(request).await; + }; + let owner = state + .instances + .lock() + .unwrap() + .values() + .find(|instance| instance.pid == peer.pid.get() && peer.uid == capsem_foundation::unix::process::current_uid()) + .map(|instance| instance.id.clone()); + let Some(owner) = owner else { + return next.run(request).await; + }; + if owner_route(&owner, request.uri().path()) { + next.run(request).await + } else { + axum::response::IntoResponse::into_response(( + axum::http::StatusCode::FORBIDDEN, + axum::Json(serde_json::json!({"error": "VM owner service route denied"})), + )) + } +} + +fn owner_route(owner: &str, path: &str) -> bool { + path == "/networks/private/resolve" || path == format!("/internal/vms/{owner}/metrics") +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/service_peer/tests.rs b/crates/capsem-service/src/service_peer/tests.rs new file mode 100644 index 000000000..519eb8890 --- /dev/null +++ b/crates/capsem-service/src/service_peer/tests.rs @@ -0,0 +1,28 @@ +use super::*; +use crate::tests::{insert_fake_instance_with_session_dir, make_test_state}; +use tower::ServiceExt; + +#[test] +fn owner_routes_are_exact_and_bound_to_the_registered_vm() { + assert!(owner_route("box", "/networks/private/resolve")); + assert!(owner_route("box", "/internal/vms/box/metrics")); + assert!(!owner_route("box", "/internal/vms/sibling/metrics")); + assert!(!owner_route("box", "/vms/list")); +} + +#[tokio::test] +async fn registered_owner_cannot_call_the_public_service_api() { + let state = make_test_state(); + insert_fake_instance_with_session_dir(&state, "box", std::process::id(), state.run_dir.join("sessions/box")); + let peer = ServicePeer(Some(capsem_foundation::unix::peer::PeerIdentity { + pid: capsem_foundation::unix::process::ProcessId::try_from(std::process::id()).unwrap(), + uid: capsem_foundation::unix::process::current_uid(), + })); + let app = crate::router_runtime::build_service_router(state); + let request = axum::http::Request::get("/version") + .extension(axum::extract::ConnectInfo(peer)) + .body(axum::body::Body::empty()) + .unwrap(); + let response = app.oneshot(request).await.unwrap(); + assert_eq!(response.status(), axum::http::StatusCode::FORBIDDEN); +} From 11d3f0ec6f724889202c782bf672044e66cdf4f7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 00:42:29 +0000 Subject: [PATCH 065/464] refactor(core): reuse prepared service channels --- crates/capsem-core/src/service_uds.rs | 110 +++++++++++++++----- crates/capsem-core/src/service_uds/tests.rs | 18 ++++ 2 files changed, 103 insertions(+), 25 deletions(-) diff --git a/crates/capsem-core/src/service_uds.rs b/crates/capsem-core/src/service_uds.rs index 148781091..ba2a7fc61 100644 --- a/crates/capsem-core/src/service_uds.rs +++ b/crates/capsem-core/src/service_uds.rs @@ -7,14 +7,92 @@ use anyhow::{ensure, Context, Result}; use http_body_util::{BodyExt, Full, Limited}; use hyper::body::Bytes; +use hyper::client::conn::http1::SendRequest; use hyper_util::rt::TokioIo; use std::path::Path; use std::time::Duration; +use tokio::net::UnixStream; const REQUEST_DEADLINE: Duration = Duration::from_secs(12); /// Maximum request or response body carried over the worker-to-service socket. pub const MAX_BODY_BYTES: usize = 4 * 1024 * 1024; +/// A persistent service connection prepared before a worker drops path and +/// connect authority. Requests are serialized because HTTP/1 has one ordered +/// response stream; response bodies are collected before the next request. +pub struct Client { + sender: tokio::sync::Mutex>>, +} + +impl Client { + pub async fn connect(socket: &Path) -> Result { + let stream = UnixStream::connect(socket) + .await + .with_context(|| format!("connect service socket {}", socket.display()))?; + Self::from_stream(stream).await + } + + pub async fn from_stream(stream: UnixStream) -> Result { + let (sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(stream)) + .await + .context("service HTTP handshake")?; + tokio::spawn(async move { + if let Err(error) = connection.await { + tracing::debug!(%error, "service connection ended"); + } + }); + Ok(Self { + sender: tokio::sync::Mutex::new(sender), + }) + } + + pub async fn post_json(&self, path: &str, body: &serde_json::Value) -> Result<(u16, serde_json::Value)> { + let response = self + .post_bytes( + path, + http::HeaderValue::from_static("application/json"), + Bytes::from(body.to_string()), + ) + .await?; + let status = response.status().as_u16(); + let bytes = response.into_body(); + let value = if bytes.is_empty() { + serde_json::Value::Null + } else { + serde_json::from_slice(&bytes) + .unwrap_or_else(|_| serde_json::Value::String(String::from_utf8_lossy(&bytes).into_owned())) + }; + Ok((status, value)) + } + + pub async fn post_bytes( + &self, + path: &str, + content_type: http::HeaderValue, + body: Bytes, + ) -> Result> { + let request = request(path, content_type, body)?; + self.send(request).await + } + + async fn send(&self, request: http::Request>) -> Result> { + tokio::time::timeout(REQUEST_DEADLINE, async { + let mut sender = self.sender.lock().await; + let response = sender.send_request(request).await.context("send service request")?; + let (parts, body) = response.into_parts(); + let bytes = Limited::new(body, MAX_BODY_BYTES) + .collect() + .await + .map_err(|error| anyhow::anyhow!("service response body exceeds limit or could not be read: {error}"))? + .to_bytes(); + drop(sender); + Ok(http::Response::from_parts(parts, bytes)) + }) + .await + .context("service request deadline exceeded")? + } +} + /// POST a JSON document and read back the status and JSON body. pub async fn post_json(socket: &Path, path: &str, body: &serde_json::Value) -> Result<(u16, serde_json::Value)> { let response = post_bytes( @@ -46,38 +124,20 @@ pub async fn post_bytes( content_type: http::HeaderValue, body: Bytes, ) -> Result> { + let request = request(path, content_type, body)?; + Client::connect(socket).await?.send(request).await +} + +fn request(path: &str, content_type: http::HeaderValue, body: Bytes) -> Result>> { ensure!( body.len() <= MAX_BODY_BYTES, "service request body exceeds {MAX_BODY_BYTES} bytes" ); - let request = http::Request::post(path) + http::Request::post(path) .header(http::header::HOST, "capsem") .header(http::header::CONTENT_TYPE, content_type) .body(Full::new(body)) - .context("build service request")?; - tokio::time::timeout(REQUEST_DEADLINE, async { - let stream = tokio::net::UnixStream::connect(socket) - .await - .with_context(|| format!("connect service socket {}", socket.display()))?; - let (mut sender, connection) = hyper::client::conn::http1::handshake(TokioIo::new(stream)) - .await - .context("service HTTP handshake")?; - tokio::spawn(async move { - if let Err(error) = connection.await { - tracing::debug!(%error, "service connection ended"); - } - }); - let response = sender.send_request(request).await.context("send service request")?; - let (parts, body) = response.into_parts(); - let bytes = Limited::new(body, MAX_BODY_BYTES) - .collect() - .await - .map_err(|error| anyhow::anyhow!("service response body exceeds limit or could not be read: {error}"))? - .to_bytes(); - Ok(http::Response::from_parts(parts, bytes)) - }) - .await - .context("service request deadline exceeded")? + .context("build service request") } #[cfg(test)] diff --git a/crates/capsem-core/src/service_uds/tests.rs b/crates/capsem-core/src/service_uds/tests.rs index 143aadd9b..99733c41e 100644 --- a/crates/capsem-core/src/service_uds/tests.rs +++ b/crates/capsem-core/src/service_uds/tests.rs @@ -41,6 +41,24 @@ async fn post_bytes_preserves_status_headers_and_body() { task.abort(); } +#[tokio::test] +async fn persistent_client_reuses_a_prepared_connection() { + let (_directory, socket, task) = server(Router::new().route("/bytes", post(echo))).await; + let client = Client::connect(&socket).await.unwrap(); + for body in [b"first".as_slice(), b"second".as_slice()] { + let response = client + .post_bytes( + "/bytes", + http::HeaderValue::from_static("application/x-protobuf"), + Bytes::copy_from_slice(body), + ) + .await + .unwrap(); + assert_eq!(response.body(), body); + } + task.abort(); +} + #[tokio::test] async fn post_bytes_refuses_an_oversized_request_before_connecting() { let socket = std::path::Path::new("/a/socket/that/must/not/be-opened"); From a76834a8d48c305e4c4ba2928564aad62b7aaab8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 00:44:02 +0000 Subject: [PATCH 066/464] refactor(process): stamp readiness sentinels --- crates/capsem-process/src/main.rs | 2 +- crates/capsem-process/src/vsock.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 7b11054dc..67790e14a 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -733,7 +733,7 @@ async fn run_async_main_loop( // separate file rather than the socket's existence, because a stale // socket from an earlier run at this path was just deleted above. let launched_path = uds_path.with_extension("launched"); - std::fs::File::create(&launched_path)?; + std::fs::write(&launched_path, b"launched\n")?; { use std::os::unix::fs::PermissionsExt; std::fs::set_permissions(&launched_path, std::fs::Permissions::from_mode(0o600))?; diff --git a/crates/capsem-process/src/vsock.rs b/crates/capsem-process/src/vsock.rs index f3d7ea41f..d946a824f 100644 --- a/crates/capsem-process/src/vsock.rs +++ b/crates/capsem-process/src/vsock.rs @@ -185,7 +185,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { ); vm_ready.store(true, Ordering::Release); let ready_path = uds_path.with_extension("ready"); - if let Err(e) = std::fs::File::create(&ready_path) { + if let Err(e) = std::fs::write(&ready_path, b"ready\n") { warn!(error = %e, "failed to create ready sentinel"); } From 07b25d3d02f0e05eebfe1fbdf1ba97aa3a8e1c49 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 00:46:21 +0000 Subject: [PATCH 067/464] refactor(service): require stamped worker readiness --- crates/capsem-service/src/vm_files.rs | 66 +++++++++++++++++++ crates/capsem-service/src/vm_files/launch.rs | 8 ++- .../src/vm_files/launch/tests.rs | 21 ++++-- 3 files changed, 89 insertions(+), 6 deletions(-) diff --git a/crates/capsem-service/src/vm_files.rs b/crates/capsem-service/src/vm_files.rs index 50a888a57..a11a509ad 100644 --- a/crates/capsem-service/src/vm_files.rs +++ b/crates/capsem-service/src/vm_files.rs @@ -1030,6 +1030,72 @@ pub(super) async fn handle_stats_summary( /// or exits during boot/restore would hang the API for `timeout_secs` (was /// reproducibly 30s under heavy suspend/resume churn). #[tracing::instrument(skip_all, fields(timeout_secs))] +pub(super) async fn wait_for_vm_ready( + uds_path: &std::path::Path, + timeout_secs: u64, + state: Option<&Arc>, + id: Option<&str>, +) -> Result<(), String> { + let ready_span = tracing::debug_span!( + target: "capsem.launch", + capsem_foundation::telemetry::LAUNCH_VSOCK_READY_SPAN, + status = tracing::field::Empty, + ); + let ready_path = uds_path.with_extension("ready"); + // Override the PollOpts::new defaults (50ms / 500ms): VM ready-time is + // sub-second in the common case and the sentinel check is a single stat, + // so 500ms max_delay overshoots readiness by ~500ms and blows the + // exec_ready / boot_ready latency gates. Peer callers (service-connect, + // gateway-ready) wait for remote processes with seconds-scale startup + // where 500ms is appropriate; this poll is different. + let opts = vm_ready_poll_opts(timeout_secs); + let died: Arc = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let res = capsem_foundation::poll::poll_until(opts, || { + let ready = ready_path.clone(); + let state = state.cloned(); + let id = id.map(|s| s.to_string()); + let died = Arc::clone(&died); + async move { + if launch::stamped(&ready, "ready") { + return Some(()); + } + if let (Some(st), Some(name)) = (state.as_ref(), id.as_ref()) { + if !st.instances.lock().unwrap().contains_key(name) { + died.store(true, std::sync::atomic::Ordering::Release); + // Returning Some short-circuits the poll loop; the + // outer caller distinguishes via `died`. + return Some(()); + } + } + None + } + }) + .instrument(ready_span.clone()) + .await; + if died.load(std::sync::atomic::Ordering::Acquire) { + ready_span.record("status", "error"); + return Err("capsem-process exited before signalling ready".into()); + } + match res { + Ok(()) => { + ready_span.record("status", "ok"); + Ok(()) + } + Err(error) => { + ready_span.record("status", "error"); + Err(format!("{error}")) + } + } +} + +pub(super) fn vm_ready_poll_opts(timeout_secs: u64) -> capsem_foundation::poll::PollOpts { + capsem_foundation::poll::PollOpts { + initial_delay: std::time::Duration::from_millis(5), + max_delay: std::time::Duration::from_millis(50), + ..capsem_foundation::poll::PollOpts::new("vm-ready", std::time::Duration::from_secs(timeout_secs)) + } +} + pub(super) fn running_uds_path(state: &ServiceState, id: &str) -> Result { let instances = state.instances.lock().unwrap(); let path = instances diff --git a/crates/capsem-service/src/vm_files/launch.rs b/crates/capsem-service/src/vm_files/launch.rs index ad6ca0e81..601e9c3fc 100644 --- a/crates/capsem-service/src/vm_files/launch.rs +++ b/crates/capsem-service/src/vm_files/launch.rs @@ -39,6 +39,10 @@ pub(crate) enum LaunchWait { TimedOut, } +pub(super) fn stamped(path: &Path, expected: &str) -> bool { + std::fs::read_to_string(path).is_ok_and(|stamp| stamp.trim() == expected) +} + /// Wait for the launch signal, readiness, or a crash, at most `ceiling`. /// `still_alive` reports whether the instance is still registered; it is /// consulted only when neither sentinel exists so a crash after launch @@ -56,10 +60,10 @@ pub(crate) async fn wait_for_launch( max_delay: Duration::from_millis(50), }; let outcome = poll_until(opts, || async { - if ready_path.exists() { + if stamped(ready_path, "ready") { return Some(LaunchWait::Ready); } - if launched_path.exists() { + if stamped(launched_path, "launched") { return Some(LaunchWait::Launched); } (!still_alive()).then_some(LaunchWait::Crashed) diff --git a/crates/capsem-service/src/vm_files/launch/tests.rs b/crates/capsem-service/src/vm_files/launch/tests.rs index 435077d5e..8b87ac975 100644 --- a/crates/capsem-service/src/vm_files/launch/tests.rs +++ b/crates/capsem-service/src/vm_files/launch/tests.rs @@ -16,7 +16,7 @@ async fn the_launch_sentinel_returns_long_before_the_ceiling() { let launched = launched.clone(); tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(60)).await; - std::fs::File::create(launched).unwrap(); + std::fs::write(launched, b"launched\n").unwrap(); }) }; let started = Instant::now(); @@ -33,8 +33,8 @@ async fn the_launch_sentinel_returns_long_before_the_ceiling() { #[tokio::test] async fn readiness_still_wins_when_the_guest_is_already_up() { let (_dir, ready, launched) = paths(); - std::fs::File::create(&ready).unwrap(); - std::fs::File::create(&launched).unwrap(); + std::fs::write(&ready, b"ready\n").unwrap(); + std::fs::write(&launched, b"launched\n").unwrap(); assert_eq!( wait_for_launch(&ready, &launched, || true, LAUNCH_CEILING).await, LaunchWait::Ready @@ -64,7 +64,7 @@ async fn a_crash_after_launch_is_not_reported_by_create() { // The launch sentinel exists; the instance dying afterwards is exec's // to surface, not create's. let (_dir, ready, launched) = paths(); - std::fs::File::create(&launched).unwrap(); + std::fs::write(&launched, b"launched\n").unwrap(); assert_eq!( wait_for_launch(&ready, &launched, || false, LAUNCH_CEILING).await, LaunchWait::Launched @@ -89,3 +89,16 @@ async fn nothing_within_the_ceiling_times_out_at_the_ceiling() { fn the_ceiling_is_the_old_fixed_wait() { assert_eq!(LAUNCH_CEILING, Duration::from_millis(500)); } + +#[test] +fn prepared_empty_sentinels_do_not_publish_readiness() { + let (_dir, ready, launched) = paths(); + std::fs::File::create(&ready).unwrap(); + std::fs::File::create(&launched).unwrap(); + assert!(!stamped(&ready, "ready")); + assert!(!stamped(&launched, "launched")); + std::fs::write(&ready, b"ready\n").unwrap(); + std::fs::write(&launched, b"launched\n").unwrap(); + assert!(stamped(&ready, "ready")); + assert!(stamped(&launched, "launched")); +} From b8d1cf30f94239788955ab4c32afe02a6371466e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 00:52:14 +0000 Subject: [PATCH 068/464] security(process): confine Linux VM owners --- CHANGELOG.md | 5 + crates/capsem-process/src/main.rs | 127 ++++++++++++++---- crates/capsem-process/src/metric_export.rs | 55 +++++--- .../capsem-process/src/metric_export/tests.rs | 3 +- crates/capsem-process/src/private_names.rs | 10 +- .../capsem-process/src/private_names/tests.rs | 22 ++- crates/capsem-process/src/private_seats.rs | 15 ++- crates/capsem-process/src/vsock.rs | 12 +- 8 files changed, 185 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3cc227d2..496d8ae67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Confine Linux VM-owner runtime threads before readiness to their session, + read-only boot assets, prepared service channels and inherited network + grants; deny sibling files, path-based control connections and direct + network dialing after startup. + - Restrict registered VM-owner service connections to private-name lookup and that owner's metrics route, preventing a compromised worker from invoking the coordinator's public API over its same-user control socket. diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 67790e14a..dcb0c6ea7 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -209,6 +209,62 @@ fn prepare_session_layout(session_dir: &Path, scratch_disk_size_gb: u32) -> Resu Ok(guest_dir) } +fn prepare_sentinel(path: &Path) -> Result { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(path) + .with_context(|| format!("prepare readiness sentinel {}", path.display()))?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; + Ok(file) +} + +#[cfg(target_os = "linux")] +fn confine_linux_owner(args: &Args, session_dir: &Path) -> Result<()> { + use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; + + let grant = |policy: Policy, path: &Path, access| -> Result { + let canonical = path + .canonicalize() + .with_context(|| format!("resolve sandbox grant {}", path.display()))?; + Ok(policy.allow(canonical, access)) + }; + let mut policy = Policy::new(Role::VmOwner); + policy = grant(policy, session_dir, Access::ReadWrite)?; + policy = grant(policy, &args.assets_dir, Access::ReadOnly)?; + policy = grant(policy, &args.rootfs, Access::ReadOnly)?; + policy = grant(policy, &args.active_policy, Access::ReadOnly)?; + for path in args.kernel.iter().chain(args.initrd.iter()) { + policy = grant(policy, path, Access::ReadOnly)?; + } + let executable = std::env::current_exe().context("locate VM-owner executable")?; + let router = executable.with_file_name("capsem-router"); + if router.exists() { + policy = grant(policy, &router, Access::Executable)?; + } + for path in ["/lib", "/lib64", "/usr/lib", "/usr/lib64"] { + let path = Path::new(path); + if path.exists() { + policy = grant(policy, path, Access::ReadOnly)?; + } + } + let linker_cache = Path::new("/etc/ld.so.cache"); + if linker_cache.exists() { + policy = grant(policy, linker_cache, Access::ReadOnly)?; + } + for path in ["/lib64/ld-linux-x86-64.so.2", "/lib/ld-linux-aarch64.so.1"] { + let path = Path::new(path); + if path.exists() { + policy = grant(policy, path, Access::Executable)?; + } + } + capsem_foundation::unix::worker_sandbox::confine(&policy)?; + Ok(()) +} + fn main() -> Result<()> { // SAFETY: process entry precedes argument parsing, telemetry, descriptor // owners and runtime threads. Broker grants will be named here explicitly. @@ -247,9 +303,6 @@ fn main() -> Result<()> { let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; info!(id = %args.id, "capsem-sandbox-process starting"); - // Held until the process exits: dropping it flushes the last measurements. - let _metric_export = metric_export::install(&args.id, args.service_socket.as_deref(), args.metric_broker); - let guest_dir = prepare_session_layout(&session_dir, args.scratch_disk_size_gb)?; // The image share is attached to every session, read-only at the device: // a device cannot be added after boot, and an image is pulled only once @@ -481,6 +534,24 @@ async fn run_async_main_loop( &job_store, ctrl_tx.clone(), )?; + let private_service = Arc::new( + capsem_core::service_uds::Client::connect(&seats.service_socket) + .await + .context("prepare private-name service channel")?, + ); + let metric_service = if args.metric_broker { + Some( + tokio::net::UnixStream::connect(&seats.service_socket) + .await + .context("prepare metric service channel")? + .into_std() + .context("adopt metric service channel")?, + ) + } else { + None + }; + // Held until the process exits: dropping it flushes the last measurements. + let _metric_export = metric_export::install(&args.id, metric_service, args.metric_broker); let restored = job_store .publisher .restore(ctrl_tx.clone()) @@ -625,7 +696,7 @@ async fn run_async_main_loop( )); // The private zone is the service's to answer, for this VM's networks. let private_names = Arc::new(private_names::ServicePrivateNames::new( - seats.service_socket, + private_service, args.id.clone(), )); let dns_handler = Arc::new( @@ -666,7 +737,6 @@ async fn run_async_main_loop( let is_restore = args.checkpoint_path.is_some(); let vm_for_vsock = Arc::clone(&vm); let vm_ready_vsock = Arc::clone(&vm_ready); - let uds_path_vsock = uds_path.clone(); let db_for_vsock = Arc::clone(&db); let shutdown_for_vsock = Arc::clone(&shutdown); let shutdown_for_vsock_error = Arc::clone(&shutdown); @@ -677,6 +747,29 @@ async fn run_async_main_loop( None } }; + + if uds_path.exists() { + std::fs::remove_file(&uds_path)?; + } + let listener = UnixListener::bind(&uds_path)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&uds_path, std::fs::Permissions::from_mode(0o600))?; + } + let launched_path = uds_path.with_extension("launched"); + let mut launched = prepare_sentinel(&launched_path)?; + let ready = prepare_sentinel(&uds_path.with_extension("ready"))?; + + #[cfg(target_os = "linux")] + confine_linux_owner(&args, &session_dir).context("install Linux VM-owner confinement")?; + + seats.start(); + use std::io::Write as _; + launched.write_all(b"launched\n")?; + launched.sync_data()?; + info!(socket = %uds_path.display(), "listening for IPC (mode 0600)"); + tokio::spawn(async move { if let Err(e) = vsock::setup_vsock(VsockOptions { vm_id: args.id.clone(), @@ -697,7 +790,7 @@ async fn run_async_main_loop( _net_state: net_state_clone, is_restore, vm_ready: vm_ready_vsock, - uds_path: uds_path_vsock, + ready, db: db_for_vsock, pty_log, shutdown: shutdown_for_vsock, @@ -717,28 +810,6 @@ async fn run_async_main_loop( } }); - if uds_path.exists() { - std::fs::remove_file(&uds_path)?; - } - let listener = UnixListener::bind(&uds_path)?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&uds_path, std::fs::Permissions::from_mode(0o600))?; - } - info!(socket = %uds_path.display(), "listening for IPC (mode 0600)"); - // The launch signal: the hypervisor has started the VM and this process - // is answering IPC. `create` returns on it instead of waiting out a timer - // (`.ready`, written after the guest handshake, comes much later). A - // separate file rather than the socket's existence, because a stale - // socket from an earlier run at this path was just deleted above. - let launched_path = uds_path.with_extension("launched"); - std::fs::write(&launched_path, b"launched\n")?; - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&launched_path, std::fs::Permissions::from_mode(0o600))?; - } - // Terminal relay: fan-out broadcast + ring buffer so a newly-attached // terminal stream sees the shell's startup banner (printed before it joined). let term_relay = terminal::TerminalRelay::new(1024); diff --git a/crates/capsem-process/src/metric_export.rs b/crates/capsem-process/src/metric_export.rs index 387da2d1e..090e59416 100644 --- a/crates/capsem-process/src/metric_export.rs +++ b/crates/capsem-process/src/metric_export.rs @@ -6,13 +6,13 @@ //! a collector address or credential. Without a broker grant, export is off. use capsem_telemetry::export::{Exporter, KeyValue}; -use std::path::{Path, PathBuf}; use std::sync::Mutex; use tracing::{info, warn}; /// OTLP transport that can reach only this process's fixed service route. struct ServiceMetricClient { - service_socket: PathBuf, + service_socket: Mutex>, + service: Mutex>, path: String, runtime: Mutex, } @@ -21,20 +21,23 @@ impl std::fmt::Debug for ServiceMetricClient { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { formatter .debug_struct("ServiceMetricClient") - .field("service_socket", &self.service_socket) .field("path", &self.path) .finish_non_exhaustive() } } impl ServiceMetricClient { - fn new(service_socket: PathBuf, vm_id: &str) -> Result { + fn new(service_socket: std::os::unix::net::UnixStream, vm_id: &str) -> Result { + service_socket + .set_nonblocking(true) + .map_err(|error| format!("prepare metric broker socket: {error}"))?; let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .map_err(|error| format!("build metric broker runtime: {error}"))?; Ok(Self { - service_socket, + service_socket: Mutex::new(Some(service_socket)), + service: Mutex::new(None), path: format!("/internal/vms/{vm_id}/metrics"), runtime: Mutex::new(runtime), }) @@ -47,23 +50,41 @@ impl opentelemetry_http::HttpClient for ServiceMetricClient { &self, request: http::Request, ) -> Result, opentelemetry_http::HttpError> { - let response = self - .runtime - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .block_on(capsem_core::service_uds::post_bytes( - &self.service_socket, - &self.path, - http::HeaderValue::from_static("application/x-protobuf"), - request.into_body(), - )); + let runtime = self.runtime.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + let mut service = self.service.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); + if service.is_none() { + let socket = self + .service_socket + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .take() + .ok_or_else(|| std::io::Error::other("metric broker socket unavailable"))?; + let client = runtime + .block_on(async move { + let stream = tokio::net::UnixStream::from_std(socket)?; + capsem_core::service_uds::Client::from_stream(stream).await + }) + .map_err(|error| std::io::Error::other(format!("prepare metric broker HTTP channel: {error:#}")))?; + *service = Some(client); + } + let response = runtime.block_on(service.as_ref().unwrap().post_bytes( + &self.path, + http::HeaderValue::from_static("application/x-protobuf"), + request.into_body(), + )); + drop(service); + drop(runtime); response.map_err(|error| Box::new(std::io::Error::other(format!("metric broker request: {error:#}"))) as _) } } /// Install export when the service granted its broker. Failure is logged and /// leaves export off; metrics never prevent a VM from booting. -pub(crate) fn install(vm_id: &str, service_socket: Option<&Path>, broker: bool) -> Option { +pub(crate) fn install( + vm_id: &str, + service_socket: Option, + broker: bool, +) -> Option { if !broker { return None; } @@ -75,7 +96,7 @@ pub(crate) fn install(vm_id: &str, service_socket: Option<&Path>, broker: bool) return None; } }; - let client = match ServiceMetricClient::new(socket.to_path_buf(), vm_id) { + let client = match ServiceMetricClient::new(socket, vm_id) { Ok(client) => client, Err(error) => { warn!(%error, "metric export not installed"); diff --git a/crates/capsem-process/src/metric_export/tests.rs b/crates/capsem-process/src/metric_export/tests.rs index d72b66c93..8ffc2ba5c 100644 --- a/crates/capsem-process/src/metric_export/tests.rs +++ b/crates/capsem-process/src/metric_export/tests.rs @@ -35,7 +35,8 @@ async fn broker_client_uses_only_the_fixed_local_route_and_protobuf_body() { .route("/internal/vms/{id}/metrics", post(receive)) .with_state(Arc::clone(&received)); let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - let client = ServiceMetricClient::new(socket, "vm-a").unwrap(); + let stream = std::os::unix::net::UnixStream::connect(socket).unwrap(); + let client = ServiceMetricClient::new(stream, "vm-a").unwrap(); let request = http::Request::post("https://attacker.example/v1/metrics") .header(http::header::CONTENT_TYPE, "text/plain") .header(http::header::AUTHORIZATION, "Bearer must-not-cross") diff --git a/crates/capsem-process/src/private_names.rs b/crates/capsem-process/src/private_names.rs index 18d3b56a2..7237b5557 100644 --- a/crates/capsem-process/src/private_names.rs +++ b/crates/capsem-process/src/private_names.rs @@ -4,22 +4,22 @@ //! is gone from the next answer. use capsem_core::net::dns::private::{Lookup, PrivateNames}; use std::net::Ipv4Addr; -use std::path::PathBuf; +use std::sync::Arc; pub(crate) struct ServicePrivateNames { - service_socket: PathBuf, + service: Arc, vm_id: String, } impl ServicePrivateNames { - pub(crate) fn new(service_socket: PathBuf, vm_id: String) -> Self { - Self { service_socket, vm_id } + pub(crate) fn new(service: Arc, vm_id: String) -> Self { + Self { service, vm_id } } async fn ask(&self, question: serde_json::Value) -> Option { let mut request = question; request["source_vm"] = serde_json::Value::String(self.vm_id.clone()); - match capsem_core::service_uds::post_json(&self.service_socket, "/networks/private/resolve", &request).await { + match self.service.post_json("/networks/private/resolve", &request).await { Ok((200, answer)) => Some(answer), Ok((404, _)) => None, Ok((status, answer)) => { diff --git a/crates/capsem-process/src/private_names/tests.rs b/crates/capsem-process/src/private_names/tests.rs index 7988298ee..ff32a1ab2 100644 --- a/crates/capsem-process/src/private_names/tests.rs +++ b/crates/capsem-process/src/private_names/tests.rs @@ -1,4 +1,5 @@ use super::*; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; /// The service's resolve route as a fake: answers from a table, records @@ -41,7 +42,8 @@ async fn names_and_addresses_are_the_services_answers_and_misses_are_none() { let socket = dir.path().join("service.sock"); let seen = Arc::new(Mutex::new(Vec::new())); fake_service(socket.clone(), Arc::clone(&seen)).await; - let names = ServicePrivateNames::new(socket, "vm-a".into()); + let service = Arc::new(capsem_core::service_uds::Client::connect(&socket).await.unwrap()); + let names = ServicePrivateNames::new(service, "vm-a".into()); assert_eq!(names.address_of("beta.team").await, Some(Ipv4Addr::new(10, 128, 0, 3))); assert_eq!(names.address_of("nobody.team").await, None); assert_eq!( @@ -58,7 +60,7 @@ async fn names_and_addresses_are_the_services_answers_and_misses_are_none() { } #[tokio::test] -async fn a_refusal_or_an_absent_service_is_a_miss_not_a_failure() { +async fn a_refusal_or_a_closed_service_is_a_miss_not_a_failure() { let dir = tempfile::tempdir().unwrap(); let socket = dir.path().join("service.sock"); let listener = tokio::net::UnixListener::bind(&socket).unwrap(); @@ -69,8 +71,18 @@ async fn a_refusal_or_an_absent_service_is_a_miss_not_a_failure() { ); let _ = axum::serve(listener, app).await; }); - let refused = ServicePrivateNames::new(socket, "vm-a".into()); + let service = Arc::new(capsem_core::service_uds::Client::connect(&socket).await.unwrap()); + let refused = ServicePrivateNames::new(service, "vm-a".into()); assert_eq!(refused.address_of("beta.team").await, None); - let absent = ServicePrivateNames::new(dir.path().join("nobody.sock"), "vm-a".into()); - assert_eq!(absent.address_of("beta.team").await, None); + + let closed_socket = dir.path().join("closed.sock"); + let listener = tokio::net::UnixListener::bind(&closed_socket).unwrap(); + let closer = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + drop(stream); + }); + let closed = Arc::new(capsem_core::service_uds::Client::connect(&closed_socket).await.unwrap()); + closer.await.unwrap(); + let closed = ServicePrivateNames::new(closed, "vm-a".into()); + assert_eq!(closed.address_of("beta.team").await, None); } diff --git a/crates/capsem-process/src/private_seats.rs b/crates/capsem-process/src/private_seats.rs index 6e63a6a8f..d6562ebb1 100644 --- a/crates/capsem-process/src/private_seats.rs +++ b/crates/capsem-process/src/private_seats.rs @@ -30,6 +30,14 @@ fn bound(path: std::path::PathBuf, what: &str) -> Result<(std::path::PathBuf, Un /// the service on its VM's behalf. pub(crate) struct Bound { pub service_socket: std::path::PathBuf, + cables: Arc, + seat_listener: UnixListener, +} + +impl Bound { + pub(crate) fn start(self) { + tokio::spawn(self.cables.serve_seat(self.seat_listener)); + } } pub(crate) fn bind( @@ -58,6 +66,9 @@ pub(crate) fn bind( "cable seat", )?; let _ = job_store.cable_seat.set(seat_path); - tokio::spawn(cables.serve_seat(seat_listener)); - Ok(Bound { service_socket }) + Ok(Bound { + service_socket, + cables, + seat_listener, + }) } diff --git a/crates/capsem-process/src/vsock.rs b/crates/capsem-process/src/vsock.rs index d946a824f..d52e78c0e 100644 --- a/crates/capsem-process/src/vsock.rs +++ b/crates/capsem-process/src/vsock.rs @@ -81,7 +81,7 @@ pub(crate) struct VsockOptions { pub(crate) _net_state: Arc, pub(crate) is_restore: bool, pub(crate) vm_ready: Arc, - pub(crate) uds_path: PathBuf, + pub(crate) ready: std::fs::File, pub(crate) db: Arc, pub(crate) pty_log: Option>, pub(crate) shutdown: Arc>, @@ -107,7 +107,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { plugin_policy, is_restore, vm_ready, - uds_path, + mut ready, db, pty_log, shutdown, @@ -184,8 +184,8 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { }) ); vm_ready.store(true, Ordering::Release); - let ready_path = uds_path.with_extension("ready"); - if let Err(e) = std::fs::write(&ready_path, b"ready\n") { + let ready_for_reader = ready.try_clone().context("clone ready sentinel")?; + if let Err(e) = ready.write_all(b"ready\n").and_then(|()| ready.sync_data()) { warn!(error = %e, "failed to create ready sentinel"); } @@ -300,7 +300,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { let js_for_teardown = Arc::clone(&job_store); let vm_ready_for_reader = Arc::clone(&vm_ready); - let ready_path_for_reader = ready_path; + let ready_for_reader = ready_for_reader; // Pending-ack map lives on `JobStore` (see job_store.rs::pending_acks) // and the bridge end here can replay-on-rekey. See the field doc on @@ -318,7 +318,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { None => { js_for_teardown.fail_all("control channel closed"); vm_ready_for_reader.store(false, Ordering::Release); - let _ = std::fs::remove_file(&ready_path_for_reader); + let _ = ready_for_reader.set_len(0).and_then(|()| ready_for_reader.sync_data()); break; } }, From 6a99fc8c37be4fde62ef50d6aa306ca5eee7a4df Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 00:53:35 +0000 Subject: [PATCH 069/464] fix(foundation): close inherited descriptors without procfs --- crates/capsem-foundation/src/unix/fd.rs | 83 +++++++++++++++++++------ tests/citadel/unix_boundary_debt.toml | 4 ++ 2 files changed, 67 insertions(+), 20 deletions(-) diff --git a/crates/capsem-foundation/src/unix/fd.rs b/crates/capsem-foundation/src/unix/fd.rs index 192e22d3f..583faa2ac 100644 --- a/crates/capsem-foundation/src/unix/fd.rs +++ b/crates/capsem-foundation/src/unix/fd.rs @@ -26,33 +26,76 @@ pub unsafe fn close_inherited_descriptors() -> io::Result<()> { /// or another thread can open and reuse them. Every descriptor in `preserved` /// must be an intentional grant whose ownership the caller establishes next. pub unsafe fn close_inherited_descriptors_except(preserved: &[RawFd]) -> io::Result<()> { - #[cfg(target_os = "macos")] - let directory = "/dev/fd"; - #[cfg(not(target_os = "macos"))] - let directory = "/proc/self/fd"; - let mut descriptors = Vec::new(); - for entry in std::fs::read_dir(directory)? { - let name = entry?.file_name(); - let fd: RawFd = name - .to_str() - .ok_or_else(|| io::Error::other("invalid descriptor name"))? - .parse() - .map_err(io::Error::other)?; - if should_close_inherited_descriptor(fd, preserved) { - descriptors.push(fd); + #[cfg(target_os = "linux")] + { + close_inherited_ranges(preserved) + } + #[cfg(not(target_os = "linux"))] + { + #[cfg(target_os = "macos")] + let directory = "/dev/fd"; + #[cfg(not(target_os = "macos"))] + let directory = "/proc/self/fd"; + let mut descriptors = Vec::new(); + for entry in std::fs::read_dir(directory)? { + let name = entry?.file_name(); + let fd: RawFd = name + .to_str() + .ok_or_else(|| io::Error::other("invalid descriptor name"))? + .parse() + .map_err(io::Error::other)?; + if should_close_inherited_descriptor(fd, preserved) { + descriptors.push(fd); + } + } + for fd in descriptors { + // The caller guarantees no live Rust owner or fd reuse. The directory + // iterator itself has closed; EBADF for that fd is expected. + match nix::unistd::close(fd) { + Ok(()) | Err(Errno::EBADF) => {} + Err(error) => return Err(errno::io(error)), + } } + Ok(()) } - for fd in descriptors { - // The caller guarantees no live Rust owner or fd reuse. The directory - // iterator itself has closed; EBADF for that fd is expected. - match nix::unistd::close(fd) { - Ok(()) | Err(Errno::EBADF) => {} - Err(error) => return Err(errno::io(error)), +} + +#[cfg(target_os = "linux")] +fn close_inherited_ranges(preserved: &[RawFd]) -> io::Result<()> { + let mut preserved: Vec = preserved + .iter() + .copied() + .filter(|descriptor| *descriptor > 2) + .map(|descriptor| descriptor as u32) + .collect(); + preserved.sort_unstable(); + preserved.dedup(); + let mut first = 3u32; + for descriptor in preserved { + if first < descriptor { + close_range(first, descriptor - 1)?; } + first = descriptor.saturating_add(1); + } + close_range(first, u32::MAX) +} + +#[cfg(target_os = "linux")] +fn close_range(first: u32, last: u32) -> io::Result<()> { + if first > last { + return Ok(()); + } + // nix 0.29 has no close_range wrapper. The zero flags select the kernel's + // atomic close behavior and avoid opening /proc/self/fd, which remains + // usable after a parent installs Landlock before exec. + let result = unsafe { libc::syscall(libc::SYS_close_range, first, last, 0u32) }; + if result < 0 { + return Err(io::Error::last_os_error()); } Ok(()) } +#[cfg(any(not(target_os = "linux"), test))] fn should_close_inherited_descriptor(fd: RawFd, preserved: &[RawFd]) -> bool { fd > 2 && !preserved.contains(&fd) } diff --git a/tests/citadel/unix_boundary_debt.toml b/tests/citadel/unix_boundary_debt.toml index 8efe94081..cb69e77d9 100644 --- a/tests/citadel/unix_boundary_debt.toml +++ b/tests/citadel/unix_boundary_debt.toml @@ -75,6 +75,10 @@ fingerprint = "003d8fd29845c26d97ce2f3c5f7d4a3ee00a3e84c0fe122c71385f8b0d650fde" reason = "Linux AF_VSOCK buffer bounds (SO_VM_SOCKETS_BUFFER_* 0/1/2) set before credit can grow" nix_gap = "nix::sys::socket::sockopt defines no AF_VSOCK buffer options, so there is no SetSockOpt type" +[foundation.raw_calls."crates/capsem-foundation/src/unix/fd.rs::syscall"] +fingerprint = "484b5c9efe57fe349ba09a8c628eeabe3c6cd0cfd9b60e592cf28fc8cf4e9662" +reason = "Linux close_range avoids procfs after a parent Landlock domain is inherited across exec" + [foundation.raw_calls."crates/capsem-foundation/src/unix/fs/extents.rs::fclonefileat"] fingerprint = "1382b8391823c31f4fbdbf23e319493504ee03a024fcd02c45f543e466af0d2d" reason = "macOS APFS clone from an open descriptor into a directory descriptor; no nix wrapper" From 8b9edce76c8b5cff342b1d59b6a2f4d485f96758 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:08:33 +0000 Subject: [PATCH 070/464] fix(core): launch images without stage chmod --- CHANGELOG.md | 3 +++ crates/capsem-core/src/container.rs | 10 ++-------- crates/capsem-core/src/container/tests.rs | 11 ++++------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 496d8ae67..94afe7a9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Launch OCI workloads without mutating the guest-owned staging file, so the + Linux VM-owner sandbox can keep host permission syscalls denied. + - KVM image workloads retain write access to their workspace after VM-root access by expiring caller-owned VirtioFS attributes across user namespaces. diff --git a/crates/capsem-core/src/container.rs b/crates/capsem-core/src/container.rs index e2daa2c43..18912eed4 100644 --- a/crates/capsem-core/src/container.rs +++ b/crates/capsem-core/src/container.rs @@ -126,19 +126,13 @@ macro_rules! launcher_in_moved_root { /// The launcher with the client attached: `capsem run --image` streams its /// stdio, so the workload's command reads and writes the client's. -pub const LAUNCH_COMMAND: &str = concat!( - "chmod 555 /root/.capsem-image/launch.py && ", - launcher_in_moved_root!("/root/.capsem-image") -); +pub const LAUNCH_COMMAND: &str = launcher_in_moved_root!("/root/.capsem-image"); /// The launcher with nothing attached: a detached create, a relaunch. The /// workload's command runs on a terminal the launcher holds, so an /// interactive one (a shell, an agent's TUI) waits for its user instead of /// reading end-of-file, and the session terminal attaches to it. -pub const DETACHED_LAUNCH_COMMAND: &str = concat!( - "chmod 555 /root/.capsem-image/launch.py && ", - launcher_in_moved_root!("--detached /root/.capsem-image") -); +pub const DETACHED_LAUNCH_COMMAND: &str = launcher_in_moved_root!("--detached /root/.capsem-image"); /// The session terminal's command in an image session: the launcher enters the /// workload with a login shell, again whenever it ends, and never falls back to diff --git a/crates/capsem-core/src/container/tests.rs b/crates/capsem-core/src/container/tests.rs index 953be5f5c..b121bed54 100644 --- a/crates/capsem-core/src/container/tests.rs +++ b/crates/capsem-core/src/container/tests.rs @@ -75,13 +75,14 @@ fn a_vm_too_small_for_a_workload_is_refused() { } #[test] -fn the_launch_command_is_unchanged_by_sharing_the_moved_root_prefix() { +fn the_launch_commands_do_not_mutate_the_guest_owned_stage() { assert_eq!( LAUNCH_COMMAND, - "chmod 555 /root/.capsem-image/launch.py && chroot /proc/1/root /bin/busybox unshare -m /bin/sh -ec \ + "chroot /proc/1/root /bin/busybox unshare -m /bin/sh -ec \ 'mount --make-rprivate /; cd /newroot; mount --move . /; exec chroot . /usr/bin/python3 \ /root/.capsem-image/launch.py /root/.capsem-image'" ); + assert!(!DETACHED_LAUNCH_COMMAND.contains("chmod"), "{DETACHED_LAUNCH_COMMAND}"); } /// The JSON request a `workload_exec_command` carries, hex-decoded. @@ -100,11 +101,7 @@ fn decode_workload_exec(payload: &str) -> serde_json::Value { fn a_workload_exec_carries_any_command_exactly_to_the_launcher() { let hostile = "printf '%s' \"$(id -u)\"; echo 'a'\\''b' `uname`\n; exit 7"; let command = workload_exec_command(hostile); - let (context, _) = LAUNCH_COMMAND - .strip_prefix("chmod 555 /root/.capsem-image/launch.py && ") - .unwrap() - .split_once("launch.py ") - .unwrap(); + let (context, _) = LAUNCH_COMMAND.split_once("launch.py ").unwrap(); assert!(command.starts_with(context), "{command}"); // What the outer shell hands `sh -ec`: run it through a real shell with // the chroot replaced by printf and read the inner script back. From 0d48d3fd9c3e1f9a2735e85f12cb63986eb02d60 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:11:31 +0000 Subject: [PATCH 071/464] feat(proto): define gateway descriptor grants --- crates/capsem-proto/src/gateway_grant.rs | 205 ++++++++++++++++++ .../capsem-proto/src/gateway_grant/tests.rs | 56 +++++ crates/capsem-proto/src/lib.rs | 1 + 3 files changed, 262 insertions(+) create mode 100644 crates/capsem-proto/src/gateway_grant.rs create mode 100644 crates/capsem-proto/src/gateway_grant/tests.rs diff --git a/crates/capsem-proto/src/gateway_grant.rs b/crates/capsem-proto/src/gateway_grant.rs new file mode 100644 index 000000000..0c2db842a --- /dev/null +++ b/crates/capsem-proto/src/gateway_grant.rs @@ -0,0 +1,205 @@ +//! Fixed records for coordinator-granted gateway Unix connections. +//! +//! The confined gateway never names a socket path. It may ask for a fresh +//! service connection, or name a VM whose current owner handoff endpoint the +//! coordinator derives and authenticates from its registry. + +use anyhow::{bail, Result}; + +pub const GATEWAY_GRANT_FRAME_SIZE: usize = 80; +pub const GATEWAY_GRANT_MAX_FDS: usize = 1; +pub const MAX_GATEWAY_VM_ID_BYTES: usize = 64; + +const MAGIC: [u8; 2] = *b"GG"; +const VERSION: u8 = 1; +const KIND_OFFSET: usize = 3; +const REQUEST_ID_RANGE: std::ops::Range = 4..12; +const DETAIL_OFFSET: usize = 12; +const VM_ID_LENGTH_OFFSET: usize = 13; +const VM_ID_RANGE: std::ops::Range = 14..14 + MAX_GATEWAY_VM_ID_BYTES; +const RESERVED_RANGE: std::ops::Range = VM_ID_RANGE.end..GATEWAY_GRANT_FRAME_SIZE; + +const OPEN_SERVICE: u8 = 1; +const OPEN_OWNER_HANDOFF: u8 = 2; +const GRANTED: u8 = 101; +const DENIED: u8 = 102; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum GatewayGrantRequest { + OpenService { request_id: u64 }, + OpenOwnerHandoff { request_id: u64, vm_id: String }, +} + +impl GatewayGrantRequest { + pub fn request_id(&self) -> u64 { + match self { + Self::OpenService { request_id } | Self::OpenOwnerHandoff { request_id, .. } => *request_id, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GatewayGrantKind { + Service, + OwnerHandoff, +} + +impl GatewayGrantKind { + fn code(self) -> u8 { + match self { + Self::Service => OPEN_SERVICE, + Self::OwnerHandoff => OPEN_OWNER_HANDOFF, + } + } + + fn decode(code: u8) -> Result { + match code { + OPEN_SERVICE => Ok(Self::Service), + OPEN_OWNER_HANDOFF => Ok(Self::OwnerHandoff), + _ => bail!("invalid gateway grant kind {code}"), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GatewayGrantDenial { + Unavailable, + NotFound, + Revoked, + InvalidRequest, +} + +impl GatewayGrantDenial { + fn code(self) -> u8 { + match self { + Self::Unavailable => 1, + Self::NotFound => 2, + Self::Revoked => 3, + Self::InvalidRequest => 4, + } + } + + fn decode(code: u8) -> Result { + match code { + 1 => Ok(Self::Unavailable), + 2 => Ok(Self::NotFound), + 3 => Ok(Self::Revoked), + 4 => Ok(Self::InvalidRequest), + _ => bail!("invalid gateway grant denial {code}"), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GatewayGrantResponse { + Granted { + request_id: u64, + kind: GatewayGrantKind, + }, + Denied { + request_id: u64, + reason: GatewayGrantDenial, + }, +} + +impl GatewayGrantResponse { + pub fn request_id(self) -> u64 { + match self { + Self::Granted { request_id, .. } | Self::Denied { request_id, .. } => request_id, + } + } +} + +pub fn encode_gateway_grant_request(request: &GatewayGrantRequest) -> Result<[u8; GATEWAY_GRANT_FRAME_SIZE]> { + let mut frame = base(match request { + GatewayGrantRequest::OpenService { .. } => OPEN_SERVICE, + GatewayGrantRequest::OpenOwnerHandoff { .. } => OPEN_OWNER_HANDOFF, + }); + frame[REQUEST_ID_RANGE].copy_from_slice(&request.request_id().to_be_bytes()); + if let GatewayGrantRequest::OpenOwnerHandoff { vm_id, .. } = request { + let id = vm_id.as_bytes(); + if id.is_empty() || id.len() > MAX_GATEWAY_VM_ID_BYTES { + bail!("gateway VM id length is out of bounds"); + } + frame[VM_ID_LENGTH_OFFSET] = id.len() as u8; + frame[VM_ID_RANGE.start..VM_ID_RANGE.start + id.len()].copy_from_slice(id); + } + Ok(frame) +} + +pub fn decode_gateway_grant_request(frame: &[u8; GATEWAY_GRANT_FRAME_SIZE]) -> Result { + validate(frame)?; + let request_id = u64::from_be_bytes(frame[REQUEST_ID_RANGE].try_into().unwrap()); + match frame[KIND_OFFSET] { + OPEN_SERVICE if frame[VM_ID_LENGTH_OFFSET] == 0 => Ok(GatewayGrantRequest::OpenService { request_id }), + OPEN_OWNER_HANDOFF => { + let length = usize::from(frame[VM_ID_LENGTH_OFFSET]); + if length == 0 || length > MAX_GATEWAY_VM_ID_BYTES { + bail!("gateway VM id length is out of bounds"); + } + if frame[VM_ID_RANGE.start + length..VM_ID_RANGE.end] + .iter() + .any(|byte| *byte != 0) + { + bail!("gateway grant frame has nonzero VM id padding"); + } + let vm_id = std::str::from_utf8(&frame[VM_ID_RANGE.start..VM_ID_RANGE.start + length])?.to_owned(); + Ok(GatewayGrantRequest::OpenOwnerHandoff { request_id, vm_id }) + } + kind => bail!("invalid gateway grant request kind {kind}"), + } +} + +pub fn encode_gateway_grant_response(response: GatewayGrantResponse) -> [u8; GATEWAY_GRANT_FRAME_SIZE] { + let (kind, request_id, detail) = match response { + GatewayGrantResponse::Granted { request_id, kind } => (GRANTED, request_id, kind.code()), + GatewayGrantResponse::Denied { request_id, reason } => (DENIED, request_id, reason.code()), + }; + let mut frame = base(kind); + frame[REQUEST_ID_RANGE].copy_from_slice(&request_id.to_be_bytes()); + frame[DETAIL_OFFSET] = detail; + frame +} + +pub fn decode_gateway_grant_response(frame: &[u8; GATEWAY_GRANT_FRAME_SIZE]) -> Result { + validate(frame)?; + if frame[VM_ID_LENGTH_OFFSET] != 0 || frame[VM_ID_RANGE].iter().any(|byte| *byte != 0) { + bail!("gateway grant response has unexpected payload"); + } + let request_id = u64::from_be_bytes(frame[REQUEST_ID_RANGE].try_into().unwrap()); + match frame[KIND_OFFSET] { + GRANTED => Ok(GatewayGrantResponse::Granted { + request_id, + kind: GatewayGrantKind::decode(frame[DETAIL_OFFSET])?, + }), + DENIED => Ok(GatewayGrantResponse::Denied { + request_id, + reason: GatewayGrantDenial::decode(frame[DETAIL_OFFSET])?, + }), + kind => bail!("invalid gateway grant response kind {kind}"), + } +} + +fn base(kind: u8) -> [u8; GATEWAY_GRANT_FRAME_SIZE] { + let mut frame = [0; GATEWAY_GRANT_FRAME_SIZE]; + frame[..2].copy_from_slice(&MAGIC); + frame[2] = VERSION; + frame[KIND_OFFSET] = kind; + frame +} + +fn validate(frame: &[u8; GATEWAY_GRANT_FRAME_SIZE]) -> Result<()> { + if frame[..2] != MAGIC { + bail!("invalid gateway grant magic"); + } + if frame[2] != VERSION { + bail!("unsupported gateway grant version {}", frame[2]); + } + if frame[RESERVED_RANGE].iter().any(|byte| *byte != 0) { + bail!("gateway grant reserved bytes are nonzero"); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/gateway_grant/tests.rs b/crates/capsem-proto/src/gateway_grant/tests.rs new file mode 100644 index 000000000..725bfe445 --- /dev/null +++ b/crates/capsem-proto/src/gateway_grant/tests.rs @@ -0,0 +1,56 @@ +use super::*; + +#[test] +fn requests_round_trip_with_bounded_identity() { + for request in [ + GatewayGrantRequest::OpenService { request_id: 7 }, + GatewayGrantRequest::OpenOwnerHandoff { + request_id: u64::MAX, + vm_id: "session-1".into(), + }, + ] { + let frame = encode_gateway_grant_request(&request).unwrap(); + assert_eq!(decode_gateway_grant_request(&frame).unwrap(), request); + } + assert!(encode_gateway_grant_request(&GatewayGrantRequest::OpenOwnerHandoff { + request_id: 1, + vm_id: "x".repeat(MAX_GATEWAY_VM_ID_BYTES + 1), + }) + .is_err()); +} + +#[test] +fn responses_round_trip_with_no_worker_chosen_path() { + for response in [ + GatewayGrantResponse::Granted { + request_id: 9, + kind: GatewayGrantKind::Service, + }, + GatewayGrantResponse::Denied { + request_id: 10, + reason: GatewayGrantDenial::Revoked, + }, + ] { + let frame = encode_gateway_grant_response(response); + assert_eq!(decode_gateway_grant_response(&frame).unwrap(), response); + } +} + +#[test] +fn malformed_and_extended_records_are_refused() { + let request = GatewayGrantRequest::OpenService { request_id: 1 }; + for offset in [0, 2, RESERVED_RANGE.start] { + let mut frame = encode_gateway_grant_request(&request).unwrap(); + frame[offset] ^= 1; + assert!( + decode_gateway_grant_request(&frame).is_err(), + "accepted corruption at {offset}" + ); + } + let mut response = encode_gateway_grant_response(GatewayGrantResponse::Granted { + request_id: 1, + kind: GatewayGrantKind::Service, + }); + response[VM_ID_LENGTH_OFFSET] = 1; + assert!(decode_gateway_grant_response(&response).is_err()); +} diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 4311da04c..43373a846 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -14,6 +14,7 @@ pub mod credential_injection; pub mod credential_reference; mod exec_stream; pub mod forensic; +pub mod gateway_grant; pub mod handshake; pub mod host_session; pub mod ipc; From 1026a0db7cb6c6a79a4de41618bd961832bb09cf Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:16:54 +0000 Subject: [PATCH 072/464] refactor(gateway): consume granted service channels --- crates/capsem-gateway/src/main.rs | 25 +++- crates/capsem-gateway/src/preview.rs | 16 +- .../src/preview/tests/admission.rs | 5 +- crates/capsem-gateway/src/service_client.rs | 51 ++++++- crates/capsem-gateway/src/service_grant.rs | 141 ++++++++++++++++++ .../capsem-gateway/src/service_grant/tests.rs | 50 +++++++ crates/capsem-gateway/src/stream.rs | 2 +- 7 files changed, 270 insertions(+), 20 deletions(-) create mode 100644 crates/capsem-gateway/src/service_grant.rs create mode 100644 crates/capsem-gateway/src/service_grant/tests.rs diff --git a/crates/capsem-gateway/src/main.rs b/crates/capsem-gateway/src/main.rs index a527970ec..695a25100 100644 --- a/crates/capsem-gateway/src/main.rs +++ b/crates/capsem-gateway/src/main.rs @@ -5,11 +5,13 @@ mod preview; mod proxy; mod schema; mod service_client; +mod service_grant; mod status; mod stream; mod surface; use std::net::SocketAddr; +use std::os::fd::FromRawFd; use std::path::PathBuf; use std::sync::Arc; @@ -57,6 +59,10 @@ struct Args { /// Path for the singleton lockfile (overrides default under run_dir). #[arg(long)] lock_path: Option, + + /// Receive coordinator-minted service and owner connections over stdin. + #[arg(long, default_value_t = false)] + service_grant_stdin: bool, } pub struct AppState { @@ -71,15 +77,15 @@ pub struct AppState { } fn main() -> Result<()> { - // SAFETY: process entry precedes argument parsing, telemetry, descriptor - // owners and runtime threads. Broker grants will be named here explicitly. + let args = Args::parse(); + // SAFETY: argument parsing has opened no descriptors or threads; process + // entry still precedes telemetry, descriptor owners and runtime threads. unsafe { capsem_foundation::unix::fd::close_inherited_descriptors()? }; let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; - runtime.block_on(run()) + runtime.block_on(run(args)) } -async fn run() -> Result<()> { - let args = Args::parse(); +async fn run(args: Args) -> Result<()> { let run_dir = gateway_run_dir(&args); let _ = std::fs::create_dir_all(&run_dir); let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { @@ -150,7 +156,14 @@ async fn run() -> Result<()> { let auth_state = AuthState::new(&run_dir, &token, bound_port, preview_port)?; let (events_tx, _) = tokio::sync::broadcast::channel::(64); - let service_client = ServiceClient::new(&uds_path); + let service_client = if args.service_grant_stdin { + // SAFETY: the service gives this child sole ownership of stdin as its + // descriptor grant socket. No standard-input owner has been created. + let grant_socket = unsafe { std::os::unix::net::UnixStream::from_raw_fd(0) }; + ServiceClient::granted(service_grant::GatewayGrantClient::start(grant_socket)?) + } else { + ServiceClient::new(&uds_path) + }; let state = Arc::new(AppState { token, uds_path, diff --git a/crates/capsem-gateway/src/preview.rs b/crates/capsem-gateway/src/preview.rs index 597cbf51e..c7619a156 100644 --- a/crates/capsem-gateway/src/preview.rs +++ b/crates/capsem-gateway/src/preview.rs @@ -224,7 +224,7 @@ struct Head { async fn handle_connection(mut stream: tokio::net::TcpStream, state: Arc) -> anyhow::Result<()> { match admit(&mut stream, &state).await { Ok(None) => Ok(()), - Ok(Some(admitted)) => handoff(stream, admitted).await, + Ok(Some((vm_id, admitted))) => handoff(stream, &vm_id, admitted, &state.service_client).await, Err(refusal) => { write_refusal(&mut stream, &refusal).await; Err(refusal.error) @@ -237,7 +237,7 @@ async fn handle_connection(mut stream: tokio::net::TcpStream, state: Arc, -) -> Result, Refusal> { +) -> Result, Refusal> { const MALFORMED: &str = "This is a Capsem preview origin and did not receive a valid HTTP request."; const UNKNOWN: &str = "This preview is no longer available. Reopen it from Capsem."; const EXPIRED: &str = "This preview session has expired. Reopen the preview from Capsem."; @@ -307,7 +307,7 @@ async fn admit( anyhow::anyhow!("preview owner generation changed"), )); } - Ok(Some(admitted)) + Ok(Some((scope.vm_id, admitted))) } async fn exchange_bootstrap( @@ -407,9 +407,15 @@ fn validate_bootstrap_head(head: &Head) -> anyhow::Result<()> { Ok(()) } -async fn handoff(stream: tokio::net::TcpStream, admitted: PreviewConnectionAdmissionResponse) -> anyhow::Result<()> { +async fn handoff( + stream: tokio::net::TcpStream, + vm_id: &str, + admitted: PreviewConnectionAdmissionResponse, + service: &crate::service_client::ServiceClient, +) -> anyhow::Result<()> { let source = stream.into_std()?; - let seat = tokio::net::UnixStream::connect(&admitted.handoff_socket) + let seat = service + .connect_owner(vm_id, &admitted.handoff_socket) .await? .into_std()?; let owner = capsem_foundation::unix::peer::PeerIdentity { diff --git a/crates/capsem-gateway/src/preview/tests/admission.rs b/crates/capsem-gateway/src/preview/tests/admission.rs index 75110c15d..ab39fb17e 100644 --- a/crates/capsem-gateway/src/preview/tests/admission.rs +++ b/crates/capsem-gateway/src/preview/tests/admission.rs @@ -27,7 +27,8 @@ async fn a_wrong_process_at_the_granted_path_receives_no_seat_or_browser_descrip owner_uid: capsem_foundation::unix::process::current_uid(), }; - let attempted = tokio::spawn(handoff(browser, admission)); + let service = crate::service_client::ServiceClient::new(dir.path().join("service.sock").as_path()); + let attempted = tokio::spawn(async move { handoff(browser, "box", admission, &service).await }); let (seat, _) = listener.accept().await.unwrap(); let receiver = Receiver::new(seat.into_std().unwrap()).unwrap(); let error = match receiver.recv().await { @@ -67,6 +68,7 @@ async fn the_authenticated_owner_receives_the_exact_browser_descriptor() { client.write_all(b"x").await.unwrap(); handoff( browser, + "box", PreviewConnectionAdmissionResponse { handoff_socket: path.to_string_lossy().into_owned(), handoff_token: 78, @@ -74,6 +76,7 @@ async fn the_authenticated_owner_receives_the_exact_browser_descriptor() { owner_pid: std::process::id(), owner_uid: capsem_foundation::unix::process::current_uid(), }, + &crate::service_client::ServiceClient::new(dir.path().join("service.sock").as_path()), ) .await .unwrap(); diff --git a/crates/capsem-gateway/src/service_client.rs b/crates/capsem-gateway/src/service_client.rs index ca428730a..5af849f28 100644 --- a/crates/capsem-gateway/src/service_client.rs +++ b/crates/capsem-gateway/src/service_client.rs @@ -12,23 +12,34 @@ use hyper_util::rt::{TokioExecutor, TokioIo}; use tokio::net::UnixStream; use tower_service::Service; +use crate::service_grant::GatewayGrantClient; + const MAX_IDLE_CONNECTIONS: usize = 8; #[derive(Clone)] pub struct ServiceClient { inner: Client, + source: ConnectionSource, } impl ServiceClient { pub fn new(uds_path: &Path) -> Self { - let connector = UdsConnector { - uds_path: Arc::new(uds_path.to_path_buf()), - }; + let source = ConnectionSource::Path(Arc::new(uds_path.to_path_buf())); + let connector = UdsConnector { source: source.clone() }; let inner = Client::builder(TokioExecutor::new()) .set_host(false) .pool_max_idle_per_host(MAX_IDLE_CONNECTIONS) .build(connector); - Self { inner } + Self { inner, source } + } + + pub(crate) fn granted(grants: GatewayGrantClient) -> Self { + let source = ConnectionSource::Grant(grants); + let inner = Client::builder(TokioExecutor::new()) + .set_host(false) + .pool_max_idle_per_host(MAX_IDLE_CONNECTIONS) + .build(UdsConnector { source: source.clone() }); + Self { inner, source } } pub async fn request( @@ -37,11 +48,37 @@ impl ServiceClient { ) -> Result, hyper_util::client::legacy::Error> { self.inner.request(request).await } + + pub(crate) async fn connect(&self) -> std::io::Result { + self.source.open_service().await + } + + pub(crate) async fn connect_owner(&self, vm_id: &str, direct_path: &str) -> std::io::Result { + match &self.source { + ConnectionSource::Path(_) => UnixStream::connect(direct_path).await, + ConnectionSource::Grant(grants) => grants.open_owner(vm_id.to_owned()).await, + } + } } #[derive(Clone)] struct UdsConnector { - uds_path: Arc, + source: ConnectionSource, +} + +#[derive(Clone)] +enum ConnectionSource { + Path(Arc), + Grant(GatewayGrantClient), +} + +impl ConnectionSource { + async fn open_service(&self) -> std::io::Result { + match self { + Self::Path(path) => UnixStream::connect(path.as_ref()).await, + Self::Grant(grants) => grants.open_service().await, + } + } } impl Service for UdsConnector { @@ -54,7 +91,7 @@ impl Service for UdsConnector { } fn call(&mut self, _uri: Uri) -> Self::Future { - let uds_path = self.uds_path.clone(); - Box::pin(async move { UnixStream::connect(uds_path.as_ref()).await.map(TokioIo::new) }) + let source = self.source.clone(); + Box::pin(async move { source.open_service().await.map(TokioIo::new) }) } } diff --git a/crates/capsem-gateway/src/service_grant.rs b/crates/capsem-gateway/src/service_grant.rs new file mode 100644 index 000000000..cd112075a --- /dev/null +++ b/crates/capsem-gateway/src/service_grant.rs @@ -0,0 +1,141 @@ +//! Serialized requests for coordinator-minted Unix connection descriptors. + +use std::io; +use std::os::fd::OwnedFd; +use std::os::unix::net::UnixStream; +use std::time::Duration; + +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::gateway_grant::{ + decode_gateway_grant_response, encode_gateway_grant_request, GatewayGrantKind, GatewayGrantRequest, + GatewayGrantResponse, GATEWAY_GRANT_FRAME_SIZE, GATEWAY_GRANT_MAX_FDS, +}; +use tokio::sync::{mpsc, oneshot}; + +const COMMAND_CAPACITY: usize = 64; +const WIRE_TIMEOUT: Duration = Duration::from_secs(5); + +type WireSender = DescriptorSender; +type WireReceiver = DescriptorReceiver; + +enum Command { + Service(oneshot::Sender>), + Owner { + vm_id: String, + reply: oneshot::Sender>, + }, +} + +#[derive(Clone)] +pub(crate) struct GatewayGrantClient { + commands: mpsc::Sender, +} + +impl GatewayGrantClient { + pub(crate) fn start(socket: UnixStream) -> io::Result { + let sender = WireSender::new(socket.try_clone()?)?; + let receiver = WireReceiver::new(socket)?; + let (commands, requests) = mpsc::channel(COMMAND_CAPACITY); + tokio::spawn(async move { + if let Err(error) = run(sender, receiver, requests).await { + tracing::warn!(%error, "gateway descriptor grant channel stopped"); + } + }); + Ok(Self { commands }) + } + + pub(crate) async fn open_service(&self) -> io::Result { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::Service(reply)) + .await + .map_err(|_| closed())?; + into_tokio(result.await.map_err(|_| closed())??) + } + + pub(crate) async fn open_owner(&self, vm_id: String) -> io::Result { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::Owner { vm_id, reply }) + .await + .map_err(|_| closed())?; + into_tokio(result.await.map_err(|_| closed())??) + } +} + +async fn run(sender: WireSender, receiver: WireReceiver, mut commands: mpsc::Receiver) -> io::Result<()> { + let mut request_id = 1_u64; + while let Some(command) = commands.recv().await { + let (request, expected, reply) = match command { + Command::Service(reply) => ( + GatewayGrantRequest::OpenService { request_id }, + GatewayGrantKind::Service, + reply, + ), + Command::Owner { vm_id, reply } => ( + GatewayGrantRequest::OpenOwnerHandoff { request_id, vm_id }, + GatewayGrantKind::OwnerHandoff, + reply, + ), + }; + request_id = request_id + .checked_add(1) + .ok_or_else(|| io::Error::other("gateway grant request id exhausted"))?; + let result = exchange(&sender, &receiver, request, expected).await; + let failed = result.is_err(); + let _ = reply.send(result); + if failed { + return Err(io::Error::other("gateway descriptor grant exchange failed")); + } + } + Ok(()) +} + +async fn exchange( + sender: &WireSender, + receiver: &WireReceiver, + request: GatewayGrantRequest, + expected: GatewayGrantKind, +) -> io::Result { + let request_id = request.request_id(); + let frame = encode_gateway_grant_request(&request).map_err(io::Error::other)?; + tokio::time::timeout(WIRE_TIMEOUT, sender.send(&frame, &[])) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "gateway grant request timed out"))??; + let response = tokio::time::timeout(WIRE_TIMEOUT, receiver.recv()) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "gateway grant response timed out"))??; + let decoded = decode_gateway_grant_response(&response.bytes).map_err(io::Error::other)?; + if decoded.request_id() != request_id { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "gateway grant response id mismatch", + )); + } + match decoded { + GatewayGrantResponse::Granted { kind, .. } if kind == expected && response.fds.len() == 1 => { + Ok(response.fds.into_iter().next().unwrap()) + } + GatewayGrantResponse::Denied { reason, .. } if response.fds.is_empty() => Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("gateway grant denied: {reason:?}"), + )), + _ => Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid gateway descriptor grant response", + )), + } +} + +fn into_tokio(descriptor: OwnedFd) -> io::Result { + let stream = UnixStream::from(descriptor); + stream.set_nonblocking(true)?; + tokio::net::UnixStream::from_std(stream) +} + +fn closed() -> io::Error { + io::Error::new(io::ErrorKind::BrokenPipe, "gateway descriptor grant channel is closed") +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-gateway/src/service_grant/tests.rs b/crates/capsem-gateway/src/service_grant/tests.rs new file mode 100644 index 000000000..fe1e76867 --- /dev/null +++ b/crates/capsem-gateway/src/service_grant/tests.rs @@ -0,0 +1,50 @@ +use super::*; +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::gateway_grant::{decode_gateway_grant_request, encode_gateway_grant_response, GatewayGrantResponse}; +use std::os::fd::AsRawFd; +use tokio::io::AsyncReadExt; + +#[tokio::test] +async fn grants_service_and_registry_named_owner_connections() { + let (client, server) = UnixStream::pair().unwrap(); + let grants = GatewayGrantClient::start(client).unwrap(); + let broker = tokio::spawn(async move { + let requests = + DescriptorReceiver::::new(server.try_clone().unwrap()) + .unwrap(); + let responses = DescriptorSender::::new(server).unwrap(); + for expected in [GatewayGrantKind::Service, GatewayGrantKind::OwnerHandoff] { + let request = requests.recv().await.unwrap(); + let request = decode_gateway_grant_request(&request.bytes).unwrap(); + if expected == GatewayGrantKind::OwnerHandoff { + assert!(matches!( + &request, + GatewayGrantRequest::OpenOwnerHandoff { vm_id, .. } if vm_id == "box" + )); + } + let (mut peer, granted) = UnixStream::pair().unwrap(); + let response = encode_gateway_grant_response(GatewayGrantResponse::Granted { + request_id: request.request_id(), + kind: expected, + }); + responses.send(&response, &[granted.as_raw_fd()]).await.unwrap(); + std::io::Write::write_all(&mut peer, &[kind_code(expected)]).unwrap(); + } + }); + + let mut service = grants.open_service().await.unwrap(); + let mut byte = [0]; + service.read_exact(&mut byte).await.unwrap(); + assert_eq!(byte, [1]); + let mut owner = grants.open_owner("box".into()).await.unwrap(); + owner.read_exact(&mut byte).await.unwrap(); + assert_eq!(byte, [2]); + broker.await.unwrap(); +} + +fn kind_code(kind: GatewayGrantKind) -> u8 { + match kind { + GatewayGrantKind::Service => 1, + GatewayGrantKind::OwnerHandoff => 2, + } +} diff --git a/crates/capsem-gateway/src/stream.rs b/crates/capsem-gateway/src/stream.rs index 40de0c1ed..f337f8efe 100644 --- a/crates/capsem-gateway/src/stream.rs +++ b/crates/capsem-gateway/src/stream.rs @@ -67,7 +67,7 @@ pub async fn handle_stream_tunnel( }; let client_upgrade = hyper::upgrade::on(&mut request); - let socket = match tokio::net::UnixStream::connect(state.uds_path.as_path()).await { + let socket = match state.service_client.connect().await { Ok(socket) => socket, Err(error) => { tracing::warn!(%id, %error, "stream tunnel: service unavailable"); From 16cd2df6ecfb81766970c683f0de31a0efbd949d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:17:52 +0000 Subject: [PATCH 073/464] security(service): broker gateway connections --- crates/capsem-service/src/gateway_grant.rs | 68 +++++++++++++++++++ .../capsem-service/src/gateway_grant/tests.rs | 53 +++++++++++++++ crates/capsem-service/src/main.rs | 1 + crates/capsem-service/src/owner_handoff.rs | 37 +++++++++- crates/capsem-service/src/service_runtime.rs | 17 +++++ 5 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 crates/capsem-service/src/gateway_grant.rs create mode 100644 crates/capsem-service/src/gateway_grant/tests.rs diff --git a/crates/capsem-service/src/gateway_grant.rs b/crates/capsem-service/src/gateway_grant.rs new file mode 100644 index 000000000..4b804e38a --- /dev/null +++ b/crates/capsem-service/src/gateway_grant.rs @@ -0,0 +1,68 @@ +//! Coordinator-minted connections for the confined HTTP gateway. + +use super::*; +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::gateway_grant::{ + decode_gateway_grant_request, encode_gateway_grant_response, GatewayGrantDenial, GatewayGrantKind, + GatewayGrantRequest, GatewayGrantResponse, GATEWAY_GRANT_FRAME_SIZE, GATEWAY_GRANT_MAX_FDS, +}; +use std::os::fd::AsRawFd; + +type WireSender = DescriptorSender; +type WireReceiver = DescriptorReceiver; + +pub(super) async fn serve(socket: std::os::unix::net::UnixStream, state: Arc) -> std::io::Result<()> { + let requests = WireReceiver::new(socket.try_clone()?)?; + let responses = WireSender::new(socket)?; + loop { + let frame = requests.recv().await?; + if !frame.fds.is_empty() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "gateway grant request carried descriptors", + )); + } + let request = decode_gateway_grant_request(&frame.bytes).map_err(std::io::Error::other)?; + let request_id = request.request_id(); + match open(&state, request).await { + Ok((kind, descriptor)) => { + let response = encode_gateway_grant_response(GatewayGrantResponse::Granted { request_id, kind }); + responses.send(&response, &[descriptor.as_raw_fd()]).await?; + } + Err(reason) => { + let response = encode_gateway_grant_response(GatewayGrantResponse::Denied { request_id, reason }); + responses.send(&response, &[]).await?; + } + } + } +} + +async fn open( + state: &Arc, + request: GatewayGrantRequest, +) -> Result<(GatewayGrantKind, std::os::unix::net::UnixStream), GatewayGrantDenial> { + match request { + GatewayGrantRequest::OpenService { .. } => tokio::net::UnixStream::connect(&state.service_socket) + .await + .and_then(|socket| socket.into_std()) + .map(|socket| (GatewayGrantKind::Service, socket)) + .map_err(|_| GatewayGrantDenial::Unavailable), + GatewayGrantRequest::OpenOwnerHandoff { vm_id, .. } => { + let handoff = crate::owner_handoff::OwnerHandoff::acquire(state, &vm_id) + .await + .map_err(|error| match error.0 { + StatusCode::NOT_FOUND => GatewayGrantDenial::NotFound, + StatusCode::CONFLICT => GatewayGrantDenial::Revoked, + _ => GatewayGrantDenial::Unavailable, + })?; + handoff + .connect(state) + .await + .map(|socket| (GatewayGrantKind::OwnerHandoff, socket)) + .map_err(|_| GatewayGrantDenial::Revoked) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/gateway_grant/tests.rs b/crates/capsem-service/src/gateway_grant/tests.rs new file mode 100644 index 000000000..f138dd89e --- /dev/null +++ b/crates/capsem-service/src/gateway_grant/tests.rs @@ -0,0 +1,53 @@ +use super::*; +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::gateway_grant::{decode_gateway_grant_response, encode_gateway_grant_request, GatewayGrantRequest}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +#[tokio::test] +async fn grants_only_the_service_and_registry_derived_owner_seats() { + let state = crate::tests::make_test_state(); + std::fs::create_dir_all(state.service_socket.parent().unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(&state.service_socket).unwrap(); + let (client, server) = std::os::unix::net::UnixStream::pair().unwrap(); + let broker = tokio::spawn(serve(server, Arc::clone(&state))); + let requests = + DescriptorSender::::new(client.try_clone().unwrap()).unwrap(); + let responses = DescriptorReceiver::::new(client).unwrap(); + + let request = encode_gateway_grant_request(&GatewayGrantRequest::OpenService { request_id: 1 }).unwrap(); + requests.send(&request, &[]).await.unwrap(); + let response = responses.recv().await.unwrap(); + assert!(matches!( + decode_gateway_grant_response(&response.bytes).unwrap(), + GatewayGrantResponse::Granted { + request_id: 1, + kind: GatewayGrantKind::Service + } + )); + let (mut service_peer, _) = listener.accept().await.unwrap(); + let mut granted = tokio::net::UnixStream::from_std(std::os::unix::net::UnixStream::from( + response.fds.into_iter().next().unwrap(), + )) + .unwrap(); + service_peer.write_all(b"s").await.unwrap(); + let mut byte = [0]; + granted.read_exact(&mut byte).await.unwrap(); + assert_eq!(byte, [b's']); + + let request = encode_gateway_grant_request(&GatewayGrantRequest::OpenOwnerHandoff { + request_id: 2, + vm_id: "not-running".into(), + }) + .unwrap(); + requests.send(&request, &[]).await.unwrap(); + let response = responses.recv().await.unwrap(); + assert!(response.fds.is_empty()); + assert!(matches!( + decode_gateway_grant_response(&response.bytes).unwrap(), + GatewayGrantResponse::Denied { + request_id: 2, + reason: GatewayGrantDenial::NotFound + } + )); + broker.abort(); +} diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 2fb7505f7..0d1939d00 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -31,6 +31,7 @@ mod asset_background; mod asset_routes; mod blocking; mod container_setup; +mod gateway_grant; mod host_ledger; mod instance; mod instance_reaper; diff --git a/crates/capsem-service/src/owner_handoff.rs b/crates/capsem-service/src/owner_handoff.rs index 5dfa21298..a4df4c5d6 100644 --- a/crates/capsem-service/src/owner_handoff.rs +++ b/crates/capsem-service/src/owner_handoff.rs @@ -1,6 +1,7 @@ //! The coordinator's session and spawn, never a worker-reported path, select //! the endpoint to which a handoff may grant descriptors. use super::*; +use std::os::fd::AsFd; pub(crate) struct OwnerHandoff { id: String, @@ -50,6 +51,40 @@ impl OwnerHandoff { "handoff endpoint does not belong to this VM owner".into(), )); } + self.validate_current(state)?; + Ok(self.endpoint.clone()) + } + + pub(crate) async fn connect(&self, state: &ServiceState) -> Result { + self.validate_current(state)?; + let socket = tokio::net::UnixStream::connect(&self.endpoint) + .await + .and_then(|socket| socket.into_std()) + .map_err(|error| { + AppError( + StatusCode::BAD_GATEWAY, + format!("VM owner handoff unavailable: {error}"), + ) + })?; + capsem_foundation::unix::peer::require( + socket.as_fd(), + capsem_foundation::unix::peer::PeerIdentity { + pid: capsem_foundation::unix::process::ProcessId::try_from(self.owner.pid()) + .map_err(|error| AppError(StatusCode::BAD_GATEWAY, error.to_string()))?, + uid: self.owner.uid(), + }, + ) + .map_err(|error| { + AppError( + StatusCode::BAD_GATEWAY, + format!("authenticate VM owner handoff: {error}"), + ) + })?; + self.validate_current(state)?; + Ok(socket) + } + + fn validate_current(&self, state: &ServiceState) -> Result<(), AppError> { if !state .instances .lock() @@ -62,7 +97,7 @@ impl OwnerHandoff { "VM owner changed during handoff admission".into(), )); } - Ok(self.endpoint.clone()) + Ok(()) } } diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index a2632a4cb..affd7e8ed 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -425,9 +425,11 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { let gateway_binary = args.gateway_binary; let gateway_port = args.gateway_port; let tray_binary = args.tray_binary; + let state_for_spawn = Arc::clone(&state); let spawn_task = tokio::spawn(async move { let spawned = spawn_companions( + state_for_spawn, &service_sock_for_spawn, &run_dir_for_spawn, gateway_binary, @@ -617,6 +619,7 @@ pub(super) fn companion_stdio(log_path: &std::path::Path) -> (std::process::Stdi /// Spawn the gateway and tray as child processes of the service. pub(super) async fn spawn_companions( + state: Arc, service_sock: &std::path::Path, run_dir: &std::path::Path, gateway_bin: Option, @@ -662,7 +665,16 @@ pub(super) async fn spawn_companions( info!(binary = %gateway_bin.display(), "spawning capsem-gateway"); let mut gw_cmd = tokio::process::Command::new(&gateway_bin); + let (grant_server, grant_client) = match std::os::unix::net::UnixStream::pair() { + Ok(pair) => pair, + Err(error) => { + warn!(%error, "failed to create gateway descriptor grant channel"); + return children; + } + }; gw_cmd.arg("--uds-path").arg(service_sock); + gw_cmd.arg("--service-grant-stdin"); + gw_cmd.stdin(std::process::Stdio::from(std::os::fd::OwnedFd::from(grant_client))); // Pin the gateway to the service's run_dir so gateway.{token,port,pid} land // in the same place we poll for them below and the same place clients read. gw_cmd.arg("--run-dir").arg(run_dir); @@ -682,6 +694,11 @@ pub(super) async fn spawn_companions( Ok(child) => { info!(pid = child.id(), "capsem-gateway spawned"); children.push(child); + tokio::spawn(async move { + if let Err(error) = crate::gateway_grant::serve(grant_server, state).await { + tracing::debug!(%error, "gateway descriptor grant channel closed"); + } + }); // Wait for gateway to write token + port files (up to 5s) let token_path = run_dir.join("gateway.token"); From 2b8531d7232108e9df13c6fffb73fda6835a0ca6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:19:42 +0000 Subject: [PATCH 074/464] fix(service): validate gateway readiness stamps --- crates/capsem-service/src/service_runtime.rs | 25 +++++++++++++++++-- .../src/service_runtime/tests.rs | 24 ++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 crates/capsem-service/src/service_runtime/tests.rs diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index affd7e8ed..2bc21880e 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -3,6 +3,8 @@ use super::*; mod orphans; mod shutdown; pub(crate) mod telemetry_export; +#[cfg(test)] +mod tests; pub(super) async fn run_service() -> Result<()> { let args = Args::parse(); @@ -715,7 +717,7 @@ pub(super) async fn spawn_companions( let pp = pp.clone(); let ppp = ppp.clone(); async move { - if tp.exists() && pp.exists() && ppp.exists() { + if gateway_runtime_ready(&tp, &pp, &ppp) { Some(()) } else { None @@ -726,7 +728,7 @@ pub(super) async fn spawn_companions( .instrument(gateway_span.clone()) .await; } - if token_path.exists() && port_path.exists() && preview_port_path.exists() { + if gateway_runtime_ready(&token_path, &port_path, &preview_port_path) { gateway_span.record("status", "ok"); } else { gateway_span.record("status", "error"); @@ -765,6 +767,25 @@ pub(super) async fn spawn_companions( children } +fn gateway_runtime_ready( + token_path: &std::path::Path, + port_path: &std::path::Path, + preview_path: &std::path::Path, +) -> bool { + let Ok(token) = std::fs::read_to_string(token_path) else { + return false; + }; + if token.len() != 64 || !token.bytes().all(|byte| byte.is_ascii_alphanumeric()) { + return false; + } + [port_path, preview_path].into_iter().all(|path| { + std::fs::read_to_string(path) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .is_some_and(|port| port != 0) + }) +} + /// Remove the sentinels capsem-process writes beside its socket: `.launched` /// once the VM is running and `.ready` once the guest handshake completed. pub(crate) fn remove_instance_sentinels(uds_path: &std::path::Path) { diff --git a/crates/capsem-service/src/service_runtime/tests.rs b/crates/capsem-service/src/service_runtime/tests.rs new file mode 100644 index 000000000..f970754b5 --- /dev/null +++ b/crates/capsem-service/src/service_runtime/tests.rs @@ -0,0 +1,24 @@ +use super::*; + +#[test] +fn gateway_readiness_requires_complete_stamped_runtime_files() { + let dir = tempfile::tempdir().unwrap(); + let token = dir.path().join("gateway.token"); + let port = dir.path().join("gateway.port"); + let preview = dir.path().join("preview.port"); + for path in [&token, &port, &preview] { + std::fs::write(path, "").unwrap(); + } + assert!(!gateway_runtime_ready(&token, &port, &preview)); + + std::fs::write(&token, "a".repeat(64)).unwrap(); + std::fs::write(&port, "19222").unwrap(); + std::fs::write(&preview, "19223").unwrap(); + assert!(gateway_runtime_ready(&token, &port, &preview)); + + std::fs::write(&token, "short").unwrap(); + assert!(!gateway_runtime_ready(&token, &port, &preview)); + std::fs::write(&token, "a".repeat(64)).unwrap(); + std::fs::write(&preview, "0").unwrap(); + assert!(!gateway_runtime_ready(&token, &port, &preview)); +} From 38d1c59975a514a4f069e9e7b2558d13859d391a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:22:49 +0000 Subject: [PATCH 075/464] security(gateway): confine Linux API worker --- CHANGELOG.md | 4 ++ crates/capsem-gateway/src/auth.rs | 80 +++++++++++++++++-------- crates/capsem-gateway/src/auth/tests.rs | 16 +++++ crates/capsem-gateway/src/main.rs | 42 ++++++++----- crates/capsem-gateway/src/tests.rs | 2 +- 5 files changed, 104 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94afe7a9c..852fdf145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Confine the Linux API gateway before readiness to its inherited listeners + and coordinator-minted service and VM-owner channels, denying filesystem + access, process execution and direct network or control-socket dialing. + - Confine Linux VM-owner runtime threads before readiness to their session, read-only boot assets, prepared service channels and inherited network grants; deny sibling files, path-based control connections and direct diff --git a/crates/capsem-gateway/src/auth.rs b/crates/capsem-gateway/src/auth.rs index b95b64df9..7a629c6ae 100644 --- a/crates/capsem-gateway/src/auth.rs +++ b/crates/capsem-gateway/src/auth.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant}; +use std::{fs::File, io::Seek, io::Write}; use anyhow::{Context, Result}; use axum::extract::State; @@ -76,48 +77,40 @@ impl AuthFailureTracker { } /// Runtime file state for cleanup on shutdown. -#[derive(Clone)] pub struct AuthState { pub token_path: PathBuf, pub port_path: PathBuf, pub pid_path: PathBuf, pub preview_port_path: PathBuf, + token_file: File, + port_file: File, + pid_file: File, + preview_port_file: File, } impl AuthState { /// Generate runtime files. The token is written last: its presence is the /// readiness marker, after both listeners and every other marker exist. + #[cfg(test)] pub fn new(run_dir: &Path, token: &str, port: u16, preview_port: u16) -> Result { + let mut state = Self::prepare(run_dir)?; + state.publish(token, port, preview_port)?; + Ok(state) + } + + /// Create empty, private marker files while filesystem authority remains. + pub fn prepare(run_dir: &Path) -> Result { std::fs::create_dir_all(run_dir).with_context(|| format!("failed to create run dir: {}", run_dir.display()))?; let token_path = run_dir.join("gateway.token"); let port_path = run_dir.join("gateway.port"); let pid_path = run_dir.join("gateway.pid"); let preview_port_path = run_dir.join("preview.port"); - - std::fs::write(&preview_port_path, preview_port.to_string()) - .with_context(|| format!("failed to write {}", preview_port_path.display()))?; - #[cfg(unix)] - std::fs::set_permissions(&preview_port_path, std::os::unix::fs::PermissionsExt::from_mode(0o600))?; - std::fs::write(&port_path, port.to_string())?; - std::fs::write(&pid_path, std::process::id().to_string())?; - std::fs::write(&token_path, token).with_context(|| format!("failed to write {}", token_path.display()))?; - - // chmod 600 on token file - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&token_path, std::fs::Permissions::from_mode(0o600))?; - } - - info!( - token_path = %token_path.display(), - port_path = %port_path.display(), - pid_path = %pid_path.display(), - "runtime files written" - ); - Ok(Self { + token_file: prepare_file(&token_path)?, + port_file: prepare_file(&port_path)?, + pid_file: prepare_file(&pid_path)?, + preview_port_file: prepare_file(&preview_port_path)?, token_path, port_path, pid_path, @@ -125,7 +118,25 @@ impl AuthState { }) } + /// Stamp listener metadata through prepared descriptors. The token is + /// written last because a complete token is the readiness marker. + pub fn publish(&mut self, token: &str, port: u16, preview_port: u16) -> Result<()> { + write_prepared(&mut self.preview_port_file, preview_port.to_string().as_bytes())?; + write_prepared(&mut self.port_file, port.to_string().as_bytes())?; + write_prepared(&mut self.pid_file, std::process::id().to_string().as_bytes())?; + write_prepared(&mut self.token_file, token.as_bytes())?; + info!( + token_path = %self.token_path.display(), + port_path = %self.port_path.display(), + pid_path = %self.pid_path.display(), + preview_port_path = %self.preview_port_path.display(), + "runtime files written" + ); + Ok(()) + } + /// Remove runtime files on shutdown. + #[cfg(test)] pub fn cleanup(&self) { for path in [ &self.token_path, @@ -143,6 +154,27 @@ impl AuthState { } } +fn prepare_file(path: &Path) -> Result { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(path) + .with_context(|| format!("failed to prepare {}", path.display()))?; + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + Ok(file) +} + +fn write_prepared(file: &mut File, value: &[u8]) -> Result<()> { + file.set_len(0)?; + file.rewind()?; + file.write_all(value)?; + file.flush()?; + Ok(()) +} + /// Generate a 64-character alphanumeric random token. pub fn generate_token() -> String { capsem_crypto::random::alphanumeric(64) diff --git a/crates/capsem-gateway/src/auth/tests.rs b/crates/capsem-gateway/src/auth/tests.rs index 340e64ef0..23138fc20 100644 --- a/crates/capsem-gateway/src/auth/tests.rs +++ b/crates/capsem-gateway/src/auth/tests.rs @@ -96,6 +96,22 @@ fn token_uniqueness_over_100_samples() { // --- AuthState file lifecycle --- +#[test] +fn prepared_runtime_files_do_not_publish_readiness() { + let dir = tempfile::tempdir().unwrap(); + let mut state = AuthState::prepare(dir.path()).unwrap(); + for path in [ + &state.token_path, + &state.port_path, + &state.pid_path, + &state.preview_port_path, + ] { + assert_eq!(std::fs::read(path).unwrap(), b""); + } + state.publish("a-token", 19222, 19223).unwrap(); + assert_eq!(std::fs::read_to_string(&state.token_path).unwrap(), "a-token"); +} + #[test] fn auth_state_lifecycle() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/capsem-gateway/src/main.rs b/crates/capsem-gateway/src/main.rs index 695a25100..deef113d3 100644 --- a/crates/capsem-gateway/src/main.rs +++ b/crates/capsem-gateway/src/main.rs @@ -86,6 +86,10 @@ fn main() -> Result<()> { } async fn run(args: Args) -> Result<()> { + anyhow::ensure!( + args.service_grant_stdin, + "capsem-gateway requires a coordinator descriptor grant channel" + ); let run_dir = gateway_run_dir(&args); let _ = std::fs::create_dir_all(&run_dir); let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { @@ -150,20 +154,16 @@ async fn run(args: Args) -> Result<()> { .context("failed to bind preview listener")?; let preview_port = preview_listener.local_addr()?.port(); - // Generate auth token and write runtime files only after both listeners - // exist. gateway.token is the final readiness marker. + // Prepare private marker descriptors before confinement. Their empty files + // are not readiness; the complete token is stamped only after confinement. let token = auth::generate_token(); - let auth_state = AuthState::new(&run_dir, &token, bound_port, preview_port)?; + let mut auth_state = AuthState::prepare(&run_dir)?; let (events_tx, _) = tokio::sync::broadcast::channel::(64); - let service_client = if args.service_grant_stdin { - // SAFETY: the service gives this child sole ownership of stdin as its - // descriptor grant socket. No standard-input owner has been created. - let grant_socket = unsafe { std::os::unix::net::UnixStream::from_raw_fd(0) }; - ServiceClient::granted(service_grant::GatewayGrantClient::start(grant_socket)?) - } else { - ServiceClient::new(&uds_path) - }; + // SAFETY: the service gives this child sole ownership of stdin as its + // descriptor grant socket. No standard-input owner has been created. + let grant_socket = unsafe { std::os::unix::net::UnixStream::from_raw_fd(0) }; + let service_client = ServiceClient::granted(service_grant::GatewayGrantClient::start(grant_socket)?); let state = Arc::new(AppState { token, uds_path, @@ -174,6 +174,8 @@ async fn run(args: Args) -> Result<()> { previews: preview::PreviewState::new(preview_port), }); + confine_linux_gateway()?; + let preview_state = state.clone(); tokio::spawn(async move { preview::serve(preview_listener, preview_state).await; @@ -201,6 +203,8 @@ async fn run(args: Args) -> Result<()> { .layer(request_trace_layer()) .with_state(state.clone()); + auth_state.publish(&state.token, bound_port, preview_port)?; + info!( port = bound_port, token_path = %auth_state.token_path.display(), @@ -210,7 +214,6 @@ async fn run(args: Args) -> Result<()> { ); // Graceful shutdown on SIGTERM/SIGINT - let shutdown_auth = auth_state.clone(); axum::serve( listener::low_latency(listener), app.into_make_service_with_connect_info::(), @@ -218,14 +221,23 @@ async fn run(args: Args) -> Result<()> { .with_graceful_shutdown(async move { shutdown_signal().await; info!("shutting down"); - shutdown_auth.cleanup(); }) .await .context("server error")?; - // Belt-and-suspenders cleanup (signal handler may not run on all exit paths) - auth_state.cleanup(); + Ok(()) +} + +#[cfg(target_os = "linux")] +fn confine_linux_gateway() -> Result<()> { + use capsem_foundation::unix::worker_sandbox::{Policy, Role}; + capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Gateway)) + .context("confine gateway before readiness")?; + Ok(()) +} +#[cfg(not(target_os = "linux"))] +fn confine_linux_gateway() -> Result<()> { Ok(()) } diff --git a/crates/capsem-gateway/src/tests.rs b/crates/capsem-gateway/src/tests.rs index 58ebb2992..5233f2370 100644 --- a/crates/capsem-gateway/src/tests.rs +++ b/crates/capsem-gateway/src/tests.rs @@ -16,7 +16,7 @@ fn entry_closes_ambient_descriptors_before_runtime_construction() { .find("close_inherited_descriptors()") .expect("gateway closes ambient descriptors at process entry"); let runtime = entry.find("tokio::runtime::Builder").unwrap(); - let run = entry.find("runtime.block_on(run())").unwrap(); + let run = entry.find("runtime.block_on(run(args))").unwrap(); assert!(close < runtime && runtime < run); } From b04776488a8dd7605df857f4e34ab87014d6cd1d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:22:58 +0000 Subject: [PATCH 076/464] fix(service): own gateway marker cleanup --- crates/capsem-service/src/service_runtime.rs | 21 ++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index 2bc21880e..5b479cb82 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -465,6 +465,7 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { } let children = std::mem::take(&mut companions.lock().unwrap().children); shutdown::stop_companions(children).await; + remove_gateway_runtime_files(&run_dir); if let Err(error) = state .record_service_event(capsem_logger::HostEventKind::ServiceStopped) .await @@ -653,14 +654,7 @@ pub(super) async fn spawn_companions( // A previous service may have exited before its gateway removed runtime // markers. Never let those stale files satisfy our readiness poll for the // replacement gateway. - for name in ["gateway.token", "gateway.port", "gateway.pid", "preview.port"] { - let path = run_dir.join(name); - if let Err(error) = std::fs::remove_file(&path) { - if error.kind() != std::io::ErrorKind::NotFound { - warn!(path = %path.display(), %error, "failed to remove stale gateway runtime file"); - } - } - } + remove_gateway_runtime_files(run_dir); let gateway_bin = gateway_bin.unwrap_or_else(|| find_sibling_binary("capsem-gateway")); let (gw_out, gw_err) = companion_stdio(&log_dir.join("gateway.log")); @@ -786,6 +780,17 @@ fn gateway_runtime_ready( }) } +fn remove_gateway_runtime_files(run_dir: &std::path::Path) { + for name in ["gateway.token", "gateway.port", "gateway.pid", "preview.port"] { + let path = run_dir.join(name); + if let Err(error) = std::fs::remove_file(&path) { + if error.kind() != std::io::ErrorKind::NotFound { + warn!(path = %path.display(), %error, "failed to remove gateway runtime file"); + } + } + } +} + /// Remove the sentinels capsem-process writes beside its socket: `.launched` /// once the VM is running and `.ready` once the guest handshake completed. pub(crate) fn remove_instance_sentinels(uds_path: &std::path::Path) { From 669b7219f85c1524278c3a2da89fbd1211fe329a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:25:00 +0000 Subject: [PATCH 077/464] security(gateway): attest denied Linux authority --- crates/capsem-gateway/src/main.rs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/capsem-gateway/src/main.rs b/crates/capsem-gateway/src/main.rs index deef113d3..3f033d77d 100644 --- a/crates/capsem-gateway/src/main.rs +++ b/crates/capsem-gateway/src/main.rs @@ -166,7 +166,7 @@ async fn run(args: Args) -> Result<()> { let service_client = ServiceClient::granted(service_grant::GatewayGrantClient::start(grant_socket)?); let state = Arc::new(AppState { token, - uds_path, + uds_path: uds_path.clone(), service_client, status_cache: StatusCache::new(), auth_failures: AuthFailureTracker::new(), @@ -174,7 +174,7 @@ async fn run(args: Args) -> Result<()> { previews: preview::PreviewState::new(preview_port), }); - confine_linux_gateway()?; + confine_linux_gateway(&uds_path, listener.local_addr()?)?; let preview_state = state.clone(); tokio::spawn(async move { @@ -229,18 +229,37 @@ async fn run(args: Args) -> Result<()> { } #[cfg(target_os = "linux")] -fn confine_linux_gateway() -> Result<()> { +fn confine_linux_gateway(uds_path: &std::path::Path, listener: SocketAddr) -> Result<()> { use capsem_foundation::unix::worker_sandbox::{Policy, Role}; capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Gateway)) .context("confine gateway before readiness")?; + require_denied(std::fs::File::open("/etc/passwd"), "read ambient host files")?; + require_denied(std::net::TcpStream::connect(listener), "dial TCP directly")?; + require_denied( + std::os::unix::net::UnixStream::connect(uds_path), + "dial control sockets by path", + )?; + require_denied( + std::process::Command::new("/bin/true").status(), + "execute processes", + )?; Ok(()) } #[cfg(not(target_os = "linux"))] -fn confine_linux_gateway() -> Result<()> { +fn confine_linux_gateway(_uds_path: &std::path::Path, _listener: SocketAddr) -> Result<()> { Ok(()) } +#[cfg(target_os = "linux")] +fn require_denied(result: std::io::Result, operation: &str) -> Result<()> { + match result { + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => Ok(()), + Err(error) => anyhow::bail!("gateway confinement did not prove it cannot {operation}: {error}"), + Ok(_) => anyhow::bail!("gateway confinement allowed it to {operation}"), + } +} + fn gateway_run_dir(args: &Args) -> PathBuf { args.run_dir .clone() From d2a5c908b59cb13fa52e25eaf4c08f189f47b8b8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:25:13 +0000 Subject: [PATCH 078/464] test(gateway): broker confined service grants --- tests/helpers/gateway.py | 75 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/tests/helpers/gateway.py b/tests/helpers/gateway.py index 9d5fb09e4..fc31086ac 100644 --- a/tests/helpers/gateway.py +++ b/tests/helpers/gateway.py @@ -6,10 +6,13 @@ import json import os +import socket import subprocess import sys import tempfile +import threading import time +from array import array from pathlib import Path from urllib.parse import quote @@ -48,6 +51,8 @@ def __init__(self, uds_path: str | Path, port: int = 0, frontend_dir: str | Path self._log_file = None self._stdio_log_path = self.tmp_dir / "gateway-stdio.log" self._log_path = self.tmp_dir / ".capsem" / "run" / "gateway.log" + self._grant_socket = None + self._grant_thread = None self.token = "" self.port = port @@ -95,16 +100,27 @@ def start(self): str(run_dir), "--parent-pid", str(os.getpid()), + "--service-grant-stdin", ] if self.frontend_dir: cmd += ["--frontend-dir", self.frontend_dir] + grant_server, grant_client = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + self._grant_socket = grant_server + self._grant_thread = threading.Thread( + target=self._serve_grants, + args=(grant_server, run_dir), + daemon=True, + ) + self._grant_thread.start() self.proc = subprocess.Popen( cmd, env=env, + stdin=grant_client, stdout=self._log_file, stderr=self._log_file, ) + grant_client.close() # Wait for gateway to start and write runtime files token_path = run_dir / "gateway.token" @@ -120,7 +136,7 @@ def start(self): _, _, body = probe.request("GET", "/health", timeout=2) if b"ok" in body.lower(): return - except Exception: + except (OSError, TimeoutError): pass finally: probe.close() @@ -146,10 +162,67 @@ def stop(self): self.proc.kill() self.proc.wait() self.proc = None + if self._grant_socket: + self._grant_socket.close() + self._grant_socket = None + if self._grant_thread: + self._grant_thread.join(timeout=2) + self._grant_thread = None if self._log_file: self._log_file.close() self._log_file = None + def _serve_grants(self, channel: socket.socket, run_dir: Path) -> None: + """Test coordinator: mint only connections to this fixture's service.""" + try: + while request := self._recv_exact(channel, 80): + if request[:3] != b"GG\x01": + return + request_id = request[4:12] + kind = request[3] + response = bytearray(80) + response[:3] = b"GG\x01" + response[4:12] = request_id + granted = None + if kind == 1: + try: + granted = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + granted.connect(self.uds_path) + except OSError: + if granted: + granted.close() + granted = None + if granted: + response[3] = 101 + response[12] = kind + fds = array("i", [granted.fileno()]) + sent = channel.sendmsg([response], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) + if sent < len(response): + channel.sendall(response[sent:]) + granted.close() + else: + response[3] = 102 + response[12] = 1 + channel.sendall(response) + except OSError: + pass + finally: + for name in ("gateway.token", "gateway.port", "gateway.pid", "preview.port"): + try: + (run_dir / name).unlink() + except FileNotFoundError: + pass + + @staticmethod + def _recv_exact(channel: socket.socket, size: int) -> bytes: + data = bytearray() + while len(data) < size: + chunk = channel.recv(size - len(data)) + if not chunk: + return b"" + data.extend(chunk) + return bytes(data) + def stop_and_read_log(self) -> str: """Stop the gateway so Rust's stdout/stderr log buffer is flushed. From c6465ffe39629de7eb159eaa8bd82f0c72eebce7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:34:43 +0000 Subject: [PATCH 079/464] fix(logger): keep flush scratch state confined --- crates/capsem-logger/src/schema/pragmas.rs | 5 +++++ crates/capsem-logger/src/schema/tests.rs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/crates/capsem-logger/src/schema/pragmas.rs b/crates/capsem-logger/src/schema/pragmas.rs index f524b9384..107fd269b 100644 --- a/crates/capsem-logger/src/schema/pragmas.rs +++ b/crates/capsem-logger/src/schema/pragmas.rs @@ -85,6 +85,11 @@ pub fn record_sqlite_mmap_telemetry(conn: &Connection, path: &Path, role: &'stat pub fn apply_pragmas(conn: &Connection) -> rusqlite::Result<()> { conn.pragma_update(None, "journal_mode", "WAL")?; conn.pragma_update(None, "synchronous", "FULL")?; + // A confined ledger owner has authority over its session directory, not + // the process-wide temporary directory SQLite otherwise selects for + // large INSERT..SELECT and sort operations. Keep that scratch state in + // the connection so a burst can flush without ambient filesystem access. + conn.pragma_update(None, "temp_store", "MEMORY")?; #[cfg(target_os = "macos")] conn.pragma_update(None, "fullfsync", "ON")?; apply_mmap_pragma(conn)?; diff --git a/crates/capsem-logger/src/schema/tests.rs b/crates/capsem-logger/src/schema/tests.rs index 09e435009..5d4efb5ed 100644 --- a/crates/capsem-logger/src/schema/tests.rs +++ b/crates/capsem-logger/src/schema/tests.rs @@ -226,8 +226,13 @@ fn writer_pragmas_use_full_wal_durability() { let journal: String = conn.query_row("PRAGMA journal_mode", [], |row| row.get(0)).unwrap(); let synchronous: i64 = conn.query_row("PRAGMA synchronous", [], |row| row.get(0)).unwrap(); + let temp_store: i64 = conn.query_row("PRAGMA temp_store", [], |row| row.get(0)).unwrap(); assert_eq!(journal.to_ascii_lowercase(), "wal"); assert_eq!(synchronous, 2, "SQLite FULL is pragma value 2"); + assert_eq!( + temp_store, 2, + "writer sort and copy scratch state must stay inside the confined process" + ); #[cfg(target_os = "macos")] assert_eq!( conn.query_row("PRAGMA fullfsync", [], |row| row.get::<_, i64>(0)) From c8fb8d31088e25b507ee4536183b6c70e86f00ee Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:35:42 +0000 Subject: [PATCH 080/464] feat(proto): broker guest metadata changes --- crates/capsem-proto/src/upstream_grant.rs | 108 +++++++++++++++++- .../capsem-proto/src/upstream_grant/tests.rs | 48 ++++++++ 2 files changed, 153 insertions(+), 3 deletions(-) diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index 83a5611f9..e1d114ed1 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -11,11 +11,13 @@ use std::net::IpAddr; use anyhow::{bail, Result}; /// Exact record size carried by the bounded SCM_RIGHTS channel. -pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 360; +pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 4480; /// A response grants at most one connected TCP or UDP descriptor. pub const UPSTREAM_GRANT_MAX_FDS: usize = 1; /// DNS names are at most 253 wire-text bytes without a root dot. pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; +/// Maximum relative path carried by a guest-share metadata request. +pub const MAX_GUEST_SHARE_PATH_BYTES: usize = 4096; const MAGIC: [u8; 2] = *b"UG"; const VERSION: u8 = 1; @@ -30,16 +32,21 @@ const NAME_LENGTH_OFFSET: usize = 24; const NAME_RANGE: std::ops::Range = 25..25 + MAX_UPSTREAM_HOST_BYTES; const POLICY_DIGEST_BYTES: usize = 71; const POLICY_DIGEST_RANGE: std::ops::Range = NAME_RANGE.end..NAME_RANGE.end + POLICY_DIGEST_BYTES; -const RESERVED_RANGE: std::ops::Range = POLICY_DIGEST_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; +const PATH_LENGTH_RANGE: std::ops::Range = POLICY_DIGEST_RANGE.end..POLICY_DIGEST_RANGE.end + 2; +const PATH_RANGE: std::ops::Range = + PATH_LENGTH_RANGE.end..PATH_LENGTH_RANGE.end + MAX_GUEST_SHARE_PATH_BYTES; +const RESERVED_RANGE: std::ops::Range = PATH_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; const RESOLVE_TCP: u8 = 1; const CONNECT_TCP: u8 = 2; const OPEN_DNS: u8 = 3; const ADOPTED: u8 = 4; const RELEASE: u8 = 5; +const SET_GUEST_MODE: u8 = 6; const TCP_RESOLVED: u8 = 101; const DESCRIPTOR_GRANTED: u8 = 102; const DENIED: u8 = 103; +const GUEST_MODE_SET: u8 = 104; /// Application protocol spoken over a granted TCP stream. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -152,6 +159,11 @@ pub enum UpstreamGrantRequest { Release { resource_id: u64, }, + SetGuestMode { + request_id: u64, + relative_path: Vec, + mode: u16, + }, } /// Coordinator-to-worker record. Only `DescriptorGranted` carries one fd. @@ -174,6 +186,9 @@ pub enum UpstreamGrantResponse { request_id: u64, reason: UpstreamGrantDenial, }, + GuestModeSet { + request_id: u64, + }, } impl UpstreamGrantRequest { @@ -188,7 +203,7 @@ impl UpstreamGrantResponse { pub const fn expected_descriptor_count(&self) -> usize { match self { Self::DescriptorGranted { .. } => 1, - Self::TcpResolved { .. } | Self::Denied { .. } => 0, + Self::TcpResolved { .. } | Self::Denied { .. } | Self::GuestModeSet { .. } => 0, } } } @@ -243,6 +258,21 @@ pub fn encode_upstream_grant_request(request: &UpstreamGrantRequest) -> Result<[ frame[KIND_OFFSET] = RELEASE; put_u64(&mut frame, RESOURCE_ID_RANGE, *resource_id); } + UpstreamGrantRequest::SetGuestMode { + request_id, + relative_path, + mode, + } => { + require_nonzero("request id", *request_id)?; + if *mode & !0o7777 != 0 { + bail!("guest mode contains non-permission bits"); + } + validate_guest_share_path(relative_path)?; + frame[KIND_OFFSET] = SET_GUEST_MODE; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u16(&mut frame, DETAIL_RANGE, *mode); + put_path(&mut frame, relative_path)?; + } } Ok(frame) } @@ -256,6 +286,7 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> let port = get_u16(frame, PORT_RANGE); let detail = get_u16(frame, DETAIL_RANGE); let name = get_name(frame)?; + let relative_path = get_path(frame)?; match frame[KIND_OFFSET] { RESOLVE_TCP => { require_nonzero("request id", request_id)?; @@ -264,6 +295,7 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> } let protocol = UpstreamProtocol::decode(detail)?; validate_normalized_host(&name)?; + require_empty_path(&relative_path)?; Ok(UpstreamGrantRequest::ResolveTcp { request_id, protocol, @@ -275,6 +307,7 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> require_nonzero("request id", request_id)?; require_nonzero("selection id", resource_id)?; require_empty_fields(port, detail, &name)?; + require_empty_path(&relative_path)?; Ok(UpstreamGrantRequest::ConnectTcp { request_id, selection_id: resource_id, @@ -285,6 +318,7 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> if resource_id != 0 || port != 0 || !name.is_empty() { bail!("invalid DNS grant fields"); } + require_empty_path(&relative_path)?; Ok(UpstreamGrantRequest::OpenDns { request_id, upstream_index: detail, @@ -296,6 +330,7 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> bail!("adoption cannot carry a request id"); } require_empty_fields(port, detail, &name)?; + require_empty_path(&relative_path)?; Ok(UpstreamGrantRequest::Adopted { grant_id: resource_id }) } RELEASE => { @@ -304,8 +339,21 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> bail!("release cannot carry a request id"); } require_empty_fields(port, detail, &name)?; + require_empty_path(&relative_path)?; Ok(UpstreamGrantRequest::Release { resource_id }) } + SET_GUEST_MODE => { + require_nonzero("request id", request_id)?; + if resource_id != 0 || port != 0 || !name.is_empty() || detail & !0o7777 != 0 { + bail!("invalid guest mode request fields"); + } + validate_guest_share_path(&relative_path)?; + Ok(UpstreamGrantRequest::SetGuestMode { + request_id, + relative_path, + mode: detail, + }) + } kind => bail!("invalid upstream request kind {kind}"), } } @@ -352,6 +400,11 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); put_u16(&mut frame, DETAIL_RANGE, reason.code()); } + UpstreamGrantResponse::GuestModeSet { request_id } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = GUEST_MODE_SET; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + } } Ok(frame) } @@ -365,6 +418,8 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - let detail = get_u16(frame, DETAIL_RANGE); let name = get_name(frame)?; let policy_digest = get_policy_digest(frame)?; + let relative_path = get_path(frame)?; + require_empty_path(&relative_path)?; require_nonzero("request id", request_id)?; if port != 0 { bail!("upstream response cannot carry a port"); @@ -414,6 +469,12 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - reason: UpstreamGrantDenial::decode(detail)?, }) } + GUEST_MODE_SET => { + if resource_id != 0 || detail != 0 || !name.is_empty() || policy_digest.is_some() { + bail!("guest mode response carries unrelated fields"); + } + Ok(UpstreamGrantResponse::GuestModeSet { request_id }) + } kind => bail!("invalid upstream response kind {kind}"), } } @@ -468,6 +529,47 @@ fn validate_normalized_host(host: &str) -> Result<()> { Ok(()) } +fn validate_guest_share_path(path: &[u8]) -> Result<()> { + if path.len() > MAX_GUEST_SHARE_PATH_BYTES || path.contains(&0) || path.first() == Some(&b'/') { + bail!("invalid guest-share path"); + } + if !path.is_empty() + && path + .split(|byte| *byte == b'/') + .any(|component| component.is_empty() || component == b"." || component == b"..") + { + bail!("guest-share path is not normalized"); + } + Ok(()) +} + +fn put_path(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], path: &[u8]) -> Result<()> { + validate_guest_share_path(path)?; + let length = u16::try_from(path.len()).map_err(|_| anyhow::anyhow!("guest-share path is too long"))?; + put_u16(frame, PATH_LENGTH_RANGE, length); + frame[PATH_RANGE.start..PATH_RANGE.start + path.len()].copy_from_slice(path); + Ok(()) +} + +fn get_path(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result> { + let length = usize::from(get_u16(frame, PATH_LENGTH_RANGE)); + if length > PATH_RANGE.len() { + bail!("guest-share path length exceeds its field"); + } + let (used, padding) = frame[PATH_RANGE].split_at(length); + if padding.iter().any(|byte| *byte != 0) { + bail!("guest-share path padding is nonzero"); + } + Ok(used.to_vec()) +} + +fn require_empty_path(path: &[u8]) -> Result<()> { + if !path.is_empty() { + bail!("record carries an unrelated guest-share path"); + } + Ok(()) +} + fn put_name(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], name: &str) -> Result<()> { let length = u8::try_from(name.len()).map_err(|_| anyhow::anyhow!("upstream name is too long"))?; if usize::from(length) > MAX_UPSTREAM_HOST_BYTES { diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index b4d4a4ed0..29f853732 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -37,6 +37,11 @@ fn every_request_round_trips_exactly() { }, UpstreamGrantRequest::Adopted { grant_id: 51 }, UpstreamGrantRequest::Release { resource_id: 61 }, + UpstreamGrantRequest::SetGuestMode { + request_id: 7, + relative_path: b"workspace/pyvenv/bin/activate".to_vec(), + mode: 0o755, + }, ] { request_roundtrip(request); } @@ -82,11 +87,53 @@ fn every_response_round_trips_exactly() { request_id: 6, reason: UpstreamGrantDenial::Revoked, }, + UpstreamGrantResponse::GuestModeSet { request_id: 7 }, ] { response_roundtrip(response); } } +#[test] +fn guest_mode_paths_are_bounded_normalized_bytes() { + request_roundtrip(UpstreamGrantRequest::SetGuestMode { + request_id: 1, + relative_path: vec![0xff, b'a'], + mode: 0o700, + }); + request_roundtrip(UpstreamGrantRequest::SetGuestMode { + request_id: 2, + relative_path: Vec::new(), + mode: 0o755, + }); + + for relative_path in [ + b"/absolute".to_vec(), + b"../escape".to_vec(), + b"a/../escape".to_vec(), + b"a//b".to_vec(), + b"trailing/".to_vec(), + b"nul\0byte".to_vec(), + vec![b'x'; MAX_GUEST_SHARE_PATH_BYTES + 1], + ] { + assert!( + encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestMode { + request_id: 3, + relative_path, + mode: 0o644, + }) + .is_err() + ); + } + assert!( + encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestMode { + request_id: 4, + relative_path: b"file".to_vec(), + mode: 0o10_000, + }) + .is_err() + ); +} + #[test] fn descriptor_expectations_are_explicit() { let request = UpstreamGrantRequest::OpenDns { @@ -195,6 +242,7 @@ fn envelope_and_trailing_bytes_are_strict() { |frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE]| frame[VERSION_OFFSET] = VERSION + 1, |frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE]| frame[RESERVED_RANGE.start] = 1, |frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE]| frame[NAME_RANGE.start + 1] = 1, + |frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE]| frame[PATH_RANGE.start + 1] = 1, ] { let mut malformed = valid; mutate(&mut malformed); From dafc181827f99dbc6182d2674a89fe52a39f5b1b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:36:39 +0000 Subject: [PATCH 081/464] security(foundation): contain guest mode changes --- .../capsem-foundation/src/unix/contained.rs | 35 +++++++++++++++++- .../src/unix/contained/tests.rs | 36 +++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/crates/capsem-foundation/src/unix/contained.rs b/crates/capsem-foundation/src/unix/contained.rs index 35d10168f..9d04bab9e 100644 --- a/crates/capsem-foundation/src/unix/contained.rs +++ b/crates/capsem-foundation/src/unix/contained.rs @@ -13,7 +13,7 @@ use std::path::{Component, Path, PathBuf}; use nix::errno::Errno; use nix::fcntl::{openat, readlinkat, renameat, AtFlags, OFlag}; -use nix::sys::stat::{fstatat, mkdirat, Mode, SFlag}; +use nix::sys::stat::{fchmod, fchmodat, fstatat, mkdirat, FchmodatFlags, Mode, SFlag}; use nix::unistd::{linkat, symlinkat}; use nix::unistd::{unlinkat, UnlinkatFlags}; @@ -292,6 +292,39 @@ impl ContainedDir { Ok(metadata.permissions().mode() & 0o7777) } + /// Change one entry's permission bits below this open directory. + /// + /// Every parent component is opened without following links and the final + /// operation uses `AT_SYMLINK_NOFOLLOW`, so a guest cannot redirect a + /// metadata request outside the share between validation and mutation. + /// An empty relative path names this directory itself. + pub fn set_relative_mode(&self, rel: &Path, mode: u32) -> io::Result<()> { + let names = rel + .components() + .map(|component| match component { + Component::Normal(name) => Ok(name), + _ => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("relative path may only contain plain names: {}", rel.display()), + )), + }) + .collect::>>()?; + let Some((name, parents)) = names.split_last() else { + return fchmod(self.fd.as_raw_fd(), permission_mode(mode)).map_err(Into::into); + }; + let mut parent = self.try_clone()?; + for component in parents { + parent = parent.descend(component)?; + } + fchmodat( + Some(parent.fd.as_raw_fd()), + *name, + permission_mode(mode), + FchmodatFlags::NoFollowSymlink, + ) + .map_err(Into::into) + } + /// Create a symlink child. An existing entry of any type is an error; the /// target is stored verbatim and never resolved. pub fn symlink(&self, name: &OsStr, target: &OsStr) -> io::Result<()> { diff --git a/crates/capsem-foundation/src/unix/contained/tests.rs b/crates/capsem-foundation/src/unix/contained/tests.rs index b9e2e4bc5..b73d203af 100644 --- a/crates/capsem-foundation/src/unix/contained/tests.rs +++ b/crates/capsem-foundation/src/unix/contained/tests.rs @@ -576,3 +576,39 @@ fn only_filesystem_limits_count_as_a_link_that_cannot_be_made() { assert!(!is_link_unsupported(&io::Error::from(errno)), "{errno}"); } } + +#[test] +fn relative_mode_changes_stay_beneath_the_open_root() { + let tree = tree(); + std::fs::create_dir_all(tree.root_path.join("nested/dir")).unwrap(); + std::fs::write(tree.root_path.join("nested/file"), b"data").unwrap(); + + tree.root.set_relative_mode(Path::new("nested/file"), 0o751).unwrap(); + tree.root.set_relative_mode(Path::new("nested/dir"), 0o710).unwrap(); + + assert_eq!( + std::fs::metadata(tree.root_path.join("nested/file")).unwrap().mode() & 0o7777, + 0o751 + ); + assert_eq!( + std::fs::metadata(tree.root_path.join("nested/dir")).unwrap().mode() & 0o7777, + 0o710 + ); +} + +#[test] +fn relative_mode_changes_refuse_links_and_traversal() { + let tree = tree(); + let secret = tree.outside.join("secret"); + std::fs::set_permissions(&secret, std::fs::Permissions::from_mode(0o600)).unwrap(); + symlink(&secret, tree.root_path.join("link")).unwrap(); + symlink(&tree.outside, tree.root_path.join("dir-link")).unwrap(); + + for path in ["link", "dir-link/secret", "../outside/secret", "/tmp/file", "a/../b"] { + assert!( + tree.root.set_relative_mode(Path::new(path), 0o777).is_err(), + "accepted {path}" + ); + } + assert_eq!(std::fs::metadata(secret).unwrap().mode() & 0o7777, 0o600); +} From f07aa36f19bcacd60ec25302957017c31812458d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:37:43 +0000 Subject: [PATCH 082/464] style(proto): format metadata grant records --- crates/capsem-proto/src/upstream_grant.rs | 3 +- .../capsem-proto/src/upstream_grant/tests.rs | 28 ++++++++----------- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index e1d114ed1..c6a5f29d3 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -33,8 +33,7 @@ const NAME_RANGE: std::ops::Range = 25..25 + MAX_UPSTREAM_HOST_BYTES; const POLICY_DIGEST_BYTES: usize = 71; const POLICY_DIGEST_RANGE: std::ops::Range = NAME_RANGE.end..NAME_RANGE.end + POLICY_DIGEST_BYTES; const PATH_LENGTH_RANGE: std::ops::Range = POLICY_DIGEST_RANGE.end..POLICY_DIGEST_RANGE.end + 2; -const PATH_RANGE: std::ops::Range = - PATH_LENGTH_RANGE.end..PATH_LENGTH_RANGE.end + MAX_GUEST_SHARE_PATH_BYTES; +const PATH_RANGE: std::ops::Range = PATH_LENGTH_RANGE.end..PATH_LENGTH_RANGE.end + MAX_GUEST_SHARE_PATH_BYTES; const RESERVED_RANGE: std::ops::Range = PATH_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; const RESOLVE_TCP: u8 = 1; diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index 29f853732..ebf517949 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -115,23 +115,19 @@ fn guest_mode_paths_are_bounded_normalized_bytes() { b"nul\0byte".to_vec(), vec![b'x'; MAX_GUEST_SHARE_PATH_BYTES + 1], ] { - assert!( - encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestMode { - request_id: 3, - relative_path, - mode: 0o644, - }) - .is_err() - ); - } - assert!( - encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestMode { - request_id: 4, - relative_path: b"file".to_vec(), - mode: 0o10_000, + assert!(encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestMode { + request_id: 3, + relative_path, + mode: 0o644, }) - .is_err() - ); + .is_err()); + } + assert!(encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestMode { + request_id: 4, + relative_path: b"file".to_vec(), + mode: 0o10_000, + }) + .is_err()); } #[test] From 76db798d4fd4a85752c063bd2bb429b795af546b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:41:23 +0000 Subject: [PATCH 083/464] security(service): broker guest mode changes --- crates/capsem-service/src/upstream_broker.rs | 51 ++++++++++++- .../src/upstream_broker/tests.rs | 76 +++++++++++++++++++ .../src/vm_lifecycle/provision.rs | 6 +- .../src/vm_lifecycle/resume_process.rs | 6 +- 4 files changed, 134 insertions(+), 5 deletions(-) diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 8df546a9e..3a071be2c 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -9,7 +9,9 @@ use std::collections::HashMap; use std::io; use std::os::fd::{AsFd as _, AsRawFd as _, OwnedFd}; +use std::os::unix::ffi::OsStrExt as _; use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; @@ -50,6 +52,7 @@ pub(crate) struct PendingBroker { worker: UnixStream, initial_policy: Arc, updates: mpsc::Receiver, + session_dir: Option, } struct PolicyUpdate { @@ -100,6 +103,7 @@ impl PendingBroker { worker, initial_policy, updates, + session_dir: None, }, PolicyPublisher { updates: Some(updates_tx), @@ -107,6 +111,15 @@ impl PendingBroker { )) } + pub(crate) fn pair_for_session( + initial_policy: Arc, + session_dir: PathBuf, + ) -> io::Result<(Self, PolicyPublisher)> { + let (mut pending, publisher) = Self::pair(initial_policy)?; + pending.session_dir = Some(session_dir); + Ok((pending, publisher)) + } + pub(crate) fn worker_stdio(&self) -> io::Result { let descriptor: OwnedFd = self.worker.try_clone()?.into(); Ok(std::process::Stdio::from(descriptor)) @@ -114,7 +127,15 @@ impl PendingBroker { pub(crate) fn start(self, authority: WorkerGrant) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { - if let Err(error) = run(self.coordinator, self.initial_policy, self.updates, authority).await { + if let Err(error) = run( + self.coordinator, + self.initial_policy, + self.updates, + authority, + self.session_dir, + ) + .await + { debug!(%error, "upstream broker stopped"); } }) @@ -126,6 +147,7 @@ async fn run( mut policy: Arc, mut updates: mpsc::Receiver, authority: WorkerGrant, + session_dir: Option, ) -> Result<(), String> { let requests = WireReceiver::new(socket.try_clone().map_err(|error| error.to_string())?).map_err(|error| error.to_string())?; @@ -408,6 +430,26 @@ async fn run( revoke_grant(resource_id, grant)?; } } + UpstreamGrantRequest::SetGuestMode { + request_id, + relative_path, + mode, + } => { + let Some(session_dir) = session_dir.as_deref() else { + send_denied(&responses, request_id, UpstreamGrantDenial::NotConfigured).await?; + continue; + }; + match set_guest_mode(session_dir, &relative_path, mode) { + Ok(()) => { + send_response(&responses, &UpstreamGrantResponse::GuestModeSet { request_id }, None) + .await?; + } + Err(error) => { + warn!(%error, "guest mode broker refused metadata change"); + send_denied(&responses, request_id, UpstreamGrantDenial::NotAllowed).await?; + } + } + } } } } @@ -422,6 +464,13 @@ async fn run( } } +fn set_guest_mode(session_dir: &Path, relative_path: &[u8], mode: u16) -> io::Result<()> { + let root = capsem_foundation::unix::contained::ContainedDir::open_root(session_dir)? + .descend(std::ffi::OsStr::new(capsem_core::GUEST_SHARE_DIR))?; + let path = Path::new(std::ffi::OsStr::from_bytes(relative_path)); + root.set_relative_mode(path, u32::from(mode)) +} + async fn send_active_grant( sender: &WireSender, active: &HashMap, diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index 5a3945aeb..43feccd26 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -2,6 +2,8 @@ use std::collections::BTreeMap; use std::fs::File; use std::net::SocketAddr; use std::os::fd::{AsRawFd as _, OwnedFd}; +use std::os::unix::fs::{symlink, PermissionsExt as _}; +use std::path::PathBuf; use capsem_core::net::policy::{UpstreamOverride, UpstreamOverrideProtocol}; use capsem_core::net::policy_config::{ActivePolicyFile, SettingsFile}; @@ -76,10 +78,84 @@ fn start_broker( pending.initial_policy, pending.updates, authority.grant(), + pending.session_dir, )); (client, publisher, authority, task) } +fn start_broker_for_session( + policy: Arc, + session_dir: PathBuf, +) -> ( + TestClient, + PolicyPublisher, + WorkerAuthority, + tokio::task::JoinHandle>, +) { + let (pending, publisher) = PendingBroker::pair_for_session(policy, session_dir).unwrap(); + let client = TestClient::new(pending.worker); + let authority = WorkerAuthority::default(); + let task = tokio::spawn(run( + pending.coordinator, + pending.initial_policy, + pending.updates, + authority.grant(), + pending.session_dir, + )); + (client, publisher, authority, task) +} + +#[tokio::test] +async fn guest_mode_requests_are_confined_to_the_session_share() { + let session = tempfile::tempdir().unwrap(); + let guest = session.path().join(capsem_core::GUEST_SHARE_DIR); + std::fs::create_dir_all(guest.join("workspace/nested")).unwrap(); + std::fs::write(guest.join("workspace/nested/tool"), b"tool").unwrap(); + let outside = tempfile::tempdir().unwrap(); + let secret = outside.path().join("secret"); + std::fs::write(&secret, b"secret").unwrap(); + std::fs::set_permissions(&secret, std::fs::Permissions::from_mode(0o600)).unwrap(); + symlink(&secret, guest.join("workspace/link")).unwrap(); + + let (client, _publisher, authority, task) = + start_broker_for_session(test_policy("policy-a", None, Vec::new()), session.path().to_path_buf()); + let (response, fds) = client + .request(&UpstreamGrantRequest::SetGuestMode { + request_id: 1, + relative_path: b"workspace/nested/tool".to_vec(), + mode: 0o751, + }) + .await; + assert_eq!(response, UpstreamGrantResponse::GuestModeSet { request_id: 1 }); + assert!(fds.is_empty()); + assert_eq!( + std::fs::metadata(guest.join("workspace/nested/tool")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o751 + ); + + let (response, fds) = client + .request(&UpstreamGrantRequest::SetGuestMode { + request_id: 2, + relative_path: b"workspace/link".to_vec(), + mode: 0o777, + }) + .await; + assert_eq!( + response, + UpstreamGrantResponse::Denied { + request_id: 2, + reason: UpstreamGrantDenial::NotAllowed, + } + ); + assert!(fds.is_empty()); + assert_eq!(std::fs::metadata(secret).unwrap().permissions().mode() & 0o7777, 0o600); + stop_broker(authority, task).await; +} + async fn resolve_override(client: &TestClient, request_id: u64) -> u64 { let (response, fds) = client .request(&UpstreamGrantRequest::ResolveTcp { diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index b8205d2b4..f2e323a7d 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -156,8 +156,10 @@ impl ServiceState { let active_policy = self.materialize_active_policy(&session_dir)?; let active_policy_path = active_policy.path.clone(); - let (upstream_broker, upstream_policy) = - crate::upstream_broker::PendingBroker::pair(active_policy.broker_policy())?; + let (upstream_broker, upstream_policy) = crate::upstream_broker::PendingBroker::pair_for_session( + active_policy.broker_policy(), + session_dir.clone(), + )?; let upstream_stdio = upstream_broker.worker_stdio()?; info!(process_binary = %self.process_binary.display(), exists = self.process_binary.exists(), "checking process_binary"); diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 8be58b454..b6d603252 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -67,8 +67,10 @@ impl ServiceState { self.validate_persistent_entry(&entry)?; let active_policy = self.materialize_active_policy(&entry.session_dir)?; let active_policy_path = active_policy.path.clone(); - let (upstream_broker, upstream_policy) = - crate::upstream_broker::PendingBroker::pair(active_policy.broker_policy())?; + let (upstream_broker, upstream_policy) = crate::upstream_broker::PendingBroker::pair_for_session( + active_policy.broker_policy(), + entry.session_dir.clone(), + )?; let upstream_stdio = upstream_broker.worker_stdio()?; let scratch_disk_size_gb = session_rootfs_size_gb(&entry)?; let resolved = self.resolve_pinned_asset_paths(&entry.asset_pins)?; From 7a31a88473dad9ae93e6eeb8e0601f4e2a0c129b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:46:58 +0000 Subject: [PATCH 084/464] security(core): broker VirtioFS mode changes --- .../examples/release_runtime_boot.rs | 1 + crates/capsem-core/src/hypervisor/kvm/mod.rs | 13 ++++ .../capsem-core/src/hypervisor/kvm/tests.rs | 32 ++++++++ .../src/hypervisor/kvm/virtio_fs/mod.rs | 48 ++++++++---- .../src/hypervisor/kvm/virtio_fs/ops_dir.rs | 15 ++-- .../src/hypervisor/kvm/virtio_fs/ops_file.rs | 14 ++-- .../src/hypervisor/kvm/virtio_fs/ops_meta.rs | 4 +- .../src/hypervisor/kvm/virtio_fs/tests.rs | 74 ++++++++++++++++++- .../kvm/virtio_fs/tests/read_only.rs | 2 +- crates/capsem-core/src/lib.rs | 2 +- crates/capsem-core/src/tests.rs | 1 + crates/capsem-core/src/vm/boot.rs | 2 +- crates/capsem-core/src/vm/boot/tests.rs | 1 + crates/capsem-core/src/vm/config.rs | 29 +++++++- 14 files changed, 202 insertions(+), 36 deletions(-) diff --git a/crates/capsem-core/examples/release_runtime_boot.rs b/crates/capsem-core/examples/release_runtime_boot.rs index 4754ece20..6d49ad4cb 100644 --- a/crates/capsem-core/examples/release_runtime_boot.rs +++ b/crates/capsem-core/examples/release_runtime_boot.rs @@ -106,6 +106,7 @@ fn run() -> Result<()> { tag: "capsem".to_string(), host_path: guest_dir, read_only: false, + metadata_authority: None, }]; let result = (|| -> Result<()> { diff --git a/crates/capsem-core/src/hypervisor/kvm/mod.rs b/crates/capsem-core/src/hypervisor/kvm/mod.rs index 2711a3d19..c2c6a14c6 100644 --- a/crates/capsem-core/src/hypervisor/kvm/mod.rs +++ b/crates/capsem-core/src/hypervisor/kvm/mod.rs @@ -40,6 +40,17 @@ use crate::vm::VmState; const KVM_PAUSE_TIMEOUT: Duration = Duration::from_secs(5); +fn require_metadata_authority(config: &VmConfig) -> Result<()> { + anyhow::ensure!( + config + .virtio_fs_shares + .iter() + .all(|share| share.read_only || share.metadata_authority.is_some()), + "writable KVM VirtioFS shares require a trusted metadata authority" + ); + Ok(()) +} + fn kvm_vsock_seed(config: &VmConfig) -> u32 { let mut hasher = capsem_crypto::blake3::Hasher::new(); hasher.update(config.kernel_path.to_string_lossy().as_bytes()); @@ -123,6 +134,7 @@ impl Hypervisor for KvmHypervisor { config: &VmConfig, vsock_ports: &[u32], ) -> Result<(Box, mpsc::UnboundedReceiver)> { + require_metadata_authority(config)?; #[cfg(not(target_arch = "x86_64"))] if config.checkpoint_path.is_some() { anyhow::bail!("KVM checkpoint restore is only implemented for x86_64; refusing to ignore checkpoint_path"); @@ -517,6 +529,7 @@ impl Hypervisor for KvmHypervisor { &share.tag, &share.host_path, share.read_only, + share.metadata_authority.clone(), fs_irq_fd.as_raw_fd(), Arc::clone(&fs_interrupt_status), )?; diff --git a/crates/capsem-core/src/hypervisor/kvm/tests.rs b/crates/capsem-core/src/hypervisor/kvm/tests.rs index 73fe8fd48..42f5c47fb 100644 --- a/crates/capsem-core/src/hypervisor/kvm/tests.rs +++ b/crates/capsem-core/src/hypervisor/kvm/tests.rs @@ -533,6 +533,7 @@ fn every_mmio_slot_up_to_the_last_share_is_announced() { tag: tag.into(), host_path: "/share".into(), read_only: tag == "capsem-image", + metadata_authority: None, }; let mut config = crate::VmConfig { cpu_count: 1, @@ -563,3 +564,34 @@ fn every_mmio_slot_up_to_the_last_share_is_announced() { config.disk_path = None; assert_eq!(virtio_mmio_device_count(&config, &[]), 1); } + +#[test] +fn writable_virtiofs_requires_brokered_metadata_authority() { + let dir = tempfile::tempdir().unwrap(); + let mut config = crate::VmConfig { + cpu_count: 1, + ram_bytes: 1 << 30, + kernel_path: "/vmlinuz".into(), + initrd_path: None, + disk_path: None, + scratch_disk_path: None, + virtio_fs_shares: vec![crate::VirtioFsShare { + tag: "capsem".into(), + host_path: dir.path().into(), + read_only: false, + metadata_authority: None, + }], + kernel_cmdline: String::new(), + expected_kernel_hash: None, + expected_initrd_hash: None, + checkpoint_path: None, + expected_disk_hash: None, + machine_identifier_path: None, + serial_log_path: None, + }; + + let error = require_metadata_authority(&config).unwrap_err(); + assert!(error.to_string().contains("trusted metadata authority")); + config.virtio_fs_shares[0].read_only = true; + require_metadata_authority(&config).unwrap(); +} diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs index 92e602c44..183cde081 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs @@ -14,6 +14,7 @@ mod ops_dir; mod ops_file; mod ops_meta; +use std::os::unix::ffi::OsStrExt as _; use std::os::unix::io::RawFd; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; @@ -29,6 +30,7 @@ use super::virtio_mmio::{QueueConfig, VirtioDevice}; use super::virtio_queue::{DescriptorChain, VirtQueue, VIRTIO_RING_F_EVENT_IDX}; use crate::hypervisor::fuse::{self, *}; +use crate::GuestMetadataAuthority; use checkpoint_state::VirtioFsBackendSnapshot; const VIRTIO_ID_FS: u32 = 26; @@ -58,21 +60,21 @@ pub(super) struct FuseProcessor { pub(super) read_only: bool, pub(super) inodes: InodeTable, pub(super) file_handles: FileHandleTable, + pub(super) metadata_authority: Option>, } impl FuseProcessor { - fn handle_request_into(&mut self, request_buf: &[u8], response: &mut Vec) { - if let Some(header) = fuse::read_struct::(request_buf) { - if header.opcode == FUSE_READ { - self.do_read_into(&header, &request_buf[std::mem::size_of::()..], response); - return; - } - } - response.clear(); - response.extend_from_slice(&self.handle_non_read_request(request_buf)); + fn set_guest_mode(&self, path: &Path, mode: u32) -> std::io::Result<()> { + let authority = self + .metadata_authority + .as_ref() + .ok_or_else(|| std::io::Error::from_raw_os_error(libc::EPERM))?; + let relative = path + .strip_prefix(&self.root_path) + .map_err(|_| std::io::Error::from_raw_os_error(libc::EACCES))?; + authority.set_mode(relative.as_os_str().as_bytes(), (mode & 0o7777) as u16) } - #[cfg(test)] fn handle_request(&mut self, request_buf: &[u8]) -> Vec { let mut response = Vec::new(); self.handle_request_into(request_buf, &mut response); @@ -145,7 +147,12 @@ impl FuseProcessor { self.encode_checkpoint_with_tag([0; TAG_LEN]) } - fn restore_snapshot(root_path: &Path, read_only: bool, snapshot: &VirtioFsBackendSnapshot) -> Result { + fn restore_snapshot( + root_path: &Path, + read_only: bool, + metadata_authority: Option>, + snapshot: &VirtioFsBackendSnapshot, + ) -> Result { ensure!( snapshot.read_only == read_only, "VirtioFS checkpoint read-only identity mismatch" @@ -157,13 +164,14 @@ impl FuseProcessor { read_only, inodes, file_handles, + metadata_authority, }) } #[cfg(test)] fn restore_checkpoint(root_path: &Path, read_only: bool, encoded: &[u8]) -> Result { let snapshot = VirtioFsBackendSnapshot::decode(encoded)?; - Self::restore_snapshot(root_path, read_only, &snapshot) + Self::restore_snapshot(root_path, read_only, None, &snapshot) } } @@ -420,9 +428,13 @@ impl VirtioFsDevice { tag: &str, root_path: &Path, read_only: bool, + metadata_authority: Option>, irq_fd: RawFd, interrupt_status: Arc, ) -> Result { + let root_path = root_path + .canonicalize() + .with_context(|| format!("canonicalize VirtioFS root: {}", root_path.display()))?; let mut tag_buf = [0u8; TAG_LEN]; let len = tag.len().min(TAG_LEN); tag_buf[..len].copy_from_slice(&tag.as_bytes()[..len]); @@ -430,10 +442,11 @@ impl VirtioFsDevice { Ok(Self { tag: tag_buf, processor: Some(FuseProcessor { - root_path: root_path.to_path_buf(), + root_path: root_path.clone(), read_only, - inodes: InodeTable::new(root_path)?, + inodes: InodeTable::new(&root_path)?, file_handles: FileHandleTable::new(), + metadata_authority, }), notify_tx: None, worker_handle: None, @@ -667,7 +680,12 @@ impl VirtioDevice for VirtioFsDevice { snapshot.read_only == current.read_only, "VirtioFS checkpoint read-only identity mismatch" ); - let restored = FuseProcessor::restore_snapshot(¤t.root_path, current.read_only, &snapshot)?; + let restored = FuseProcessor::restore_snapshot( + ¤t.root_path, + current.read_only, + current.metadata_authority.clone(), + &snapshot, + )?; self.processor = Some(restored); self.checkpoint_state = Some(encoded.to_vec()); Ok(()) diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs index afd51489c..430145583 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs @@ -1,7 +1,7 @@ //! Directory and namespace FUSE operations: OPENDIR, READDIR, RELEASEDIR, //! MKDIR, RMDIR, UNLINK, RENAME, MKNOD, SYMLINK, READLINK, LINK. -use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::os::unix::fs::{DirBuilderExt as _, MetadataExt}; use super::FuseProcessor; use crate::hypervisor::fuse::{self, *}; @@ -125,13 +125,16 @@ impl FuseProcessor { None => return fuse::error_response(header.unique, -libc::EINVAL), }; - if let Err(e) = std::fs::create_dir(&child_path) { + if let Err(e) = std::fs::DirBuilder::new() + .mode(mkdir_in.mode & !mkdir_in.umask) + .create(&child_path) + { return fuse::error_response(header.unique, -fuse::io_error_to_errno(&e)); } - let _ = std::fs::set_permissions( - &child_path, - std::fs::Permissions::from_mode(mkdir_in.mode & !mkdir_in.umask), - ); + if let Err(error) = self.set_guest_mode(&child_path, mkdir_in.mode & !mkdir_in.umask) { + let _ = std::fs::remove_dir(&child_path); + return fuse::error_response(header.unique, -fuse::io_error_to_errno(&error)); + } let ino = match self.inodes.lookup(header.nodeid, name) { Some(i) => i, diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs index 0b3da08f0..f119b8b83 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs @@ -2,9 +2,7 @@ use capsem_foundation::unix::fd; use std::io::{Seek, SeekFrom, Write}; -use std::os::fd::AsFd; -use std::os::unix::fs::PermissionsExt; - +use std::os::unix::fs::FileExt; use std::os::unix::fs::OpenOptionsExt; use super::FuseProcessor; @@ -187,20 +185,24 @@ impl FuseProcessor { let flags = create_in.flags as i32; let accmode = flags & libc::O_ACCMODE; let readable = accmode == libc::O_RDONLY || accmode == libc::O_RDWR; + let mode = create_in.mode & !create_in.umask; let file = match std::fs::OpenOptions::new() .read(readable) .write(true) .append(flags & libc::O_APPEND != 0) .create_new(true) + .mode(mode) .custom_flags(NO_FOLLOW_FLAGS) .open(&child_path) { Ok(f) => f, Err(e) => return fuse::error_response(header.unique, -fuse::io_error_to_errno(&e)), }; - let mode = create_in.mode & !create_in.umask; - let _ = std::fs::set_permissions(&child_path, std::fs::Permissions::from_mode(mode)); - + if let Err(error) = self.set_guest_mode(&child_path, mode) { + drop(file); + let _ = std::fs::remove_file(&child_path); + return fuse::error_response(header.unique, -fuse::io_error_to_errno(&error)); + } let ino = match self.inodes.lookup(header.nodeid, name) { Some(i) => i, None => return fuse::error_response(header.unique, -libc::EIO), diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs index 69f1ae0cb..1748974db 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs @@ -1,6 +1,6 @@ //! Metadata FUSE operations: INIT, LOOKUP, GETATTR, SETATTR, STATFS, FORGET. -use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::os::unix::fs::OpenOptionsExt; use super::FuseProcessor; use crate::hypervisor::fuse::{self, *}; @@ -127,7 +127,7 @@ impl FuseProcessor { if is_symlink { return fuse::error_response(header.unique, -libc::EPERM); } - if let Err(e) = std::fs::set_permissions(&path, std::fs::Permissions::from_mode(attr_in.mode)) { + if let Err(e) = self.set_guest_mode(&path, attr_in.mode) { return fuse::error_response(header.unique, -fuse::io_error_to_errno(&e)); } } diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs index d35085cbf..9e503cfe2 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs @@ -5,6 +5,7 @@ use crate::hypervisor::kvm::virtio_queue::VirtqDesc; use std::collections::BTreeMap; use std::io::{Seek, SeekFrom}; use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; +use std::os::unix::ffi::OsStrExt as _; use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::sync::atomic::AtomicU32; use std::sync::{Arc, Mutex, OnceLock}; @@ -122,6 +123,19 @@ fn host_file_type(metadata: &std::fs::Metadata) -> u32 { metadata.mode() & MASK } +struct LocalMetadataAuthority(PathBuf); + +impl crate::GuestMetadataAuthority for LocalMetadataAuthority { + fn set_mode(&self, relative_path: &[u8], mode: u16) -> std::io::Result<()> { + let root = capsem_foundation::unix::contained::ContainedDir::open_root(&self.0)?; + root.set_relative_mode(Path::new(std::ffi::OsStr::from_bytes(relative_path)), u32::from(mode)) + } +} + +fn metadata_authority(dir: &Path) -> Arc { + Arc::new(LocalMetadataAuthority(dir.to_path_buf())) +} + /// Helper: create a FuseProcessor for testing (no queues needed). pub(super) fn test_processor(dir: &Path) -> FuseProcessor { FuseProcessor { @@ -129,11 +143,20 @@ pub(super) fn test_processor(dir: &Path) -> FuseProcessor { read_only: false, inodes: InodeTable::new(dir).unwrap(), file_handles: FileHandleTable::new(), + metadata_authority: Some(metadata_authority(dir)), } } pub(super) fn test_device(dir: &Path) -> VirtioFsDevice { - VirtioFsDevice::new("capsem", dir, false, -1, Arc::new(AtomicU32::new(0))).unwrap() + VirtioFsDevice::new( + "capsem", + dir, + false, + Some(metadata_authority(dir)), + -1, + Arc::new(AtomicU32::new(0)), + ) + .unwrap() } const TEST_QUEUE_SIZE: u16 = 8; @@ -842,11 +865,19 @@ fn virtiofs_device_restore_rejects_tag_and_read_only_identity_mismatch() { source.quiesce().unwrap(); let state = source.checkpoint_state().unwrap(); - let mut wrong_tag = VirtioFsDevice::new("other", &dir, false, -1, Arc::new(AtomicU32::new(0))).unwrap(); + let mut wrong_tag = VirtioFsDevice::new( + "other", + &dir, + false, + Some(metadata_authority(&dir)), + -1, + Arc::new(AtomicU32::new(0)), + ) + .unwrap(); let tag_err = wrong_tag.restore_checkpoint_state(&state).unwrap_err(); assert!(tag_err.to_string().contains("tag identity mismatch"), "{tag_err:#}"); - let mut wrong_mode = VirtioFsDevice::new("capsem", &dir, true, -1, Arc::new(AtomicU32::new(0))).unwrap(); + let mut wrong_mode = VirtioFsDevice::new("capsem", &dir, true, None, -1, Arc::new(AtomicU32::new(0))).unwrap(); let mode_err = wrong_mode.restore_checkpoint_state(&state).unwrap_err(); assert!( mode_err.to_string().contains("read-only identity mismatch"), @@ -994,6 +1025,43 @@ fn setattr_chmod() { assert_eq!(perms.mode() & 0o777, 0o755); } +#[test] +fn setattr_mode_never_falls_back_to_owner_authority() { + let dir = temp_share("setattr-mode-no-broker"); + std::fs::write(dir.join("f.txt"), b"x").unwrap(); + std::fs::set_permissions(dir.join("f.txt"), std::fs::Permissions::from_mode(0o640)).unwrap(); + let mut proc = test_processor(&dir); + proc.metadata_authority = None; + let ino = lookup(&mut proc, 1, "f.txt").unwrap(); + let attr_in = FuseSetAttrIn { + valid: FATTR_MODE, + padding: 0, + fh: 0, + size: 0, + lock_owner: 0, + atime: 0, + mtime: 0, + ctime: 0, + atimensec: 0, + mtimensec: 0, + ctimensec: 0, + mode: 0o777, + unused4: 0, + uid: 0, + gid: 0, + unused5: 0, + }; + + let h = make_header(FUSE_SETATTR, ino, 4); + let resp = proc.handle_request(&build_request(&h, fuse::as_bytes(&attr_in))); + + assert_eq!(response_error(&resp), -libc::EPERM); + assert_eq!( + std::fs::metadata(dir.join("f.txt")).unwrap().permissions().mode() & 0o777, + 0o640 + ); +} + #[test] fn setattr_truncate() { let dir = temp_share("setattr-trunc"); diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/read_only.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/read_only.rs index c1ab03990..ccae03781 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/read_only.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/read_only.rs @@ -26,7 +26,7 @@ fn a_read_only_share_refuses_every_mutating_request() { let dir = temp_share("image-share-read-only"); std::fs::write(dir.join("blob"), b"verified").unwrap(); std::fs::create_dir(dir.join("directory")).unwrap(); - let mut device = VirtioFsDevice::new("capsem-image", &dir, true, -1, Arc::new(AtomicU32::new(0))).unwrap(); + let mut device = VirtioFsDevice::new("capsem-image", &dir, true, None, -1, Arc::new(AtomicU32::new(0))).unwrap(); let mut proc = device .processor .take() diff --git a/crates/capsem-core/src/lib.rs b/crates/capsem-core/src/lib.rs index d396b13b8..7ad00eadf 100644 --- a/crates/capsem-core/src/lib.rs +++ b/crates/capsem-core/src/lib.rs @@ -24,7 +24,7 @@ pub use vm::boot::{ boot_vm, create_net_state, create_net_state_with_policy, read_control_msg, send_boot_config, write_control_msg, BootOptions, ControlFrameTooLarge, }; -pub use vm::config::{VirtioFsShare, VmConfig}; +pub use vm::config::{GuestMetadataAuthority, VirtioFsShare, VmConfig}; pub use vm::registry::{SandboxInstance, SandboxNetworkState}; pub use vm::terminal::TerminalOutputQueue; pub use vm::vsock::{self, CoalesceBuffer}; diff --git a/crates/capsem-core/src/tests.rs b/crates/capsem-core/src/tests.rs index 9a7adb613..c15ea25b9 100644 --- a/crates/capsem-core/src/tests.rs +++ b/crates/capsem-core/src/tests.rs @@ -224,6 +224,7 @@ fn reexport_surface_compiles() { tag: "test".into(), host_path: PathBuf::from("/tmp"), read_only: false, + metadata_authority: None, }; // VmState diff --git a/crates/capsem-core/src/vm/boot.rs b/crates/capsem-core/src/vm/boot.rs index 6ccd3770c..aa5f79d8d 100644 --- a/crates/capsem-core/src/vm/boot.rs +++ b/crates/capsem-core/src/vm/boot.rs @@ -209,7 +209,7 @@ pub fn boot_vm( share.tag, share.host_path.display() ); - builder = builder.virtio_fs_share(&share.tag, &share.host_path, share.read_only); + builder = builder.virtio_fs_share_config(share.clone()); } info!("[boot-audit] calling VmConfig::build()"); diff --git a/crates/capsem-core/src/vm/boot/tests.rs b/crates/capsem-core/src/vm/boot/tests.rs index 6c05ad7a3..36e7dd38e 100644 --- a/crates/capsem-core/src/vm/boot/tests.rs +++ b/crates/capsem-core/src/vm/boot/tests.rs @@ -100,6 +100,7 @@ fn virtiofs_cmdline_append() { tag: "capsem".into(), host_path: "/tmp/session".into(), read_only: false, + metadata_authority: None, }]; let effective = effective_kernel_cmdline(base, &shares, None); assert!(effective.contains("capsem.storage=virtiofs")); diff --git a/crates/capsem-core/src/vm/config.rs b/crates/capsem-core/src/vm/config.rs index a14a04992..f35e47486 100644 --- a/crates/capsem-core/src/vm/config.rs +++ b/crates/capsem-core/src/vm/config.rs @@ -1,5 +1,6 @@ use std::io::Read; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::UNIX_EPOCH; use thiserror::Error; @@ -69,8 +70,14 @@ pub enum ConfigError { ArchMismatch(String), } +/// Trusted authority for metadata operations a confined KVM owner cannot +/// safely perform itself. +pub trait GuestMetadataAuthority: Send + Sync { + fn set_mode(&self, relative_path: &[u8], mode: u16) -> std::io::Result<()>; +} + /// A VirtioFS shared directory to expose to the guest via virtio-fs. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct VirtioFsShare { /// Mount tag visible in guest (e.g., "capsem"). Max 36 ASCII bytes. pub tag: String, @@ -78,6 +85,20 @@ pub struct VirtioFsShare { pub host_path: PathBuf, /// If true, guest cannot write to this share. pub read_only: bool, + /// Coordinator broker for permission changes on a writable KVM share. + pub metadata_authority: Option>, +} + +impl std::fmt::Debug for VirtioFsShare { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("VirtioFsShare") + .field("tag", &self.tag) + .field("host_path", &self.host_path) + .field("read_only", &self.read_only) + .field("metadata_authority", &self.metadata_authority.is_some()) + .finish() + } } #[derive(Debug, Clone)] @@ -196,10 +217,16 @@ impl VmConfigBuilder { tag: tag.into(), host_path: path.as_ref().to_path_buf(), read_only, + metadata_authority: None, }); self } + pub fn virtio_fs_share_config(mut self, share: VirtioFsShare) -> Self { + self.virtio_fs_shares.push(share); + self + } + pub fn kernel_cmdline(mut self, cmdline: impl Into) -> Self { self.kernel_cmdline = cmdline.into(); self From c8796fa7582a0de0b3bb5e32fc669b2e3820b744 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:49:48 +0000 Subject: [PATCH 085/464] security(process): delegate guest metadata authority --- crates/capsem-process/src/main.rs | 29 ++++++--- crates/capsem-process/src/upstream_grant.rs | 64 +++++++++++++++++++ .../src/upstream_grant/tests.rs | 28 ++++++++ 3 files changed, 111 insertions(+), 10 deletions(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index dcb0c6ea7..52a168121 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -301,6 +301,15 @@ fn main() -> Result<()> { .context("this session already has a running VM owner")?; let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + let runtime_source = runtime_config::RuntimePolicySource::new(args.active_policy.clone()); + let runtime_config = runtime_source.load()?; + let upstream_grants = { + let _runtime = rt.enter(); + Arc::new(upstream_grant::UpstreamGrantClient::start( + upstream_socket, + runtime_config.active_policy_digest.clone(), + )?) + }; info!(id = %args.id, "capsem-sandbox-process starting"); let guest_dir = prepare_session_layout(&session_dir, args.scratch_disk_size_gb)?; @@ -314,11 +323,13 @@ fn main() -> Result<()> { tag: "capsem".into(), host_path: guest_dir, read_only: false, + metadata_authority: Some(upstream_grants.clone()), }, VirtioFsShare { tag: capsem_core::session::IMAGE_SHARE_TAG.into(), host_path: capsem_core::session::prepare_image_share(&session_dir)?, read_only: true, + metadata_authority: None, }, ]; @@ -387,7 +398,7 @@ fn main() -> Result<()> { let vm_for_signal = Arc::clone(&vm_arc); let vm_for_exit = Arc::clone(&vm_arc); rt.spawn(async move { - if let Err(e) = run_async_main_loop( + if let Err(e) = Box::pin(run_async_main_loop( args, controller, vm_arc, @@ -395,8 +406,10 @@ fn main() -> Result<()> { trace_id_for_loop, session_dir_for_loop, shutdown_for_loop, - upstream_socket, - ) + runtime_source, + runtime_config, + upstream_grants, + )) .await { error!(error = format!("{e:#}"), "async loop failed"); @@ -467,14 +480,10 @@ async fn run_async_main_loop( trace_id: String, session_dir: std::path::PathBuf, shutdown: Arc>, - upstream_socket: std::os::unix::net::UnixStream, + runtime_source: runtime_config::RuntimePolicySource, + runtime_config: runtime_config::RuntimePolicyConfig, + upstream_grants: Arc, ) -> Result<()> { - let runtime_source = runtime_config::RuntimePolicySource::new(args.active_policy.clone()); - let runtime_config = runtime_source.load()?; - let upstream_grants = Arc::new(upstream_grant::UpstreamGrantClient::start( - upstream_socket, - runtime_config.active_policy_digest.clone(), - )?); let terminal_output = Arc::new(capsem_core::TerminalOutputQueue::new()); // 1024 queued events: a guest resolving and fetching in parallel enqueues diff --git a/crates/capsem-process/src/upstream_grant.rs b/crates/capsem-process/src/upstream_grant.rs index d037b9f30..0b1cb29ab 100644 --- a/crates/capsem-process/src/upstream_grant.rs +++ b/crates/capsem-process/src/upstream_grant.rs @@ -29,6 +29,11 @@ type WireSender = DescriptorSender; enum Command { + SetGuestMode { + relative_path: Vec, + mode: u16, + reply: oneshot::Sender>, + }, OpenDns { upstream_index: u16, policy_digest: String, @@ -95,6 +100,23 @@ impl UpstreamGrantClient { } } +impl capsem_core::GuestMetadataAuthority for UpstreamGrantClient { + fn set_mode(&self, relative_path: &[u8], mode: u16) -> io::Result<()> { + let (reply, result) = oneshot::channel(); + self.commands + .blocking_send(Command::SetGuestMode { + relative_path: relative_path.to_vec(), + mode, + reply, + }) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; + result + .blocking_recv() + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? + .map_err(io::Error::other) + } +} + impl DnsUpstreamGrants for UpstreamGrantClient { fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_> { Box::pin(async move { @@ -199,6 +221,14 @@ async fn dispatch( command: Command, ) -> Result<(), String> { match command { + Command::SetGuestMode { + relative_path, + mode, + reply, + } => finish( + reply, + set_guest_mode(sender, receiver, request_id, relative_path, mode).await, + ), Command::OpenDns { upstream_index, policy_digest, @@ -238,6 +268,40 @@ async fn dispatch( } } +async fn set_guest_mode( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, + relative_path: Vec, + mode: u16, +) -> Result, String> { + send_request( + sender, + &UpstreamGrantRequest::SetGuestMode { + request_id, + relative_path, + mode, + }, + ) + .await?; + let (response, fds) = receive_response(receiver).await?; + if !fds.is_empty() { + return Err("guest mode response carried a descriptor".into()); + } + match response { + UpstreamGrantResponse::GuestModeSet { + request_id: response_id, + } if response_id == request_id => Ok(Ok(())), + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("guest mode change denied: {reason:?}"))), + response => Err(format!( + "unexpected guest mode response for request {request_id}: {response:?}" + )), + } +} + fn finish( reply: oneshot::Sender>, result: Result, String>, diff --git a/crates/capsem-process/src/upstream_grant/tests.rs b/crates/capsem-process/src/upstream_grant/tests.rs index 992b1165d..24a7ebafa 100644 --- a/crates/capsem-process/src/upstream_grant/tests.rs +++ b/crates/capsem-process/src/upstream_grant/tests.rs @@ -3,6 +3,7 @@ use std::os::fd::{AsFd as _, AsRawFd as _, RawFd}; use capsem_core::net::dns::{DnsResolver, DnsUpstreamGrants}; use capsem_core::net::mitm_proxy::{TcpUpstreamGrants, UpstreamTarget}; +use capsem_core::GuestMetadataAuthority as _; use capsem_proto::upstream_grant::{ decode_upstream_grant_request, encode_upstream_grant_response, UpstreamDescriptorKind, UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, @@ -46,6 +47,33 @@ fn query() -> Vec { query } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guest_mode_changes_use_the_serialized_coordinator_channel() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = Arc::new(UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap()); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::SetGuestMode { + request_id: 1, + relative_path: b"workspace/pyvenv/bin/activate".to_vec(), + mode: 0o755, + } + ); + send_response(&responses, &UpstreamGrantResponse::GuestModeSet { request_id: 1 }, None).await; + }); + let client_for_mode = Arc::clone(&client); + tokio::task::spawn_blocking(move || { + client_for_mode + .set_mode(b"workspace/pyvenv/bin/activate", 0o755) + .unwrap(); + }) + .await + .unwrap(); + broker_task.await.unwrap(); +} + #[tokio::test] async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); From 88d850aa891a62401ceb8db4eb8cba114534469a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:54:16 +0000 Subject: [PATCH 086/464] security(process): attest confined owner authority --- crates/capsem-process/src/main.rs | 98 ++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 52a168121..ffdf1e671 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -265,6 +265,96 @@ fn confine_linux_owner(args: &Args, session_dir: &Path) -> Result<()> { Ok(()) } +#[cfg(target_os = "linux")] +struct OwnerSandboxAttestation { + direct_path: PathBuf, + direct_file: std::fs::File, + guest_path: PathBuf, + guest_relative: Vec, +} + +#[cfg(target_os = "linux")] +fn prepare_owner_sandbox_attestation(session_dir: &Path) -> Result { + use std::os::unix::ffi::OsStrExt as _; + use std::os::unix::fs::OpenOptionsExt as _; + + let create = |path: &Path| { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .with_context(|| format!("prepare owner sandbox attestation {}", path.display())) + }; + let nonce = format!( + "{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + let direct_path = session_dir.join(format!(".owner-sandbox-attestation-{nonce}")); + let guest_relative = format!("workspace/.owner-metadata-attestation-{nonce}").into_bytes(); + let guest_path = session_dir + .join(capsem_core::GUEST_SHARE_DIR) + .join(std::ffi::OsStr::from_bytes(&guest_relative)); + let direct_file = create(&direct_path)?; + drop(create(&guest_path)?); + Ok(OwnerSandboxAttestation { + direct_path, + direct_file, + guest_path, + guest_relative, + }) +} + +#[cfg(target_os = "linux")] +async fn attest_linux_owner( + attestation: OwnerSandboxAttestation, + service_socket: PathBuf, + upstream_grants: Arc, +) -> Result<()> { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + + let require_denied = |result: std::io::Result<()>, authority: &str| -> Result<()> { + match result { + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => Ok(()), + Err(error) => anyhow::bail!("VM owner sandbox returned {error} while denying {authority}"), + Ok(()) => anyhow::bail!("VM owner sandbox retained authority to {authority}"), + } + }; + require_denied( + std::fs::File::open("/etc/passwd").map(|_| ()), + "read unrelated host files", + )?; + require_denied( + std::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, 9)).map(|_| ()), + "dial TCP sockets directly", + )?; + require_denied( + std::os::unix::net::UnixStream::connect(service_socket).map(|_| ()), + "dial control sockets by path", + )?; + require_denied( + attestation + .direct_file + .set_permissions(std::fs::Permissions::from_mode(0o777)), + "change host modes directly", + )?; + + tokio::task::spawn_blocking(move || { + capsem_core::GuestMetadataAuthority::set_mode(upstream_grants.as_ref(), &attestation.guest_relative, 0o701) + }) + .await + .context("join brokered metadata attestation")??; + let mode = std::fs::metadata(&attestation.guest_path)?.mode() & 0o7777; + anyhow::ensure!(mode == 0o701, "brokered metadata attestation returned mode {mode:o}"); + std::fs::remove_file(&attestation.guest_path)?; + std::fs::remove_file(&attestation.direct_path)?; + Ok(()) +} + fn main() -> Result<()> { // SAFETY: process entry precedes argument parsing, telemetry, descriptor // owners and runtime threads. Broker grants will be named here explicitly. @@ -771,7 +861,13 @@ async fn run_async_main_loop( let ready = prepare_sentinel(&uds_path.with_extension("ready"))?; #[cfg(target_os = "linux")] - confine_linux_owner(&args, &session_dir).context("install Linux VM-owner confinement")?; + { + let attestation = prepare_owner_sandbox_attestation(&session_dir)?; + confine_linux_owner(&args, &session_dir).context("install Linux VM-owner confinement")?; + attest_linux_owner(attestation, seats.service_socket.clone(), Arc::clone(&upstream_grants)) + .await + .context("attest Linux VM-owner confinement")?; + } seats.start(); use std::io::Write as _; From 10a44fdfba09d53b9ae96a543c95918cf0aca283 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 01:59:49 +0000 Subject: [PATCH 087/464] docs(changelog): record confined metadata fixes --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 852fdf145..f4f2ce3b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 grants; deny sibling files, path-based control connections and direct network dialing after startup. +- Broker guest VirtioFS permission changes through the coordinator beneath + the session's guest share, keeping direct host permission syscalls denied + and attesting that authority before the VM owner publishes readiness. + - Restrict registered VM-owner service connections to private-name lookup and that owner's metrics route, preventing a compromised worker from invoking the coordinator's public API over its same-user control socket. @@ -65,6 +69,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Keep SQLite writer scratch state in memory so a confined VM owner can flush + high-volume session-ledger bursts without ambient temporary-directory access. + - Launch OCI workloads without mutating the guest-owned staging file, so the Linux VM-owner sandbox can keep host permission syscalls denied. From 0774a37f313c93f341bc8189a67e5beaa28f4ef9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:05:08 +0000 Subject: [PATCH 088/464] security(foundation): confine macOS workers --- .../src/unix/worker_sandbox.rs | 10 +- .../src/unix/worker_sandbox/macos.rs | 56 ++++++++ .../src/unix/worker_sandbox/seatbelt.rs | 89 ++++++++++++ .../src/unix/worker_sandbox/tests.rs | 136 ++++++++++++++++++ 4 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 crates/capsem-foundation/src/unix/worker_sandbox/macos.rs create mode 100644 crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs diff --git a/crates/capsem-foundation/src/unix/worker_sandbox.rs b/crates/capsem-foundation/src/unix/worker_sandbox.rs index 35a025946..3def396e9 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox.rs @@ -78,7 +78,15 @@ pub fn confine(policy: &Policy) -> io::Result<()> { #[path = "worker_sandbox/linux.rs"] mod platform; -#[cfg(not(target_os = "linux"))] +#[cfg(target_os = "macos")] +#[path = "worker_sandbox/macos.rs"] +mod platform; + +#[cfg(any(target_os = "macos", test))] +#[path = "worker_sandbox/seatbelt.rs"] +mod seatbelt; + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] mod platform { use super::*; diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/macos.rs b/crates/capsem-foundation/src/unix/worker_sandbox/macos.rs new file mode 100644 index 000000000..af2e13894 --- /dev/null +++ b/crates/capsem-foundation/src/unix/worker_sandbox/macos.rs @@ -0,0 +1,56 @@ +use super::{Access, Policy}; +use std::ffi::CStr; +use std::io; + +unsafe extern "C" { + fn sandbox_init_with_parameters( + profile: *const libc::c_char, + flags: u64, + parameters: *const *const libc::c_char, + error: *mut *mut libc::c_char, + ) -> libc::c_int; + fn sandbox_free_error(error: *mut libc::c_char); +} + +pub(super) fn confine(policy: &Policy) -> io::Result<()> { + validate_paths(policy)?; + let compiled = super::seatbelt::compile(policy)?; + let parameters = compiled.parameter_ptrs(); + let mut error = std::ptr::null_mut(); + // SAFETY: source and every alternating key/value parameter are NUL + // terminated and remain alive for the synchronous initialization call. + let status = + unsafe { sandbox_init_with_parameters(compiled.source().as_ptr(), 0, parameters.as_ptr(), &mut error) }; + if status == 0 { + return Ok(()); + } + let message = if error.is_null() { + "worker sandbox initialization failed".into() + } else { + // SAFETY: sandbox_init initialized this error allocation; its paired + // API releases it after the bytes have been copied. + let message = unsafe { CStr::from_ptr(error) }.to_string_lossy().into_owned(); + unsafe { sandbox_free_error(error) }; + message + }; + Err(io::Error::other(message)) +} + +fn validate_paths(policy: &Policy) -> io::Result<()> { + for rule in policy.paths() { + let metadata = std::fs::symlink_metadata(rule.path())?; + if metadata.file_type().is_symlink() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("sandbox grant is a symlink: {}", rule.path().display()), + )); + } + if rule.access() == Access::Executable && !metadata.is_file() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("sandbox executable grant is not a file: {}", rule.path().display()), + )); + } + } + Ok(()) +} diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs b/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs new file mode 100644 index 000000000..8bd88c9bb --- /dev/null +++ b/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs @@ -0,0 +1,89 @@ +use super::{Access, Policy, Role}; +use std::ffi::CString; +use std::io; +use std::os::unix::ffi::OsStrExt; + +pub(super) struct Compiled { + source: CString, + parameters: Vec<(CString, CString)>, +} + +pub(super) fn compile(policy: &Policy) -> io::Result { + let mut source = String::from( + "(version 1)\n\ + (allow default)\n\ + (deny file-read*)\n\ + (deny file-write*)\n\ + (deny network-outbound)\n\ + (deny network-bind)\n\ + (deny network-inbound)\n\ + (deny process-exec)\n\ + (deny signal)\n\ + (allow signal (target same-sandbox))\n", + ); + if policy.role() == Role::Gateway { + source.push_str("(allow network-inbound (local ip \"*:*\"))\n"); + } + + let mut parameters = Vec::with_capacity(policy.paths().len()); + for (index, rule) in policy.paths().iter().enumerate() { + let name = format!("PATH_{index}"); + let filter = format!("(require-any (literal (param \"{name}\")) (subpath (param \"{name}\")))"); + source.push_str(&format!( + "(allow file-read-metadata (path-ancestors (param \"{name}\")))\n" + )); + match rule.access() { + Access::ReadOnly => source.push_str(&format!("(allow file-read* {filter})\n")), + Access::ReadWrite => { + source.push_str(&format!("(allow file-read* file-write* {filter})\n")); + } + Access::Executable => { + source.push_str(&format!( + "(allow file-read* (literal (param \"{name}\")))\n\ + (allow process-exec (with no-sandbox) (literal (param \"{name}\")))\n" + )); + } + } + parameters.push(( + cstring(name.as_bytes(), "sandbox parameter name")?, + cstring(rule.path().as_os_str().as_bytes(), "sandbox path")?, + )); + } + // Permission and ownership changes stay brokered even beneath writable + // grants. These final rules win over the broader file-write allowance. + source.push_str("(deny file-write-mode file-write-owner file-write-setugid)\n"); + + Ok(Compiled { + source: cstring(source.as_bytes(), "sandbox profile")?, + parameters, + }) +} + +fn cstring(value: &[u8], name: &str) -> io::Result { + CString::new(value).map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, format!("{name} contains NUL"))) +} + +impl Compiled { + pub(super) fn source(&self) -> &std::ffi::CStr { + &self.source + } + + #[cfg(target_os = "macos")] + pub(super) fn parameter_ptrs(&self) -> Vec<*const libc::c_char> { + let mut pointers = Vec::with_capacity(self.parameters.len() * 2 + 1); + for (name, value) in &self.parameters { + pointers.push(name.as_ptr()); + pointers.push(value.as_ptr()); + } + pointers.push(std::ptr::null()); + pointers + } + + #[cfg(test)] + pub(super) fn parameters(&self) -> Vec<(&str, &str)> { + self.parameters + .iter() + .map(|(name, value)| (name.to_str().unwrap(), value.to_str().unwrap())) + .collect() + } +} diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index bb9ebd8c1..7aebe3738 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -6,6 +6,45 @@ use std::os::unix::net::{UnixListener, UnixStream}; use std::process::{Command, Stdio}; use std::time::Duration; +#[test] +fn seatbelt_profile_keeps_grant_values_out_of_policy_source() { + let policy = Policy::new(Role::VmOwner) + .allow("/session/with \"quotes\"", Access::ReadWrite) + .allow("/assets", Access::ReadOnly) + .allow("/opt/capsem-router", Access::Executable); + let compiled = super::seatbelt::compile(&policy).unwrap(); + let source = compiled.source().to_str().unwrap(); + assert!(!source.contains("/session")); + assert!(!source.contains("/assets")); + assert!(!source.contains("/opt/capsem-router")); + assert!(source.contains("(deny file-read*)")); + assert!(source.contains("(deny file-write*)")); + assert!(source.contains("(deny network-outbound)")); + assert!(source.contains("(deny network-bind)")); + assert!(source.contains("(deny process-exec)")); + assert!(source.contains("(deny signal)")); + assert!(source.contains("(allow signal (target same-sandbox))")); + assert!(source.contains("(deny file-write-mode file-write-owner file-write-setugid)")); + assert!(source.contains("(allow process-exec (with no-sandbox) (literal (param \"PATH_2\")))")); + assert_eq!( + compiled.parameters(), + [ + ("PATH_0", "/session/with \"quotes\""), + ("PATH_1", "/assets"), + ("PATH_2", "/opt/capsem-router"), + ] + ); +} + +#[test] +fn seatbelt_gateway_accepts_only_prebound_inbound_network() { + let compiled = super::seatbelt::compile(&Policy::new(Role::Gateway)).unwrap(); + let source = compiled.source().to_str().unwrap(); + assert!(source.contains("(allow network-inbound (local ip \"*:*\"))")); + assert!(!source.contains("allow network-outbound")); + assert!(!source.contains("allow network-bind")); +} + #[cfg(target_os = "linux")] #[test] fn linux_policy_preserves_grants_and_denies_ambient_authority() { @@ -105,3 +144,100 @@ fn linux_sandbox_child() { inherited.write_all(b"pong").unwrap(); std::process::exit(0); } + +#[cfg(target_os = "macos")] +#[test] +fn macos_policy_preserves_grants_and_denies_ambient_authority() { + let directory = tempfile::tempdir().unwrap(); + let allowed = directory.path().join("allowed"); + let readonly = directory.path().join("readonly"); + let denied = directory.path().join("denied"); + std::fs::create_dir_all(&allowed).unwrap(); + std::fs::create_dir_all(&readonly).unwrap(); + std::fs::create_dir_all(&denied).unwrap(); + std::fs::write(readonly.join("value"), b"readable").unwrap(); + std::fs::write(denied.join("secret"), b"secret").unwrap(); + let control = denied.join("control.sock"); + let _control = UnixListener::bind(&control).unwrap(); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + let mut client = TcpStream::connect((Ipv4Addr::LOCALHOST, port)).unwrap(); + let (socket, _) = listener.accept().unwrap(); + + let mut child = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "unix::worker_sandbox::tests::macos_sandbox_child", + "--nocapture", + ]) + .env_clear() + .env("WORKER_SANDBOX_ALLOWED", &allowed) + .env("WORKER_SANDBOX_READONLY", &readonly) + .env("WORKER_SANDBOX_DENIED", &denied) + .env("WORKER_SANDBOX_CONTROL", &control) + .env("WORKER_SANDBOX_PORT", port.to_string()) + .stdin(Stdio::from(std::os::fd::OwnedFd::from(socket))) + .spawn() + .unwrap(); + client.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + client.write_all(b"ping").unwrap(); + let mut reply = [0; 4]; + let read = client.read_exact(&mut reply); + let status = child.wait().unwrap(); + assert!(status.success(), "sandbox child failed: {status}"); + read.unwrap(); + assert_eq!(&reply, b"pong"); +} + +#[cfg(target_os = "macos")] +#[test] +fn macos_sandbox_child() { + use std::os::unix::fs::PermissionsExt; + + let Some(allowed) = std::env::var_os("WORKER_SANDBOX_ALLOWED") else { + return; + }; + let allowed = std::path::PathBuf::from(allowed); + let readonly = std::path::PathBuf::from(std::env::var_os("WORKER_SANDBOX_READONLY").unwrap()); + let denied = std::path::PathBuf::from(std::env::var_os("WORKER_SANDBOX_DENIED").unwrap()); + let control = std::env::var_os("WORKER_SANDBOX_CONTROL").unwrap(); + let port: u16 = std::env::var("WORKER_SANDBOX_PORT").unwrap().parse().unwrap(); + let stdin = std::io::stdin(); + let mut inherited = TcpStream::from(super::super::fd::duplicate(stdin.as_fd()).unwrap()); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + + confine( + &Policy::new(Role::Gateway) + .allow(&allowed, Access::ReadWrite) + .allow(&readonly, Access::ReadOnly), + ) + .unwrap(); + + runtime.block_on(async { + tokio::task::spawn_blocking(move || { + assert_eq!(std::fs::read(readonly.join("value")).unwrap(), b"readable"); + assert!(std::fs::write(readonly.join("value"), b"changed").is_err()); + std::fs::write(allowed.join("created"), b"ok").unwrap(); + assert!(std::fs::set_permissions(&allowed, std::fs::Permissions::from_mode(0o777)).is_err()); + assert!(std::fs::read(denied.join("secret")).is_err()); + assert!(UnixStream::connect(control).is_err()); + assert!(TcpStream::connect((Ipv4Addr::LOCALHOST, port)).is_err()); + assert!(Command::new("/usr/bin/true").status().is_err()); + let parent = unsafe { libc::getppid() }; + assert_eq!(unsafe { libc::kill(parent, 0) }, -1, "worker signalled its parent"); + }) + .await + .unwrap(); + tokio::task::spawn_blocking(|| 7usize).await.unwrap() + }); + + let mut request = [0; 4]; + inherited.read_exact(&mut request).unwrap(); + assert_eq!(&request, b"ping"); + inherited.write_all(b"pong").unwrap(); + std::process::exit(0); +} From 4ce18e8ba316e2b288d32935270870bb574e46d5 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:07:35 +0000 Subject: [PATCH 089/464] security(gateway): install macOS worker policy --- crates/capsem-gateway/src/main.rs | 32 +++++++++++++++++++++--------- crates/capsem-gateway/src/tests.rs | 12 +++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/crates/capsem-gateway/src/main.rs b/crates/capsem-gateway/src/main.rs index 3f033d77d..5424eb7b3 100644 --- a/crates/capsem-gateway/src/main.rs +++ b/crates/capsem-gateway/src/main.rs @@ -174,7 +174,7 @@ async fn run(args: Args) -> Result<()> { previews: preview::PreviewState::new(preview_port), }); - confine_linux_gateway(&uds_path, listener.local_addr()?)?; + confine_gateway(&auth_state, &uds_path, listener.local_addr()?)?; let preview_state = state.clone(); tokio::spawn(async move { @@ -229,29 +229,43 @@ async fn run(args: Args) -> Result<()> { } #[cfg(target_os = "linux")] -fn confine_linux_gateway(uds_path: &std::path::Path, listener: SocketAddr) -> Result<()> { +fn confine_gateway(_auth_state: &AuthState, uds_path: &std::path::Path, listener: SocketAddr) -> Result<()> { use capsem_foundation::unix::worker_sandbox::{Policy, Role}; capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Gateway)) .context("confine gateway before readiness")?; + attest_gateway(uds_path, listener) +} + +#[cfg(target_os = "macos")] +fn confine_gateway(auth_state: &AuthState, uds_path: &std::path::Path, listener: SocketAddr) -> Result<()> { + use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; + let policy = Policy::new(Role::Gateway) + .allow(&auth_state.token_path, Access::ReadWrite) + .allow(&auth_state.port_path, Access::ReadWrite) + .allow(&auth_state.pid_path, Access::ReadWrite) + .allow(&auth_state.preview_port_path, Access::ReadWrite); + capsem_foundation::unix::worker_sandbox::confine(&policy).context("confine gateway before readiness")?; + attest_gateway(uds_path, listener) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn attest_gateway(uds_path: &std::path::Path, listener: SocketAddr) -> Result<()> { require_denied(std::fs::File::open("/etc/passwd"), "read ambient host files")?; require_denied(std::net::TcpStream::connect(listener), "dial TCP directly")?; require_denied( std::os::unix::net::UnixStream::connect(uds_path), "dial control sockets by path", )?; - require_denied( - std::process::Command::new("/bin/true").status(), - "execute processes", - )?; + require_denied(std::process::Command::new("/bin/true").status(), "execute processes")?; Ok(()) } -#[cfg(not(target_os = "linux"))] -fn confine_linux_gateway(_uds_path: &std::path::Path, _listener: SocketAddr) -> Result<()> { +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn confine_gateway(_auth_state: &AuthState, _uds_path: &std::path::Path, _listener: SocketAddr) -> Result<()> { Ok(()) } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn require_denied(result: std::io::Result, operation: &str) -> Result<()> { match result { Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => Ok(()), diff --git a/crates/capsem-gateway/src/tests.rs b/crates/capsem-gateway/src/tests.rs index 5233f2370..16d2d0f17 100644 --- a/crates/capsem-gateway/src/tests.rs +++ b/crates/capsem-gateway/src/tests.rs @@ -20,6 +20,18 @@ fn entry_closes_ambient_descriptors_before_runtime_construction() { assert!(close < runtime && runtime < run); } +#[test] +fn platform_confinement_precedes_gateway_readiness() { + let source = include_str!("main.rs"); + let run = source.split_once("async fn run(args: Args) -> Result<()> {").unwrap().1; + let prepare = run.find("AuthState::prepare").unwrap(); + let confine = run + .find("confine_gateway(&auth_state") + .expect("all supported platforms install the worker policy"); + let publish = run.find("auth_state.publish").unwrap(); + assert!(prepare < confine && confine < publish); +} + struct EnvGuard { key: &'static str, prev: Option, From 9e24a969834ae6d18888bc6e303d0c54f06f6de1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:13:21 +0000 Subject: [PATCH 090/464] security(process): install macOS owner policy --- crates/capsem-process/src/main.rs | 97 ++++++++++++++++++++++-------- crates/capsem-process/src/tests.rs | 16 +++++ 2 files changed, 89 insertions(+), 24 deletions(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index ffdf1e671..9f9d74b16 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -222,50 +222,95 @@ fn prepare_sentinel(path: &Path) -> Result { Ok(file) } +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn grant_owner_path( + policy: capsem_foundation::unix::worker_sandbox::Policy, + path: &Path, + access: capsem_foundation::unix::worker_sandbox::Access, +) -> Result { + let canonical = path + .canonicalize() + .with_context(|| format!("resolve sandbox grant {}", path.display()))?; + Ok(policy.allow(canonical, access)) +} + #[cfg(target_os = "linux")] -fn confine_linux_owner(args: &Args, session_dir: &Path) -> Result<()> { +fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; - let grant = |policy: Policy, path: &Path, access| -> Result { - let canonical = path - .canonicalize() - .with_context(|| format!("resolve sandbox grant {}", path.display()))?; - Ok(policy.allow(canonical, access)) - }; let mut policy = Policy::new(Role::VmOwner); - policy = grant(policy, session_dir, Access::ReadWrite)?; - policy = grant(policy, &args.assets_dir, Access::ReadOnly)?; - policy = grant(policy, &args.rootfs, Access::ReadOnly)?; - policy = grant(policy, &args.active_policy, Access::ReadOnly)?; + policy = grant_owner_path(policy, session_dir, Access::ReadWrite)?; + policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; + policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; + policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; for path in args.kernel.iter().chain(args.initrd.iter()) { - policy = grant(policy, path, Access::ReadOnly)?; + policy = grant_owner_path(policy, path, Access::ReadOnly)?; } let executable = std::env::current_exe().context("locate VM-owner executable")?; let router = executable.with_file_name("capsem-router"); if router.exists() { - policy = grant(policy, &router, Access::Executable)?; + policy = grant_owner_path(policy, &router, Access::Executable)?; } for path in ["/lib", "/lib64", "/usr/lib", "/usr/lib64"] { let path = Path::new(path); if path.exists() { - policy = grant(policy, path, Access::ReadOnly)?; + policy = grant_owner_path(policy, path, Access::ReadOnly)?; } } let linker_cache = Path::new("/etc/ld.so.cache"); if linker_cache.exists() { - policy = grant(policy, linker_cache, Access::ReadOnly)?; + policy = grant_owner_path(policy, linker_cache, Access::ReadOnly)?; } for path in ["/lib64/ld-linux-x86-64.so.2", "/lib/ld-linux-aarch64.so.1"] { let path = Path::new(path); if path.exists() { - policy = grant(policy, path, Access::Executable)?; + policy = grant_owner_path(policy, path, Access::Executable)?; } } capsem_foundation::unix::worker_sandbox::confine(&policy)?; Ok(()) } -#[cfg(target_os = "linux")] +#[cfg(target_os = "macos")] +fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { + use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; + + let mut policy = Policy::new(Role::VmOwner); + policy = grant_owner_path(policy, session_dir, Access::ReadWrite)?; + policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; + policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; + policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; + for path in args.kernel.iter().chain(args.initrd.iter()) { + policy = grant_owner_path(policy, path, Access::ReadOnly)?; + } + let router = std::env::current_exe() + .context("locate VM-owner executable")? + .with_file_name("capsem-router"); + if router.exists() { + policy = grant_owner_path(policy, &router, Access::Executable)?; + } + for path in [ + "/System/Library", + "/usr/lib", + "/private/var/db/dyld", + "/private/var/db/timezone", + "/etc/localtime", + "/dev/urandom", + ] { + let path = Path::new(path); + if path.exists() { + policy = grant_owner_path(policy, path, Access::ReadOnly)?; + } + } + let null = Path::new("/dev/null"); + if null.exists() { + policy = grant_owner_path(policy, null, Access::ReadWrite)?; + } + capsem_foundation::unix::worker_sandbox::confine(&policy)?; + Ok(()) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] struct OwnerSandboxAttestation { direct_path: PathBuf, direct_file: std::fs::File, @@ -273,7 +318,7 @@ struct OwnerSandboxAttestation { guest_relative: Vec, } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "macos"))] fn prepare_owner_sandbox_attestation(session_dir: &Path) -> Result { use std::os::unix::ffi::OsStrExt as _; use std::os::unix::fs::OpenOptionsExt as _; @@ -309,8 +354,8 @@ fn prepare_owner_sandbox_attestation(session_dir: &Path) -> Result, @@ -342,6 +387,10 @@ async fn attest_linux_owner( .set_permissions(std::fs::Permissions::from_mode(0o777)), "change host modes directly", )?; + require_denied( + std::process::Command::new("/bin/true").status().map(|_| ()), + "execute arbitrary processes", + )?; tokio::task::spawn_blocking(move || { capsem_core::GuestMetadataAuthority::set_mode(upstream_grants.as_ref(), &attestation.guest_relative, 0o701) @@ -860,13 +909,13 @@ async fn run_async_main_loop( let mut launched = prepare_sentinel(&launched_path)?; let ready = prepare_sentinel(&uds_path.with_extension("ready"))?; - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "macos"))] { let attestation = prepare_owner_sandbox_attestation(&session_dir)?; - confine_linux_owner(&args, &session_dir).context("install Linux VM-owner confinement")?; - attest_linux_owner(attestation, seats.service_socket.clone(), Arc::clone(&upstream_grants)) + confine_owner(&args, &session_dir).context("install VM-owner confinement")?; + attest_owner(attestation, seats.service_socket.clone(), Arc::clone(&upstream_grants)) .await - .context("attest Linux VM-owner confinement")?; + .context("attest VM-owner confinement")?; } seats.start(); diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index d88effb26..5ba0e1a50 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -13,6 +13,22 @@ fn entry_closes_ambient_descriptors_before_runtime_initialization() { assert!(close < telemetry && close < arguments); } +#[test] +fn platform_confinement_and_attestation_precede_owner_readiness() { + let source = include_str!("main.rs"); + let run = source.split_once("async fn run_async_main_loop(").unwrap().1; + let prepare = run.find("prepare_owner_sandbox_attestation").unwrap(); + let confine = run + .find("confine_owner(&args") + .expect("all supported platforms install the VM-owner policy"); + let attest = run + .find("attest_owner(") + .expect("all supported platforms attest the installed policy"); + let seats = run.find("seats.start()").unwrap(); + let launched = run.find("launched.write_all").unwrap(); + assert!(prepare < confine && confine < attest && attest < seats && seats < launched); +} + #[test] fn private_name_broker_carries_no_session_bearer() { for source in [include_str!("private_names.rs"), include_str!("private_seats.rs")] { From ff2a58a6da083acce81209251c8fc0e1daf9b079 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:16:36 +0000 Subject: [PATCH 091/464] docs(changelog): record macOS worker confinement --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4f2ce3b7..149d7ce92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and coordinator-minted service and VM-owner channels, denying filesystem access, process execution and direct network or control-socket dialing. +- Confine the macOS API gateway and VM owner with role-specific Seatbelt + policies before readiness, retaining only explicit session, asset, listener, + runtime-file and broker grants while denying ambient files, execution, + signals and direct network or control-socket dialing. + - Confine Linux VM-owner runtime threads before readiness to their session, read-only boot assets, prepared service channels and inherited network grants; deny sibling files, path-based control connections and direct From d580c497689654e5f653e633a03dad57eedf353f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:24:16 +0000 Subject: [PATCH 092/464] feat(proto): define ledger channel authority --- crates/capsem-proto/build.rs | 17 +- crates/capsem-proto/src/ledger.rs | 384 ++++++++++++++++++++++++ crates/capsem-proto/src/ledger/tests.rs | 112 +++++++ crates/capsem-proto/src/lib.rs | 1 + 4 files changed, 504 insertions(+), 10 deletions(-) create mode 100644 crates/capsem-proto/src/ledger.rs create mode 100644 crates/capsem-proto/src/ledger/tests.rs diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index 40575fe51..ebf9aa689 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -21,14 +21,12 @@ impl VisitMut for StripDocs { } fn main() { - let files = [ - "lib.rs", - "ipc.rs", - "handshake.rs", - "router.rs", - "exec_stream.rs", - "credential_injection.rs", - ]; + let files = ["lib.rs", "ipc.rs", "handshake.rs", "router.rs", "exec_stream.rs"]; + write_hash("schema_hash.txt", &files); + write_hash("ledger_schema_hash.txt", &["ledger.rs"]); +} + +fn write_hash(output: &str, files: &[&str]) { let mut hash = 0xcbf29ce484222325_u64; for f in files { @@ -56,6 +54,5 @@ fn main() { } let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set"); - std::fs::write(format!("{out_dir}/schema_hash.txt"), format!("{hash}u64")) - .expect("schema hash: write generated constant"); + std::fs::write(format!("{out_dir}/{output}"), format!("{hash}u64")).expect("schema hash: write generated constant"); } diff --git a/crates/capsem-proto/src/ledger.rs b/crates/capsem-proto/src/ledger.rs new file mode 100644 index 000000000..969f27939 --- /dev/null +++ b/crates/capsem-proto/src/ledger.rs @@ -0,0 +1,384 @@ +//! Generation-bound authority and envelopes for one session ledger channel. +//! +//! A trusted coordinator creates a connected channel and gives both endpoints +//! the same grant. The grant deliberately has no session id or path: the +//! worker already owns one pre-opened session directory, and possession of a +//! generation-bound descriptor selects it. After the hello exchange, ordinary +//! requests carry only a request id and a transport-independent operation. + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +/// Version of the ledger handshake, request envelopes and logger-owned +/// operation/reply enums. Bump it when any of those wire shapes changes. +pub const LEDGER_PROTOCOL_VERSION: u16 = 1; +/// Hash of ledger wire declarations, separate from the guest protocol hash. +pub const LEDGER_SCHEMA_HASH: u64 = include!(concat!(env!("OUT_DIR"), "/ledger_schema_hash.txt")); +/// Largest diagnostic admitted into a response. +pub const MAX_LEDGER_ERROR_BYTES: usize = 4 * 1024; + +/// Fresh identity of one supervised ledger-owner lifetime. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerGeneration([u8; 16]); + +impl LedgerGeneration { + pub const fn new(bytes: [u8; 16]) -> Self { + Self(bytes) + } + + pub const fn as_bytes(self) -> [u8; 16] { + self.0 + } + + const fn is_zero(self) -> bool { + let mut index = 0; + while index < self.0.len() { + if self.0[index] != 0 { + return false; + } + index += 1; + } + true + } +} + +/// Trusted identity and authority stamped on one connected client. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerClientRole { + VmOwner, + Proxy, + Coordinator, + Reader, + Maintainer, + Supervisor, +} + +/// Classes used to authorize an operation before storage dispatch. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerCapability { + Admit, + Flush, + Read, + Retain, + Export, + Shutdown, +} + +impl LedgerClientRole { + pub const fn permits(self, capability: LedgerCapability) -> bool { + match self { + Self::VmOwner | Self::Proxy | Self::Coordinator => { + matches!(capability, LedgerCapability::Admit | LedgerCapability::Flush) + } + Self::Reader => matches!(capability, LedgerCapability::Read | LedgerCapability::Export), + Self::Maintainer => matches!( + capability, + LedgerCapability::Read | LedgerCapability::Retain | LedgerCapability::Export + ), + Self::Supervisor => matches!(capability, LedgerCapability::Shutdown), + } + } + + /// Stable producer identity the worker stamps on accepted operations. + pub const fn producer_name(self) -> Option<&'static str> { + match self { + Self::VmOwner => Some("vm_owner"), + Self::Proxy => Some("proxy"), + Self::Coordinator => Some("coordinator"), + Self::Reader | Self::Maintainer | Self::Supervisor => None, + } + } +} + +/// Coordinator-minted authority for one already connected channel. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LedgerChannelGrant { + generation: LedgerGeneration, + client_id: u64, + role: LedgerClientRole, +} + +impl LedgerChannelGrant { + pub fn new( + generation: LedgerGeneration, + client_id: u64, + role: LedgerClientRole, + ) -> Result { + if generation.is_zero() { + return Err(LedgerProtocolError::ZeroGeneration); + } + if client_id == 0 { + return Err(LedgerProtocolError::ZeroClientId); + } + Ok(Self { + generation, + client_id, + role, + }) + } + + pub const fn generation(self) -> LedgerGeneration { + self.generation + } + + pub const fn client_id(self) -> u64 { + self.client_id + } + + pub const fn role(self) -> LedgerClientRole { + self.role + } + + pub fn validate_hello(self, hello: &LedgerHello) -> Result<(), LedgerHandshakeError> { + validate_compatibility(hello.protocol_version, hello.schema_hash)?; + if hello.generation != self.generation { + return Err(LedgerHandshakeError::Generation); + } + if hello.client_id != self.client_id { + return Err(LedgerHandshakeError::ClientId); + } + Ok(()) + } + + pub fn validate_welcome(self, welcome: &LedgerWelcome) -> Result<(), LedgerHandshakeError> { + validate_compatibility(welcome.protocol_version, welcome.schema_hash)?; + if welcome.generation != self.generation { + return Err(LedgerHandshakeError::Generation); + } + if welcome.client_id != self.client_id { + return Err(LedgerHandshakeError::ClientId); + } + if welcome.role != self.role { + return Err(LedgerHandshakeError::Role); + } + Ok(()) + } +} + +/// First client-to-worker value on a freshly granted channel. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerHello { + protocol_version: u16, + schema_hash: u64, + generation: LedgerGeneration, + client_id: u64, +} + +impl LedgerHello { + pub const fn for_grant(grant: &LedgerChannelGrant) -> Self { + Self { + protocol_version: LEDGER_PROTOCOL_VERSION, + schema_hash: LEDGER_SCHEMA_HASH, + generation: grant.generation, + client_id: grant.client_id, + } + } +} + +/// Worker-to-client confirmation of the exact generation and granted role. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerWelcome { + protocol_version: u16, + schema_hash: u64, + generation: LedgerGeneration, + client_id: u64, + role: LedgerClientRole, +} + +impl LedgerWelcome { + pub const fn for_grant(grant: &LedgerChannelGrant) -> Self { + Self { + protocol_version: LEDGER_PROTOCOL_VERSION, + schema_hash: LEDGER_SCHEMA_HASH, + generation: grant.generation, + client_id: grant.client_id, + role: grant.role, + } + } +} + +fn validate_compatibility(protocol_version: u16, schema_hash: u64) -> Result<(), LedgerHandshakeError> { + if protocol_version != LEDGER_PROTOCOL_VERSION { + return Err(LedgerHandshakeError::ProtocolVersion { + expected: LEDGER_PROTOCOL_VERSION, + actual: protocol_version, + }); + } + if schema_hash != LEDGER_SCHEMA_HASH { + return Err(LedgerHandshakeError::SchemaHash { + expected: LEDGER_SCHEMA_HASH, + actual: schema_hash, + }); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +pub enum LedgerHandshakeError { + #[error("ledger protocol version mismatch: expected {expected}, got {actual}")] + ProtocolVersion { expected: u16, actual: u16 }, + #[error("ledger schema hash mismatch: expected {expected:#x}, got {actual:#x}")] + SchemaHash { expected: u64, actual: u64 }, + #[error("ledger generation does not match the granted channel")] + Generation, + #[error("ledger client id does not match the granted channel")] + ClientId, + #[error("ledger role does not match the granted channel")] + Role, +} + +/// An operation declares the authority class checked before it is dispatched. +pub trait LedgerOperation { + fn capability(&self) -> LedgerCapability; + + fn validate(&self) -> Result<(), LedgerProtocolError> { + Ok(()) + } +} + +/// One post-handshake operation. Session and generation are channel state. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerRequest { + request_id: u64, + operation: O, +} + +impl LedgerRequest { + pub fn new(request_id: u64, operation: O) -> Result { + if request_id == 0 { + return Err(LedgerProtocolError::ZeroRequestId); + } + Ok(Self { request_id, operation }) + } + + pub const fn request_id(&self) -> u64 { + self.request_id + } + + pub const fn operation(&self) -> &O { + &self.operation + } + + pub fn into_operation(self) -> O { + self.operation + } +} + +impl LedgerRequest { + pub fn authorize(&self, grant: &LedgerChannelGrant) -> Result<(), LedgerProtocolError> { + if self.request_id == 0 { + return Err(LedgerProtocolError::ZeroRequestId); + } + self.operation.validate()?; + let capability = self.operation.capability(); + if !grant.role.permits(capability) { + return Err(LedgerProtocolError::Unauthorized { + role: grant.role, + capability, + }); + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerFailureCode { + InvalidRequest, + UnauthorizedOperation, + Busy, + Unavailable, + Storage, + CorruptLedger, + LimitExceeded, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerFailure { + code: LedgerFailureCode, + message: String, +} + +impl LedgerFailure { + pub fn new(code: LedgerFailureCode, message: impl Into) -> Result { + let message = message.into(); + if message.len() > MAX_LEDGER_ERROR_BYTES { + return Err(LedgerProtocolError::ErrorMessageTooLong); + } + Ok(Self { code, message }) + } + + pub const fn code(&self) -> LedgerFailureCode { + self.code + } + + pub fn message(&self) -> &str { + &self.message + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerOutcome { + Success { reply: R }, + Failure { failure: LedgerFailure }, +} + +/// One correlated reply. Streaming operations use several replies with the +/// same request id and bounded operation-specific chunks. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerResponse { + request_id: u64, + outcome: LedgerOutcome, +} + +impl LedgerResponse { + pub fn success(request_id: u64, reply: R) -> Result { + Self::new(request_id, LedgerOutcome::Success { reply }) + } + + pub fn failure(request_id: u64, failure: LedgerFailure) -> Result { + Self::new(request_id, LedgerOutcome::Failure { failure }) + } + + fn new(request_id: u64, outcome: LedgerOutcome) -> Result { + if request_id == 0 { + return Err(LedgerProtocolError::ZeroRequestId); + } + Ok(Self { request_id, outcome }) + } + + pub const fn request_id(&self) -> u64 { + self.request_id + } + + pub const fn outcome(&self) -> &LedgerOutcome { + &self.outcome + } +} + +#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +pub enum LedgerProtocolError { + #[error("ledger generation must not be zero")] + ZeroGeneration, + #[error("ledger client id must not be zero")] + ZeroClientId, + #[error("ledger request id must not be zero")] + ZeroRequestId, + #[error("ledger error message exceeds its bound")] + ErrorMessageTooLong, + #[error("ledger operation fields are invalid")] + InvalidOperation, + #[error("encoded ledger operation exceeds its bound")] + OperationTooLarge, + #[error("ledger role {role:?} cannot perform {capability:?}")] + Unauthorized { + role: LedgerClientRole, + capability: LedgerCapability, + }, +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/ledger/tests.rs b/crates/capsem-proto/src/ledger/tests.rs new file mode 100644 index 000000000..af9fba756 --- /dev/null +++ b/crates/capsem-proto/src/ledger/tests.rs @@ -0,0 +1,112 @@ +use serde::{Deserialize, Serialize}; + +use super::*; + +const GENERATION: LedgerGeneration = LedgerGeneration::new([7; 16]); + +fn grant(role: LedgerClientRole) -> LedgerChannelGrant { + LedgerChannelGrant::new(GENERATION, 41, role).expect("valid grant") +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +enum TestOperation { + Admit, + Read, +} + +impl LedgerOperation for TestOperation { + fn capability(&self) -> LedgerCapability { + match self { + Self::Admit => LedgerCapability::Admit, + Self::Read => LedgerCapability::Read, + } + } +} + +#[test] +fn named_messagepack_round_trip_carries_only_generation_identity() { + let hello = LedgerHello::for_grant(&grant(LedgerClientRole::Proxy)); + let encoded = rmp_serde::to_vec_named(&hello).expect("encode hello"); + let value: serde_json::Value = rmp_serde::from_slice(&encoded).expect("decode named map"); + let keys = value + .as_object() + .expect("named map") + .keys() + .cloned() + .collect::>(); + assert_eq!(keys, ["client_id", "generation", "protocol_version", "schema_hash"]); + assert_eq!(rmp_serde::from_slice::(&encoded).unwrap(), hello); + assert!(!String::from_utf8_lossy(&encoded).contains("session")); + assert!(!String::from_utf8_lossy(&encoded).contains("path")); +} + +#[test] +fn server_and_client_reject_wrong_generation() { + let expected = grant(LedgerClientRole::VmOwner); + let wrong = LedgerChannelGrant::new(LedgerGeneration::new([9; 16]), 41, LedgerClientRole::VmOwner).unwrap(); + assert_eq!( + expected.validate_hello(&LedgerHello::for_grant(&wrong)).unwrap_err(), + LedgerHandshakeError::Generation + ); + assert_eq!( + expected + .validate_welcome(&LedgerWelcome::for_grant(&wrong)) + .unwrap_err(), + LedgerHandshakeError::Generation + ); +} + +#[test] +fn role_rejects_wrong_operation_before_dispatch() { + let request = LedgerRequest::new(1, TestOperation::Read).unwrap(); + assert_eq!( + request.authorize(&grant(LedgerClientRole::Proxy)).unwrap_err(), + LedgerProtocolError::Unauthorized { + role: LedgerClientRole::Proxy, + capability: LedgerCapability::Read, + } + ); + request.authorize(&grant(LedgerClientRole::Reader)).unwrap(); +} + +#[test] +fn compatibility_mismatch_fails_closed() { + let expected = grant(LedgerClientRole::Reader); + let mut hello = LedgerHello::for_grant(&expected); + hello.protocol_version += 1; + assert!(matches!( + expected.validate_hello(&hello), + Err(LedgerHandshakeError::ProtocolVersion { .. }) + )); + let mut hello = LedgerHello::for_grant(&expected); + hello.schema_hash ^= 1; + assert!(matches!( + expected.validate_hello(&hello), + Err(LedgerHandshakeError::SchemaHash { .. }) + )); +} + +#[test] +fn malformed_grants_and_oversized_failures_are_rejected() { + assert_eq!( + LedgerChannelGrant::new(LedgerGeneration::new([0; 16]), 1, LedgerClientRole::Reader).unwrap_err(), + LedgerProtocolError::ZeroGeneration + ); + assert_eq!( + LedgerChannelGrant::new(GENERATION, 0, LedgerClientRole::Reader).unwrap_err(), + LedgerProtocolError::ZeroClientId + ); + assert_eq!( + LedgerFailure::new(LedgerFailureCode::Storage, "x".repeat(MAX_LEDGER_ERROR_BYTES + 1)).unwrap_err(), + LedgerProtocolError::ErrorMessageTooLong + ); +} + +#[test] +fn response_round_trip_keeps_request_correlation_and_named_outcome() { + let response = LedgerResponse::success(73, "durable".to_owned()).unwrap(); + let encoded = rmp_serde::to_vec_named(&response).unwrap(); + let decoded: LedgerResponse = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!(decoded, response); + assert!(String::from_utf8_lossy(&encoded).contains("success")); +} diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 43373a846..d3a40e803 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -18,6 +18,7 @@ pub mod gateway_grant; pub mod handshake; pub mod host_session; pub mod ipc; +pub mod ledger; pub mod ledger_counters; pub mod mcp; pub mod mcp_aggregator; From 673ac8c74f928881b6e78b2b8cb1d8bf6f92dd8b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:33:30 +0000 Subject: [PATCH 093/464] feat(logger): define typed ledger operations --- Cargo.lock | 2 + crates/capsem-logger/Cargo.toml | 3 + crates/capsem-logger/src/db/bodies.rs | 4 +- crates/capsem-logger/src/events.rs | 6 + crates/capsem-logger/src/events/host.rs | 1 + crates/capsem-logger/src/events/network.rs | 11 +- crates/capsem-logger/src/events/transport.rs | 7 +- crates/capsem-logger/src/ledger_protocol.rs | 312 ++++++++++++++++++ .../src/ledger_protocol/tests.rs | 148 +++++++++ crates/capsem-logger/src/lib.rs | 2 + crates/capsem-logger/src/wire_bytes.rs | 23 ++ crates/capsem-logger/src/writer.rs | 3 +- crates/capsem-logger/src/writer/retention.rs | 3 +- 13 files changed, 515 insertions(+), 10 deletions(-) create mode 100644 crates/capsem-logger/src/ledger_protocol.rs create mode 100644 crates/capsem-logger/src/ledger_protocol/tests.rs create mode 100644 crates/capsem-logger/src/wire_bytes.rs diff --git a/Cargo.lock b/Cargo.lock index aa0c573a1..3f37beedd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1054,8 +1054,10 @@ dependencies = [ "memmap2", "metrics", "metrics-util", + "rmp-serde", "rusqlite", "serde", + "serde_bytes", "serde_json", "tempfile", "tokio", diff --git a/crates/capsem-logger/Cargo.toml b/crates/capsem-logger/Cargo.toml index c87fed9ab..c458da981 100644 --- a/crates/capsem-logger/Cargo.toml +++ b/crates/capsem-logger/Cargo.toml @@ -21,7 +21,10 @@ humantime = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } serde = { workspace = true } +serde_bytes = { workspace = true } serde_json = { workspace = true } +rmp-serde = { workspace = true } +blake3 = "1" uuid = { version = "1", features = ["v4"] } metrics = "0.24" tempfile = "3" diff --git a/crates/capsem-logger/src/db/bodies.rs b/crates/capsem-logger/src/db/bodies.rs index 8002b6720..76535554f 100644 --- a/crates/capsem-logger/src/db/bodies.rs +++ b/crates/capsem-logger/src/db/bodies.rs @@ -12,13 +12,15 @@ use capsem_archive::{ArchiveError, BlockExtent, BodyLogReader, BodyRef}; use capsem_foundation::unix::contained::ContainedDir; use capsem_foundation::unix::lock::{self, LockMode}; use rusqlite::{Connection, Row}; +use serde::{Deserialize, Serialize}; use serde_json::Value; use super::{DbHandle, DbResult}; use crate::writer::RetainOutcome; /// Which side of an exchange a stored body is. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum BodyDirection { Request, Response, diff --git a/crates/capsem-logger/src/events.rs b/crates/capsem-logger/src/events.rs index a2dc9ba4a..8ea18616c 100644 --- a/crates/capsem-logger/src/events.rs +++ b/crates/capsem-logger/src/events.rs @@ -626,8 +626,10 @@ pub struct NetEvent { /// display preview from it at insert and stages the bytes into the /// session archive; nothing else carries a second copy. #[serde(default)] + #[serde(with = "crate::wire_bytes::option")] pub request_body: Option>, #[serde(default)] + #[serde(with = "crate::wire_bytes::option")] pub response_body: Option>, pub conn_type: Option, #[serde(default)] @@ -752,6 +754,7 @@ pub struct ModelCall { pub request_bytes: u64, /// The request body as it was captured, once; see `NetEvent::request_body`. #[serde(default)] + #[serde(with = "crate::wire_bytes::option")] pub request_body: Option>, // Response metadata pub message_id: Option, @@ -759,6 +762,7 @@ pub struct ModelCall { pub text_content: Option, pub thinking_content: Option, #[serde(default)] + #[serde(with = "crate::wire_bytes::option")] pub response_body: Option>, pub stop_reason: Option, pub input_tokens: Option, @@ -810,7 +814,9 @@ pub struct ExecEventComplete { pub duration_ms: u64, /// Each lane's leading bytes exactly as the guest wrote them, not /// necessarily UTF-8. The archive stores at most `MAX_BODY_BLOB_BYTES`. + #[serde(with = "serde_bytes")] pub stdout: Vec, + #[serde(with = "serde_bytes")] pub stderr: Vec, /// What the guest wrote per lane. More than the lane's bytes means /// capture or storage cut it, and the archived body says so. diff --git a/crates/capsem-logger/src/events/host.rs b/crates/capsem-logger/src/events/host.rs index 6c3016fc6..113583daf 100644 --- a/crates/capsem-logger/src/events/host.rs +++ b/crates/capsem-logger/src/events/host.rs @@ -87,6 +87,7 @@ pub struct HostEvent { pub session_id: Option, pub actor: String, #[serde(default)] + #[serde(with = "serde_bytes")] pub detail: Vec, #[serde(default)] pub trace_id: Option, diff --git a/crates/capsem-logger/src/events/network.rs b/crates/capsem-logger/src/events/network.rs index ca714e789..2255559cf 100644 --- a/crates/capsem-logger/src/events/network.rs +++ b/crates/capsem-logger/src/events/network.rs @@ -3,13 +3,15 @@ //! Validated at construction so a malformed record never reaches SQLite: the //! registry that owns these is the sole authority, and a CHECK failure inside //! a batched write would be a producer bug reported far from its cause. +use serde::{Deserialize, Serialize}; use std::net::Ipv4Addr; use uuid::Uuid; pub const NETWORK_NAME_MAX: usize = 64; pub const VM_ID_MAX: usize = 128; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum NetworkState { Active, Retired, @@ -24,7 +26,8 @@ impl NetworkState { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum MembershipState { Declared, Attaching, @@ -71,7 +74,7 @@ impl MembershipState { /// One named network. Written once at creation and again when retired; /// the row is upserted by id, so a later write is a state change, never a /// second network. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct NetworkRecord { pub(crate) id: String, pub(crate) name: String, @@ -134,7 +137,7 @@ impl NetworkRecord { /// One VM's membership of one network, keyed by both. Written on every state /// change; the row is upserted, so the table holds the current state of each /// membership and the ledger holds its history. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct NetworkMembership { pub(crate) network_id: String, pub(crate) vm_id: String, diff --git a/crates/capsem-logger/src/events/transport.rs b/crates/capsem-logger/src/events/transport.rs index 9d7b6482c..42fafb186 100644 --- a/crates/capsem-logger/src/events/transport.rs +++ b/crates/capsem-logger/src/events/transport.rs @@ -1,8 +1,9 @@ //! Bounded primary routing records. Core owns policy facts and log sanitization. -use serde::Serialize; +use serde::{Deserialize, Serialize}; use uuid::Uuid; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] pub enum TransportEventKind { Connect, ConnectResult, @@ -25,7 +26,7 @@ impl TransportEventKind { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct TransportEvent { pub(crate) event_id: String, pub(crate) timestamp_unix_ms: i64, diff --git a/crates/capsem-logger/src/ledger_protocol.rs b/crates/capsem-logger/src/ledger_protocol.rs new file mode 100644 index 000000000..012d21535 --- /dev/null +++ b/crates/capsem-logger/src/ledger_protocol.rs @@ -0,0 +1,312 @@ +//! Transport-independent operations and replies for one session ledger. + +use std::collections::BTreeMap; + +use capsem_foundation::ipc_channel::MAX_IPC_FRAME_SIZE; +use capsem_proto::ledger::{LedgerCapability, LedgerOperation, LedgerProtocolError}; +use capsem_proto::ledger_counters::LedgerCounters; +use serde::{Deserialize, Serialize}; + +use crate::{BodyDirection, RetainOutcome, WriteOp}; + +/// Leave room inside the 16 MiB frame for request map keys and ids. +pub const MAX_LEDGER_OPERATION_BYTES: usize = MAX_IPC_FRAME_SIZE as usize - 4 * 1024; +/// Every large result streams in independently bounded pieces. +pub const MAX_LEDGER_STREAM_CHUNK_BYTES: usize = 256 * 1024; +pub const MAX_LEDGER_QUERY_ROWS: usize = 2_000; +const MAX_QUERY_TEXT_BYTES: usize = 4 * 1024; +const MAX_LEDGER_COLUMNS: usize = 128; +const MAX_LEDGER_COLUMN_BYTES: usize = 128; +const MAX_LEDGER_CELL_BYTES: usize = 1024 * 1024; +const MAX_LEDGER_RESULT_SETS: usize = 16; +const MAX_EVENT_ID_BYTES: usize = 12; +const MAX_TIMESTAMP_BYTES: usize = 64; + +pub type LedgerResponse = capsem_proto::ledger::LedgerResponse; + +/// Every operation accepted by the ledger worker. None selects a session or +/// filesystem path; the authenticated descriptor already selects both. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerCommand { + Admit { + event: Box, + }, + /// Make every accepted event through the returned admission id durable. + Flush, + Query { + query: LedgerQuery, + }, + ReadBodies { + event_id: String, + }, + Counters, + Retain { + cutoff: String, + }, + /// Stream a WARC as bounded `WarcChunk` replies followed by `WarcComplete`. + ExportWarc, + Shutdown, +} + +impl LedgerOperation for LedgerCommand { + fn capability(&self) -> LedgerCapability { + match self { + Self::Admit { .. } => LedgerCapability::Admit, + Self::Flush => LedgerCapability::Flush, + Self::Query { .. } | Self::ReadBodies { .. } | Self::Counters => LedgerCapability::Read, + Self::Retain { .. } => LedgerCapability::Retain, + Self::ExportWarc => LedgerCapability::Export, + Self::Shutdown => LedgerCapability::Shutdown, + } + } + + fn validate(&self) -> Result<(), LedgerProtocolError> { + match self { + Self::Query { query } => query.validate()?, + Self::ReadBodies { event_id } if !valid_event_id(event_id) => { + return Err(LedgerProtocolError::InvalidOperation) + } + Self::Retain { cutoff } + if cutoff.is_empty() || cutoff.len() > MAX_TIMESTAMP_BYTES || cutoff.chars().any(char::is_control) => + { + return Err(LedgerProtocolError::InvalidOperation) + } + _ => {} + } + let encoded = rmp_serde::to_vec_named(self).map_err(|_| LedgerProtocolError::InvalidOperation)?; + if encoded.len() > MAX_LEDGER_OPERATION_BYTES { + return Err(LedgerProtocolError::OperationTooLarge); + } + Ok(()) + } +} + +fn valid_event_id(value: &str) -> bool { + value.len() == MAX_EVENT_ID_BYTES + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +/// Named read intents. SQL stays inside `capsem-logger`; clients can select +/// filters and bounds, but cannot submit statements. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerQuery { + SecurityLatest { + limit: u16, + detection_only: bool, + }, + Timeline { + layers: Vec, + cutoff: String, + trace_id: Option, + limit: u16, + }, + History { + layers: Vec, + search: Option, + limit: u16, + offset: u64, + }, + StatsDetail, + Triage { + limit: u16, + }, +} + +impl LedgerQuery { + pub fn validate(&self) -> Result<(), LedgerProtocolError> { + let valid_limit = |limit: u16| limit > 0 && usize::from(limit) <= MAX_LEDGER_QUERY_ROWS; + let bounded_text = |value: &str| value.len() <= MAX_QUERY_TEXT_BYTES; + let valid = match self { + Self::SecurityLatest { limit, .. } | Self::Triage { limit } => valid_limit(*limit), + Self::Timeline { + layers, + cutoff, + trace_id, + limit, + } => { + !layers.is_empty() + && valid_limit(*limit) + && cutoff.len() <= MAX_TIMESTAMP_BYTES + && !cutoff.chars().any(char::is_control) + && trace_id.as_deref().is_none_or(bounded_text) + } + Self::History { + layers, search, limit, .. + } => !layers.is_empty() && valid_limit(*limit) && search.as_deref().is_none_or(bounded_text), + Self::StatsDetail => true, + }; + if valid { + Ok(()) + } else { + Err(LedgerProtocolError::InvalidOperation) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerTimelineLayer { + Exec, + Tool, + Net, + File, + Model, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerHistoryLayer { + Exec, + Audit, +} + +/// SQLite values leave the worker as an explicit Rust value, not JSON text. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerValue { + Null, + Integer(i64), + Real(f64), + Text(String), + Blob(#[serde(with = "serde_bytes")] Vec), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct LedgerRows { + pub columns: Vec, + pub rows: Vec>, +} + +impl LedgerRows { + pub fn validate(&self) -> Result<(), LedgerProtocolError> { + let columns = self.columns.len(); + let valid = columns <= MAX_LEDGER_COLUMNS + && self.rows.len() <= MAX_LEDGER_QUERY_ROWS + && self + .columns + .iter() + .all(|column| !column.is_empty() && column.len() <= MAX_LEDGER_COLUMN_BYTES) + && self.rows.iter().all(|row| { + row.len() == columns + && row.iter().all(|value| match value { + LedgerValue::Text(value) => value.len() <= MAX_LEDGER_CELL_BYTES, + LedgerValue::Blob(value) => value.len() <= MAX_LEDGER_CELL_BYTES, + LedgerValue::Null | LedgerValue::Integer(_) | LedgerValue::Real(_) => true, + }) + }); + if valid { + Ok(()) + } else { + Err(LedgerProtocolError::InvalidOperation) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerBodyMetadata { + pub event_id: String, + pub source_table: String, + pub direction: BodyDirection, + pub content_type: Option, + pub original_bytes: u64, + pub stored_bytes: u64, + pub truncated: bool, + pub body_hash: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerExportSummary { + pub records: u64, + pub bytes_written: u64, + pub skipped_count: u64, + pub skipped_by_reason: BTreeMap, +} + +/// A request may yield several replies with its same request id. Body and +/// WARC bytes are always binary chunks below the frame limit. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerReply { + Accepted { + admission_id: u64, + }, + Durable { + through_admission_id: u64, + }, + Query { + sets: Vec, + }, + Counters { + counters: Box, + }, + BodyStart { + body_id: u32, + metadata: LedgerBodyMetadata, + }, + BodyChunk { + body_id: u32, + offset: u64, + #[serde(with = "serde_bytes")] + bytes: Vec, + }, + BodiesComplete, + Retained { + outcome: RetainOutcome, + }, + WarcChunk { + offset: u64, + #[serde(with = "serde_bytes")] + bytes: Vec, + }, + WarcComplete { + summary: LedgerExportSummary, + }, + ShuttingDown, +} + +impl LedgerReply { + pub fn validate(&self) -> Result<(), LedgerProtocolError> { + let valid = match self { + Self::Accepted { admission_id } => *admission_id != 0, + Self::Durable { .. } | Self::Counters { .. } | Self::BodiesComplete | Self::Retained { .. } => true, + Self::Query { sets } => { + sets.len() <= MAX_LEDGER_RESULT_SETS && sets.iter().all(|rows| rows.validate().is_ok()) + } + Self::BodyStart { body_id, metadata } => { + *body_id != 0 + && valid_event_id(&metadata.event_id) + && !metadata.source_table.is_empty() + && metadata.source_table.len() <= MAX_LEDGER_COLUMN_BYTES + && metadata + .content_type + .as_deref() + .is_none_or(|value| value.len() <= MAX_LEDGER_COLUMN_BYTES) + && metadata.body_hash.len() <= MAX_LEDGER_COLUMN_BYTES + } + Self::BodyChunk { body_id, bytes, .. } => { + *body_id != 0 && !bytes.is_empty() && bytes.len() <= MAX_LEDGER_STREAM_CHUNK_BYTES + } + Self::WarcChunk { bytes, .. } => !bytes.is_empty() && bytes.len() <= MAX_LEDGER_STREAM_CHUNK_BYTES, + Self::WarcComplete { summary } => { + summary.skipped_by_reason.len() <= 32 + && summary + .skipped_by_reason + .keys() + .all(|reason| !reason.is_empty() && reason.len() <= MAX_LEDGER_COLUMN_BYTES) + } + Self::ShuttingDown => true, + }; + if valid { + Ok(()) + } else { + Err(LedgerProtocolError::InvalidOperation) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-logger/src/ledger_protocol/tests.rs b/crates/capsem-logger/src/ledger_protocol/tests.rs new file mode 100644 index 000000000..a34f41d7c --- /dev/null +++ b/crates/capsem-logger/src/ledger_protocol/tests.rs @@ -0,0 +1,148 @@ +use std::os::unix::net::UnixStream; +use std::time::SystemTime; + +use capsem_foundation::ipc_channel; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerRequest}; + +use super::*; +use crate::{ExecEventComplete, FileAction, FileEvent, FileKind, WriteOp}; + +fn grant(role: LedgerClientRole) -> LedgerChannelGrant { + LedgerChannelGrant::new(LedgerGeneration::new([3; 16]), 9, role).unwrap() +} + +fn file_write() -> LedgerCommand { + LedgerCommand::Admit { + event: Box::new(WriteOp::FileEvent(FileEvent { + event_id: None, + timestamp: SystemTime::UNIX_EPOCH, + action: FileAction::Modified, + path: "/root/report.txt".into(), + size: Some(12), + kind: FileKind::File, + trace_id: None, + credential_ref: None, + })), + } +} + +#[tokio::test] +async fn request_round_trips_over_the_landed_bounded_channel() { + let request = LedgerRequest::new(1, file_write()).unwrap(); + request.authorize(&grant(LedgerClientRole::VmOwner)).unwrap(); + let (left, right) = UnixStream::pair().unwrap(); + let (sender, _) = ipc_channel::channel_from_std::, LedgerResponse>(left).unwrap(); + let (_, receiver) = ipc_channel::channel_from_std::>(right).unwrap(); + sender.send(request).await.unwrap(); + let decoded = receiver.recv().await.unwrap(); + assert_eq!(decoded.request_id(), 1); + assert!(matches!( + decoded.operation(), + LedgerCommand::Admit { + event + } if matches!(event.as_ref(), WriteOp::FileEvent(event) if event.path == "/root/report.txt") + )); +} + +#[test] +fn roles_admit_only_their_operation_classes() { + for (role, operation) in [ + (LedgerClientRole::VmOwner, file_write()), + (LedgerClientRole::Reader, LedgerCommand::Counters), + ( + LedgerClientRole::Maintainer, + LedgerCommand::Retain { + cutoff: "2026-10-09T00:00:00.000000Z".into(), + }, + ), + (LedgerClientRole::Reader, LedgerCommand::ExportWarc), + (LedgerClientRole::Supervisor, LedgerCommand::Shutdown), + ] { + LedgerRequest::new(1, operation) + .unwrap() + .authorize(&grant(role)) + .unwrap(); + } +} + +#[test] +fn oversized_event_and_malformed_query_are_rejected_before_dispatch() { + let huge = LedgerCommand::Admit { + event: Box::new(WriteOp::ExecEventComplete(ExecEventComplete { + exec_id: 1, + exit_code: 0, + duration_ms: 1, + stdout: vec![b'x'; MAX_LEDGER_OPERATION_BYTES], + stderr: Vec::new(), + stdout_bytes: MAX_LEDGER_OPERATION_BYTES as u64, + stderr_bytes: 0, + pid: None, + })), + }; + assert_eq!( + LedgerRequest::new(1, huge) + .unwrap() + .authorize(&grant(LedgerClientRole::VmOwner)) + .unwrap_err(), + capsem_proto::ledger::LedgerProtocolError::OperationTooLarge + ); + let empty_timeline = LedgerCommand::Query { + query: LedgerQuery::Timeline { + layers: Vec::new(), + cutoff: String::new(), + trace_id: None, + limit: 10, + }, + }; + assert_eq!( + LedgerRequest::new(2, empty_timeline) + .unwrap() + .authorize(&grant(LedgerClientRole::Reader)) + .unwrap_err(), + capsem_proto::ledger::LedgerProtocolError::InvalidOperation + ); +} + +#[test] +fn event_bytes_use_messagepack_binary_inside_the_request() { + let command = LedgerCommand::Admit { + event: Box::new(WriteOp::ExecEventComplete(ExecEventComplete { + exec_id: 1, + exit_code: 0, + duration_ms: 1, + stdout: vec![0xa5; 1024 * 1024], + stderr: Vec::new(), + stdout_bytes: 1024 * 1024, + stderr_bytes: 0, + pid: None, + })), + }; + let encoded = rmp_serde::to_vec_named(&LedgerRequest::new(1, command).unwrap()).unwrap(); + assert!( + encoded.len() < 1024 * 1024 + 1024, + "binary payload expanded to {}", + encoded.len() + ); +} + +#[test] +fn streamed_replies_enforce_chunk_and_row_bounds() { + assert!(LedgerReply::WarcChunk { + offset: 0, + bytes: vec![0; MAX_LEDGER_STREAM_CHUNK_BYTES], + } + .validate() + .is_ok()); + assert!(LedgerReply::WarcChunk { + offset: 0, + bytes: vec![0; MAX_LEDGER_STREAM_CHUNK_BYTES + 1], + } + .validate() + .is_err()); + assert!(LedgerRows { + columns: vec!["one".into()], + rows: vec![vec![]], + } + .validate() + .is_err()); +} diff --git a/crates/capsem-logger/src/lib.rs b/crates/capsem-logger/src/lib.rs index 833416b8f..ade870d9a 100644 --- a/crates/capsem-logger/src/lib.rs +++ b/crates/capsem-logger/src/lib.rs @@ -1,10 +1,12 @@ pub mod counters; pub mod db; pub mod events; +pub mod ledger_protocol; pub mod network_db; pub mod reader; pub mod schema; pub mod session_types; +mod wire_bytes; pub mod writer; pub use db::{ diff --git a/crates/capsem-logger/src/wire_bytes.rs b/crates/capsem-logger/src/wire_bytes.rs new file mode 100644 index 000000000..f6cc06e71 --- /dev/null +++ b/crates/capsem-logger/src/wire_bytes.rs @@ -0,0 +1,23 @@ +//! MessagePack binary encoding for optional byte vectors. + +pub(crate) mod option { + use serde::{Deserialize, Deserializer, Serializer}; + use serde_bytes::ByteBuf; + + pub fn serialize(value: &Option>, serializer: S) -> Result + where + S: Serializer, + { + match value { + Some(bytes) => serializer.serialize_some(serde_bytes::Bytes::new(bytes)), + None => serializer.serialize_none(), + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: Deserializer<'de>, + { + Option::::deserialize(deserializer).map(|bytes| bytes.map(ByteBuf::into_vec)) + } +} diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 7266c3607..9470a4624 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -6,6 +6,7 @@ use std::time::{Duration, Instant, SystemTime}; use capsem_telemetry::db::{DB_MEMORY_UNFLUSHED_OPS, DB_SHUTDOWN_FLUSH_MS}; use rusqlite::{params, Connection, ErrorCode, OpenFlags, OptionalExtension}; +use serde::{Deserialize, Serialize}; use tracing::{error, warn}; use crate::counters::{self, LedgerCounters, LedgerTally}; @@ -206,7 +207,7 @@ fn blake3_bytes_ref(value: &[u8]) -> String { } /// Typed write operations sent to the writer thread. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub enum WriteOp { TransportEvent(TransportEvent), NetEvent(NetEvent), diff --git a/crates/capsem-logger/src/writer/retention.rs b/crates/capsem-logger/src/writer/retention.rs index f398b2f7e..e7d4751f9 100644 --- a/crates/capsem-logger/src/writer/retention.rs +++ b/crates/capsem-logger/src/writer/retention.rs @@ -14,12 +14,13 @@ use capsem_archive::{format, BodyLogWriter, FileHeader, GenerationId, FILE_HEADE use capsem_foundation::unix::contained::{ContainedDir, ContainedOpenOptions}; use capsem_foundation::unix::lock::{self, LockMode}; use rusqlite::{params, Connection, OptionalExtension, Transaction}; +use serde::{Deserialize, Serialize}; use tracing::warn; use super::bodies::{archive_lock_path_for_db, BodyArchive}; use super::retention_faults::{take_retention_failure_for_tests, RetentionFault}; -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct RetainOutcome { pub blocks_dropped: u64, pub blocks_kept: u64, From 6d4d2d0e93548783d8102ea1ad030b5d88b6367f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:45:30 +0000 Subject: [PATCH 094/464] feat(logger): serve typed ledger operations --- crates/capsem-logger/src/ledger_protocol.rs | 18 +- crates/capsem-logger/src/ledger_server.rs | 310 ++++++++++++++++++ .../src/ledger_server/queries.rs | 230 +++++++++++++ .../capsem-logger/src/ledger_server/tests.rs | 229 +++++++++++++ crates/capsem-logger/src/lib.rs | 1 + 5 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 crates/capsem-logger/src/ledger_server.rs create mode 100644 crates/capsem-logger/src/ledger_server/queries.rs create mode 100644 crates/capsem-logger/src/ledger_server/tests.rs diff --git a/crates/capsem-logger/src/ledger_protocol.rs b/crates/capsem-logger/src/ledger_protocol.rs index 012d21535..7c2401734 100644 --- a/crates/capsem-logger/src/ledger_protocol.rs +++ b/crates/capsem-logger/src/ledger_protocol.rs @@ -3,7 +3,9 @@ use std::collections::BTreeMap; use capsem_foundation::ipc_channel::MAX_IPC_FRAME_SIZE; -use capsem_proto::ledger::{LedgerCapability, LedgerOperation, LedgerProtocolError}; +use capsem_proto::ledger::{ + LedgerCapability, LedgerHello, LedgerOperation, LedgerProtocolError, LedgerRequest, LedgerWelcome, +}; use capsem_proto::ledger_counters::LedgerCounters; use serde::{Deserialize, Serialize}; @@ -24,6 +26,20 @@ const MAX_TIMESTAMP_BYTES: usize = 64; pub type LedgerResponse = capsem_proto::ledger::LedgerResponse; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerClientMessage { + Hello { hello: LedgerHello }, + Request { request: LedgerRequest }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LedgerServerMessage { + Welcome { welcome: LedgerWelcome }, + Response { response: LedgerResponse }, +} + /// Every operation accepted by the ledger worker. None selects a session or /// filesystem path; the authenticated descriptor already selects both. #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs new file mode 100644 index 000000000..0a39943df --- /dev/null +++ b/crates/capsem-logger/src/ledger_server.rs @@ -0,0 +1,310 @@ +//! Storage-owning dispatch for one session ledger process. + +use std::io; +use std::os::unix::net::UnixStream; +use std::path::Path; +use std::sync::Arc; + +use capsem_foundation::ipc_channel; +use capsem_proto::ledger::{ + LedgerChannelGrant, LedgerFailure, LedgerFailureCode, LedgerRequest, LedgerResponse, LedgerWelcome, +}; + +use crate::ledger_protocol::{ + LedgerBodyMetadata, LedgerClientMessage, LedgerCommand, LedgerExportSummary, LedgerReply, LedgerServerMessage, + MAX_LEDGER_STREAM_CHUNK_BYTES, +}; +use crate::{DbHandle, StoredBody}; + +mod queries; + +type ServerSender = ipc_channel::Sender; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LedgerClientExit { + Disconnected, + ShutdownRequested, +} + +#[derive(Default)] +struct AdmissionState { + last_accepted: u64, +} + +/// One process-owned session database shared by its authenticated clients. +pub struct LedgerServer { + db: Arc, + admission: tokio::sync::Mutex, +} + +impl LedgerServer { + pub fn open(path: &Path) -> rusqlite::Result { + Ok(Self { + db: Arc::new(DbHandle::open(path)?), + admission: tokio::sync::Mutex::new(AdmissionState::default()), + }) + } + + /// Serve one descriptor whose expected authority came from the trusted + /// coordinator. Other clients may call this concurrently on the same + /// server; SQLite and archive ownership remain in this process. + pub async fn serve_client( + &self, + stream: UnixStream, + grant: LedgerChannelGrant, + ) -> Result { + let (sender, receiver) = + ipc_channel::channel_from_std::(stream).map_err(io_string)?; + let first = receiver.recv().await.map_err(io_string)?; + let LedgerClientMessage::Hello { hello } = first else { + return Err("ledger client sent an operation before hello".into()); + }; + grant.validate_hello(&hello).map_err(|error| error.to_string())?; + sender + .send(LedgerServerMessage::Welcome { + welcome: LedgerWelcome::for_grant(&grant), + }) + .await + .map_err(io_string)?; + + loop { + let message = match receiver.recv().await { + Ok(message) => message, + Err(error) if disconnected(&error) => return Ok(LedgerClientExit::Disconnected), + Err(error) => return Err(error.to_string()), + }; + let LedgerClientMessage::Request { request } = message else { + return Err("ledger client repeated hello".into()); + }; + let request_id = request.request_id(); + if let Err(error) = request.authorize(&grant) { + let code = if matches!(error, capsem_proto::ledger::LedgerProtocolError::Unauthorized { .. }) { + LedgerFailureCode::UnauthorizedOperation + } else { + LedgerFailureCode::InvalidRequest + }; + send_failure(&sender, request_id, code, error.to_string()).await?; + continue; + } + if self.dispatch(&sender, request).await? { + return Ok(LedgerClientExit::ShutdownRequested); + } + } + } + + async fn dispatch(&self, sender: &ServerSender, request: LedgerRequest) -> Result { + let request_id = request.request_id(); + let result = match request.into_operation() { + LedgerCommand::Admit { event } => { + let admission_id = { + let mut admission = self.admission.lock().await; + match self.db.write(*event).await { + Ok(()) => { + admission.last_accepted = admission + .last_accepted + .checked_add(1) + .ok_or_else(|| "ledger admission id exhausted".to_string())?; + Ok(admission.last_accepted) + } + Err(error) => Err(storage(error)), + } + }; + match admission_id { + Ok(admission_id) => send_success(sender, request_id, LedgerReply::Accepted { admission_id }).await, + Err(error) => Err(error), + } + } + LedgerCommand::Flush => { + // The lock orders the writer enqueue and this barrier against + // every client, so the returned id cannot name a later write. + let through_admission_id = { + let admission = self.admission.lock().await; + match self.db.flush().await { + Ok(()) => Ok(admission.last_accepted), + Err(error) => Err(storage(error)), + } + }; + match through_admission_id { + Ok(through_admission_id) => { + send_success(sender, request_id, LedgerReply::Durable { through_admission_id }).await + } + Err(error) => Err(error), + } + } + LedgerCommand::Query { query } => match queries::execute(&self.db, query).await { + Ok(sets) => send_success(sender, request_id, LedgerReply::Query { sets }).await, + Err(error) => Err(storage(error)), + }, + LedgerCommand::ReadBodies { event_id } => match self.db.read_bodies(&event_id).await { + Ok(bodies) => self.send_bodies(sender, request_id, bodies).await, + Err(error) => Err(storage(error)), + }, + LedgerCommand::Counters => match self.db.ledger_counters().await { + Ok(counters) => { + send_success( + sender, + request_id, + LedgerReply::Counters { + counters: Box::new((*counters).clone()), + }, + ) + .await + } + Err(error) => Err(storage(error)), + }, + LedgerCommand::Retain { cutoff } => match self.db.retain_bodies_since(&cutoff).await { + Ok(outcome) => send_success(sender, request_id, LedgerReply::Retained { outcome }).await, + Err(error) => Err(storage(error)), + }, + LedgerCommand::ExportWarc => self.send_warc(sender, request_id).await, + LedgerCommand::Shutdown => { + send_success(sender, request_id, LedgerReply::ShuttingDown).await?; + return Ok(true); + } + }; + if let Err(error) = result { + send_failure(sender, request_id, LedgerFailureCode::Storage, &error).await?; + } + Ok(false) + } + + async fn send_bodies(&self, sender: &ServerSender, request_id: u64, bodies: Vec) -> Result<(), String> { + for (index, body) in bodies.into_iter().enumerate() { + let body_id = u32::try_from(index + 1).map_err(|_| "too many bodies in one ledger reply")?; + let metadata = LedgerBodyMetadata { + event_id: body.event_id, + source_table: body.source_table, + direction: body.direction, + content_type: body.content_type, + original_bytes: body.original_bytes, + stored_bytes: body.bytes.len() as u64, + truncated: body.truncated, + body_hash: body.body_hash, + }; + send_success(sender, request_id, LedgerReply::BodyStart { body_id, metadata }).await?; + let mut offset = 0_u64; + for bytes in body.bytes.chunks(MAX_LEDGER_STREAM_CHUNK_BYTES) { + send_success( + sender, + request_id, + LedgerReply::BodyChunk { + body_id, + offset, + bytes: bytes.to_vec(), + }, + ) + .await?; + offset += bytes.len() as u64; + } + } + send_success(sender, request_id, LedgerReply::BodiesComplete).await + } + + async fn send_warc(&self, sender: &ServerSender, request_id: u64) -> Result<(), String> { + let (chunks, mut receiver) = tokio::sync::mpsc::channel(4); + let db = Arc::clone(&self.db); + let export = tokio::spawn(async move { db.export_warc(WarcChunkWriter { chunks }).await }); + let mut offset = 0_u64; + while let Some(bytes) = receiver.recv().await { + let chunk_len = bytes.len() as u64; + if let Err(error) = send_success(sender, request_id, LedgerReply::WarcChunk { offset, bytes }).await { + drop(receiver); + let _ = export.await; + return Err(error); + } + offset += chunk_len; + } + let summary = export + .await + .map_err(|error| format!("WARC export task failed: {error}"))? + .map_err(storage)?; + let skipped_by_reason = summary + .counts_by_reason() + .into_iter() + .map(|(reason, count)| (reason.to_string(), count)) + .collect(); + send_success( + sender, + request_id, + LedgerReply::WarcComplete { + summary: LedgerExportSummary { + records: summary.records, + bytes_written: summary.bytes_written, + skipped_count: summary.skipped_count, + skipped_by_reason, + }, + }, + ) + .await + } +} + +struct WarcChunkWriter { + chunks: tokio::sync::mpsc::Sender>, +} + +impl io::Write for WarcChunkWriter { + fn write(&mut self, buffer: &[u8]) -> io::Result { + for chunk in buffer.chunks(MAX_LEDGER_STREAM_CHUNK_BYTES) { + self.chunks + .blocking_send(chunk.to_vec()) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "ledger WARC client disconnected"))?; + } + Ok(buffer.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +async fn send_success(sender: &ServerSender, request_id: u64, reply: LedgerReply) -> Result<(), String> { + reply.validate().map_err(|error| error.to_string())?; + let response = LedgerResponse::success(request_id, reply).map_err(|error| error.to_string())?; + sender + .send(LedgerServerMessage::Response { response }) + .await + .map_err(io_string) +} + +async fn send_failure( + sender: &ServerSender, + request_id: u64, + code: LedgerFailureCode, + message: impl AsRef, +) -> Result<(), String> { + let message = message.as_ref(); + let bounded = &message[..floor_char_boundary(message, capsem_proto::ledger::MAX_LEDGER_ERROR_BYTES)]; + let failure = LedgerFailure::new(code, bounded).map_err(|error| error.to_string())?; + let response = LedgerResponse::failure(request_id, failure).map_err(|error| error.to_string())?; + sender + .send(LedgerServerMessage::Response { response }) + .await + .map_err(io_string) +} + +fn floor_char_boundary(value: &str, at: usize) -> usize { + let mut at = at.min(value.len()); + while !value.is_char_boundary(at) { + at -= 1; + } + at +} + +fn disconnected(error: &io::Error) -> bool { + matches!( + error.kind(), + io::ErrorKind::UnexpectedEof | io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset + ) +} + +fn io_string(error: impl std::fmt::Display) -> String { + error.to_string() +} + +fn storage(error: impl std::fmt::Display) -> String { + error.to_string() +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-logger/src/ledger_server/queries.rs b/crates/capsem-logger/src/ledger_server/queries.rs new file mode 100644 index 000000000..1255a16ef --- /dev/null +++ b/crates/capsem-logger/src/ledger_server/queries.rs @@ -0,0 +1,230 @@ +//! Named query intent executed only by the ledger owner. + +use serde::Deserialize; +use serde_json::{json, Value}; + +use crate::ledger_protocol::{LedgerHistoryLayer, LedgerQuery, LedgerRows, LedgerTimelineLayer, LedgerValue}; +use crate::DbHandle; + +type Statement = (String, Vec); + +pub(super) async fn execute(db: &DbHandle, query: LedgerQuery) -> Result, String> { + let statements = statements(query)?; + let raw = db.query_many(statements).await?; + raw.into_iter().map(|rows| decode_rows(&rows)).collect() +} + +fn statements(query: LedgerQuery) -> Result, String> { + match query { + LedgerQuery::SecurityLatest { limit, detection_only } => { + let detection = if detection_only { + "WHERE event.detection_level != 'none'" + } else { + "" + }; + Ok(vec![( + format!( + "SELECT event.timestamp_unix_ms, event.event_id, event.event_type, event.rule_id, \ + event.rule_action, event.detection_level, COALESCE(event.rule_json, run.rule_json) AS rule_json, \ + event.trace_id, event.turn_id, event.credential_ref FROM security_rule_events AS event \ + LEFT JOIN security_rule_runs AS run ON run.id = event.run_id {detection} \ + ORDER BY event.timestamp_unix_ms DESC, event.id DESC LIMIT ?1" + ), + vec![json!(limit)], + )]) + } + LedgerQuery::Timeline { + layers, + cutoff, + trace_id, + limit, + } => { + let tool_cutoff = if cutoff.as_str() <= "1970-01-01T00:00:00Z" { + String::new() + } else { + cutoff.clone() + }; + Ok(vec![( + timeline_sql(&layers), + vec![ + json!(limit), + json!(cutoff), + json!(tool_cutoff), + trace_id.map_or(Value::Null, Value::String), + ], + )]) + } + LedgerQuery::History { + layers, + search, + limit, + offset, + } => history_statements(&layers, search, limit, offset), + LedgerQuery::StatsDetail => Ok(stats_detail_statements()), + LedgerQuery::Triage { limit } => Ok(triage_statements(limit)), + } +} + +const EXEC_WINDOW: &str = "SELECT timestamp, 'exec' AS layer, exec_id AS ref, command AS summary, \ + exit_code AS status, duration_ms, trace_id FROM exec_events WHERE timestamp >= ?2 \ + AND (?4 IS NULL OR trace_id = ?4 OR trace_id IS NULL) ORDER BY timestamp ASC LIMIT ?1"; +const TOOL_WINDOW: &str = "SELECT COALESCE(NULLIF(tc.timestamp, ''), '1970-01-01T00:00:00Z') AS timestamp, \ + 'tool' AS layer, tc.event_id AS ref, COALESCE(tc.server_name, tc.origin) || '/' || tc.tool_name || \ + COALESCE(' (call_id=' || tc.call_id || ')', '') AS summary, tc.decision AS status, tc.duration_ms, tc.trace_id \ + FROM tool_calls tc WHERE +tc.origin IN ('model', 'native', 'mcp', 'builtin', 'local', 'mcp_proxy') \ + AND tc.timestamp >= ?3 AND (?4 IS NULL OR tc.trace_id = ?4 OR tc.trace_id IS NULL) \ + ORDER BY tc.timestamp ASC LIMIT ?1"; +const NET_WINDOW: &str = "SELECT timestamp, 'net' AS layer, id AS ref, COALESCE(method, 'GET') || ' ' || \ + domain || COALESCE(path, '') AS summary, status_code AS status, duration_ms, trace_id FROM net_events \ + WHERE timestamp >= ?2 AND (?4 IS NULL OR trace_id = ?4 OR trace_id IS NULL) \ + ORDER BY timestamp ASC LIMIT ?1"; +const FILE_WINDOW: &str = "SELECT timestamp, 'fs' AS layer, id AS ref, action || ' ' || path AS summary, \ + NULL AS status, NULL AS duration_ms, trace_id FROM fs_events WHERE timestamp >= ?2 \ + AND (?4 IS NULL OR trace_id = ?4 OR trace_id IS NULL) ORDER BY timestamp ASC LIMIT ?1"; +const MODEL_WINDOW: &str = "SELECT timestamp, 'model' AS layer, id AS ref, provider || '/' || COALESCE(model, '?') \ + AS summary, status_code AS status, duration_ms, trace_id FROM model_calls WHERE timestamp >= ?2 \ + AND (?4 IS NULL OR trace_id = ?4 OR trace_id IS NULL) ORDER BY timestamp ASC LIMIT ?1"; + +fn timeline_sql(layers: &[LedgerTimelineLayer]) -> String { + let windows = layers + .iter() + .map(|layer| match layer { + LedgerTimelineLayer::Exec => EXEC_WINDOW, + LedgerTimelineLayer::Tool => TOOL_WINDOW, + LedgerTimelineLayer::Net => NET_WINDOW, + LedgerTimelineLayer::File => FILE_WINDOW, + LedgerTimelineLayer::Model => MODEL_WINDOW, + }) + .map(|window| format!("SELECT * FROM ({window})")) + .collect::>(); + format!("{} ORDER BY timestamp ASC LIMIT ?1", windows.join(" UNION ALL ")) +} + +const EXEC_HISTORY: &str = "SELECT timestamp, 'exec' AS layer, command, exit_code, duration_ms, stdout_preview, \ + stderr_preview, json_object('source', source, 'trace_id', trace_id, 'process_name', process_name, \ + 'exec_id', exec_id) AS details FROM exec_events"; +const AUDIT_HISTORY: &str = "SELECT timestamp, 'audit' AS layer, argv AS command, exit_code, NULL AS duration_ms, \ + NULL AS stdout_preview, NULL AS stderr_preview, json_object('pid', pid, 'ppid', ppid, 'uid', uid, 'exe', exe, \ + 'comm', comm, 'cwd', cwd, 'tty', tty, 'session_id', session_id, 'audit_id', audit_id, 'parent_exe', parent_exe) \ + AS details FROM audit_events"; +const HISTORY_SEARCH: &str = " WHERE instr(command, ?1) > 0 OR instr(stdout_preview, ?1) > 0 \ + OR instr(stderr_preview, ?1) > 0 OR instr(details, ?1) > 0"; + +fn history_statements( + layers: &[LedgerHistoryLayer], + search: Option, + limit: u16, + offset: u64, +) -> Result, String> { + let has_search = search.is_some(); + let arms = layers + .iter() + .map(|layer| match layer { + LedgerHistoryLayer::Exec => EXEC_HISTORY, + LedgerHistoryLayer::Audit => AUDIT_HISTORY, + }) + .collect::>(); + let filtered = |rows: &str| format!("SELECT * FROM ({rows}){}", if has_search { HISTORY_SEARCH } else { "" }); + let page = arms + .iter() + .map(|rows| format!("SELECT * FROM ({} ORDER BY timestamp DESC LIMIT ?2)", filtered(rows))) + .collect::>() + .join(" UNION ALL "); + let params = vec![ + search.clone().map_or(Value::Null, Value::String), + json!(offset.saturating_add(u64::from(limit))), + json!(limit), + json!(offset), + ]; + let mut statements = vec![(format!("{page} ORDER BY timestamp DESC LIMIT ?3 OFFSET ?4"), params)]; + if let Some(search) = search { + let total = arms + .iter() + .map(|rows| filtered(rows)) + .collect::>() + .join(" UNION ALL "); + statements.push((format!("SELECT COUNT(*) AS total FROM ({total})"), vec![json!(search)])); + } + Ok(statements) +} + +const BODY_BLOBS: &str = "SELECT event_id, source_table, direction, content_type, original_bytes, stored_bytes, \ + truncated, body_hash FROM event_body_blobs WHERE source_table NOT IN ('security_decision_events', \ + 'security_ask_events') AND (event_id IN (SELECT event_id FROM net_events WHERE event_id IS NOT NULL ORDER BY id \ + DESC LIMIT 200) OR event_id IN (SELECT event_id FROM model_calls WHERE event_id IS NOT NULL ORDER BY id DESC \ + LIMIT 200) OR event_id IN (SELECT event_id FROM tool_calls WHERE event_id IS NOT NULL ORDER BY id DESC LIMIT 200) \ + OR event_id IN (SELECT event_id FROM tool_responses WHERE call_id IN (SELECT call_id FROM tool_calls WHERE event_id \ + IS NOT NULL ORDER BY id DESC LIMIT 200)) OR event_id IN (SELECT event_id FROM exec_events ORDER BY id DESC LIMIT \ + ?1) OR event_id IN (SELECT event_id FROM security_rule_events ORDER BY id DESC LIMIT 200)) ORDER BY event_id, direction"; +const MODEL_ITEMS: &str = "SELECT mi.event_id, mi.timestamp, mi.model_call_id, mc.event_id AS model_event_id, \ + mi.trace_id, mi.turn_id, mi.item_index, mi.kind, mi.call_id, substr(mi.content, 1, ?1) AS content, \ + COALESCE(length(mi.content) > ?1, 0) AS content_truncated, (SELECT CASE WHEN MAX(tr.is_error NOT IN (0, 1)) = 1 \ + THEN 2 WHEN COUNT(DISTINCT tr.is_error) = 1 THEN MIN(tr.is_error) ELSE NULL END FROM tool_responses tr WHERE \ + tr.model_call_id = mi.model_call_id AND tr.call_id = mi.call_id) AS is_error FROM model_items mi LEFT JOIN \ + model_calls mc ON mc.id = mi.model_call_id WHERE mi.kind != 'tool_call' ORDER BY mi.id DESC LIMIT 200"; +const TOOL_ITEMS: &str = "SELECT tc.event_id, COALESCE(NULLIF(tc.timestamp, ''), mc.timestamp, '') AS timestamp, \ + tc.model_call_id, mc.event_id AS model_event_id, tc.trace_id, tc.turn_id, tc.call_id, tc.tool_name, tc.server_name, \ + tc.origin, tc.decision, tc.method, substr(tc.arguments, 1, ?1) AS arguments, COALESCE(length(tc.arguments) > ?1, 0) \ + AS arguments_truncated, tc.response_preview, tc.error_message FROM tool_calls tc LEFT JOIN model_calls mc ON mc.id \ + = tc.model_call_id ORDER BY tc.id DESC LIMIT 200"; + +fn stats_detail_statements() -> Vec { + let field = || vec![json!(16 * 1024)]; + vec![ + (BODY_BLOBS.into(), vec![json!(100)]), + (MODEL_ITEMS.into(), field()), + (TOOL_ITEMS.into(), field()), + ("SELECT event_id, timestamp, provider, model, method, path, status_code, input_tokens, output_tokens, duration_ms, response_bytes, stop_reason, trace_id, credential_ref FROM model_calls ORDER BY id DESC LIMIT 200".into(), vec![]), + ("SELECT tc.event_id, COALESCE(NULLIF(tc.timestamp, ''), mc.timestamp) AS timestamp, tc.process_name, COALESCE(tc.server_name, 'model') AS server_name, tc.tool_name, tc.method, tc.call_id, tc.model_call_id, CASE WHEN tc.model_call_id IS NOT NULL AND mc.id IS NULL THEN 1 ELSE 0 END AS model_parent_missing, tc.decision, COALESCE(tc.duration_ms, mc.duration_ms, 0) AS duration_ms, COALESCE(LENGTH(tc.arguments), 0) + COALESCE(LENGTH(COALESCE(tc.response_preview, tr.content_preview)), 0) AS bytes, substr(tc.arguments, 1, ?1) AS arguments, COALESCE(tc.response_preview, tr.content_preview) AS response_preview, tr.event_id AS response_event_id, tc.error_message, tc.origin AS source, COALESCE(tc.credential_ref, tr.credential_ref) AS credential_ref FROM tool_calls AS tc NOT INDEXED LEFT JOIN model_calls mc ON tc.model_call_id = mc.id LEFT JOIN tool_responses tr ON tc.call_id = tr.call_id WHERE tc.origin IN ('model', 'native', 'mcp', 'builtin', 'local', 'mcp_proxy') ORDER BY tc.id DESC LIMIT 200".into(), field()), + ("SELECT event_id, timestamp, domain, port, method, path, query, status_code, decision, duration_ms, bytes_sent, bytes_received, matched_rule, policy_rule, trace_id, credential_ref, request_headers, response_headers FROM net_events ORDER BY id DESC LIMIT 200".into(), vec![]), + ("SELECT event_id, timestamp, qname, qtype, qclass, rcode, decision, matched_rule, policy_rule, source_proto, process_name, upstream_resolver_ms, trace_id, credential_ref FROM dns_events ORDER BY id DESC LIMIT 200".into(), vec![]), + ("SELECT event_id, timestamp, action, path, size, trace_id, credential_ref FROM fs_events ORDER BY id DESC LIMIT 200".into(), vec![]), + ("SELECT event_id, timestamp, exec_id, command, exit_code, duration_ms, stdout_bytes, stderr_bytes, source, process_name, pid, trace_id, credential_ref FROM exec_events ORDER BY id DESC LIMIT 100".into(), vec![]), + ("SELECT event_id, timestamp, pid, ppid, uid, exe, comm, argv, cwd, exit_code, session_id, tty, audit_id, exec_event_id, parent_exe, trace_id, credential_ref FROM audit_events ORDER BY id DESC LIMIT 100".into(), vec![]), + ("SELECT event_id, timestamp, material_class, source, event_type, event_type AS origin, outcome AS verb, provider, trace_id, context_json FROM substitution_events ORDER BY id DESC LIMIT 100".into(), vec![]), + ] +} + +fn triage_statements(limit: u16) -> Vec { + let limit = u64::from(limit); + vec![ + (format!("SELECT timestamp, domain, decision, status_code, duration_ms FROM net_events WHERE decision = 'denied' OR status_code >= 500 ORDER BY timestamp DESC LIMIT {limit}"), vec![]), + (format!("SELECT timestamp, server_name, method, decision, policy_mode, policy_action, policy_rule, policy_reason, error_message, duration_ms FROM tool_calls NOT INDEXED WHERE origin IN ('native','mcp','builtin','local') AND (decision IN ('denied','error') OR error_message IS NOT NULL) ORDER BY id DESC LIMIT {limit}"), vec![]), + (format!("SELECT timestamp, exec_id, command, exit_code, duration_ms FROM exec_events WHERE exit_code IS NOT NULL AND exit_code != 0 ORDER BY timestamp DESC LIMIT {limit}"), vec![]), + ] +} + +#[derive(Deserialize)] +struct JsonRows { + columns: Vec, + rows: Vec>, +} + +fn decode_rows(raw: &str) -> Result { + let JsonRows { columns, rows } = serde_json::from_str(raw).map_err(|error| error.to_string())?; + let rows = rows + .into_iter() + .map(|row| row.into_iter().map(value).collect()) + .collect::, _>>()?; + let rows = LedgerRows { columns, rows }; + rows.validate().map_err(|error| error.to_string())?; + Ok(rows) +} + +fn value(value: Value) -> Result { + match value { + Value::Null => Ok(LedgerValue::Null), + Value::Bool(value) => Ok(LedgerValue::Integer(i64::from(value))), + Value::Number(value) => { + if let Some(value) = value.as_i64() { + Ok(LedgerValue::Integer(value)) + } else if let Some(value) = value.as_f64() { + Ok(LedgerValue::Real(value)) + } else { + Err(format!("ledger query returned unsupported JSON number {value}")) + } + } + Value::String(value) => Ok(LedgerValue::Text(value)), + other => Err(format!("ledger query returned unsupported JSON value {other}")), + } +} diff --git a/crates/capsem-logger/src/ledger_server/tests.rs b/crates/capsem-logger/src/ledger_server/tests.rs new file mode 100644 index 000000000..0007149f4 --- /dev/null +++ b/crates/capsem-logger/src/ledger_server/tests.rs @@ -0,0 +1,229 @@ +use std::os::unix::net::UnixStream; +use std::sync::Arc; +use std::time::SystemTime; + +use capsem_foundation::ipc_channel; +use capsem_proto::ledger::{ + LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerHello, LedgerOutcome, LedgerRequest, +}; + +use super::*; +use crate::ledger_protocol::{ + LedgerClientMessage, LedgerCommand, LedgerQuery, LedgerReply, LedgerServerMessage, LedgerTimelineLayer, +}; +use crate::{Decision, NetEvent, WriteOp}; + +type ClientSender = ipc_channel::Sender; +type ClientReceiver = ipc_channel::Receiver; + +fn grant(role: LedgerClientRole) -> LedgerChannelGrant { + LedgerChannelGrant::new(LedgerGeneration::new([4; 16]), 17, role).unwrap() +} + +async fn connect( + server: Arc, + authority: LedgerChannelGrant, +) -> ( + ClientSender, + ClientReceiver, + tokio::task::JoinHandle>, +) { + let (client, worker) = UnixStream::pair().unwrap(); + let task = tokio::spawn(async move { server.serve_client(worker, authority).await }); + let (sender, receiver) = ipc_channel::channel_from_std::(client).unwrap(); + sender + .send(LedgerClientMessage::Hello { + hello: LedgerHello::for_grant(&authority), + }) + .await + .unwrap(); + assert!(matches!( + receiver.recv().await.unwrap(), + LedgerServerMessage::Welcome { .. } + )); + (sender, receiver, task) +} + +async fn request(sender: &ClientSender, receiver: &ClientReceiver, id: u64, operation: LedgerCommand) -> LedgerReply { + sender + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(id, operation).unwrap(), + }) + .await + .unwrap(); + let LedgerServerMessage::Response { response } = receiver.recv().await.unwrap() else { + panic!("expected response") + }; + assert_eq!(response.request_id(), id); + match response.outcome() { + LedgerOutcome::Success { reply } => reply.clone(), + LedgerOutcome::Failure { failure } => panic!("request failed: {}", failure.message()), + } +} + +fn body_event(body: Vec) -> WriteOp { + WriteOp::NetEvent(NetEvent { + event_id: Some("0123456789ab".into()), + timestamp: SystemTime::UNIX_EPOCH, + domain: "example.test".into(), + port: 443, + decision: Decision::Allowed, + process_name: None, + pid: None, + method: Some("POST".into()), + path: Some("/v1".into()), + query: None, + status_code: Some(200), + bytes_sent: 2, + bytes_received: body.len() as u64, + duration_ms: 1, + matched_rule: None, + request_headers: None, + response_headers: None, + request_body: None, + response_body: Some(body), + conn_type: Some("https".into()), + policy_mode: None, + policy_action: None, + policy_rule: None, + policy_reason: None, + trace_id: None, + credential_ref: None, + }) +} + +#[tokio::test] +async fn accepted_durable_query_body_retention_and_warc_run_through_one_owner() { + let dir = tempfile::tempdir().unwrap(); + let server = Arc::new(LedgerServer::open(&dir.path().join("session.db")).unwrap()); + let producer_grant = grant(LedgerClientRole::VmOwner); + let (producer, producer_replies, producer_task) = connect(Arc::clone(&server), producer_grant).await; + let body = vec![b'z'; 300_000]; + assert!(matches!( + request( + &producer, + &producer_replies, + 1, + LedgerCommand::Admit { + event: Box::new(body_event(body.clone())), + }, + ) + .await, + LedgerReply::Accepted { admission_id: 1 } + )); + assert!(matches!( + request(&producer, &producer_replies, 2, LedgerCommand::Flush).await, + LedgerReply::Durable { + through_admission_id: 1 + } + )); + + let reader_grant = LedgerChannelGrant::new(producer_grant.generation(), 18, LedgerClientRole::Maintainer).unwrap(); + let (reader, reader_replies, reader_task) = connect(Arc::clone(&server), reader_grant).await; + let query = request( + &reader, + &reader_replies, + 3, + LedgerCommand::Query { + query: LedgerQuery::Timeline { + layers: vec![LedgerTimelineLayer::Net], + cutoff: String::new(), + trace_id: None, + limit: 10, + }, + }, + ) + .await; + assert!(matches!(query, LedgerReply::Query { ref sets } if sets.len() == 1 && sets[0].rows.len() == 1)); + + reader + .send(LedgerClientMessage::Request { + request: LedgerRequest::new( + 4, + LedgerCommand::ReadBodies { + event_id: "0123456789ab".into(), + }, + ) + .unwrap(), + }) + .await + .unwrap(); + let mut restored = Vec::new(); + loop { + let LedgerServerMessage::Response { response } = reader_replies.recv().await.unwrap() else { + panic!("expected body response") + }; + let LedgerOutcome::Success { reply } = response.outcome() else { + panic!("body read failed") + }; + match reply { + LedgerReply::BodyStart { metadata, .. } => assert_eq!(metadata.stored_bytes, body.len() as u64), + LedgerReply::BodyChunk { bytes, .. } => restored.extend_from_slice(bytes), + LedgerReply::BodiesComplete => break, + other => panic!("unexpected body reply {other:?}"), + } + } + assert_eq!(restored, body); + + reader + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(5, LedgerCommand::ExportWarc).unwrap(), + }) + .await + .unwrap(); + let mut warc = Vec::new(); + loop { + let LedgerServerMessage::Response { response } = reader_replies.recv().await.unwrap() else { + panic!("expected WARC response") + }; + let LedgerOutcome::Success { reply } = response.outcome() else { + panic!("WARC failed") + }; + match reply { + LedgerReply::WarcChunk { bytes, .. } => warc.extend_from_slice(bytes), + LedgerReply::WarcComplete { summary } => { + assert_eq!(summary.records, 1); + break; + } + other => panic!("unexpected WARC reply {other:?}"), + } + } + assert!(warc.starts_with(&[0x1f, 0x8b])); + assert!(matches!( + request( + &reader, + &reader_replies, + 6, + LedgerCommand::Retain { + cutoff: "2999-01-01T00:00:00Z".into(), + }, + ) + .await, + LedgerReply::Retained { .. } + )); + + drop(producer); + drop(reader); + assert_eq!(producer_task.await.unwrap().unwrap(), LedgerClientExit::Disconnected); + assert_eq!(reader_task.await.unwrap().unwrap(), LedgerClientExit::Disconnected); +} + +#[tokio::test] +async fn wrong_generation_never_reaches_storage_dispatch() { + let dir = tempfile::tempdir().unwrap(); + let server = Arc::new(LedgerServer::open(&dir.path().join("session.db")).unwrap()); + let expected = grant(LedgerClientRole::Reader); + let (client, worker) = UnixStream::pair().unwrap(); + let task = tokio::spawn(async move { server.serve_client(worker, expected).await }); + let (sender, _receiver) = + ipc_channel::channel_from_std::(client).unwrap(); + let wrong = LedgerChannelGrant::new(LedgerGeneration::new([8; 16]), expected.client_id(), expected.role()).unwrap(); + sender + .send(LedgerClientMessage::Hello { + hello: LedgerHello::for_grant(&wrong), + }) + .await + .unwrap(); + drop(sender); + assert!(task.await.unwrap().unwrap_err().contains("generation")); +} diff --git a/crates/capsem-logger/src/lib.rs b/crates/capsem-logger/src/lib.rs index ade870d9a..48a532e52 100644 --- a/crates/capsem-logger/src/lib.rs +++ b/crates/capsem-logger/src/lib.rs @@ -2,6 +2,7 @@ pub mod counters; pub mod db; pub mod events; pub mod ledger_protocol; +pub mod ledger_server; pub mod network_db; pub mod reader; pub mod schema; From f3d2921a5a977b1e9f84d8e3196bf82c50b7ede4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:47:58 +0000 Subject: [PATCH 095/464] feat(proto): frame ledger descriptor grants --- crates/capsem-proto/build.rs | 2 +- crates/capsem-proto/src/ledger.rs | 3 +- crates/capsem-proto/src/ledger_control.rs | 291 ++++++++++++++++++ .../capsem-proto/src/ledger_control/tests.rs | 78 +++++ crates/capsem-proto/src/lib.rs | 1 + 5 files changed, 373 insertions(+), 2 deletions(-) create mode 100644 crates/capsem-proto/src/ledger_control.rs create mode 100644 crates/capsem-proto/src/ledger_control/tests.rs diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index ebf9aa689..7333cbb4f 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -23,7 +23,7 @@ impl VisitMut for StripDocs { fn main() { let files = ["lib.rs", "ipc.rs", "handshake.rs", "router.rs", "exec_stream.rs"]; write_hash("schema_hash.txt", &files); - write_hash("ledger_schema_hash.txt", &["ledger.rs"]); + write_hash("ledger_schema_hash.txt", &["ledger.rs", "ledger_control.rs"]); } fn write_hash(output: &str, files: &[&str]) { diff --git a/crates/capsem-proto/src/ledger.rs b/crates/capsem-proto/src/ledger.rs index 969f27939..22bc8bba0 100644 --- a/crates/capsem-proto/src/ledger.rs +++ b/crates/capsem-proto/src/ledger.rs @@ -11,7 +11,8 @@ use thiserror::Error; /// Version of the ledger handshake, request envelopes and logger-owned /// operation/reply enums. Bump it when any of those wire shapes changes. -pub const LEDGER_PROTOCOL_VERSION: u16 = 1; +/// Version 2 adds the fixed coordinator-to-worker descriptor handoff record. +pub const LEDGER_PROTOCOL_VERSION: u16 = 2; /// Hash of ledger wire declarations, separate from the guest protocol hash. pub const LEDGER_SCHEMA_HASH: u64 = include!(concat!(env!("OUT_DIR"), "/ledger_schema_hash.txt")); /// Largest diagnostic admitted into a response. diff --git a/crates/capsem-proto/src/ledger_control.rs b/crates/capsem-proto/src/ledger_control.rs new file mode 100644 index 000000000..171d1e4df --- /dev/null +++ b/crates/capsem-proto/src/ledger_control.rs @@ -0,0 +1,291 @@ +//! Fixed coordinator control records for one session ledger worker. +//! +//! The trusted coordinator launches a worker with one generation, then sends +//! connected client descriptors beside `Attach` records. The record contains +//! no session id or path: the process already owns exactly one ledger. + +use thiserror::Error; + +use crate::ledger::{ + LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerProtocolError, LEDGER_PROTOCOL_VERSION, +}; + +pub const LEDGER_CONTROL_FRAME_SIZE: usize = 32; +pub const LEDGER_CONTROL_MAX_FDS: usize = 1; + +const MAGIC: [u8; 2] = *b"LG"; +const VERSION_RANGE: std::ops::Range = 2..4; +const KIND_OFFSET: usize = 4; +const ROLE_OFFSET: usize = 5; +const DETAIL_RANGE: std::ops::Range = 6..8; +const GENERATION_RANGE: std::ops::Range = 8..24; +const CLIENT_ID_RANGE: std::ops::Range = 24..32; + +const ATTACH: u8 = 1; +const SHUTDOWN: u8 = 2; +const READY: u8 = 101; +const ADOPTED: u8 = 102; +const REJECTED: u8 = 103; +const STOPPED: u8 = 104; +const CLOSED: u8 = 105; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LedgerControlRequest { + Attach(LedgerChannelGrant), + Shutdown { generation: LedgerGeneration }, +} + +impl LedgerControlRequest { + pub const fn generation(self) -> LedgerGeneration { + match self { + Self::Attach(grant) => grant.generation(), + Self::Shutdown { generation } => generation, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LedgerControlEvent { + Ready { + generation: LedgerGeneration, + }, + Adopted { + generation: LedgerGeneration, + client_id: u64, + }, + Rejected { + generation: LedgerGeneration, + client_id: u64, + reason: LedgerControlRejection, + }, + Stopped { + generation: LedgerGeneration, + }, + Closed { + generation: LedgerGeneration, + client_id: u64, + reason: LedgerClientCloseReason, + }, +} + +impl LedgerControlEvent { + pub const fn generation(self) -> LedgerGeneration { + match self { + Self::Ready { generation } + | Self::Adopted { generation, .. } + | Self::Rejected { generation, .. } + | Self::Stopped { generation } + | Self::Closed { generation, .. } => generation, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u16)] +pub enum LedgerControlRejection { + Capacity = 1, + ShuttingDown = 2, + DuplicateClient = 3, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u16)] +pub enum LedgerClientCloseReason { + Disconnected = 1, + ShutdownRequested = 2, + ProtocolError = 3, +} + +pub fn encode_ledger_control_request(request: LedgerControlRequest) -> [u8; LEDGER_CONTROL_FRAME_SIZE] { + match request { + LedgerControlRequest::Attach(grant) => encode( + ATTACH, + role_code(grant.role()), + 0, + grant.generation(), + grant.client_id(), + ), + LedgerControlRequest::Shutdown { generation } => encode(SHUTDOWN, 0, 0, generation, 0), + } +} + +pub fn decode_ledger_control_request( + frame: &[u8; LEDGER_CONTROL_FRAME_SIZE], +) -> Result { + validate_header(frame)?; + let generation = decode_generation(frame); + let client_id = decode_u64(frame, CLIENT_ID_RANGE); + let detail = decode_u16(frame, DETAIL_RANGE); + match frame[KIND_OFFSET] { + ATTACH if detail == 0 => { + let role = decode_role(frame[ROLE_OFFSET])?; + Ok(LedgerControlRequest::Attach( + LedgerChannelGrant::new(generation, client_id, role).map_err(LedgerControlError::InvalidGrant)?, + )) + } + SHUTDOWN if frame[ROLE_OFFSET] == 0 && detail == 0 && client_id == 0 => { + validate_generation(generation)?; + Ok(LedgerControlRequest::Shutdown { generation }) + } + kind => Err(LedgerControlError::InvalidRequest(kind)), + } +} + +pub fn encode_ledger_control_event(event: LedgerControlEvent) -> [u8; LEDGER_CONTROL_FRAME_SIZE] { + match event { + LedgerControlEvent::Ready { generation } => encode(READY, 0, 0, generation, 0), + LedgerControlEvent::Adopted { generation, client_id } => encode(ADOPTED, 0, 0, generation, client_id), + LedgerControlEvent::Rejected { + generation, + client_id, + reason, + } => encode(REJECTED, 0, reason as u16, generation, client_id), + LedgerControlEvent::Stopped { generation } => encode(STOPPED, 0, 0, generation, 0), + LedgerControlEvent::Closed { + generation, + client_id, + reason, + } => encode(CLOSED, 0, reason as u16, generation, client_id), + } +} + +pub fn decode_ledger_control_event( + frame: &[u8; LEDGER_CONTROL_FRAME_SIZE], +) -> Result { + validate_header(frame)?; + if frame[ROLE_OFFSET] != 0 { + return Err(LedgerControlError::ReservedBytes); + } + let generation = decode_generation(frame); + validate_generation(generation)?; + let client_id = decode_u64(frame, CLIENT_ID_RANGE); + let detail = decode_u16(frame, DETAIL_RANGE); + match frame[KIND_OFFSET] { + READY if detail == 0 && client_id == 0 => Ok(LedgerControlEvent::Ready { generation }), + ADOPTED if detail == 0 && client_id != 0 => Ok(LedgerControlEvent::Adopted { generation, client_id }), + REJECTED if client_id != 0 => Ok(LedgerControlEvent::Rejected { + generation, + client_id, + reason: match detail { + 1 => LedgerControlRejection::Capacity, + 2 => LedgerControlRejection::ShuttingDown, + 3 => LedgerControlRejection::DuplicateClient, + _ => return Err(LedgerControlError::InvalidRejection(detail)), + }, + }), + STOPPED if detail == 0 && client_id == 0 => Ok(LedgerControlEvent::Stopped { generation }), + CLOSED if client_id != 0 => Ok(LedgerControlEvent::Closed { + generation, + client_id, + reason: match detail { + 1 => LedgerClientCloseReason::Disconnected, + 2 => LedgerClientCloseReason::ShutdownRequested, + 3 => LedgerClientCloseReason::ProtocolError, + _ => return Err(LedgerControlError::InvalidCloseReason(detail)), + }, + }), + kind => Err(LedgerControlError::InvalidEvent(kind)), + } +} + +fn encode( + kind: u8, + role: u8, + detail: u16, + generation: LedgerGeneration, + client_id: u64, +) -> [u8; LEDGER_CONTROL_FRAME_SIZE] { + let mut frame = [0; LEDGER_CONTROL_FRAME_SIZE]; + frame[..2].copy_from_slice(&MAGIC); + frame[VERSION_RANGE].copy_from_slice(&LEDGER_PROTOCOL_VERSION.to_be_bytes()); + frame[KIND_OFFSET] = kind; + frame[ROLE_OFFSET] = role; + frame[DETAIL_RANGE].copy_from_slice(&detail.to_be_bytes()); + frame[GENERATION_RANGE].copy_from_slice(&generation.as_bytes()); + frame[CLIENT_ID_RANGE].copy_from_slice(&client_id.to_be_bytes()); + frame +} + +fn validate_header(frame: &[u8; LEDGER_CONTROL_FRAME_SIZE]) -> Result<(), LedgerControlError> { + if frame[..2] != MAGIC { + return Err(LedgerControlError::Magic); + } + let version = decode_u16(frame, VERSION_RANGE); + if version != LEDGER_PROTOCOL_VERSION { + return Err(LedgerControlError::Version { + expected: LEDGER_PROTOCOL_VERSION, + actual: version, + }); + } + Ok(()) +} + +fn validate_generation(generation: LedgerGeneration) -> Result<(), LedgerControlError> { + LedgerChannelGrant::new(generation, 1, LedgerClientRole::Reader) + .map(|_| ()) + .map_err(LedgerControlError::InvalidGrant) +} + +const fn role_code(role: LedgerClientRole) -> u8 { + match role { + LedgerClientRole::VmOwner => 1, + LedgerClientRole::Proxy => 2, + LedgerClientRole::Coordinator => 3, + LedgerClientRole::Reader => 4, + LedgerClientRole::Maintainer => 5, + LedgerClientRole::Supervisor => 6, + } +} + +fn decode_role(role: u8) -> Result { + match role { + 1 => Ok(LedgerClientRole::VmOwner), + 2 => Ok(LedgerClientRole::Proxy), + 3 => Ok(LedgerClientRole::Coordinator), + 4 => Ok(LedgerClientRole::Reader), + 5 => Ok(LedgerClientRole::Maintainer), + 6 => Ok(LedgerClientRole::Supervisor), + _ => Err(LedgerControlError::InvalidRole(role)), + } +} + +fn decode_u16(frame: &[u8; LEDGER_CONTROL_FRAME_SIZE], range: std::ops::Range) -> u16 { + u16::from_be_bytes(frame[range].try_into().expect("fixed ledger control range")) +} + +fn decode_u64(frame: &[u8; LEDGER_CONTROL_FRAME_SIZE], range: std::ops::Range) -> u64 { + u64::from_be_bytes(frame[range].try_into().expect("fixed ledger control range")) +} + +fn decode_generation(frame: &[u8; LEDGER_CONTROL_FRAME_SIZE]) -> LedgerGeneration { + LedgerGeneration::new( + frame[GENERATION_RANGE] + .try_into() + .expect("fixed ledger generation range"), + ) +} + +#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +pub enum LedgerControlError { + #[error("ledger control magic mismatch")] + Magic, + #[error("ledger control version mismatch: expected {expected}, got {actual}")] + Version { expected: u16, actual: u16 }, + #[error("ledger control request kind {0} is invalid")] + InvalidRequest(u8), + #[error("ledger control event kind {0} is invalid")] + InvalidEvent(u8), + #[error("ledger control role {0} is invalid")] + InvalidRole(u8), + #[error("ledger control rejection {0} is invalid")] + InvalidRejection(u16), + #[error("ledger client close reason {0} is invalid")] + InvalidCloseReason(u16), + #[error("ledger control reserved bytes are nonzero")] + ReservedBytes, + #[error("ledger control grant is invalid: {0}")] + InvalidGrant(LedgerProtocolError), +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/ledger_control/tests.rs b/crates/capsem-proto/src/ledger_control/tests.rs new file mode 100644 index 000000000..74606ac22 --- /dev/null +++ b/crates/capsem-proto/src/ledger_control/tests.rs @@ -0,0 +1,78 @@ +use super::*; + +const GENERATION: LedgerGeneration = LedgerGeneration::new([9; 16]); + +#[test] +fn requests_round_trip_authority_without_session_or_path() { + for role in [ + LedgerClientRole::VmOwner, + LedgerClientRole::Proxy, + LedgerClientRole::Coordinator, + LedgerClientRole::Reader, + LedgerClientRole::Maintainer, + LedgerClientRole::Supervisor, + ] { + let grant = LedgerChannelGrant::new(GENERATION, 71, role).unwrap(); + let frame = encode_ledger_control_request(LedgerControlRequest::Attach(grant)); + assert_eq!( + decode_ledger_control_request(&frame).unwrap(), + LedgerControlRequest::Attach(grant) + ); + assert!(!String::from_utf8_lossy(&frame).contains("session")); + assert!(!String::from_utf8_lossy(&frame).contains("path")); + } + let shutdown = LedgerControlRequest::Shutdown { generation: GENERATION }; + assert_eq!( + decode_ledger_control_request(&encode_ledger_control_request(shutdown)).unwrap(), + shutdown + ); +} + +#[test] +fn events_round_trip_generation_identity_and_rejection() { + for event in [ + LedgerControlEvent::Ready { generation: GENERATION }, + LedgerControlEvent::Adopted { + generation: GENERATION, + client_id: 72, + }, + LedgerControlEvent::Rejected { + generation: GENERATION, + client_id: 73, + reason: LedgerControlRejection::Capacity, + }, + LedgerControlEvent::Rejected { + generation: GENERATION, + client_id: 74, + reason: LedgerControlRejection::DuplicateClient, + }, + LedgerControlEvent::Stopped { generation: GENERATION }, + LedgerControlEvent::Closed { + generation: GENERATION, + client_id: 75, + reason: LedgerClientCloseReason::ProtocolError, + }, + ] { + assert_eq!( + decode_ledger_control_event(&encode_ledger_control_event(event)).unwrap(), + event + ); + } +} + +#[test] +fn malformed_records_fail_closed() { + let grant = LedgerChannelGrant::new(GENERATION, 71, LedgerClientRole::Reader).unwrap(); + let valid = encode_ledger_control_request(LedgerControlRequest::Attach(grant)); + for (index, value) in [(0, 0), (3, 0), (4, 99), (5, 99)] { + let mut frame = valid; + frame[index] = value; + assert!(decode_ledger_control_request(&frame).is_err()); + } + let mut zero_generation = valid; + zero_generation[GENERATION_RANGE].fill(0); + assert!(decode_ledger_control_request(&zero_generation).is_err()); + let mut zero_client = valid; + zero_client[CLIENT_ID_RANGE].fill(0); + assert!(decode_ledger_control_request(&zero_client).is_err()); +} diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index d3a40e803..0a14816c8 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -19,6 +19,7 @@ pub mod handshake; pub mod host_session; pub mod ipc; pub mod ledger; +pub mod ledger_control; pub mod ledger_counters; pub mod mcp; pub mod mcp_aggregator; From 787c2c87efd4706dd98e466eaf5c76376140cee9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 02:57:14 +0000 Subject: [PATCH 096/464] feat(ledger): own each session database --- Cargo.lock | 15 ++ Cargo.toml | 1 + crates/capsem-ledger/Cargo.toml | 26 +++ crates/capsem-ledger/src/main.rs | 232 ++++++++++++++++++++++ crates/capsem-ledger/src/tests.rs | 238 +++++++++++++++++++++++ crates/capsem-ledger/tests/subprocess.rs | 92 +++++++++ 6 files changed, 604 insertions(+) create mode 100644 crates/capsem-ledger/Cargo.toml create mode 100644 crates/capsem-ledger/src/main.rs create mode 100644 crates/capsem-ledger/src/tests.rs create mode 100644 crates/capsem-ledger/tests/subprocess.rs diff --git a/Cargo.lock b/Cargo.lock index 3f37beedd..1dca91dee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1038,6 +1038,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "capsem-ledger" +version = "0.7.0" +dependencies = [ + "anyhow", + "capsem-foundation", + "capsem-guard", + "capsem-logger", + "capsem-proto", + "clap", + "tempfile", + "tokio", + "tracing", +] + [[package]] name = "capsem-logger" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 3d441537a..9ec45afb2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/capsem-app", "crates/capsem-agent", "crates/capsem-logger", + "crates/capsem-ledger", "crates/capsem-process", "crates/capsem-service", "crates/capsem-admin", diff --git a/crates/capsem-ledger/Cargo.toml b/crates/capsem-ledger/Cargo.toml new file mode 100644 index 000000000..bc424e334 --- /dev/null +++ b/crates/capsem-ledger/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "capsem-ledger" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +description = "Dedicated per-session Capsem ledger owner." +homepage.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +anyhow.workspace = true +capsem-foundation = { path = "../capsem-foundation" } +capsem-guard = { path = "../capsem-guard" } +capsem-logger = { path = "../capsem-logger" } +capsem-proto = { path = "../capsem-proto" } +clap = { workspace = true, features = ["derive"] } +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/capsem-ledger/src/main.rs b/crates/capsem-ledger/src/main.rs new file mode 100644 index 000000000..52359ff3d --- /dev/null +++ b/crates/capsem-ledger/src/main.rs @@ -0,0 +1,232 @@ +use std::collections::HashSet; +use std::io; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::{bail, Context, Result}; +use capsem_foundation::unix::{fd, router_channel}; +use capsem_logger::ledger_server::{LedgerClientExit, LedgerServer}; +use capsem_proto::ledger::LedgerGeneration; +use capsem_proto::ledger_control::{ + decode_ledger_control_request, encode_ledger_control_event, LedgerClientCloseReason, LedgerControlEvent, + LedgerControlRejection, LedgerControlRequest, LEDGER_CONTROL_FRAME_SIZE, LEDGER_CONTROL_MAX_FDS, +}; +use clap::Parser; +use tokio::task::{JoinError, JoinSet}; + +const CLIENT_LIMIT: usize = 64; + +type ControlSender = router_channel::DescriptorSender; +type ControlReceiver = router_channel::DescriptorReceiver; +type ClientResult = (u64, Result); + +#[derive(Parser)] +#[command(version, about = "Dedicated owner of one Capsem session ledger")] +struct Args { + #[arg(long)] + parent_pid: u32, + #[arg(long)] + database: PathBuf, + /// Sixteen-byte worker generation as 32 lowercase hexadecimal digits. + #[arg(long, value_parser = parse_generation)] + generation: LedgerGeneration, +} + +fn main() -> Result<()> { + let _telemetry = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { + service: "capsem-ledger", + sink: capsem_foundation::telemetry::LogSink::Stderr, + default_filter: "capsem_ledger=info,capsem_logger=info,capsem_foundation=warn", + })?; + let args = Args::parse(); + capsem_guard::watch_parent_or_exit(Some(args.parent_pid))?; + let stdin = io::stdin(); + let control = UnixStream::from(fd::duplicate(stdin.as_fd())?); + let server = Arc::new(LedgerServer::open(&args.database).context("open session ledger")?); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build()?; + runtime.block_on(run_control(control, server, args.generation, CLIENT_LIMIT)) +} + +async fn run_control( + control: UnixStream, + server: Arc, + generation: LedgerGeneration, + client_limit: usize, +) -> Result<()> { + let shutdown = control.try_clone()?; + let sender = ControlSender::new(control.try_clone()?)?; + let receiver = ControlReceiver::new(control)?; + let (frames_sender, mut frames) = tokio::sync::mpsc::unbounded_channel(); + let reader = tokio::spawn(read_control(receiver, frames_sender)); + + let outcome = async { + send_event(&sender, LedgerControlEvent::Ready { generation }).await?; + + let mut clients = JoinSet::::new(); + let mut active_ids = HashSet::new(); + let mut seen_ids = HashSet::new(); + loop { + let next = if clients.is_empty() { + ControlInput::Frame(frames.recv().await.context("ledger control channel closed")?) + } else { + tokio::select! { + frame = frames.recv() => ControlInput::Frame(frame.context("ledger control channel closed")?), + result = clients.join_next() => ControlInput::Client(result.expect("nonempty ledger client set")), + } + }; + match next { + ControlInput::Client(result) => { + let (event, shutdown) = client_finished(result, generation, &mut active_ids)?; + send_event(&sender, event).await?; + if shutdown { + break; + } + } + ControlInput::Frame(frame) => { + let request = decode_ledger_control_request(&frame.bytes)?; + if request.generation() != generation { + bail!("ledger control request names a stale generation"); + } + match request { + LedgerControlRequest::Attach(grant) => { + if frame.fds.len() != 1 { + bail!("ledger attach requires exactly one descriptor"); + } + let client_id = grant.client_id(); + let rejected = if seen_ids.contains(&client_id) { + Some(LedgerControlRejection::DuplicateClient) + } else if active_ids.len() >= client_limit { + Some(LedgerControlRejection::Capacity) + } else { + None + }; + if let Some(reason) = rejected { + send_event( + &sender, + LedgerControlEvent::Rejected { + generation, + client_id, + reason, + }, + ) + .await?; + continue; + } + let descriptor = frame.fds.into_iter().next().expect("one checked descriptor"); + let stream = UnixStream::from(descriptor); + let client_server = Arc::clone(&server); + active_ids.insert(client_id); + seen_ids.insert(client_id); + clients.spawn(async move { (client_id, client_server.serve_client(stream, grant).await) }); + send_event(&sender, LedgerControlEvent::Adopted { generation, client_id }).await?; + } + LedgerControlRequest::Shutdown { .. } => { + if !frame.fds.is_empty() { + bail!("ledger shutdown must not carry descriptors"); + } + break; + } + } + } + } + } + + clients.abort_all(); + while let Some(result) = clients.join_next().await { + if let Ok((client_id, _)) = result { + active_ids.remove(&client_id); + } + } + drop(active_ids); + drop(seen_ids); + let server = Arc::try_unwrap(server).map_err(|_| anyhow::anyhow!("ledger client owner survived shutdown"))?; + drop(server); + send_event(&sender, LedgerControlEvent::Stopped { generation }).await + } + .await; + + let _ = shutdown.shutdown(std::net::Shutdown::Both); + let reader_outcome = reader.await.context("ledger control reader task failed")?; + outcome?; + reader_outcome?; + Ok(()) +} + +async fn read_control( + receiver: ControlReceiver, + frames: tokio::sync::mpsc::UnboundedSender>, +) -> io::Result<()> { + loop { + match receiver.recv().await { + Ok(frame) => { + if frames.send(frame).is_err() { + return Ok(()); + } + } + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(()), + Err(error) => return Err(error), + } + } +} + +enum ControlInput { + Frame(router_channel::DescriptorFrame), + Client(std::result::Result), +} + +fn client_finished( + result: std::result::Result, + generation: LedgerGeneration, + active_ids: &mut HashSet, +) -> Result<(LedgerControlEvent, bool)> { + let (client_id, result) = result.context("ledger client task failed")?; + active_ids.remove(&client_id); + let (reason, shutdown) = match result { + Ok(LedgerClientExit::Disconnected) => (LedgerClientCloseReason::Disconnected, false), + Ok(LedgerClientExit::ShutdownRequested) => (LedgerClientCloseReason::ShutdownRequested, true), + Err(error) => { + tracing::warn!(client_id, %error, "ledger client stopped"); + (LedgerClientCloseReason::ProtocolError, false) + } + }; + Ok(( + LedgerControlEvent::Closed { + generation, + client_id, + reason, + }, + shutdown, + )) +} + +async fn send_event(sender: &ControlSender, event: LedgerControlEvent) -> Result<()> { + sender.send(&encode_ledger_control_event(event), &[]).await?; + Ok(()) +} + +fn parse_generation(value: &str) -> std::result::Result { + if value.len() != 32 + || !value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err("generation must be 32 lowercase hexadecimal digits".into()); + } + let mut bytes = [0; 16]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).map_err(|_| "invalid generation")?; + } + let generation = LedgerGeneration::new(bytes); + if generation.as_bytes() == [0; 16] { + return Err("generation must not be zero".into()); + } + Ok(generation) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-ledger/src/tests.rs b/crates/capsem-ledger/src/tests.rs new file mode 100644 index 000000000..371779931 --- /dev/null +++ b/crates/capsem-ledger/src/tests.rs @@ -0,0 +1,238 @@ +use std::os::fd::AsRawFd; +use std::time::SystemTime; + +use capsem_foundation::ipc_channel; +use capsem_logger::ledger_protocol::{LedgerClientMessage, LedgerCommand, LedgerReply, LedgerServerMessage}; +use capsem_logger::{Decision, NetEvent, WriteOp}; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerHello, LedgerOutcome, LedgerRequest}; +use capsem_proto::ledger_control::{ + decode_ledger_control_event, encode_ledger_control_request, LedgerControlEvent, LedgerControlRequest, +}; + +use super::*; + +const GENERATION: LedgerGeneration = LedgerGeneration::new([5; 16]); + +fn event() -> WriteOp { + WriteOp::NetEvent(NetEvent { + event_id: Some("0123456789ab".into()), + timestamp: SystemTime::UNIX_EPOCH, + domain: "worker.test".into(), + port: 443, + decision: Decision::Allowed, + process_name: None, + pid: None, + method: Some("GET".into()), + path: Some("/".into()), + query: None, + status_code: Some(200), + bytes_sent: 0, + bytes_received: 0, + duration_ms: 1, + matched_rule: None, + request_headers: None, + response_headers: None, + request_body: None, + response_body: None, + conn_type: Some("https".into()), + policy_mode: None, + policy_action: None, + policy_rule: None, + policy_reason: None, + trace_id: None, + credential_ref: None, + }) +} + +async fn control_event(receiver: &ControlReceiver) -> LedgerControlEvent { + let frame = receiver.recv().await.unwrap(); + assert!(frame.fds.is_empty()); + decode_ledger_control_event(&frame.bytes).unwrap() +} + +#[tokio::test] +async fn worker_adopts_one_client_flushes_and_stops() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("session.db"); + let server = Arc::new(LedgerServer::open(&database).unwrap()); + let (coordinator, worker) = UnixStream::pair().unwrap(); + let requests = ControlSender::new(coordinator.try_clone().unwrap()).unwrap(); + let events = ControlReceiver::new(coordinator).unwrap(); + let task = tokio::spawn(run_control(worker, server, GENERATION, 1)); + assert_eq!( + control_event(&events).await, + LedgerControlEvent::Ready { generation: GENERATION } + ); + + let grant = LedgerChannelGrant::new(GENERATION, 11, LedgerClientRole::VmOwner).unwrap(); + let (client, ledger_end) = UnixStream::pair().unwrap(); + requests + .send( + &encode_ledger_control_request(LedgerControlRequest::Attach(grant)), + &[ledger_end.as_raw_fd()], + ) + .await + .unwrap(); + assert_eq!( + control_event(&events).await, + LedgerControlEvent::Adopted { + generation: GENERATION, + client_id: 11, + } + ); + drop(ledger_end); + + let (sender, receiver) = ipc_channel::channel_from_std::(client).unwrap(); + sender + .send(LedgerClientMessage::Hello { + hello: LedgerHello::for_grant(&grant), + }) + .await + .unwrap(); + assert!(matches!( + receiver.recv().await.unwrap(), + LedgerServerMessage::Welcome { .. } + )); + for (request_id, operation) in [ + ( + 1, + LedgerCommand::Admit { + event: Box::new(event()), + }, + ), + (2, LedgerCommand::Flush), + ] { + sender + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(request_id, operation).unwrap(), + }) + .await + .unwrap(); + let LedgerServerMessage::Response { response } = receiver.recv().await.unwrap() else { + panic!("expected ledger response") + }; + assert!(matches!(response.outcome(), LedgerOutcome::Success { .. })); + assert!(matches!( + response.outcome(), + LedgerOutcome::Success { + reply: LedgerReply::Accepted { .. } | LedgerReply::Durable { .. } + } + )); + } + + requests + .send( + &encode_ledger_control_request(LedgerControlRequest::Shutdown { generation: GENERATION }), + &[], + ) + .await + .unwrap(); + assert_eq!( + control_event(&events).await, + LedgerControlEvent::Stopped { generation: GENERATION } + ); + task.await.unwrap().unwrap(); + + let reader = capsem_logger::DbHandle::open_external_reader(&database).unwrap(); + reader.ready().await.unwrap(); + let rows = reader.query("SELECT domain FROM net_events", &[]).await.unwrap(); + assert!(rows.contains("worker.test")); +} + +#[tokio::test] +async fn duplicate_client_id_is_rejected_without_adopting_its_descriptor() { + let dir = tempfile::tempdir().unwrap(); + let server = Arc::new(LedgerServer::open(&dir.path().join("session.db")).unwrap()); + let (coordinator, worker) = UnixStream::pair().unwrap(); + let requests = ControlSender::new(coordinator.try_clone().unwrap()).unwrap(); + let events = ControlReceiver::new(coordinator).unwrap(); + let task = tokio::spawn(run_control(worker, server, GENERATION, 2)); + assert!(matches!(control_event(&events).await, LedgerControlEvent::Ready { .. })); + + let grant = LedgerChannelGrant::new(GENERATION, 12, LedgerClientRole::Reader).unwrap(); + let (first_client, first_worker) = UnixStream::pair().unwrap(); + requests + .send( + &encode_ledger_control_request(LedgerControlRequest::Attach(grant)), + &[first_worker.as_raw_fd()], + ) + .await + .unwrap(); + assert!(matches!( + control_event(&events).await, + LedgerControlEvent::Adopted { .. } + )); + drop(first_worker); + + let (_duplicate_client, duplicate_worker) = UnixStream::pair().unwrap(); + requests + .send( + &encode_ledger_control_request(LedgerControlRequest::Attach(grant)), + &[duplicate_worker.as_raw_fd()], + ) + .await + .unwrap(); + assert!(matches!( + control_event(&events).await, + LedgerControlEvent::Rejected { + reason: LedgerControlRejection::DuplicateClient, + .. + } + )); + drop(duplicate_worker); + + drop(first_client); + assert_eq!( + control_event(&events).await, + LedgerControlEvent::Closed { + generation: GENERATION, + client_id: 12, + reason: LedgerClientCloseReason::ProtocolError, + } + ); + let (_replayed_client, replayed_worker) = UnixStream::pair().unwrap(); + requests + .send( + &encode_ledger_control_request(LedgerControlRequest::Attach(grant)), + &[replayed_worker.as_raw_fd()], + ) + .await + .unwrap(); + assert!(matches!( + control_event(&events).await, + LedgerControlEvent::Rejected { + reason: LedgerControlRejection::DuplicateClient, + .. + } + )); + drop(replayed_worker); + requests + .send( + &encode_ledger_control_request(LedgerControlRequest::Shutdown { generation: GENERATION }), + &[], + ) + .await + .unwrap(); + assert!(matches!( + control_event(&events).await, + LedgerControlEvent::Stopped { .. } + )); + task.await.unwrap().unwrap(); +} + +#[test] +fn generation_parser_is_exact_and_nonzero() { + assert_eq!( + parse_generation("01010101010101010101010101010101").unwrap().as_bytes(), + [1; 16] + ); + for invalid in [ + "", + "00", + "00000000000000000000000000000000", + "0101010101010101010101010101010G", + "0101010101010101010101010101010A", + ] { + assert!(parse_generation(invalid).is_err(), "accepted {invalid}"); + } +} diff --git a/crates/capsem-ledger/tests/subprocess.rs b/crates/capsem-ledger/tests/subprocess.rs new file mode 100644 index 000000000..47b0e87dc --- /dev/null +++ b/crates/capsem-ledger/tests/subprocess.rs @@ -0,0 +1,92 @@ +use std::os::fd::OwnedFd; +use std::os::unix::net::UnixStream; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::ledger::LedgerGeneration; +use capsem_proto::ledger_control::{ + decode_ledger_control_event, encode_ledger_control_request, LedgerControlEvent, LedgerControlRequest, + LEDGER_CONTROL_FRAME_SIZE, LEDGER_CONTROL_MAX_FDS, +}; + +type ControlSender = DescriptorSender; +type ControlReceiver = DescriptorReceiver; + +const GENERATION: LedgerGeneration = LedgerGeneration::new([6; 16]); + +struct ChildGuard(Option); + +impl ChildGuard { + fn take(&mut self) -> Child { + self.0.take().expect("child present") + } +} + +impl Drop for ChildGuard { + fn drop(&mut self) { + if let Some(child) = &mut self.0 { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +async fn event(receiver: &ControlReceiver) -> LedgerControlEvent { + let frame = tokio::time::timeout(Duration::from_secs(5), receiver.recv()) + .await + .expect("ledger control event timed out") + .expect("ledger control event"); + assert!(frame.fds.is_empty()); + decode_ledger_control_event(&frame.bytes).expect("valid ledger control event") +} + +#[tokio::test] +async fn executable_opens_one_ledger_and_reports_durable_stop() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("session.db"); + let (coordinator, child_control) = UnixStream::pair().unwrap(); + let mut child = ChildGuard(Some( + Command::new(env!("CARGO_BIN_EXE_capsem-ledger")) + .arg("--parent-pid") + .arg(std::process::id().to_string()) + .arg("--database") + .arg(&database) + .arg("--generation") + .arg("06060606060606060606060606060606") + .stdin(Stdio::from(OwnedFd::from(child_control))) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(), + )); + let requests = ControlSender::new(coordinator.try_clone().unwrap()).unwrap(); + let events = ControlReceiver::new(coordinator).unwrap(); + assert_eq!( + event(&events).await, + LedgerControlEvent::Ready { generation: GENERATION } + ); + requests + .send( + &encode_ledger_control_request(LedgerControlRequest::Shutdown { generation: GENERATION }), + &[], + ) + .await + .unwrap(); + assert_eq!( + event(&events).await, + LedgerControlEvent::Stopped { generation: GENERATION } + ); + + let child = child.take(); + let output = tokio::task::spawn_blocking(move || child.wait_with_output()) + .await + .unwrap() + .unwrap(); + assert!( + output.status.success(), + "ledger worker failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(database.is_file()); +} From 4e4b53fb5181629570a421d927ae3e1a2380de97 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 03:01:01 +0000 Subject: [PATCH 097/464] security(foundation): confine ledger workers --- crates/capsem-foundation/src/unix/process.rs | 72 +++++++++++++++++++ .../src/unix/process/tests.rs | 43 +++++++++++ .../src/unix/worker_sandbox.rs | 1 + .../src/unix/worker_sandbox/linux.rs | 13 +++- .../src/unix/worker_sandbox/seatbelt.rs | 28 ++++---- .../src/unix/worker_sandbox/tests.rs | 20 +++++- 6 files changed, 162 insertions(+), 15 deletions(-) diff --git a/crates/capsem-foundation/src/unix/process.rs b/crates/capsem-foundation/src/unix/process.rs index d1ba6aafd..d5b2cb115 100644 --- a/crates/capsem-foundation/src/unix/process.rs +++ b/crates/capsem-foundation/src/unix/process.rs @@ -21,6 +21,55 @@ mod pinned_macos; #[cfg(target_os = "macos")] pub use pinned_macos::ProcessHandle; +/// Erase the inherited environment and detach it from the C process image. +/// +/// `remove_var` alone leaves the original stack bytes visible through Linux +/// `/proc//environ`. This scrubs those bytes before emptying `environ`. +/// +/// # Safety +/// Call only at process entry, before any library can replace environment +/// pointers and before another thread can read or mutate the environment. +pub unsafe fn clear_inherited_environment() { + let environment = inherited_environment(); + if environment.is_null() { + return; + } + let mut entry = environment; + while !unsafe { *entry }.is_null() { + let value = unsafe { *entry }; + // SAFETY: process-entry `environ` is a null-terminated array of + // writable, NUL-terminated inherited strings. No other thread exists. + let length = unsafe { std::ffi::CStr::from_ptr(value) }.to_bytes().len(); + unsafe { std::ptr::write_bytes(value.cast::(), 0, length) }; + entry = unsafe { entry.add(1) }; + } + // libc and Rust observe an empty list without retaining scrubbed entries. + unsafe { *environment = std::ptr::null_mut() }; +} + +#[cfg(target_os = "linux")] +fn inherited_environment() -> *mut *mut libc::c_char { + unsafe extern "C" { + static mut environ: *mut *mut libc::c_char; + } + // SAFETY: the caller upholds process-entry exclusivity. + unsafe { environ } +} + +#[cfg(target_os = "macos")] +fn inherited_environment() -> *mut *mut libc::c_char { + unsafe extern "C" { + fn _NSGetEnviron() -> *mut *mut *mut libc::c_char; + } + // SAFETY: Darwin returns the address of this process's environ pointer. + unsafe { *_NSGetEnviron() } +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn inherited_environment() -> *mut *mut libc::c_char { + std::ptr::null_mut() +} + /// A positive process identifier representable by the host kernel ABI. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub struct ProcessId(NonZeroI32); @@ -55,6 +104,15 @@ pub enum ProcessState { Gone, } +/// Whether `kill(pid, 0)` is permitted, denied, or names no process. +/// Unlike [`probe`], this preserves `EPERM` for sandbox attestation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum SignalProbe { + Allowed, + Denied, + Gone, +} + /// Signals used by host-side lifecycle orchestration. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Signal { @@ -110,6 +168,20 @@ pub fn probe(pid: ProcessId) -> io::Result { classify_probe(kill(pid.as_nix(), None)) } +/// Probe signal authority without folding permission denial into liveness. +pub fn probe_signal_authority(pid: ProcessId) -> io::Result { + classify_signal_probe(kill(pid.as_nix(), None)) +} + +fn classify_signal_probe(result: Result<(), Errno>) -> io::Result { + match result { + Ok(()) => Ok(SignalProbe::Allowed), + Err(Errno::EPERM) => Ok(SignalProbe::Denied), + Err(Errno::ESRCH) => Ok(SignalProbe::Gone), + Err(error) => Err(errno::io(error)), + } +} + fn classify_probe(result: Result<(), Errno>) -> io::Result { match result { Ok(()) | Err(Errno::EPERM) => Ok(ProcessState::Alive), diff --git a/crates/capsem-foundation/src/unix/process/tests.rs b/crates/capsem-foundation/src/unix/process/tests.rs index 94f33345f..424dc2069 100644 --- a/crates/capsem-foundation/src/unix/process/tests.rs +++ b/crates/capsem-foundation/src/unix/process/tests.rs @@ -28,6 +28,17 @@ fn probe_classifies_only_esrch_as_gone() { ); } +#[test] +fn signal_authority_probe_preserves_permission_denial() { + assert_eq!(classify_signal_probe(Ok(())).unwrap(), SignalProbe::Allowed); + assert_eq!(classify_signal_probe(Err(Errno::EPERM)).unwrap(), SignalProbe::Denied); + assert_eq!(classify_signal_probe(Err(Errno::ESRCH)).unwrap(), SignalProbe::Gone); + assert_eq!( + classify_signal_probe(Err(Errno::EINVAL)).unwrap_err().raw_os_error(), + Some(Errno::EINVAL as i32) + ); +} + #[test] fn signal_classifies_disappearance_without_hiding_other_errno() { assert_eq!(classify_signal(Ok(())).unwrap(), SignalOutcome::Delivered); @@ -44,6 +55,38 @@ fn current_identity_is_numeric_and_has_a_parent() { assert!(parent_process_id().is_some()); } +#[test] +fn inherited_environment_is_erased_in_an_isolated_process() { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "unix::process::tests::clear_inherited_environment_child", + "--nocapture", + ]) + .env_clear() + .env("CAPSEM_ENV_SECRET", "must-not-survive") + .status() + .unwrap(); + assert!(status.success()); +} + +#[test] +fn clear_inherited_environment_child() { + if std::env::var("CAPSEM_ENV_SECRET").is_err() { + return; + } + // SAFETY: isolated child entry before it starts any thread or library. + unsafe { clear_inherited_environment() }; + assert!(std::env::vars_os().next().is_none()); + #[cfg(target_os = "linux")] + { + let bytes = std::fs::read("/proc/self/environ").unwrap(); + assert!(!bytes + .windows(b"must-not-survive".len()) + .any(|value| value == b"must-not-survive")); + } +} + #[test] fn a_real_child_moves_from_alive_to_gone() { let mut child = std::process::Command::new("sleep").arg("30").spawn().unwrap(); diff --git a/crates/capsem-foundation/src/unix/worker_sandbox.rs b/crates/capsem-foundation/src/unix/worker_sandbox.rs index 3def396e9..5a0704223 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Role { Gateway, + Ledger, VmOwner, } diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs index 858fc978d..e4e0417d0 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs @@ -253,9 +253,20 @@ fn install_seccomp(role: Role) -> io::Result<()> { libc::SYS_futimesat, libc::SYS_utimensat, ]; - if role == Role::Gateway { + if matches!(role, Role::Gateway | Role::Ledger) { denied.extend([libc::SYS_execve, libc::SYS_execveat]); } + if role == Role::Ledger { + denied.extend([ + libc::SYS_socket, + libc::SYS_socketpair, + libc::SYS_accept, + libc::SYS_accept4, + libc::SYS_kill, + libc::SYS_tkill, + libc::SYS_tgkill, + ]); + } for syscall in denied { filter.push(instruction(0x15, 0, 1, syscall as u32)); filter.push(instruction(0x06, 0, 0, deny)); diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs b/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs index 8bd88c9bb..4026c0d04 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs @@ -9,18 +9,22 @@ pub(super) struct Compiled { } pub(super) fn compile(policy: &Policy) -> io::Result { - let mut source = String::from( - "(version 1)\n\ - (allow default)\n\ - (deny file-read*)\n\ - (deny file-write*)\n\ - (deny network-outbound)\n\ - (deny network-bind)\n\ - (deny network-inbound)\n\ - (deny process-exec)\n\ - (deny signal)\n\ - (allow signal (target same-sandbox))\n", - ); + let mut source = if policy.role() == Role::Ledger { + String::from("(version 1)\n(deny default)\n") + } else { + String::from( + "(version 1)\n\ + (allow default)\n\ + (deny file-read*)\n\ + (deny file-write*)\n\ + (deny network-outbound)\n\ + (deny network-bind)\n\ + (deny network-inbound)\n\ + (deny process-exec)\n\ + (deny signal)\n\ + (allow signal (target same-sandbox))\n", + ) + }; if policy.role() == Role::Gateway { source.push_str("(allow network-inbound (local ip \"*:*\"))\n"); } diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index 7aebe3738..7e3304c72 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -45,6 +45,20 @@ fn seatbelt_gateway_accepts_only_prebound_inbound_network() { assert!(!source.contains("allow network-bind")); } +#[test] +fn seatbelt_ledger_is_deny_by_default_with_only_its_directory() { + let compiled = + super::seatbelt::compile(&Policy::new(Role::Ledger).allow("/sessions/one", Access::ReadWrite)).unwrap(); + let source = compiled.source().to_str().unwrap(); + assert!(source.starts_with("(version 1)\n(deny default)\n")); + assert!(!source.contains("allow network")); + assert!(!source.contains("allow process-exec")); + assert!(!source.contains("allow signal")); + assert!(!source.contains("allow mach-lookup")); + assert!(source.contains("(allow file-read* file-write*")); + assert_eq!(compiled.parameters(), [("PATH_0", "/sessions/one")]); +} + #[cfg(target_os = "linux")] #[test] fn linux_policy_preserves_grants_and_denies_ambient_authority() { @@ -113,7 +127,7 @@ fn linux_sandbox_child() { .unwrap(); confine( - &Policy::new(Role::Gateway) + &Policy::new(Role::Ledger) .allow(&allowed, Access::ReadWrite) .allow(&readonly, Access::ReadOnly) .allow(&exact, Access::ReadOnly), @@ -128,6 +142,7 @@ fn linux_sandbox_child() { std::fs::write(allowed.join("created"), b"ok").unwrap(); assert!(std::fs::read(denied.join("secret")).is_err()); assert!(UnixStream::connect(control).is_err()); + assert!(UnixStream::pair().is_err()); assert!(TcpStream::connect((Ipv4Addr::LOCALHOST, port)).is_err()); assert!(Command::new("/usr/bin/true").status().is_err()); let parent = unsafe { libc::getppid() }; @@ -211,7 +226,7 @@ fn macos_sandbox_child() { .unwrap(); confine( - &Policy::new(Role::Gateway) + &Policy::new(Role::Ledger) .allow(&allowed, Access::ReadWrite) .allow(&readonly, Access::ReadOnly), ) @@ -225,6 +240,7 @@ fn macos_sandbox_child() { assert!(std::fs::set_permissions(&allowed, std::fs::Permissions::from_mode(0o777)).is_err()); assert!(std::fs::read(denied.join("secret")).is_err()); assert!(UnixStream::connect(control).is_err()); + assert!(UnixStream::pair().is_err()); assert!(TcpStream::connect((Ipv4Addr::LOCALHOST, port)).is_err()); assert!(Command::new("/usr/bin/true").status().is_err()); let parent = unsafe { libc::getppid() }; From 6673f87e58ec83ac18378e685c089dd336ac2c0a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 03:08:14 +0000 Subject: [PATCH 098/464] security(ledger): confine session owners --- CHANGELOG.md | 5 ++ crates/capsem-ledger/src/main.rs | 75 +++++++++++++++++++++++- crates/capsem-ledger/src/tests.rs | 23 ++++++++ crates/capsem-ledger/tests/subprocess.rs | 46 +++++++++++++++ 4 files changed, 148 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 149d7ce92..596b56ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Confine each session-ledger owner before readiness to its one database and + inherited channels, denying sibling sessions, socket creation, execution, + signals and ambient files; erase inherited environment bytes and reject + symlinked session grants before serving clients. + - Confine the Linux API gateway before readiness to its inherited listeners and coordinator-minted service and VM-owner channels, denying filesystem access, process execution and direct network or control-socket dialing. diff --git a/crates/capsem-ledger/src/main.rs b/crates/capsem-ledger/src/main.rs index 52359ff3d..f202ee3fd 100644 --- a/crates/capsem-ledger/src/main.rs +++ b/crates/capsem-ledger/src/main.rs @@ -1,11 +1,15 @@ use std::collections::HashSet; +use std::fmt::Write as FmtWrite; use std::io; +use std::io::Write; use std::os::fd::AsFd; use std::os::unix::net::UnixStream; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; +use std::process::Command; use std::sync::Arc; use anyhow::{bail, Context, Result}; +use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; use capsem_foundation::unix::{fd, router_channel}; use capsem_logger::ledger_server::{LedgerClientExit, LedgerServer}; use capsem_proto::ledger::LedgerGeneration; @@ -35,6 +39,12 @@ struct Args { } fn main() -> Result<()> { + // SAFETY: process entry, before telemetry, runtimes, database handles or + // parent-watch threads can observe environment state or own descriptors. + unsafe { + capsem_foundation::unix::process::clear_inherited_environment(); + fd::close_inherited_descriptors()?; + } let _telemetry = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { service: "capsem-ledger", sink: capsem_foundation::telemetry::LogSink::Stderr, @@ -44,14 +54,77 @@ fn main() -> Result<()> { capsem_guard::watch_parent_or_exit(Some(args.parent_pid))?; let stdin = io::stdin(); let control = UnixStream::from(fd::duplicate(stdin.as_fd())?); + let session_dir = session_directory(&args.database)?; + let denied_file = std::env::current_exe().context("resolve ledger executable before confinement")?; let server = Arc::new(LedgerServer::open(&args.database).context("open session ledger")?); let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) .enable_all() .build()?; + capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Ledger).allow(&session_dir, Access::ReadWrite)) + .context("confine ledger worker before readiness")?; + attest_confinement(&session_dir, &denied_file, args.parent_pid, args.generation)?; runtime.block_on(run_control(control, server, args.generation, CLIENT_LIMIT)) } +fn session_directory(database: &Path) -> Result { + if !database.is_absolute() || database.file_name().and_then(|name| name.to_str()) != Some("session.db") { + bail!("ledger database must be an absolute session.db path"); + } + let directory = database + .parent() + .filter(|directory| *directory != Path::new("/")) + .context("ledger database must have a session directory")?; + Ok(directory.to_path_buf()) +} + +fn attest_confinement( + session_dir: &Path, + denied_file: &Path, + parent_pid: u32, + generation: LedgerGeneration, +) -> Result<()> { + let mut generation_hex = String::with_capacity(32); + for byte in generation.as_bytes() { + write!(&mut generation_hex, "{byte:02x}").expect("write to string"); + } + let marker = session_dir.join(format!(".capsem-ledger-confinement-{generation_hex}")); + if std::env::vars_os().next().is_some() { + bail!("ledger worker retained inherited environment state"); + } + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&marker) + .context("ledger confinement denied its session directory")?; + file.write_all(b"confined\n")?; + file.sync_all()?; + drop(file); + std::fs::remove_file(&marker)?; + + require_denied(std::fs::read(denied_file), "read outside its session")?; + require_denied(UnixStream::pair(), "create a Unix socket")?; + require_denied( + std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)), + "create a TCP listener", + )?; + require_denied(Command::new(denied_file).arg("--version").status(), "execute a process")?; + let parent = capsem_foundation::unix::process::ProcessId::try_from(parent_pid)?; + if capsem_foundation::unix::process::probe_signal_authority(parent)? + != capsem_foundation::unix::process::SignalProbe::Denied + { + bail!("ledger confinement allowed it to signal its parent"); + } + Ok(()) +} + +fn require_denied(result: io::Result, operation: &str) -> Result<()> { + if result.is_ok() { + bail!("ledger confinement allowed it to {operation}"); + } + Ok(()) +} + async fn run_control( control: UnixStream, server: Arc, diff --git a/crates/capsem-ledger/src/tests.rs b/crates/capsem-ledger/src/tests.rs index 371779931..681a3764f 100644 --- a/crates/capsem-ledger/src/tests.rs +++ b/crates/capsem-ledger/src/tests.rs @@ -236,3 +236,26 @@ fn generation_parser_is_exact_and_nonzero() { assert!(parse_generation(invalid).is_err(), "accepted {invalid}"); } } + +#[test] +fn database_path_is_one_absolute_session_directory() { + assert_eq!( + session_directory(Path::new("/run/capsem/sessions/one/session.db")).unwrap(), + Path::new("/run/capsem/sessions/one") + ); + for invalid in ["session.db", "/session.db", "/run/capsem/other.db"] { + assert!(session_directory(Path::new(invalid)).is_err(), "accepted {invalid}"); + } +} + +#[test] +fn inherited_authority_is_removed_and_confinement_precedes_readiness() { + let source = include_str!("main.rs"); + let clear = source.find("clear_inherited_environment();").unwrap(); + let close = source.find("close_inherited_descriptors()?").unwrap(); + let telemetry = source.find("telemetry::init").unwrap(); + let confine = source.find("worker_sandbox::confine").unwrap(); + let attest = source.find("attest_confinement(&session_dir").unwrap(); + let run = source.find("runtime.block_on(run_control").unwrap(); + assert!(clear < close && close < telemetry && telemetry < confine && confine < attest && attest < run); +} diff --git a/crates/capsem-ledger/tests/subprocess.rs b/crates/capsem-ledger/tests/subprocess.rs index 47b0e87dc..63eef51c8 100644 --- a/crates/capsem-ledger/tests/subprocess.rs +++ b/crates/capsem-ledger/tests/subprocess.rs @@ -54,6 +54,7 @@ async fn executable_opens_one_ledger_and_reports_durable_stop() { .arg(&database) .arg("--generation") .arg("06060606060606060606060606060606") + .env("CAPSEM_LEDGER_SECRET", "must-not-survive") .stdin(Stdio::from(OwnedFd::from(child_control))) .stdout(Stdio::null()) .stderr(Stdio::piped()) @@ -66,6 +67,13 @@ async fn executable_opens_one_ledger_and_reports_durable_stop() { event(&events).await, LedgerControlEvent::Ready { generation: GENERATION } ); + #[cfg(target_os = "linux")] + { + let environ = std::fs::read(format!("/proc/{}/environ", child.0.as_ref().unwrap().id())).unwrap(); + assert!(!environ + .windows(b"CAPSEM_LEDGER_SECRET".len()) + .any(|value| value == b"CAPSEM_LEDGER_SECRET")); + } requests .send( &encode_ledger_control_request(LedgerControlRequest::Shutdown { generation: GENERATION }), @@ -90,3 +98,41 @@ async fn executable_opens_one_ledger_and_reports_durable_stop() { ); assert!(database.is_file()); } + +#[tokio::test] +async fn executable_refuses_a_symlinked_session_before_readiness() { + let dir = tempfile::tempdir().unwrap(); + let actual = dir.path().join("actual"); + let linked = dir.path().join("linked"); + std::fs::create_dir(&actual).unwrap(); + std::os::unix::fs::symlink(&actual, &linked).unwrap(); + let (coordinator, child_control) = UnixStream::pair().unwrap(); + let child = Command::new(env!("CARGO_BIN_EXE_capsem-ledger")) + .arg("--parent-pid") + .arg(std::process::id().to_string()) + .arg("--database") + .arg(linked.join("session.db")) + .arg("--generation") + .arg("06060606060606060606060606060606") + .stdin(Stdio::from(OwnedFd::from(child_control))) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + drop(coordinator); + let output = tokio::time::timeout( + Duration::from_secs(5), + tokio::task::spawn_blocking(move || child.wait_with_output()), + ) + .await + .expect("unconfined ledger worker did not exit") + .unwrap() + .unwrap(); + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("open session ledger") || stderr.contains("confine ledger worker before readiness"), + "unexpected refusal: {}", + stderr + ); +} From b14dee079b27ce67534da26b7c4038e5ab03ee98 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 03:22:42 +0000 Subject: [PATCH 099/464] security(proxy): apply atomic policy revisions --- CHANGELOG.md | 4 + crates/capsem-core/benches/dns_handler.rs | 19 +++-- crates/capsem-core/benches/mitm_request.rs | 21 +++-- crates/capsem-core/src/net/dns/mod.rs | 2 +- crates/capsem-core/src/net/dns/resolver.rs | 24 ++++-- .../capsem-core/src/net/dns/resolver/tests.rs | 14 +++- crates/capsem-core/src/net/dns/server.rs | 71 ++++------------ .../capsem-core/src/net/dns/server/tests.rs | 50 ++++------- .../capsem-core/src/net/mitm_proxy/config.rs | 14 +--- crates/capsem-core/src/net/mitm_proxy/mod.rs | 74 +++++++--------- .../src/net/mitm_proxy/telemetry_hook.rs | 11 +-- .../net/mitm_proxy/telemetry_hook/tests.rs | 21 ++--- .../telemetry_hook/tests/headers.rs | 2 - .../mitm_proxy/telemetry_hook/tests/ledger.rs | 20 ++--- .../capsem-core/src/net/mitm_proxy/tests.rs | 1 + .../capsem-core/src/net/mitm_proxy/upgrade.rs | 15 ++-- .../src/net/mitm_proxy/upstream.rs | 48 +++++++++-- .../src/net/mitm_proxy/upstream/tests.rs | 62 +++++++++++--- crates/capsem-core/src/net/mod.rs | 1 + crates/capsem-core/src/net/proxy_engine.rs | 84 +++++++++++++++++++ .../capsem-core/src/net/proxy_engine/tests.rs | 70 ++++++++++++++++ crates/capsem-core/tests/mitm_integration.rs | 23 ++--- .../tests/mitm_integration/grants.rs | 14 ++-- .../mitm_integration/mcp_gateway_route.rs | 6 +- .../mitm_integration/resolved_upstream.rs | 2 +- 25 files changed, 430 insertions(+), 243 deletions(-) create mode 100644 crates/capsem-core/src/net/proxy_engine.rs create mode 100644 crates/capsem-core/src/net/proxy_engine/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 596b56ecf..bb59d5d7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Apply each proxy policy reload as one digest-keyed immutable revision across + HTTP routing, security plugins, provider identity, response telemetry and + guest DNS, preventing a request from combining authority from two revisions. + - Confine each session-ledger owner before readiness to its one database and inherited channels, denying sibling sessions, socket creation, execution, signals and ambient files; erase inherited environment bytes and reject diff --git a/crates/capsem-core/benches/dns_handler.rs b/crates/capsem-core/benches/dns_handler.rs index ca845a9ce..f3c88c5b5 100644 --- a/crates/capsem-core/benches/dns_handler.rs +++ b/crates/capsem-core/benches/dns_handler.rs @@ -1,9 +1,12 @@ use std::collections::BTreeMap; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; use capsem_core::net::dns::{DnsHandler, DnsResolver}; use capsem_core::net::policy::NetworkMechanics; -use capsem_core::net::policy_config::{SecurityPluginConfig, SecurityRuleProfile, SecurityRuleSet, SecurityRuleSource}; +use capsem_core::net::policy_config::{ + ModelEndpointRegistry, SecurityRuleProfile, SecurityRuleSet, SecurityRuleSource, +}; +use capsem_core::net::proxy_engine::{ProxyPolicyHandle, ProxyPolicySnapshot}; use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; use hickory_proto::op::{Message, MessageType, OpCode, Query}; use hickory_proto::rr::{Name, RecordType}; @@ -17,14 +20,18 @@ fn build_query_bytes(name: &str, qtype: RecordType, id: u16) -> Vec { } fn local_nxdomain_handler() -> DnsHandler { - let policy = Arc::new(RwLock::new(Arc::new(NetworkMechanics::new()))); let profile = SecurityRuleProfile::parse_toml("").expect("empty rule profile parses"); let rules = SecurityRuleSet::compile_profile(&profile, SecurityRuleSource::User).expect("empty rule profile compiles"); - let security_rules = Arc::new(RwLock::new(Arc::new(rules))); - let plugin_policy = Arc::new(RwLock::new(Arc::new(BTreeMap::::new()))); + let policy = ProxyPolicyHandle::new(ProxyPolicySnapshot::new( + "blake3:benchmark".into(), + NetworkMechanics::new(), + rules, + BTreeMap::new(), + ModelEndpointRegistry::default(), + )); let resolver = Arc::new(DnsResolver::with_upstreams(Vec::new())); - DnsHandler::new(policy, security_rules, plugin_policy, resolver) + DnsHandler::new(policy, resolver) } fn bench_local_nxdomain(c: &mut Criterion) { diff --git a/crates/capsem-core/benches/mitm_request.rs b/crates/capsem-core/benches/mitm_request.rs index 809efa5c8..7a72376ba 100644 --- a/crates/capsem-core/benches/mitm_request.rs +++ b/crates/capsem-core/benches/mitm_request.rs @@ -71,28 +71,31 @@ fn proxy_config(upstream_port: u16) -> Arc { let ca = Arc::new(CertAuthority::load(CA_KEY, CA_CERT).unwrap()); let mut mechanics = NetworkMechanics::new(); mechanics.http_upstream_ports = vec![80, upstream_port]; - let policy = Arc::new(std::sync::RwLock::new(Arc::new(mechanics))); let dir = tempfile::tempdir().unwrap(); let db = Arc::new(DbWriter::open(&dir.path().join("bench.db"), 1024).unwrap()); std::mem::forget(dir); let security_rules = capsem_core::net::policy_config::SecurityRuleSet::new(Vec::new()); + let policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new( + capsem_core::net::proxy_engine::ProxyPolicySnapshot::new( + "blake3:benchmark".into(), + mechanics, + security_rules, + BTreeMap::new(), + capsem_core::net::policy_config::ProviderRuleProfile::builtin_defaults() + .endpoint_registry() + .expect("builtin provider endpoint registry"), + ), + ); let telemetry = Arc::new(mitm_proxy::telemetry_hook::TelemetryDeps { db: db.clone(), pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), trace_state: Arc::new(std::sync::Mutex::new(capsem_core::net::ai_traffic::TraceState::new())), - security_rules: Arc::new(std::sync::RwLock::new(Arc::new(security_rules))), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); - let pipeline = mitm_proxy::make_production_pipeline(Arc::clone(&policy), Arc::clone(&telemetry)); + let pipeline = mitm_proxy::make_production_pipeline(Arc::clone(&telemetry)); Arc::new(MitmProxyConfig { server_tls: mitm_proxy::make_server_tls_config(&ca), ca, policy, - model_endpoints: Arc::new(std::sync::RwLock::new(Arc::new( - capsem_core::net::policy_config::ProviderRuleProfile::builtin_defaults() - .endpoint_registry() - .expect("builtin provider endpoint registry"), - ))), db, upstream_tls: mitm_proxy::make_upstream_tls_config(), telemetry, diff --git a/crates/capsem-core/src/net/dns/mod.rs b/crates/capsem-core/src/net/dns/mod.rs index 9a4286bcc..a2651387e 100644 --- a/crates/capsem-core/src/net/dns/mod.rs +++ b/crates/capsem-core/src/net/dns/mod.rs @@ -46,5 +46,5 @@ pub mod telemetry; pub use cache::{DnsAnswerCache, DEFAULT_CAPACITY, DEFAULT_MAX_TTL_SECS}; pub use resolver::{DnsDatagram, DnsGrantFuture, DnsResolver, DnsUpstreamGrants, DEFAULT_UPSTREAMS}; -pub use server::{DnsHandler, DnsHandlerResult, SharedPolicy}; +pub use server::{DnsHandler, DnsHandlerResult}; pub use telemetry::{build_dns_event, security_event_from_dns_event}; diff --git a/crates/capsem-core/src/net/dns/resolver.rs b/crates/capsem-core/src/net/dns/resolver.rs index 14f23edb6..5ced889c9 100644 --- a/crates/capsem-core/src/net/dns/resolver.rs +++ b/crates/capsem-core/src/net/dns/resolver.rs @@ -73,13 +73,13 @@ pub type DnsGrantFuture<'a> = Pin> + /// Supplies a connected UDP socket for one configured upstream index. pub trait DnsUpstreamGrants: Send + Sync { - fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_>; + fn open(&self, upstream_index: u16, policy_digest: &str) -> DnsGrantFuture<'_>; } struct DirectDnsUpstreams(Arc>>); impl DnsUpstreamGrants for DirectDnsUpstreams { - fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_> { + fn open(&self, upstream_index: u16, _policy_digest: &str) -> DnsGrantFuture<'_> { Box::pin(async move { let upstream = self .0 @@ -175,6 +175,11 @@ impl DnsResolver { /// On total failure (every upstream timed out or errored) returns /// the cumulative error so the caller can synthesize a SERVFAIL. pub async fn resolve(&self, query_bytes: &[u8]) -> Result<(Vec, Duration)> { + self.resolve_for_policy(query_bytes, "direct").await + } + + /// Forward under the exact active-policy revision that admitted the query. + pub async fn resolve_for_policy(&self, query_bytes: &[u8], policy_digest: &str) -> Result<(Vec, Duration)> { let upstreams = self.upstreams(); if upstreams.is_empty() { return Err(anyhow!("no upstream nameservers configured")); @@ -183,7 +188,10 @@ impl DnsResolver { for (index, upstream) in upstreams.iter().enumerate() { let t0 = Instant::now(); let upstream_index = u16::try_from(index).map_err(|_| anyhow!("too many DNS upstreams configured"))?; - match self.try_one(upstream_index, *upstream, query_bytes).await { + match self + .try_one(upstream_index, *upstream, query_bytes, policy_digest) + .await + { Ok(resp) => { let elapsed = t0.elapsed(); debug!( @@ -203,9 +211,15 @@ impl DnsResolver { Err(last_err.unwrap_or_else(|| anyhow!("all DNS upstreams failed"))) } - async fn try_one(&self, upstream_index: u16, upstream: SocketAddr, query_bytes: &[u8]) -> Result> { + async fn try_one( + &self, + upstream_index: u16, + upstream: SocketAddr, + query_bytes: &[u8], + policy_digest: &str, + ) -> Result> { let expected = ExpectedAnswer::for_query(query_bytes)?; - let grant = self.grants.open(upstream_index).await?; + let grant = self.grants.open(upstream_index, policy_digest).await?; let sock = grant.socket(); sock.send(query_bytes).await?; let deadline = Instant::now() + self.per_attempt_timeout; diff --git a/crates/capsem-core/src/net/dns/resolver/tests.rs b/crates/capsem-core/src/net/dns/resolver/tests.rs index bcf7a8fae..8a6151865 100644 --- a/crates/capsem-core/src/net/dns/resolver/tests.rs +++ b/crates/capsem-core/src/net/dns/resolver/tests.rs @@ -196,11 +196,13 @@ async fn a_hundred_concurrent_resolves_complete_against_one_upstream() { struct IndexedGrants { upstreams: Arc>, requested: Arc>>, + policy_digests: Arc>>, refuse: bool, } impl DnsUpstreamGrants for IndexedGrants { - fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_> { + fn open(&self, upstream_index: u16, policy_digest: &str) -> DnsGrantFuture<'_> { + self.policy_digests.lock().unwrap().push(policy_digest.to_string()); Box::pin(async move { self.requested.lock().unwrap().push(upstream_index); if self.refuse { @@ -228,19 +230,25 @@ async fn injected_grants_preserve_configured_failover_order() { }) .await; let requested = Arc::new(Mutex::new(Vec::new())); + let policy_digests = Arc::new(Mutex::new(Vec::new())); let grants = IndexedGrants { upstreams: Arc::new(vec![primary, failover]), requested: Arc::clone(&requested), + policy_digests: Arc::clone(&policy_digests), refuse: false, }; let resolver = DnsResolver::with_grants(vec![primary, failover], Arc::new(grants)).with_timeout(Duration::from_millis(50)); resolver - .resolve(&query_bytes(7, "ordered.example.")) + .resolve_for_policy(&query_bytes(7, "ordered.example."), "blake3:dns") .await .expect("the second configured grant answers"); assert_eq!(*requested.lock().unwrap(), vec![0, 1]); + assert_eq!( + *policy_digests.lock().unwrap(), + vec!["blake3:dns".to_string(), "blake3:dns".to_string()] + ); } #[tokio::test] @@ -254,6 +262,7 @@ async fn refused_grant_never_falls_back_to_direct_udp() { let grants = IndexedGrants { upstreams: Arc::new(vec![upstream]), requested: Arc::clone(&requested), + policy_digests: Arc::new(Mutex::new(Vec::new())), refuse: true, }; let error = DnsResolver::with_grants(vec![upstream], Arc::new(grants)) @@ -275,6 +284,7 @@ async fn a_policy_reload_replaces_the_configured_grant_order() { let grants = IndexedGrants { upstreams: Arc::new(vec![upstream]), requested: Arc::clone(&requested), + policy_digests: Arc::new(Mutex::new(Vec::new())), refuse: false, }; let resolver = DnsResolver::with_grants(Vec::new(), Arc::new(grants)); diff --git a/crates/capsem-core/src/net/dns/server.rs b/crates/capsem-core/src/net/dns/server.rs index 1e1ac2902..0c00adcef 100644 --- a/crates/capsem-core/src/net/dns/server.rs +++ b/crates/capsem-core/src/net/dns/server.rs @@ -31,8 +31,7 @@ use crate::net::dns::resolver::DnsResolver; use crate::net::parsers::dns_parser::{ build_nxdomain, build_ptr_response, build_redirect_response, build_servfail, parse_query, DnsQuery, }; -use crate::net::policy::NetworkMechanics; -use crate::net::policy_config::{snapshot_plugin_policy, SecurityRuleSet, SharedPluginPolicy}; +use crate::net::proxy_engine::ProxyPolicyHandle; use crate::security_engine::{ evaluate_security_boundary, DnsSecurityEvent, RuntimeSecurityEventType, SecurityEnforcementDecision, SecurityEvent, }; @@ -168,15 +167,6 @@ fn apply_security_enforcement_fields(result: &mut DnsHandlerResult, enforcement: result.policy_reason = enforcement.reason.clone(); } -/// Hot-swappable network policy snapshot for DNS resolver mechanics. -/// -/// The outer `Arc>` lets admins edit the policy at runtime -/// (frontend's policy editor → service → write lock); the inner -/// `Arc` is what each request snapshots before redirect/cache -/// checks so we never hold the read lock across an await point. -pub type SharedPolicy = Arc>>; -pub type SharedSecurityRules = Arc>>; - /// Async DNS handler shared across vsock connections. /// /// `policy` is shared (not cloned) with the MITM proxy via the same @@ -190,9 +180,7 @@ pub type SharedSecurityRules = Arc>>; /// runs use `new(policy, resolver)` which leaves cache=None. #[derive(Clone)] pub struct DnsHandler { - policy: SharedPolicy, - security_rules: SharedSecurityRules, - plugin_policy: SharedPluginPolicy, + policy: ProxyPolicyHandle, resolver: Arc, cache: Option>, /// Identical lookups on their way upstream, shared by every clone so @@ -206,16 +194,9 @@ impl DnsHandler { /// Build a handler with no answer cache. Tests use this so a /// cache hit can't accidentally hide an upstream-path /// regression. - pub fn new( - policy: SharedPolicy, - security_rules: SharedSecurityRules, - plugin_policy: SharedPluginPolicy, - resolver: Arc, - ) -> Self { + pub fn new(policy: ProxyPolicyHandle, resolver: Arc) -> Self { Self { policy, - security_rules, - plugin_policy, resolver, cache: None, in_flight: Arc::default(), @@ -224,17 +205,9 @@ impl DnsHandler { } /// Build a handler with an explicit answer cache. - pub fn with_cache( - policy: SharedPolicy, - security_rules: SharedSecurityRules, - plugin_policy: SharedPluginPolicy, - resolver: Arc, - cache: Arc, - ) -> Self { + pub fn with_cache(policy: ProxyPolicyHandle, resolver: Arc, cache: Arc) -> Self { Self { policy, - security_rules, - plugin_policy, resolver, cache: Some(cache), in_flight: Arc::default(), @@ -245,15 +218,9 @@ impl DnsHandler { /// Build a production handler: default UDP forwarder /// (DEFAULT_UPSTREAMS, 5s timeout) + default-sized /// TTL-honoring answer cache. - pub fn with_default_resolver( - policy: SharedPolicy, - security_rules: SharedSecurityRules, - plugin_policy: SharedPluginPolicy, - ) -> Self { + pub fn with_default_resolver(policy: ProxyPolicyHandle) -> Self { Self::with_cache( policy, - security_rules, - plugin_policy, Arc::new(DnsResolver::new()), Arc::new(DnsAnswerCache::default()), ) @@ -315,13 +282,6 @@ impl DnsHandler { self.cache.as_ref() } - /// Snapshot the current `NetworkMechanics` under the read lock, - /// release the lock immediately, and return the cheap-Arc snapshot - /// for use across the rest of the request lifecycle. - fn policy_snapshot(&self) -> Arc { - self.policy.read().unwrap().clone() - } - /// Process one DNS query message. Pure async, no background tasks. /// /// The contract: every input produces a `DnsHandlerResult`, even @@ -394,9 +354,12 @@ impl DnsHandler { qname: Some(query.qname.clone()), qtype: Some(query.qtype.to_string()), }); - let rules = self.security_rules.read().unwrap().clone(); - let plugin_policy = snapshot_plugin_policy(&self.plugin_policy); - let dns_evaluation = match evaluate_security_boundary(&rules, plugin_policy, dns_security_event) { + let policy = self.policy.snapshot(); + let dns_evaluation = match evaluate_security_boundary( + policy.security_rules(), + Arc::clone(policy.plugins()), + dns_security_event, + ) { Ok(evaluation) => evaluation, Err(error) => { warn!(error = %error, qname = %query.qname, "dns handler: security engine failed"); @@ -440,8 +403,6 @@ impl DnsHandler { return result; } - let policy = self.policy_snapshot(); - if is_capsem_local_nxdomain_name(&query.qname) { debug!( qname = %query.qname, @@ -466,7 +427,7 @@ impl DnsHandler { // (a blocked query stays NXDOMAIN; redirect never weakens a block) // and BEFORE the upstream forward (no network round trip when an // admin has pinned the answer locally). - if let Some(redirect) = policy.find_dns_redirect(&query.qname, query.qtype) { + if let Some(redirect) = policy.network().find_dns_redirect(&query.qname, query.qtype) { let matched_rule = format!("redirect:{}", redirect.matcher.pattern_str()); debug!( qname = %query.qname, @@ -500,7 +461,7 @@ impl DnsHandler { // from cache. See `dns/cache.rs` for the full invariant. let key = super::coalesce::LookupKey::new(query_bytes); if let Some(cache) = &self.cache { - if let Some(cached) = cache.get(&query, &key, &policy) { + if let Some(cached) = cache.get(&query, &key, policy.network()) { let rcode = response_rcode(&cached); debug!( qname = %query.qname, @@ -519,7 +480,11 @@ impl DnsHandler { let t0 = Instant::now(); let (outcome, led) = match self.in_flight.join_or_lead(key.clone()) { super::coalesce::Role::Lead(lease) => { - let outcome = self.resolver.resolve(query_bytes).await.map_err(|e| format!("{e:#}")); + let outcome = self + .resolver + .resolve_for_policy(query_bytes, policy.digest()) + .await + .map_err(|e| format!("{e:#}")); (lease.finish(outcome), true) } super::coalesce::Role::Follow(rx) => { diff --git a/crates/capsem-core/src/net/dns/server/tests.rs b/crates/capsem-core/src/net/dns/server/tests.rs index 5aef31dc0..2c935b820 100644 --- a/crates/capsem-core/src/net/dns/server/tests.rs +++ b/crates/capsem-core/src/net/dns/server/tests.rs @@ -2,6 +2,8 @@ use super::*; use std::collections::BTreeMap; use std::sync::atomic::{AtomicUsize, Ordering}; +use crate::net::policy::NetworkMechanics; +use crate::net::policy_config::SecurityRuleSet; use hickory_proto::op::{Message, MessageType, OpCode, Query}; use hickory_proto::rr::{Name, RecordType}; @@ -13,30 +15,23 @@ fn build_query_bytes(name: &str, qtype: RecordType, id: u16) -> Vec { msg.to_vec().unwrap() } -fn shared_policy() -> SharedPolicy { - Arc::new(std::sync::RwLock::new(Arc::new(NetworkMechanics::new()))) -} - -fn security_rules(toml: &str) -> SharedSecurityRules { +fn proxy_policy(toml: &str) -> ProxyPolicyHandle { let profile = crate::net::policy_config::SecurityRuleProfile::parse_toml(toml).unwrap(); let rules = SecurityRuleSet::compile_profile(&profile, crate::net::policy_config::SecurityRuleSource::User).unwrap(); - Arc::new(std::sync::RwLock::new(Arc::new(rules))) -} - -fn plugin_policy() -> SharedPluginPolicy { - Arc::new(std::sync::RwLock::new(Arc::new(BTreeMap::new()))) + ProxyPolicyHandle::new(crate::net::proxy_engine::ProxyPolicySnapshot::new( + "blake3:test".into(), + NetworkMechanics::new(), + rules, + BTreeMap::new(), + crate::net::policy_config::ModelEndpointRegistry::default(), + )) } #[tokio::test] async fn only_a_single_standard_question_can_reach_upstream() { let (addr, seen) = counting_upstream(hickory_proto::op::ResponseCode::NoError, std::time::Duration::ZERO).await; - let handler = DnsHandler::new( - shared_policy(), - security_rules(""), - plugin_policy(), - Arc::new(DnsResolver::with_upstreams(vec![addr])), - ); + let handler = DnsHandler::new(proxy_policy(""), Arc::new(DnsResolver::with_upstreams(vec![addr]))); let ordinary = build_query_bytes("allowed.example.", RecordType::A, 123); let mut multiple = Message::from_vec(&ordinary).unwrap(); multiple.add_query(Query::query( @@ -61,8 +56,7 @@ async fn only_a_single_standard_question_can_reach_upstream() { #[tokio::test] async fn dns_handler_blocks_query_through_security_event_rules() { let handler = DnsHandler::new( - shared_policy(), - security_rules( + proxy_policy( r#" [profiles.rules.block_dns_example] name = "block_dns_example" @@ -71,7 +65,6 @@ async fn dns_handler_blocks_query_through_security_event_rules() { match = 'dns.qname == "blocked.example.com"' "#, ), - plugin_policy(), Arc::new(DnsResolver::new()), ); @@ -91,12 +84,7 @@ async fn dns_handler_blocks_query_through_security_event_rules() { #[tokio::test] async fn dns_handler_returns_local_nxdomain_for_capsem_bogus_without_upstream() { - let handler = DnsHandler::new( - shared_policy(), - security_rules(""), - plugin_policy(), - Arc::new(DnsResolver::with_upstreams(Vec::new())), - ); + let handler = DnsHandler::new(proxy_policy(""), Arc::new(DnsResolver::with_upstreams(Vec::new()))); let result = handler .handle(&build_query_bytes("load-test.capsem-bogus.", RecordType::A, 0xBEEF)) @@ -165,8 +153,8 @@ async fn counting_upstream( fn handler_with(upstream: std::net::SocketAddr, rules: &str, cache: Option>) -> DnsHandler { let resolver = Arc::new(DnsResolver::with_upstreams(vec![upstream])); match cache { - Some(cache) => DnsHandler::with_cache(shared_policy(), security_rules(rules), plugin_policy(), resolver, cache), - None => DnsHandler::new(shared_policy(), security_rules(rules), plugin_policy(), resolver), + Some(cache) => DnsHandler::with_cache(proxy_policy(rules), resolver, cache), + None => DnsHandler::new(proxy_policy(rules), resolver), } } @@ -317,13 +305,7 @@ async fn a_failed_leader_gives_every_follower_its_own_servfail_and_caches_nothin let cache = Arc::new(DnsAnswerCache::default()); let resolver = Arc::new(DnsResolver::with_upstreams(vec![upstream]).with_timeout(std::time::Duration::from_millis(100))); - let handler = Arc::new(DnsHandler::with_cache( - shared_policy(), - security_rules(""), - plugin_policy(), - resolver, - Arc::clone(&cache), - )); + let handler = Arc::new(DnsHandler::with_cache(proxy_policy(""), resolver, Arc::clone(&cache))); let mut tasks = Vec::new(); for id in 1..=5u16 { let handler = Arc::clone(&handler); diff --git a/crates/capsem-core/src/net/mitm_proxy/config.rs b/crates/capsem-core/src/net/mitm_proxy/config.rs index 8ea1f6c68..998a7b69a 100644 --- a/crates/capsem-core/src/net/mitm_proxy/config.rs +++ b/crates/capsem-core/src/net/mitm_proxy/config.rs @@ -7,23 +7,15 @@ use super::pipeline; use super::telemetry_hook; use super::upstream::TcpUpstreamGrants; use crate::net::cert_authority::CertAuthority; -use crate::net::policy::NetworkMechanics; +use crate::net::proxy_engine::ProxyPolicyHandle; /// Configuration for the MITM proxy. pub struct MitmProxyConfig { pub ca: Arc, /// Guest-facing TLS config, built once (`make_server_tls_config`): one session cache for all. pub server_tls: Arc, - /// Live policy, swappable via RwLock so settings changes take effect - /// without restarting the VM. Each HTTP request snapshots the Arc so - /// that disabling a provider blocks the next request even on an - /// existing keep-alive connection. - pub policy: Arc>>, - /// Live model endpoint registry from settings and corp provider blocks. - /// MITM resolves host -> model protocol once per request and then passes - /// that typed metadata to enforcement, hooks, broker substitution, and - /// telemetry. Provider hooks must not infer protocol from domains. - pub model_endpoints: Arc>>, + /// One digest-keyed active-policy revision, snapshotted per request. + pub policy: ProxyPolicyHandle, pub db: Arc, /// Cached upstream TLS config (shared across all connections). pub upstream_tls: Arc, diff --git a/crates/capsem-core/src/net/mitm_proxy/mod.rs b/crates/capsem-core/src/net/mitm_proxy/mod.rs index f98e00947..7a1f1c688 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mod.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mod.rs @@ -55,7 +55,6 @@ trait TokioReadWrite: AsyncRead + AsyncWrite {} impl TokioReadWrite for T where T: AsyncRead + AsyncWrite {} -use super::policy::NetworkMechanics; use crate::net::ai_traffic::provider::{route_provider, ModelProtocol, ProviderKind}; use crate::security_engine::{HttpSecurityEvent, IpSecurityEvent, ModelSecurityEvent, SecurityEvent, TcpSecurityEvent}; use body::{BodyStats, ProxyBoxBody, TrackedBody}; @@ -63,7 +62,7 @@ use fd_stream::{AsyncFdStream, ReplayReader}; use mcp_observe::{observed_mcp_http_request_for_body, should_sniff_mcp_http_body, ObservedMcpHttpRequest}; use protocol::Protocol; use telemetry_hook::TelemetryRequestContext; -use upstream::{CachedUpstream, UpstreamCache}; +use upstream::{CachedUpstream, UpstreamCache, UpstreamPolicy}; use util::{ current_unix_ms, format_headers, format_headers_for_domain, http_upstream_port_allowed, is_anthropic_model_name, is_google_model_name, is_llm_api_path, is_openai_model_name, materialize_collected_response_headers, @@ -119,11 +118,7 @@ impl Drop for ConnectionGauge { /// (Content-Encoding / Content-Length strip) happen inline in /// `handle_request` before chunk dispatch begins -- the chunk hooks /// themselves never see the head. -pub fn make_production_pipeline( - policy: Arc>>, - telemetry: Arc, -) -> Arc { - let _ = policy; +pub fn make_production_pipeline(telemetry: Arc) -> Arc { let p = pipeline::Pipeline::builder() // Chunk-hook order is load-bearing: // 1. DecompressionHook -- gzip detection on first chunk's @@ -146,20 +141,6 @@ pub fn make_production_pipeline( Arc::new(p) } -fn ai_provider_for_domain(config: &MitmProxyConfig, domain: &str) -> Option { - config.model_endpoints.read().unwrap().provider_for_host(domain) -} - -fn ai_provider_for_target( - config: &MitmProxyConfig, - domain: &str, - upstream_port: u16, - path: &str, -) -> Option { - let registry = config.model_endpoints.read().unwrap(); - ai_identity_for_target_or_path(®istry, domain, upstream_port, path).provider -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct ModelTrafficIdentity { /// Endpoint owner used for policy/logging. Example: `ollama` for @@ -700,10 +681,13 @@ async fn serve_pipeline( let (domain, port) = (&request_domain, upstream_port); return Ok(mcp_http::serve(req, route, domain, port, protocol, &config_arc, &process_name).await); } - let ai_identity = { - let registry = config_arc.model_endpoints.read().unwrap(); - ai_identity_for_target_or_path(®istry, &request_domain, upstream_port, req.uri().path()) - }; + let policy_snapshot = config_arc.policy.snapshot(); + let ai_identity = ai_identity_for_target_or_path( + policy_snapshot.model_endpoints(), + &request_domain, + upstream_port, + req.uri().path(), + ); handle_request( req, &request_domain, @@ -715,6 +699,7 @@ async fn serve_pipeline( ai_identity.provider, ai_identity.protocol, &cached_upstream, + policy_snapshot, ) .await } @@ -793,7 +778,7 @@ fn security_event_with_transport(mut event: SecurityEvent, ip: Option, u /// Handle a single HTTP request within a MITM-proxied connection /// (TLS or plain HTTP). /// -/// Reads the live policy from `config.policy` RwLock per-request so that +/// Reads one live policy revision per request so that /// settings changes (e.g. disabling a provider) take effect immediately, /// even for in-flight keep-alive connections. #[allow(clippy::too_many_arguments)] @@ -820,6 +805,7 @@ async fn handle_request( ai_provider: Option, ai_protocol: Option, cached_upstream: &UpstreamCache, + policy_snapshot: Arc, ) -> Result, anyhow::Error> { use http_body_util::BodyExt; @@ -835,15 +821,12 @@ async fn handle_request( None }; - // Snapshot the live policy for this request (not per-connection) so that - // hot-reloaded settings take effect for subsequent requests on the same - // keep-alive connection. - let policy: Arc = config.policy.read().unwrap().clone(); + let policy = policy_snapshot.network(); // Resolve before the rules run, so they judge the address the dial reaches. let resolver = &config.upstream_resolver; let target = UpstreamTarget::select( resolver, - &policy, + UpstreamPolicy::from(policy_snapshot.as_ref()), protocol, domain, upstream_port, @@ -951,7 +934,7 @@ async fn handle_request( upstream_tls, config, process_name, - policy: &policy, + policy_snapshot: &policy_snapshot, ai_provider, ai_protocol, method: method.clone(), @@ -986,6 +969,7 @@ async fn handle_request( warn!(domain, method, path, error = %error, "MITM proxy: upstream error"); let body_text = format!("Capsem: upstream error ({error})\n"); let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(&policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider, @@ -1149,7 +1133,7 @@ async fn handle_request( if let Some(trace_id) = capsem_foundation::telemetry::ambient_capsem_trace_id() { http_security_event = http_security_event.with_trace_id(trace_id); } - let rules = config.telemetry.security_rules.read().unwrap().clone(); + let rules = Arc::clone(policy_snapshot.security_rules()); let actions_span = tracing::debug_span!( target: "capsem.mitm", spans::MITM_SECURITY_ACTIONS, @@ -1162,7 +1146,7 @@ async fn handle_request( let http_evaluation = match actions_span.in_scope(|| { crate::security_engine::evaluate_security_boundary( &rules, - config.telemetry.plugin_policy.read().unwrap().clone(), + Arc::clone(policy_snapshot.plugins()), http_security_event, ) }) { @@ -1202,6 +1186,7 @@ async fn handle_request( format!("capsem: HTTP request blocked by security rule: {rule_id}\n") }; let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(&policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider, @@ -1239,12 +1224,13 @@ async fn handle_request( // Host-side plain-HTTP port allowlist. The port came from the guest's Host // header; enforce it before any upstream dial so a guest reaching the proxy // directly cannot make the host connect to an arbitrary port. - if !http_upstream_port_allowed(&policy, protocol, upstream_port) { + if !http_upstream_port_allowed(policy, protocol, upstream_port) { actions_span.record("decision", "deny"); actions_span.record("status", "ok"); let matched = "security.web.http_upstream_ports"; let body_text = format!("capsem: HTTP upstream port {upstream_port} blocked by {matched}\n"); let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(&policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider, @@ -1330,11 +1316,7 @@ async fn handle_request( upstream_port, ); let mcp_evaluation = match mcp_span.in_scope(|| { - crate::security_engine::evaluate_security_boundary( - &rules, - config.telemetry.plugin_policy.read().unwrap().clone(), - mcp_event, - ) + crate::security_engine::evaluate_security_boundary(&rules, Arc::clone(policy_snapshot.plugins()), mcp_event) }) { Ok(evaluation) => evaluation, Err(error) => { @@ -1414,6 +1396,7 @@ async fn handle_request( let mut scrubbed_stats = BodyStats::new(0); scrubbed_stats.bytes = observed.bytes_sent; let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(&policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider: effective_ai_provider, @@ -1550,7 +1533,7 @@ async fn handle_request( let model_event = security_event_with_transport(model_event, upstream_ip, upstream_port); let model_evaluation = match crate::security_engine::evaluate_security_boundary( &rules, - config.telemetry.plugin_policy.read().unwrap().clone(), + Arc::clone(policy_snapshot.plugins()), model_event, ) { Ok(evaluation) => evaluation, @@ -1580,6 +1563,7 @@ async fn handle_request( let mut scrubbed_stats = BodyStats::new(0); scrubbed_stats.bytes = body_bytes.len() as u64; let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(&policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider: effective_ai_provider, @@ -1716,7 +1700,7 @@ async fn handle_request( let dial_start = Instant::now(); let tcp_start = Instant::now(); let upstream_tcp = match target - .connect_with_grants(config.upstream_grants.as_deref()) + .connect_with_grants(config.upstream_grants.as_deref(), policy_snapshot.digest()) .instrument(upstream_prepare_span.clone()) .await { @@ -1964,7 +1948,7 @@ async fn handle_request( "cached upstream sender failed on send; reconnecting replayable request" ); let upstream_tcp = match target - .connect_with_grants(config.upstream_grants.as_deref()) + .connect_with_grants(config.upstream_grants.as_deref(), policy_snapshot.digest()) .instrument(upstream_send_span.clone()) .await { @@ -2202,7 +2186,7 @@ async fn handle_request( let model_event = security_event_with_transport(model_event, upstream_ip, upstream_port); let model_evaluation = match crate::security_engine::evaluate_security_boundary( &rules, - config.telemetry.plugin_policy.read().unwrap().clone(), + Arc::clone(policy_snapshot.plugins()), model_event, ) { Ok(evaluation) => evaluation, @@ -2232,6 +2216,7 @@ async fn handle_request( model_evaluation.enforcement.rule_id.as_deref().unwrap_or("unknown") ); let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(&policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider: effective_ai_provider, @@ -2350,6 +2335,7 @@ async fn handle_request( }; let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(&policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider: effective_ai_provider, diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs index 1f5e24996..12432f50a 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs @@ -40,7 +40,8 @@ use crate::net::ai_traffic::events::{ use crate::net::ai_traffic::pricing::PricingTable; use crate::net::ai_traffic::provider::{extract_model_from_path, tool_origin, ModelProtocol, ProviderKind}; use crate::net::ai_traffic::{request_parser, TraceState}; -use crate::net::policy_config::{snapshot_plugin_policy, PluginPolicySnapshot, SecurityRuleSet, SharedPluginPolicy}; +use crate::net::policy_config::{PluginPolicySnapshot, SecurityRuleSet}; +use crate::net::proxy_engine::ProxyPolicySnapshot; use crate::security_engine::{ emit_evaluated_security_rules, emit_security_write, HttpSecurityEvent, IpSecurityEvent, ModelSecurityEvent, RuntimeSecurityEventType, SecurityEvent, TcpSecurityEvent, @@ -53,6 +54,8 @@ use capsem_telemetry::mitm as m; /// the request head and upstream response head have been observed, /// before the body wrapper begins iterating chunks. pub struct TelemetryRequestContext { + /// The exact active-policy revision that admitted this request. + pub policy_snapshot: Arc, pub domain: String, pub process_name: Option, pub ai_provider: Option, @@ -109,8 +112,6 @@ pub struct TelemetryDeps { pub db: Arc, pub pricing: Arc, pub trace_state: Arc>, - pub security_rules: Arc>>, - pub plugin_policy: SharedPluginPolicy, } /// Sync `ChunkHook` that tracks response bytes/preview and, on @@ -309,8 +310,8 @@ impl ChunkHook for TelemetryHook { let stage_started = Instant::now(); let db = Arc::clone(&self.deps.db); - let rules = Arc::clone(&self.deps.security_rules.read().unwrap()); - let plugin_policy = snapshot_plugin_policy(&self.deps.plugin_policy); + let rules = Arc::clone(req_ctx.policy_snapshot.security_rules()); + let plugin_policy = Arc::clone(req_ctx.policy_snapshot.plugins()); let credential_injections = req_ctx.credential_injections; record_telemetry_stage(stage_started, "ledger_deps_clone"); diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs index e18017ee6..2201b1668 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs @@ -128,6 +128,7 @@ impl Drop for TraceEnvGuard { /// Returns a generic request context for an allowed Anthropic POST. fn anthropic_req_ctx() -> TelemetryRequestContext { TelemetryRequestContext { + policy_snapshot: policy_snapshot(SecurityRuleSet::new(Vec::new())), domain: "api.anthropic.com".into(), process_name: Some("agent".into()), ai_provider: Some(ProviderKind::Anthropic), @@ -156,6 +157,16 @@ fn anthropic_req_ctx() -> TelemetryRequestContext { } } +fn policy_snapshot(rules: SecurityRuleSet) -> Arc { + Arc::new(crate::net::proxy_engine::ProxyPolicySnapshot::new( + "blake3:telemetry-test".into(), + crate::net::policy::NetworkMechanics::new(), + rules, + BTreeMap::new(), + crate::net::policy_config::ModelEndpointRegistry::default(), + )) +} + fn empty_resp_stats() -> TelemetryResponseStats { TelemetryResponseStats::default() } @@ -655,15 +666,9 @@ fn fake_deps() -> Arc { db, pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }) } -fn empty_security_rules() -> Arc>> { - Arc::new(std::sync::RwLock::new(Arc::new(SecurityRuleSet::new(Vec::new())))) -} - /// Without a seeded request context, the hook is shadow-mode: it /// doesn't allocate the response stats slot and doesn't emit on end. #[test] @@ -724,8 +729,6 @@ async fn hook_accepts_primary_net_event_before_completing_response() { db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); let hook = TelemetryHook::new(deps); @@ -783,8 +786,6 @@ async fn bounded_logger_backpressure_keeps_every_completed_response_event() { db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), })); let conn = ConnMeta { domain: "127.0.0.1".to_string(), diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/headers.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/headers.rs index 661e7c8ca..f427366cf 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/headers.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/headers.rs @@ -23,8 +23,6 @@ async fn an_observed_credential_is_redacted_from_every_stored_header_and_body() db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), })); let mut req_ctx = anthropic_req_ctx(); diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/ledger.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/ledger.rs index 73801411d..fa20a4f31 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/ledger.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/ledger.rs @@ -23,8 +23,6 @@ async fn hook_writes_substitution_event_and_shared_credential_ref() { db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); let hook = TelemetryHook::new(deps); let raw = "sk-ant-hook-test"; @@ -102,8 +100,6 @@ async fn hook_does_not_repay_capture_ledger_for_repeated_identical_credential() db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); let hook = TelemetryHook::new(deps); let raw = "sk-ant-hook-repeat-test"; @@ -206,8 +202,6 @@ match = 'http.host == "api.anthropic.com" && http.path == "/v1/messages" && tcp. db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: Arc::new(std::sync::RwLock::new(Arc::new(rules))), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); let hook = TelemetryHook::new(deps); @@ -215,7 +209,9 @@ match = 'http.host == "api.anthropic.com" && http.path == "/v1/messages" && tcp. let conn = any_conn(); { let mut c = ctx_for(&mut state, &conn); - *c.state::>(|| None) = Some(anthropic_req_ctx()); + let mut request = anthropic_req_ctx(); + request.policy_snapshot = policy_snapshot(rules); + *c.state::>(|| None) = Some(request); } complete_response(&hook, &mut state, &conn).await; @@ -268,8 +264,6 @@ match = 'model.provider == "anthropic" && model.name == "claude-test"' db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: Arc::new(std::sync::RwLock::new(Arc::new(rules))), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); let hook = TelemetryHook::new(deps); @@ -277,7 +271,9 @@ match = 'model.provider == "anthropic" && model.name == "claude-test"' let conn = any_conn(); { let mut c = ctx_for(&mut state, &conn); - *c.state::>(|| None) = Some(anthropic_req_ctx()); + let mut request = anthropic_req_ctx(); + request.policy_snapshot = policy_snapshot(rules); + *c.state::>(|| None) = Some(request); } complete_response(&hook, &mut state, &conn).await; @@ -328,8 +324,6 @@ async fn hook_writes_injected_substitution_event_for_broker_ref_replay() { db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); let hook = TelemetryHook::new(deps); let raw = "sk-ant-replayed-hook-test"; @@ -402,8 +396,6 @@ async fn hook_detects_response_body_token_exchange_and_redacts_preview() { db: Arc::clone(&db), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); let hook = TelemetryHook::new(deps); let raw = "github_pat_exchange_secret"; diff --git a/crates/capsem-core/src/net/mitm_proxy/tests.rs b/crates/capsem-core/src/net/mitm_proxy/tests.rs index 345c36da8..d6de4aa36 100644 --- a/crates/capsem-core/src/net/mitm_proxy/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::net::policy::NetworkMechanics; use crate::net::policy_config::{SecurityRuleAction, SecurityRuleProfile, SecurityRuleSet}; #[test] diff --git a/crates/capsem-core/src/net/mitm_proxy/upgrade.rs b/crates/capsem-core/src/net/mitm_proxy/upgrade.rs index 377fd1129..de0c9549b 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upgrade.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upgrade.rs @@ -19,7 +19,7 @@ pub(super) struct UpgradeRequest<'a> { pub upstream_tls: &'a Arc, pub config: &'a Arc, pub process_name: &'a Option, - pub policy: &'a NetworkMechanics, + pub policy_snapshot: &'a Arc, pub ai_provider: Option, pub ai_protocol: Option, pub method: String, @@ -53,7 +53,7 @@ pub(super) async fn handle_upgrade( upstream_tls, config, process_name, - policy, + policy_snapshot, ai_provider, ai_protocol, method, @@ -84,6 +84,7 @@ pub(super) async fn handle_upgrade( let make_ws_error = |error: &dyn std::fmt::Display| -> hyper::Response { let body_text = format!("Capsem: websocket upstream error ({error})\n"); let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider, @@ -129,6 +130,7 @@ pub(super) async fn handle_upgrade( decision: &SecurityBoundaryDecisionFields| -> hyper::Response { let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider, @@ -163,7 +165,7 @@ pub(super) async fn handle_upgrade( .body(seal_with_telemetry(body, req_ctx, ai_provider, ai_protocol)) .unwrap() }; - if !http_upstream_port_allowed(policy, protocol, upstream_port) { + if !http_upstream_port_allowed(policy_snapshot.network(), protocol, upstream_port) { let matched = "security.web.http_upstream_ports"; tracing::Span::current().record("decision", "deny"); return Ok(refuse( @@ -186,10 +188,10 @@ pub(super) async fn handle_upgrade( if let Some(trace_id) = capsem_foundation::telemetry::ambient_capsem_trace_id() { upgrade_event = upgrade_event.with_trace_id(trace_id); } - let rules = config.telemetry.security_rules.read().unwrap().clone(); + let rules = Arc::clone(policy_snapshot.security_rules()); let upgrade_evaluation = crate::security_engine::evaluate_security_boundary( &rules, - config.telemetry.plugin_policy.read().unwrap().clone(), + Arc::clone(policy_snapshot.plugins()), upgrade_event, ); let upgrade_decision = match upgrade_evaluation { @@ -213,7 +215,7 @@ pub(super) async fn handle_upgrade( .unwrap_or_else(|| matched_rule.clone()); let upstream_tcp = match target - .connect_with_grants(config.upstream_grants.as_deref()) + .connect_with_grants(config.upstream_grants.as_deref(), policy_snapshot.digest()) .instrument(ws_span.clone()) .await { @@ -362,6 +364,7 @@ pub(super) async fn handle_upgrade( } let req_ctx = TelemetryRequestContext { + policy_snapshot: Arc::clone(policy_snapshot), domain: domain.to_string(), process_name: process_name.clone(), ai_provider, diff --git a/crates/capsem-core/src/net/mitm_proxy/upstream.rs b/crates/capsem-core/src/net/mitm_proxy/upstream.rs index 5e3f7f861..50ea159df 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upstream.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upstream.rs @@ -27,6 +27,7 @@ use tokio::net::TcpStream; use super::body::ProxyBoxBody; use super::protocol::Protocol; use crate::net::policy::{NetworkMechanics, UpstreamOverrideProtocol}; +use crate::net::proxy_engine::ProxyPolicySnapshot; use crate::net::upstream_address::{judged_address, UpstreamResolver}; /// Future returned by a trusted TCP selection provider. @@ -36,8 +37,30 @@ pub type TcpConnectGrantFuture<'a> = Pin TcpResolveGrantFuture<'_>; - fn connect(&self, selection_id: u64) -> TcpConnectGrantFuture<'_>; + fn resolve(&self, protocol: Protocol, host: &str, port: u16, policy_digest: &str) -> TcpResolveGrantFuture<'_>; + fn connect(&self, selection_id: u64, policy_digest: &str) -> TcpConnectGrantFuture<'_>; +} + +/// The policy authority carried into one upstream selection. +pub(super) struct UpstreamPolicy<'a> { + network: &'a NetworkMechanics, + digest: &'a str, +} + +impl<'a> UpstreamPolicy<'a> { + #[cfg(test)] + fn new(network: &'a NetworkMechanics, digest: &'a str) -> Self { + Self { network, digest } + } +} + +impl<'a> From<&'a ProxyPolicySnapshot> for UpstreamPolicy<'a> { + fn from(snapshot: &'a ProxyPolicySnapshot) -> Self { + Self { + network: snapshot.network(), + digest: snapshot.digest(), + } + } } struct SelectionLease { @@ -193,7 +216,7 @@ impl UpstreamTarget { /// connection was judged and opened with instead of resolving again. pub(super) async fn select( resolver: &UpstreamResolver, - policy: &NetworkMechanics, + policy: UpstreamPolicy<'_>, guest_protocol: Protocol, domain: &str, port: u16, @@ -201,7 +224,7 @@ impl UpstreamTarget { grants: Option<&dyn TcpUpstreamGrants>, ) -> Self { if grants.is_none() { - if let Some(target) = Self::override_target(policy, domain, port) { + if let Some(target) = Self::override_target(policy.network, domain, port) { return target; } } @@ -214,7 +237,7 @@ impl UpstreamTarget { return pinned; } if let Some(grants) = grants { - return Self::resolve_granted(grants, guest_protocol, domain, port).await; + return Self::resolve_granted(grants, guest_protocol, domain, port, policy.digest).await; } match resolver.resolve(domain, port).await { Ok(addresses) => Self::Resolved(addresses), @@ -222,8 +245,14 @@ impl UpstreamTarget { } } - async fn resolve_granted(grants: &dyn TcpUpstreamGrants, protocol: Protocol, host: &str, port: u16) -> Self { - match grants.resolve(protocol, host, port).await { + async fn resolve_granted( + grants: &dyn TcpUpstreamGrants, + protocol: Protocol, + host: &str, + port: u16, + policy_digest: &str, + ) -> Self { + match grants.resolve(protocol, host, port, policy_digest).await { Ok(selection) => Self::Granted { host: host.to_owned(), port, @@ -296,6 +325,7 @@ impl UpstreamTarget { pub async fn connect_with_grants( &self, grants: Option<&dyn TcpUpstreamGrants>, + policy_digest: &str, ) -> io::Result<(GrantedTcpStream, Self)> { let Some(grants) = grants else { let (stream, pinned) = self.connect().await?; @@ -319,7 +349,7 @@ impl UpstreamTarget { let selection = match selection { Some(selection) => selection, None => { - refreshed = grants.resolve(*guest_protocol, host, *port).await?; + refreshed = grants.resolve(*guest_protocol, host, *port, policy_digest).await?; if refreshed.protocol != *protocol || refreshed.judged_ip != *judged_ip { return Err(io::Error::new( io::ErrorKind::PermissionDenied, @@ -329,7 +359,7 @@ impl UpstreamTarget { &refreshed } }; - let stream = grants.connect(selection.claim()?).await?; + let stream = grants.connect(selection.claim()?, policy_digest).await?; Ok(( stream, Self::Granted { diff --git a/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs b/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs index cba198c59..18df6e03a 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs @@ -17,10 +17,12 @@ struct TestGrants { next_id: AtomicU64, selection_releases: Arc, stream_releases: Arc, + policy_digests: Arc>>, } impl TcpUpstreamGrants for TestGrants { - fn resolve(&self, _protocol: Protocol, _host: &str, _port: u16) -> TcpResolveGrantFuture<'_> { + fn resolve(&self, _protocol: Protocol, _host: &str, _port: u16, policy_digest: &str) -> TcpResolveGrantFuture<'_> { + self.policy_digests.lock().unwrap().push(policy_digest.to_string()); let id = self.next_id.fetch_add(1, Ordering::Relaxed); let releases = Arc::clone(&self.selection_releases); Box::pin(async move { @@ -35,7 +37,8 @@ impl TcpUpstreamGrants for TestGrants { }) } - fn connect(&self, _selection_id: u64) -> TcpConnectGrantFuture<'_> { + fn connect(&self, _selection_id: u64, policy_digest: &str) -> TcpConnectGrantFuture<'_> { + self.policy_digests.lock().unwrap().push(policy_digest.to_string()); let address = self.address; let releases = Arc::clone(&self.stream_releases); Box::pin(async move { @@ -75,7 +78,7 @@ fn decision(target: &UpstreamTarget, domain: &str, port: u16) -> (String, Securi async fn select(resolver: &UpstreamResolver, policy: &NetworkMechanics, domain: &str, port: u16) -> UpstreamTarget { UpstreamTarget::select( resolver, - policy, + UpstreamPolicy::new(policy, "blake3:test"), Protocol::Tls, domain, port, @@ -231,12 +234,38 @@ async fn keep_alive_requests_reuse_only_their_own_pinned_upstream() { .with_fixed_answer("other.example", vec!["127.0.0.1".parse().unwrap()]); let policy = NetworkMechanics::new(); - let same = UpstreamTarget::select(&rebound, &policy, Protocol::Http, "site.example", 80, &cache, None).await; + let same = UpstreamTarget::select( + &rebound, + UpstreamPolicy::new(&policy, "blake3:test"), + Protocol::Http, + "site.example", + 80, + &cache, + None, + ) + .await; assert_eq!(same, pinned, "the same host stays on the address it was judged with"); - let other_port = - UpstreamTarget::select(&rebound, &policy, Protocol::Http, "site.example", 8080, &cache, None).await; - let other_host = UpstreamTarget::select(&rebound, &policy, Protocol::Http, "other.example", 80, &cache, None).await; + let other_port = UpstreamTarget::select( + &rebound, + UpstreamPolicy::new(&policy, "blake3:test"), + Protocol::Http, + "site.example", + 8080, + &cache, + None, + ) + .await; + let other_host = UpstreamTarget::select( + &rebound, + UpstreamPolicy::new(&policy, "blake3:test"), + Protocol::Http, + "other.example", + 80, + &cache, + None, + ) + .await; assert_eq!( other_host.judged_ip("other.example"), Some("127.0.0.1".parse().unwrap()) @@ -267,7 +296,7 @@ async fn a_trusted_override_preempts_an_existing_resolved_connection() { let selected = UpstreamTarget::select( &UpstreamResolver::system(), - &policy, + UpstreamPolicy::new(&policy, "blake3:test"), Protocol::Tls, "site.example", 443, @@ -295,11 +324,12 @@ async fn brokered_selection_is_judged_connected_pinned_and_released() { next_id: AtomicU64::new(1), selection_releases: Arc::clone(&selection_releases), stream_releases: Arc::clone(&stream_releases), + policy_digests: Arc::new(Mutex::new(Vec::new())), }; let cache = UpstreamCache::default(); let target = UpstreamTarget::select( &UpstreamResolver::system(), - &NetworkMechanics::new(), + UpstreamPolicy::new(&NetworkMechanics::new(), "blake3:test"), Protocol::Tls, "brokered.example", 443, @@ -310,17 +340,24 @@ async fn brokered_selection_is_judged_connected_pinned_and_released() { assert_eq!(target.judged_ip("brokered.example"), Some("127.0.0.1".parse().unwrap())); assert_eq!(target.protocol(Protocol::Tls), Protocol::Http); - let (connected, accepted) = tokio::join!(target.connect_with_grants(Some(&grants)), listener.accept()); + let (connected, accepted) = tokio::join!( + target.connect_with_grants(Some(&grants), "blake3:test"), + listener.accept() + ); let (stream, pinned) = connected.unwrap(); let (_peer, _) = accepted.unwrap(); assert!(matches!(pinned, UpstreamTarget::Granted { selection: None, .. })); assert_eq!(selection_releases.load(Ordering::Relaxed), 0); drop(stream); assert_eq!(stream_releases.load(Ordering::Relaxed), 1); + assert_eq!( + *grants.policy_digests.lock().unwrap(), + vec!["blake3:test".to_string(), "blake3:test".to_string()] + ); let unused = UpstreamTarget::select( &UpstreamResolver::system(), - &NetworkMechanics::new(), + UpstreamPolicy::new(&NetworkMechanics::new(), "blake3:test"), Protocol::Tls, "blocked.example", 443, @@ -340,9 +377,10 @@ async fn installed_grants_never_fall_back_to_a_direct_target() { next_id: AtomicU64::new(1), selection_releases: Arc::new(AtomicUsize::new(0)), stream_releases: Arc::new(AtomicUsize::new(0)), + policy_digests: Arc::new(Mutex::new(Vec::new())), }; let direct = UpstreamTarget::Resolved(vec![listener.local_addr().unwrap()]); - let error = match direct.connect_with_grants(Some(&grants)).await { + let error = match direct.connect_with_grants(Some(&grants), "blake3:test").await { Ok(_) => panic!("direct connection bypassed installed grants"), Err(error) => error, }; diff --git a/crates/capsem-core/src/net/mod.rs b/crates/capsem-core/src/net/mod.rs index 37e344458..9aba0d1da 100644 --- a/crates/capsem-core/src/net/mod.rs +++ b/crates/capsem-core/src/net/mod.rs @@ -10,6 +10,7 @@ pub mod network_registry; pub mod parsers; pub mod policy; pub mod policy_config; +pub mod proxy_engine; pub mod router_process; pub mod switch_host; pub mod upstream_address; diff --git a/crates/capsem-core/src/net/proxy_engine.rs b/crates/capsem-core/src/net/proxy_engine.rs new file mode 100644 index 000000000..02f6bbf37 --- /dev/null +++ b/crates/capsem-core/src/net/proxy_engine.rs @@ -0,0 +1,84 @@ +//! Transport-independent state for one session's HTTP and DNS proxy. +//! +//! A request clones one [`ProxyPolicySnapshot`] and keeps it through routing, +//! enforcement, credential handling, and ledger completion. Reload replaces +//! the outer `Arc` once, so no request can combine pieces from two active +//! policy files. + +use std::collections::BTreeMap; +use std::sync::{Arc, RwLock}; + +use super::policy::NetworkMechanics; +use super::policy_config::{ModelEndpointRegistry, SecurityPluginConfig, SecurityRuleSet}; + +/// One fully compiled active-policy revision. +pub struct ProxyPolicySnapshot { + digest: String, + network: NetworkMechanics, + security_rules: Arc, + plugins: Arc>, + model_endpoints: ModelEndpointRegistry, +} + +impl ProxyPolicySnapshot { + pub fn new( + digest: String, + network: NetworkMechanics, + security_rules: SecurityRuleSet, + plugins: BTreeMap, + model_endpoints: ModelEndpointRegistry, + ) -> Self { + Self { + digest, + network, + security_rules: Arc::new(security_rules), + plugins: Arc::new(plugins), + model_endpoints, + } + } + + pub fn digest(&self) -> &str { + &self.digest + } + + pub fn network(&self) -> &NetworkMechanics { + &self.network + } + + pub fn security_rules(&self) -> &Arc { + &self.security_rules + } + + pub fn plugins(&self) -> &Arc> { + &self.plugins + } + + pub fn model_endpoints(&self) -> &ModelEndpointRegistry { + &self.model_endpoints + } +} + +/// Live handle whose sole mutable operation atomically replaces a revision. +#[derive(Clone)] +pub struct ProxyPolicyHandle { + current: Arc>>, +} + +impl ProxyPolicyHandle { + pub fn new(snapshot: ProxyPolicySnapshot) -> Self { + Self { + current: Arc::new(RwLock::new(Arc::new(snapshot))), + } + } + + pub fn snapshot(&self) -> Arc { + self.current.read().unwrap_or_else(|error| error.into_inner()).clone() + } + + pub fn replace(&self, snapshot: ProxyPolicySnapshot) { + *self.current.write().unwrap_or_else(|error| error.into_inner()) = Arc::new(snapshot); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-core/src/net/proxy_engine/tests.rs b/crates/capsem-core/src/net/proxy_engine/tests.rs new file mode 100644 index 000000000..58d151917 --- /dev/null +++ b/crates/capsem-core/src/net/proxy_engine/tests.rs @@ -0,0 +1,70 @@ +use std::collections::BTreeMap; +use std::sync::Arc; + +use super::*; +use crate::net::policy::NetworkMechanics; +use crate::net::policy_config::{ + ModelEndpointRegistry, SecurityPluginConfig, SecurityRuleProfile, SecurityRuleSet, SecurityRuleSource, +}; + +fn revision(marker: usize) -> ProxyPolicySnapshot { + let profile = SecurityRuleProfile::parse_toml(&format!( + r#" +[profiles.rules.marker_{marker}] +name = "marker_{marker}" +action = "allow" +match = 'http.host == "marker-{marker}.example"' +"# + )) + .unwrap(); + let rules = SecurityRuleSet::compile_profile(&profile, SecurityRuleSource::User).unwrap(); + let mut network = NetworkMechanics::new(); + network.max_body_capture = marker; + let plugins = BTreeMap::::new(); + + ProxyPolicySnapshot::new( + format!("blake3:{marker:064x}"), + network, + rules, + plugins, + ModelEndpointRegistry::default(), + ) +} + +#[test] +fn policy_handle_replaces_one_immutable_revision_atomically() { + let handle = ProxyPolicyHandle::new(revision(0)); + + for marker in 1..=256 { + handle.replace(revision(marker)); + let snapshot = handle.snapshot(); + assert_eq!(snapshot.network().max_body_capture, marker); + assert_eq!(snapshot.digest(), format!("blake3:{marker:064x}")); + assert_eq!( + snapshot.security_rules().rules()[0].rule_id, + format!("profiles.rules.marker_{marker}") + ); + } +} + +#[test] +fn request_snapshot_cannot_mix_concurrent_policy_revisions() { + let handle = Arc::new(ProxyPolicyHandle::new(revision(0))); + let writer_handle = Arc::clone(&handle); + let writer = std::thread::spawn(move || { + for marker in 1..=10_000 { + writer_handle.replace(revision(marker % 257)); + } + }); + + for _ in 0..10_000 { + let snapshot = handle.snapshot(); + let marker = snapshot.network().max_body_capture; + assert_eq!(snapshot.digest(), format!("blake3:{marker:064x}")); + assert_eq!( + snapshot.security_rules().rules()[0].rule_id, + format!("profiles.rules.marker_{marker}") + ); + } + writer.join().unwrap(); +} diff --git a/crates/capsem-core/tests/mitm_integration.rs b/crates/capsem-core/tests/mitm_integration.rs index 0b6544946..a60efba57 100644 --- a/crates/capsem-core/tests/mitm_integration.rs +++ b/crates/capsem-core/tests/mitm_integration.rs @@ -176,7 +176,17 @@ fn make_proxy_config_with_mechanics( policy_inner: NetworkMechanics, ) -> (Arc, Arc) { let ca = Arc::new(CertAuthority::load(CA_KEY, CA_CERT).unwrap()); - let policy = Arc::new(std::sync::RwLock::new(Arc::new(policy_inner))); + let policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new( + capsem_core::net::proxy_engine::ProxyPolicySnapshot::new( + "blake3:integration".into(), + policy_inner, + security_rules, + BTreeMap::new(), + capsem_core::net::policy_config::ProviderRuleProfile::builtin_defaults() + .endpoint_registry() + .expect("builtin provider endpoint registry"), + ), + ); let dir = tempfile::tempdir().unwrap(); let db = Arc::new(DbWriter::open(&dir.path().join("test.db"), 256).unwrap()); std::mem::forget(dir); // the tempdir lives as long as the test @@ -184,20 +194,13 @@ fn make_proxy_config_with_mechanics( db: db.clone(), pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), trace_state: Arc::new(std::sync::Mutex::new(capsem_core::net::ai_traffic::TraceState::new())), - security_rules: Arc::new(std::sync::RwLock::new(Arc::new(security_rules))), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); - let pipeline = mitm_proxy::make_production_pipeline(Arc::clone(&policy), Arc::clone(&telemetry)); - let upstream_grants = Arc::new(IntegrationGrants::new(Arc::clone(&policy), UpstreamResolver::system())); + let pipeline = mitm_proxy::make_production_pipeline(Arc::clone(&telemetry)); + let upstream_grants = Arc::new(IntegrationGrants::new(policy.clone(), UpstreamResolver::system())); let config = Arc::new(MitmProxyConfig { server_tls: mitm_proxy::make_server_tls_config(&ca), ca, policy, - model_endpoints: Arc::new(std::sync::RwLock::new(Arc::new( - capsem_core::net::policy_config::ProviderRuleProfile::builtin_defaults() - .endpoint_registry() - .expect("builtin provider endpoint registry"), - ))), db: db.clone(), upstream_tls: mitm_proxy::make_upstream_tls_config(), telemetry, diff --git a/crates/capsem-core/tests/mitm_integration/grants.rs b/crates/capsem-core/tests/mitm_integration/grants.rs index 24200c190..29c05d85f 100644 --- a/crates/capsem-core/tests/mitm_integration/grants.rs +++ b/crates/capsem-core/tests/mitm_integration/grants.rs @@ -7,18 +7,18 @@ use capsem_core::net::mitm_proxy::{ protocol::Protocol, GrantedTcpStream, TcpConnectGrantFuture, TcpGrantSelection, TcpResolveGrantFuture, TcpUpstreamGrants, UpstreamTarget, }; -use capsem_core::net::policy::NetworkMechanics; +use capsem_core::net::proxy_engine::ProxyPolicyHandle; use capsem_core::net::upstream_address::UpstreamResolver; pub(super) struct IntegrationGrants { - policy: Arc>>, + policy: ProxyPolicyHandle, resolver: UpstreamResolver, selections: Arc>>, next_id: AtomicU64, } impl IntegrationGrants { - pub(super) fn new(policy: Arc>>, resolver: UpstreamResolver) -> Self { + pub(super) fn new(policy: ProxyPolicyHandle, resolver: UpstreamResolver) -> Self { Self { policy, resolver, @@ -29,11 +29,11 @@ impl IntegrationGrants { } impl TcpUpstreamGrants for IntegrationGrants { - fn resolve(&self, protocol: Protocol, host: &str, port: u16) -> TcpResolveGrantFuture<'_> { + fn resolve(&self, protocol: Protocol, host: &str, port: u16, _policy_digest: &str) -> TcpResolveGrantFuture<'_> { let host = host.to_owned(); Box::pin(async move { - let policy = self.policy.read().unwrap().clone(); - let target = UpstreamTarget::resolve(&self.resolver, &policy, &host, port).await; + let policy = self.policy.snapshot(); + let target = UpstreamTarget::resolve(&self.resolver, policy.network(), &host, port).await; if let UpstreamTarget::Unresolved(error) = &target { return Err(io::Error::new(io::ErrorKind::NotFound, error.clone())); } @@ -48,7 +48,7 @@ impl TcpUpstreamGrants for IntegrationGrants { }) } - fn connect(&self, selection_id: u64) -> TcpConnectGrantFuture<'_> { + fn connect(&self, selection_id: u64, _policy_digest: &str) -> TcpConnectGrantFuture<'_> { Box::pin(async move { let target = self .selections diff --git a/crates/capsem-core/tests/mitm_integration/mcp_gateway_route.rs b/crates/capsem-core/tests/mitm_integration/mcp_gateway_route.rs index dbbc0ecdd..249cf822a 100644 --- a/crates/capsem-core/tests/mitm_integration/mcp_gateway_route.rs +++ b/crates/capsem-core/tests/mitm_integration/mcp_gateway_route.rs @@ -59,8 +59,10 @@ fn proxy_with_gateway() -> (Arc, Arc, Called) { }); config.mcp_endpoint = Some(Arc::new(McpEndpointState::new( aggregator, - Arc::clone(&config.telemetry.security_rules), - Arc::clone(&config.telemetry.plugin_policy), + Arc::new(std::sync::RwLock::new(Arc::clone( + config.policy.snapshot().security_rules(), + ))), + Arc::new(std::sync::RwLock::new(Arc::clone(config.policy.snapshot().plugins()))), Arc::new(tokio::sync::Semaphore::new(4)), McpTimeouts::default(), ))); diff --git a/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs b/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs index 716fbe1f1..c071e181c 100644 --- a/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs +++ b/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs @@ -178,7 +178,7 @@ fn resolving(config: Arc, name: &str, address: &str) -> Arc Date: Fri, 9 Oct 2026 03:33:14 +0000 Subject: [PATCH 100/464] security(process): bind grants to request policy --- crates/capsem-process/src/ipc.rs | 2 - crates/capsem-process/src/ipc/tests.rs | 33 +++++++++------- crates/capsem-process/src/main.rs | 28 ++++--------- crates/capsem-process/src/mcp_runtime.rs | 5 ++- crates/capsem-process/src/runtime_config.rs | 12 +++++- .../src/runtime_config/tests.rs | 19 +++++++++ crates/capsem-process/src/upstream_grant.rs | 39 ++++--------------- .../src/upstream_grant/tests.rs | 36 ++++++++++------- crates/capsem-process/src/vsock/dns/tests.rs | 18 ++++----- 9 files changed, 98 insertions(+), 94 deletions(-) diff --git a/crates/capsem-process/src/ipc.rs b/crates/capsem-process/src/ipc.rs index 46ef9af6f..458d09b60 100644 --- a/crates/capsem-process/src/ipc.rs +++ b/crates/capsem-process/src/ipc.rs @@ -143,7 +143,6 @@ pub(crate) async fn handle_ipc_connection( mcp_builtin_env: HashMap, vm_ready: Arc, dns_resolver: Arc, - upstream_grants: Arc, ) -> Result<()> { // First frame on every IPC connection is a Hello -- detect cross-version // mixes (capsem-service built before X, capsem-process built after) in @@ -686,7 +685,6 @@ pub(crate) async fn handle_ipc_connection( let reply = match runtime_source.load() { Ok(runtime_config) => { let digest = runtime_config.active_policy_digest.clone(); - upstream_grants.replace_policy_digest(digest.clone()); dns_resolver.replace_upstreams(runtime_config.dns_upstreams.clone()); runtime_config.apply(&net_state, &mcp_runtime); ProcessToService::ConfigReloadResult { diff --git a/crates/capsem-process/src/ipc/tests.rs b/crates/capsem-process/src/ipc/tests.rs index 62e70def1..8b28b2cc4 100644 --- a/crates/capsem-process/src/ipc/tests.rs +++ b/crates/capsem-process/src/ipc/tests.rs @@ -101,7 +101,6 @@ pub(in crate::ipc) struct Dispatcher { runtime_source: RuntimePolicySource, pub(in crate::ipc) ready: Arc, dns_resolver: Arc, - upstream_grants: Arc, } impl Dispatcher { @@ -115,6 +114,8 @@ impl Dispatcher { ) -> (Self, mpsc::Receiver) { let active_policy = temp.join("active_policy.toml"); std::fs::write(&active_policy, "[user_rules]\n[corp_rules]\n[network]\n").unwrap(); + let initial_digest = + capsem_core::net::policy_config::active_policy_digest(&std::fs::read(&active_policy).unwrap()); let net_state = Arc::new( capsem_core::create_net_state_with_policy( "ipc-dispatch-test", @@ -123,13 +124,20 @@ impl Dispatcher { ) .unwrap(), ); - let security_rules = Arc::new(std::sync::RwLock::new(Arc::new( - capsem_core::net::policy_config::SecurityRuleSet::new(Vec::new()), - ))); - let plugin_policy = Arc::new(std::sync::RwLock::new(Arc::new(BTreeMap::new()))); - let model_endpoints = Arc::new(std::sync::RwLock::new(Arc::new( - capsem_core::net::policy_config::ModelEndpointRegistry::default(), - ))); + let rule_set = capsem_core::net::policy_config::SecurityRuleSet::new(Vec::new()); + let plugins = BTreeMap::new(); + let model_endpoints = capsem_core::net::policy_config::ModelEndpointRegistry::default(); + let security_rules = Arc::new(std::sync::RwLock::new(Arc::new(rule_set.clone()))); + let plugin_policy = Arc::new(std::sync::RwLock::new(Arc::new(plugins.clone()))); + let proxy_policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new( + capsem_core::net::proxy_engine::ProxyPolicySnapshot::new( + initial_digest, + capsem_core::net::policy::NetworkMechanics::default(), + rule_set, + plugins, + model_endpoints, + ), + ); let (aggregator, mut aggregator_rx) = AggregatorClient::channel(8); tokio::spawn(async move { while let Some((request, response_tx)) = aggregator_rx.recv().await { @@ -187,16 +195,13 @@ impl Dispatcher { db: Arc::clone(&db), security_rules, plugin_policy, - model_endpoints, + proxy_policy, }); let term_relay = TerminalRelay::new(8); let job_store = Arc::new(JobStore::new()); let (ctrl_tx, ctrl_rx) = mpsc::channel(16); let (events_tx, _) = broadcast::channel(16); - let initial_digest = - capsem_core::net::policy_config::active_policy_digest(&std::fs::read(&active_policy).unwrap()); let dns_resolver = Arc::new(capsem_core::net::dns::DnsResolver::with_upstreams(Vec::new())); - let upstream_grants = Arc::new(crate::upstream_grant::UpstreamGrantClient::test_handle(initial_digest)); ( Self { job_store, @@ -208,7 +213,6 @@ impl Dispatcher { runtime_source: RuntimePolicySource::new(active_policy), ready: Arc::new(AtomicBool::new(true)), dns_resolver, - upstream_grants, }, ctrl_rx, ) @@ -230,7 +234,6 @@ impl Dispatcher { HashMap::new(), Arc::clone(&self.ready), Arc::clone(&self.dns_resolver), - Arc::clone(&self.upstream_grants), )) } } @@ -586,7 +589,7 @@ async fn negotiated_dispatcher_covers_stream_jobs_queries_and_lifecycle() { ProcessToService::ConfigReloadResult { id: 30, active_policy_digest: Some(digest), error: None } if digest == applied )); - assert_eq!(dispatcher.upstream_grants.policy_digest(), applied); + assert_eq!(dispatcher.mcp_runtime.proxy_policy.snapshot().digest(), applied); assert_eq!( dispatcher.dns_resolver.upstreams(), vec!["127.0.0.1:5353".parse().unwrap()] diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 9f9d74b16..2adc7bc1b 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -444,10 +444,7 @@ fn main() -> Result<()> { let runtime_config = runtime_source.load()?; let upstream_grants = { let _runtime = rt.enter(); - Arc::new(upstream_grant::UpstreamGrantClient::start( - upstream_socket, - runtime_config.active_policy_digest.clone(), - )?) + Arc::new(upstream_grant::UpstreamGrantClient::start(upstream_socket)?) }; info!(id = %args.id, "capsem-sandbox-process starting"); @@ -651,6 +648,7 @@ async fn run_async_main_loop( let guest_config = capsem_core::net::policy_config::GuestConfig::default(); let security_rules = Arc::new(std::sync::RwLock::new(Arc::new(runtime_config.security_rules.clone()))); let plugin_policy = Arc::new(std::sync::RwLock::new(Arc::new(runtime_config.plugins.clone()))); + let proxy_policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new(runtime_config.proxy_policy_snapshot()); let job_store = Arc::new(JobStore { publisher: Arc::new( capsem_core::container::publish::Publisher::for_session( @@ -786,7 +784,6 @@ async fn run_async_main_loop( let inflight_cap = capsem_core::mcp::resolve_inflight_cap(); info!(inflight_cap, "MITM MCP endpoint in-flight handler cap"); - let model_endpoints = Arc::new(std::sync::RwLock::new(Arc::new(runtime_config.model_endpoints.clone()))); let mcp_inflight = Arc::new(tokio::sync::Semaphore::new(inflight_cap)); let mcp_endpoint = Arc::new( capsem_core::net::mitm_proxy::McpEndpointState::new( @@ -808,25 +805,19 @@ async fn run_async_main_loop( db: Arc::clone(&db), security_rules: Arc::clone(&security_rules), plugin_policy: Arc::clone(&plugin_policy), - model_endpoints: Arc::clone(&model_endpoints), + proxy_policy: proxy_policy.clone(), }); let telemetry_deps = Arc::new(capsem_core::net::mitm_proxy::telemetry_hook::TelemetryDeps { db: Arc::clone(&db), pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), trace_state: Arc::clone(&model_trace_state), - security_rules: Arc::clone(&security_rules), - plugin_policy: Arc::clone(&plugin_policy), }); - let mitm_pipeline = capsem_core::net::mitm_proxy::make_production_pipeline( - Arc::clone(&net_state.policy), - Arc::clone(&telemetry_deps), - ); + let mitm_pipeline = capsem_core::net::mitm_proxy::make_production_pipeline(Arc::clone(&telemetry_deps)); let mitm_config = Arc::new(capsem_core::net::mitm_proxy::MitmProxyConfig { ca: Arc::clone(&net_state.ca), server_tls: capsem_core::net::mitm_proxy::make_server_tls_config(&net_state.ca), - policy: Arc::clone(&net_state.policy), - model_endpoints, + policy: proxy_policy.clone(), db: Arc::clone(&db), upstream_tls: Arc::clone(&net_state.upstream_tls), telemetry: telemetry_deps, @@ -836,8 +827,7 @@ async fn run_async_main_loop( upstream_grants: Some(Arc::clone(&upstream_grants) as Arc), }); - // DNS handler shares the same security rule/plugin handles as MITM - // so admin enforcement edits take effect across protocols at once. + // DNS and HTTP clone the same immutable revision once per request. let dns_resolver = Arc::new(DnsResolver::with_grants( runtime_config.dns_upstreams.clone(), Arc::clone(&upstream_grants) as Arc, @@ -849,9 +839,7 @@ async fn run_async_main_loop( )); let dns_handler = Arc::new( capsem_core::net::dns::DnsHandler::with_cache( - Arc::clone(&net_state.policy), - Arc::clone(&security_rules), - Arc::clone(&plugin_policy), + proxy_policy, Arc::clone(&dns_resolver), Arc::new(DnsAnswerCache::default()), ) @@ -988,7 +976,6 @@ async fn run_async_main_loop( let builtin_env_c = builtin_env.clone(); let ready_c = Arc::clone(&vm_ready); let dns_resolver_c = Arc::clone(&dns_resolver); - let upstream_grants_c = Arc::clone(&upstream_grants); tokio::spawn(async move { if let Err(e) = ipc::handle_ipc_connection( @@ -1005,7 +992,6 @@ async fn run_async_main_loop( builtin_env_c, ready_c, dns_resolver_c, - upstream_grants_c, ) .await { diff --git a/crates/capsem-process/src/mcp_runtime.rs b/crates/capsem-process/src/mcp_runtime.rs index 74fe3e087..3a103e0af 100644 --- a/crates/capsem-process/src/mcp_runtime.rs +++ b/crates/capsem-process/src/mcp_runtime.rs @@ -2,7 +2,8 @@ use std::sync::Arc; use capsem_core::container::publish::Publisher; use capsem_core::net::mitm_proxy::{McpEndpointState, ScopedMcpTools}; -use capsem_core::net::policy_config::{ModelEndpointRegistry, SecurityRuleSet, SharedPluginPolicy}; +use capsem_core::net::policy_config::{SecurityRuleSet, SharedPluginPolicy}; +use capsem_core::net::proxy_engine::ProxyPolicyHandle; use capsem_logger::DbWriter; use capsem_proto::mcp_aggregator::AggregatorClient; use capsem_proto::mcp_contracts::{McpToolDef, ToolAnnotations}; @@ -135,7 +136,7 @@ pub(crate) struct McpRuntime { pub(crate) db: Arc, pub(crate) security_rules: Arc>>, pub(crate) plugin_policy: SharedPluginPolicy, - pub(crate) model_endpoints: Arc>>, + pub(crate) proxy_policy: ProxyPolicyHandle, } #[cfg(test)] diff --git a/crates/capsem-process/src/runtime_config.rs b/crates/capsem-process/src/runtime_config.rs index 00597425e..b0e8138ab 100644 --- a/crates/capsem-process/src/runtime_config.rs +++ b/crates/capsem-process/src/runtime_config.rs @@ -82,10 +82,10 @@ impl RuntimePolicyConfig { .iter() .map(|rule| rule.rule_id.clone()) .collect::>(); + mcp_runtime.proxy_policy.replace(self.proxy_policy_snapshot()); *net_state.policy.write().unwrap() = std::sync::Arc::new(self.network); *mcp_runtime.security_rules.write().unwrap() = std::sync::Arc::new(self.security_rules); *mcp_runtime.plugin_policy.write().unwrap() = std::sync::Arc::new(self.plugins); - *mcp_runtime.model_endpoints.write().unwrap() = std::sync::Arc::new(self.model_endpoints); tracing::info!( active_policy_digest = %self.active_policy_digest, security_rule_count = security_rule_ids.len(), @@ -94,6 +94,16 @@ impl RuntimePolicyConfig { ); } + pub(crate) fn proxy_policy_snapshot(&self) -> capsem_core::net::proxy_engine::ProxyPolicySnapshot { + capsem_core::net::proxy_engine::ProxyPolicySnapshot::new( + self.active_policy_digest.clone(), + self.network.clone(), + self.security_rules.clone(), + self.plugins.clone(), + self.model_endpoints.clone(), + ) + } + pub(crate) fn mcp_servers( &self, builtin_binary: Option<&Path>, diff --git a/crates/capsem-process/src/runtime_config/tests.rs b/crates/capsem-process/src/runtime_config/tests.rs index 0e8f26177..6d8719cd6 100644 --- a/crates/capsem-process/src/runtime_config/tests.rs +++ b/crates/capsem-process/src/runtime_config/tests.rs @@ -39,6 +39,25 @@ local = false assert_eq!(runtime.plugins["credential_broker"].mode, SecurityPluginMode::Rewrite); assert!(!runtime.mcp.server_enabled["local"]); assert_eq!(runtime.network.http_upstream_ports, vec![80, 3128, 3713, 8080, 11434]); + let snapshot = runtime.proxy_policy_snapshot(); + assert_eq!(snapshot.digest(), runtime.active_policy_digest); + assert_eq!( + snapshot.network().http_upstream_ports, + runtime.network.http_upstream_ports + ); + assert!(snapshot + .security_rules() + .rules() + .iter() + .any(|rule| rule.rule_id == "profiles.rules.runtime_http")); + assert_eq!( + snapshot.plugins()["credential_broker"].mode, + SecurityPluginMode::Rewrite + ); + assert!(snapshot + .model_endpoints() + .provider_for_host("api.anthropic.com") + .is_some()); } #[test] diff --git a/crates/capsem-process/src/upstream_grant.rs b/crates/capsem-process/src/upstream_grant.rs index 0b1cb29ab..b5a5c5da9 100644 --- a/crates/capsem-process/src/upstream_grant.rs +++ b/crates/capsem-process/src/upstream_grant.rs @@ -3,7 +3,6 @@ use std::io; use std::os::fd::{AsFd as _, BorrowedFd, OwnedFd}; use std::os::unix::net::UnixStream; -use std::sync::{Arc, RwLock}; use std::time::Duration; use anyhow::{anyhow, Result}; @@ -57,11 +56,10 @@ enum Command { #[derive(Clone)] pub(crate) struct UpstreamGrantClient { commands: mpsc::Sender, - policy_digest: Arc>, } impl UpstreamGrantClient { - pub(crate) fn start(socket: UnixStream, policy_digest: String) -> io::Result { + pub(crate) fn start(socket: UnixStream) -> io::Result { let sender = WireSender::new(socket.try_clone()?)?; let receiver = WireReceiver::new(socket.try_clone()?)?; let (commands_tx, commands) = mpsc::channel(COMMAND_CAPACITY); @@ -75,28 +73,7 @@ impl UpstreamGrantClient { tracing::debug!(%error, "failed to shut down upstream grant channel"); } }); - Ok(Self { - commands: commands_tx, - policy_digest: Arc::new(RwLock::new(policy_digest)), - }) - } - - pub(crate) fn replace_policy_digest(&self, policy_digest: String) { - *self.policy_digest.write().unwrap() = policy_digest; - } - - #[cfg(test)] - pub(crate) fn policy_digest(&self) -> String { - self.policy_digest.read().unwrap().clone() - } - - #[cfg(test)] - pub(crate) fn test_handle(policy_digest: String) -> Self { - let (commands, _requests) = mpsc::channel(COMMAND_CAPACITY); - Self { - commands, - policy_digest: Arc::new(RwLock::new(policy_digest)), - } + Ok(Self { commands: commands_tx }) } } @@ -118,9 +95,9 @@ impl capsem_core::GuestMetadataAuthority for UpstreamGrantClient { } impl DnsUpstreamGrants for UpstreamGrantClient { - fn open(&self, upstream_index: u16) -> DnsGrantFuture<'_> { + fn open(&self, upstream_index: u16, policy_digest: &str) -> DnsGrantFuture<'_> { + let policy_digest = policy_digest.to_owned(); Box::pin(async move { - let policy_digest = self.policy_digest.read().unwrap().clone(); let (reply, result) = oneshot::channel(); self.commands .send(Command::OpenDns { @@ -139,10 +116,10 @@ impl DnsUpstreamGrants for UpstreamGrantClient { } impl TcpUpstreamGrants for UpstreamGrantClient { - fn resolve(&self, protocol: Protocol, host: &str, port: u16) -> TcpResolveGrantFuture<'_> { + fn resolve(&self, protocol: Protocol, host: &str, port: u16, policy_digest: &str) -> TcpResolveGrantFuture<'_> { let host = host.to_owned(); + let policy_digest = policy_digest.to_owned(); Box::pin(async move { - let policy_digest = self.policy_digest.read().unwrap().clone(); let (reply, result) = oneshot::channel(); self.commands .send(Command::ResolveTcp { @@ -161,9 +138,9 @@ impl TcpUpstreamGrants for UpstreamGrantClient { }) } - fn connect(&self, selection_id: u64) -> TcpConnectGrantFuture<'_> { + fn connect(&self, selection_id: u64, policy_digest: &str) -> TcpConnectGrantFuture<'_> { + let policy_digest = policy_digest.to_owned(); Box::pin(async move { - let policy_digest = self.policy_digest.read().unwrap().clone(); let (reply, result) = oneshot::channel(); self.commands .send(Command::ConnectTcp { diff --git a/crates/capsem-process/src/upstream_grant/tests.rs b/crates/capsem-process/src/upstream_grant/tests.rs index 24a7ebafa..c3d7d77bb 100644 --- a/crates/capsem-process/src/upstream_grant/tests.rs +++ b/crates/capsem-process/src/upstream_grant/tests.rs @@ -1,5 +1,6 @@ use std::io::{Read as _, Write as _}; use std::os::fd::{AsFd as _, AsRawFd as _, RawFd}; +use std::sync::Arc; use capsem_core::net::dns::{DnsResolver, DnsUpstreamGrants}; use capsem_core::net::mitm_proxy::{TcpUpstreamGrants, UpstreamTarget}; @@ -16,7 +17,7 @@ const POLICY_A: &str = "blake3:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa const POLICY_B: &str = "blake3:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; async fn open_error(client: &UpstreamGrantClient) -> String { - match client.open(0).await { + match client.open(0, POLICY_A).await { Ok(_) => panic!("unexpected DNS grant"), Err(error) => error.to_string(), } @@ -50,7 +51,7 @@ fn query() -> Vec { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn guest_mode_changes_use_the_serialized_coordinator_channel() { let (broker, worker) = UnixStream::pair().unwrap(); - let client = Arc::new(UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap()); + let client = Arc::new(UpstreamGrantClient::start(worker).unwrap()); let (requests, responses) = channels(broker); let broker_task = tokio::spawn(async move { assert_eq!( @@ -79,7 +80,7 @@ async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); let upstream_address = upstream.local_addr().unwrap(); let (broker, worker) = UnixStream::pair().unwrap(); - let client = Arc::new(UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap()); + let client = Arc::new(UpstreamGrantClient::start(worker).unwrap()); let (requests, responses) = channels(broker); let expected_query = query(); let broker_task = tokio::spawn(async move { @@ -121,7 +122,7 @@ async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { }); let resolver = DnsResolver::with_grants(vec![upstream_address], client); - let (answer, _) = resolver.resolve(&query()).await.unwrap(); + let (answer, _) = resolver.resolve_for_policy(&query(), POLICY_A).await.unwrap(); assert_eq!(&answer[..2], &[0x12, 0x34]); assert_eq!(&answer[2..4], &[0x81, 0x80]); broker_task.await.unwrap(); @@ -130,7 +131,7 @@ async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { #[tokio::test] async fn stale_policy_grant_is_adopted_released_and_channel_remains_usable() { let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); let (requests, responses) = channels(broker); let broker_task = tokio::spawn(async move { assert!(matches!( @@ -184,7 +185,7 @@ async fn stale_policy_grant_is_adopted_released_and_channel_remains_usable() { #[tokio::test] async fn malformed_or_revoked_grant_channel_fails_closed() { let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); let (requests, responses) = channels(broker); let broker_task = tokio::spawn(async move { assert!(matches!( @@ -210,7 +211,7 @@ async fn malformed_or_revoked_grant_channel_fails_closed() { assert!(error.contains("closed") || error.contains("stopped"), "{error}"); let (broker, worker) = UnixStream::pair().unwrap(); - let revoked = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let revoked = UpstreamGrantClient::start(worker).unwrap(); drop(broker); assert!(!open_error(&revoked).await.is_empty()); } @@ -220,7 +221,7 @@ async fn tcp_selection_and_connected_stream_are_brokered_and_released() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let upstream_address = listener.local_addr().unwrap(); let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); let (requests, responses) = channels(broker); let broker_task = tokio::spawn(async move { assert_eq!( @@ -282,7 +283,10 @@ async fn tcp_selection_and_connected_stream_are_brokered_and_released() { ); }); - let selection = client.resolve(Protocol::Tls, "api.example", 443).await.unwrap(); + let selection = client + .resolve(Protocol::Tls, "api.example", 443, POLICY_A) + .await + .unwrap(); assert_eq!(selection.protocol, Protocol::Http); assert_eq!(selection.judged_ip, Some("127.0.0.1".parse().unwrap())); let target = UpstreamTarget::Granted { @@ -293,7 +297,7 @@ async fn tcp_selection_and_connected_stream_are_brokered_and_released() { judged_ip: selection.judged_ip, selection: Some(selection), }; - let (mut stream, pinned) = target.connect_with_grants(Some(&client)).await.unwrap(); + let (mut stream, pinned) = target.connect_with_grants(Some(&client), POLICY_A).await.unwrap(); assert!(matches!(pinned, UpstreamTarget::Granted { selection: None, .. })); stream.write_all(b"ping").await.unwrap(); let mut response = [0_u8; 4]; @@ -306,7 +310,7 @@ async fn tcp_selection_and_connected_stream_are_brokered_and_released() { #[tokio::test] async fn unused_or_stale_tcp_selections_are_released_without_connecting() { let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker, POLICY_A.into()).unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); let (requests, responses) = channels(broker); let broker_task = tokio::spawn(async move { for (request_id, selection_id, digest) in [(1, 81, POLICY_A), (2, 82, POLICY_B)] { @@ -338,9 +342,15 @@ async fn unused_or_stale_tcp_selections_are_released_without_connecting() { } }); - let unused = client.resolve(Protocol::Tls, "unused.example", 443).await.unwrap(); + let unused = client + .resolve(Protocol::Tls, "unused.example", 443, POLICY_A) + .await + .unwrap(); drop(unused); - let error = client.resolve(Protocol::Tls, "stale.example", 443).await.unwrap_err(); + let error = client + .resolve(Protocol::Tls, "stale.example", 443, POLICY_A) + .await + .unwrap_err(); assert!(error.to_string().contains("policy mismatch"), "{error}"); broker_task.await.unwrap(); } diff --git a/crates/capsem-process/src/vsock/dns/tests.rs b/crates/capsem-process/src/vsock/dns/tests.rs index 44a4a8e51..bef047c7d 100644 --- a/crates/capsem-process/src/vsock/dns/tests.rs +++ b/crates/capsem-process/src/vsock/dns/tests.rs @@ -4,17 +4,17 @@ use std::os::fd::AsRawFd; use std::os::unix::net::UnixStream; fn handler() -> Arc { - let policy = Arc::new(std::sync::RwLock::new(Arc::new( - capsem_core::net::policy::NetworkMechanics::default(), - ))); - let rules = Arc::new(std::sync::RwLock::new(Arc::new( - capsem_core::net::policy_config::SecurityRuleSet::new(Vec::new()), - ))); - let plugins = Arc::new(std::sync::RwLock::new(std::collections::BTreeMap::new().into())); + let policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new( + capsem_core::net::proxy_engine::ProxyPolicySnapshot::new( + "blake3:test".into(), + capsem_core::net::policy::NetworkMechanics::default(), + capsem_core::net::policy_config::SecurityRuleSet::new(Vec::new()), + std::collections::BTreeMap::new(), + capsem_core::net::policy_config::ModelEndpointRegistry::default(), + ), + ); Arc::new(DnsHandler::new( policy, - rules, - plugins, Arc::new(DnsResolver::with_upstreams(Vec::new())), )) } From dcaed00de3ad40c183cbce96f286add2378d16d0 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 03:47:50 +0000 Subject: [PATCH 101/464] security(proxy): scope engine capabilities --- CHANGELOG.md | 5 + crates/capsem-core/benches/mitm_request.rs | 7 +- crates/capsem-core/src/credential_broker.rs | 18 +- .../capsem-core/src/net/mitm_proxy/config.rs | 6 +- crates/capsem-core/src/net/mitm_proxy/mod.rs | 64 +++---- .../src/net/mitm_proxy/telemetry_hook.rs | 26 ++- .../net/mitm_proxy/telemetry_hook/tests.rs | 3 + .../telemetry_hook/tests/headers.rs | 1 + .../mitm_proxy/telemetry_hook/tests/ledger.rs | 6 + .../capsem-core/src/net/mitm_proxy/upgrade.rs | 7 +- crates/capsem-core/src/net/proxy_engine.rs | 136 +++++++++++++ .../capsem-core/src/net/proxy_engine/tests.rs | 179 +++++++++++++++++- .../src/security_engine/builtin_actions.rs | 4 +- crates/capsem-core/src/security_engine/mod.rs | 47 ++++- .../src/security_engine/plugins/pre.rs | 20 +- crates/capsem-core/tests/mitm_integration.rs | 9 +- .../mitm_integration/mcp_gateway_route.rs | 6 +- .../mitm_integration/resolved_upstream.rs | 2 +- 18 files changed, 471 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb59d5d7e..c30991303 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 HTTP routing, security plugins, provider identity, response telemetry and guest DNS, preventing a request from combining authority from two revisions. +- Route proxy policy evaluation, credential capture, upstream substitution and + redaction through a transport-independent engine with typed ledger and + credential capabilities, so a confined proxy needs no credential-store or + session-database path authority. + - Confine each session-ledger owner before readiness to its one database and inherited channels, denying sibling sessions, socket creation, execution, signals and ambient files; erase inherited environment bytes and reject diff --git a/crates/capsem-core/benches/mitm_request.rs b/crates/capsem-core/benches/mitm_request.rs index 7a72376ba..bfed5b0fc 100644 --- a/crates/capsem-core/benches/mitm_request.rs +++ b/crates/capsem-core/benches/mitm_request.rs @@ -86,8 +86,13 @@ fn proxy_config(upstream_port: u16) -> Arc { .expect("builtin provider endpoint registry"), ), ); + let engine = Arc::new(capsem_core::net::proxy_engine::ProxyEngine::local( + policy, + Arc::clone(&db), + )); let telemetry = Arc::new(mitm_proxy::telemetry_hook::TelemetryDeps { db: db.clone(), + credentials: engine.credentials(), pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), trace_state: Arc::new(std::sync::Mutex::new(capsem_core::net::ai_traffic::TraceState::new())), }); @@ -95,7 +100,7 @@ fn proxy_config(upstream_port: u16) -> Arc { Arc::new(MitmProxyConfig { server_tls: mitm_proxy::make_server_tls_config(&ca), ca, - policy, + engine, db, upstream_tls: mitm_proxy::make_upstream_tls_config(), telemetry, diff --git a/crates/capsem-core/src/credential_broker.rs b/crates/capsem-core/src/credential_broker.rs index 4f6d43b38..9f8111a2c 100644 --- a/crates/capsem-core/src/credential_broker.rs +++ b/crates/capsem-core/src/credential_broker.rs @@ -292,6 +292,21 @@ pub async fn broker_and_log_observations( db: &DbWriter, rules: &SecurityRuleSet, observations: Vec, +) -> Option { + broker_and_log_observations_with_credentials( + db, + rules, + std::sync::Arc::new(crate::net::proxy_engine::LocalProxyCredentials), + observations, + ) + .await +} + +pub async fn broker_and_log_observations_with_credentials( + db: &DbWriter, + rules: &SecurityRuleSet, + credentials: std::sync::Arc, + observations: Vec, ) -> Option { let mut first_ref = None; let mut seen = HashSet::new(); @@ -313,7 +328,8 @@ pub async fn broker_and_log_observations( } let brokered = match tokio::task::spawn_blocking({ let observation = observation.clone(); - move || broker_observed_credential(&observation) + let credentials = std::sync::Arc::clone(&credentials); + move || credentials.capture(&observation) }) .await { diff --git a/crates/capsem-core/src/net/mitm_proxy/config.rs b/crates/capsem-core/src/net/mitm_proxy/config.rs index 998a7b69a..7d9c95727 100644 --- a/crates/capsem-core/src/net/mitm_proxy/config.rs +++ b/crates/capsem-core/src/net/mitm_proxy/config.rs @@ -7,15 +7,15 @@ use super::pipeline; use super::telemetry_hook; use super::upstream::TcpUpstreamGrants; use crate::net::cert_authority::CertAuthority; -use crate::net::proxy_engine::ProxyPolicyHandle; +use crate::net::proxy_engine::ProxyEngine; /// Configuration for the MITM proxy. pub struct MitmProxyConfig { pub ca: Arc, /// Guest-facing TLS config, built once (`make_server_tls_config`): one session cache for all. pub server_tls: Arc, - /// One digest-keyed active-policy revision, snapshotted per request. - pub policy: ProxyPolicyHandle, + /// Transport-independent policy and scoped host capabilities. + pub engine: Arc, pub db: Arc, /// Cached upstream TLS config (shared across all connections). pub upstream_tls: Arc, diff --git a/crates/capsem-core/src/net/mitm_proxy/mod.rs b/crates/capsem-core/src/net/mitm_proxy/mod.rs index 7a1f1c688..92c60ec11 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mod.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mod.rs @@ -681,7 +681,7 @@ async fn serve_pipeline( let (domain, port) = (&request_domain, upstream_port); return Ok(mcp_http::serve(req, route, domain, port, protocol, &config_arc, &process_name).await); } - let policy_snapshot = config_arc.policy.snapshot(); + let policy_snapshot = config_arc.engine.policy().snapshot(); let ai_identity = ai_identity_for_target_or_path( policy_snapshot.model_endpoints(), &request_domain, @@ -1143,13 +1143,8 @@ async fn handle_request( status = tracing::field::Empty, error_kind = tracing::field::Empty, ); - let http_evaluation = match actions_span.in_scope(|| { - crate::security_engine::evaluate_security_boundary( - &rules, - Arc::clone(policy_snapshot.plugins()), - http_security_event, - ) - }) { + let http_evaluation = match actions_span.in_scope(|| config.engine.evaluate(&policy_snapshot, http_security_event)) + { Ok(evaluation) => evaluation, Err(error) => { actions_span.record("decision", "error"); @@ -1267,25 +1262,24 @@ async fn handle_request( } actions_span.record("decision", "allow"); actions_span.record("status", "ok"); - let upstream_materialized = match actions_span - .in_scope(|| crate::security_engine::materialize_http_request_for_upstream(&http_evaluation.event)) - { - Ok(materialized) => materialized, - Err(error) => { - actions_span.record("decision", "error"); - actions_span.record("status", "error"); - actions_span.record("error_kind", "materialize_http_request"); - return Ok(make_502( - &anyhow::anyhow!(error), - &method, - &path, - &query, - &req_hdrs, - start_time, - &request_security_decision, - )); - } - }; + let upstream_materialized = + match actions_span.in_scope(|| config.engine.materialize_http_request(&http_evaluation.event)) { + Ok(materialized) => materialized, + Err(error) => { + actions_span.record("decision", "error"); + actions_span.record("status", "error"); + actions_span.record("error_kind", "materialize_http_request"); + return Ok(make_502( + &anyhow::anyhow!(error), + &method, + &path, + &query, + &req_hdrs, + start_time, + &request_security_decision, + )); + } + }; original_headers = upstream_materialized.headers; let credential_ref = credential_ref .clone() @@ -1315,9 +1309,7 @@ async fn handle_request( upstream_ip, upstream_port, ); - let mcp_evaluation = match mcp_span.in_scope(|| { - crate::security_engine::evaluate_security_boundary(&rules, Arc::clone(policy_snapshot.plugins()), mcp_event) - }) { + let mcp_evaluation = match mcp_span.in_scope(|| config.engine.evaluate(&policy_snapshot, mcp_event)) { Ok(evaluation) => evaluation, Err(error) => { mcp_span.record("decision", "error"); @@ -1531,11 +1523,7 @@ async fn handle_request( body: Some(String::from_utf8_lossy(&body_bytes).to_string()), }); let model_event = security_event_with_transport(model_event, upstream_ip, upstream_port); - let model_evaluation = match crate::security_engine::evaluate_security_boundary( - &rules, - Arc::clone(policy_snapshot.plugins()), - model_event, - ) { + let model_evaluation = match config.engine.evaluate(&policy_snapshot, model_event) { Ok(evaluation) => evaluation, Err(error) => { model_request_span.record("decision", "error"); @@ -2184,11 +2172,7 @@ async fn handle_request( body: Some(String::from_utf8_lossy(&response_body).to_string()), }); let model_event = security_event_with_transport(model_event, upstream_ip, upstream_port); - let model_evaluation = match crate::security_engine::evaluate_security_boundary( - &rules, - Arc::clone(policy_snapshot.plugins()), - model_event, - ) { + let model_evaluation = match config.engine.evaluate(&policy_snapshot, model_event) { Ok(evaluation) => evaluation, Err(error) => { model_response_span.record("decision", "error"); diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs index 12432f50a..7a8eea44b 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs @@ -29,9 +29,8 @@ use super::hooks::{ChunkCtx, ChunkEndFuture, ChunkHook}; use super::interpreter_hook::LlmEventStream; use super::util::is_llm_api_path; use crate::credential_broker::{ - broker_and_log_observations, detect_http_body_credentials, log_brokered_injections, - redact_observed_credentials_in_bytes, redact_observed_credentials_in_text, CredentialInjection, - CredentialObservation, + broker_and_log_observations_with_credentials, detect_http_body_credentials, log_brokered_injections, + CredentialInjection, CredentialObservation, }; use crate::net::ai_traffic::events::{ collect_summary, parse_non_streaming_response_summary, parse_non_streaming_tool_calls, parse_non_streaming_usage, @@ -41,7 +40,7 @@ use crate::net::ai_traffic::pricing::PricingTable; use crate::net::ai_traffic::provider::{extract_model_from_path, tool_origin, ModelProtocol, ProviderKind}; use crate::net::ai_traffic::{request_parser, TraceState}; use crate::net::policy_config::{PluginPolicySnapshot, SecurityRuleSet}; -use crate::net::proxy_engine::ProxyPolicySnapshot; +use crate::net::proxy_engine::{ProxyCredentials, ProxyPolicySnapshot}; use crate::security_engine::{ emit_evaluated_security_rules, emit_security_write, HttpSecurityEvent, IpSecurityEvent, ModelSecurityEvent, RuntimeSecurityEventType, SecurityEvent, TcpSecurityEvent, @@ -110,6 +109,7 @@ pub struct TelemetryResponseStats { /// derivable from the per-request context. pub struct TelemetryDeps { pub db: Arc, + pub credentials: Arc, pub pricing: Arc, pub trace_state: Arc>, } @@ -123,6 +123,7 @@ pub struct TelemetryHook { struct PendingTelemetryCompletion { db: Arc, + credentials: Arc, rules: Arc, plugin_policy: PluginPolicySnapshot, net_event: NetEvent, @@ -136,6 +137,7 @@ impl PendingTelemetryCompletion { async fn run(self) { let Self { db, + credentials, rules, plugin_policy, net_event, @@ -181,7 +183,7 @@ impl PendingTelemetryCompletion { if !credential_observations.is_empty() || !credential_injections.is_empty() { log_brokered_injections(&db, &rules, credential_injections).await; - broker_and_log_observations(&db, &rules, credential_observations).await; + broker_and_log_observations_with_credentials(&db, &rules, credentials, credential_observations).await; } } } @@ -272,15 +274,21 @@ impl ChunkHook for TelemetryHook { if !credential_observations.is_empty() { let redact = |text: &mut Option| { if let Some(text) = text { - *text = redact_observed_credentials_in_text(text, &credential_observations); + *text = self.deps.credentials.redact_text(text, &credential_observations); } }; redact(&mut req_ctx.request_headers); redact(&mut req_ctx.response_headers); let mut stats = req_ctx.request_body_stats.lock().expect("req body stats lock"); - stats.preview = redact_observed_credentials_in_bytes(&stats.preview, &credential_observations); + stats.preview = self + .deps + .credentials + .redact_bytes(&stats.preview, &credential_observations); drop(stats); - resp_stats.preview = redact_observed_credentials_in_bytes(&resp_stats.preview, &credential_observations); + resp_stats.preview = self + .deps + .credentials + .redact_bytes(&resp_stats.preview, &credential_observations); } record_telemetry_stage(stage_started, "credential_detect_and_redact"); @@ -310,6 +318,7 @@ impl ChunkHook for TelemetryHook { let stage_started = Instant::now(); let db = Arc::clone(&self.deps.db); + let credentials = Arc::clone(&self.deps.credentials); let rules = Arc::clone(req_ctx.policy_snapshot.security_rules()); let plugin_policy = Arc::clone(req_ctx.policy_snapshot.plugins()); let credential_injections = req_ctx.credential_injections; @@ -325,6 +334,7 @@ impl ChunkHook for TelemetryHook { let has_model_call = model_call.is_some(); *ctx.state::>(|| None) = Some(PendingTelemetryCompletion { db, + credentials, rules, plugin_policy, net_event, diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs index 2201b1668..4609740ba 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs @@ -664,6 +664,7 @@ fn fake_deps() -> Arc { let db = Arc::new(DbWriter::open_in_memory(64).expect("in-memory db")); Arc::new(TelemetryDeps { db, + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), }) @@ -727,6 +728,7 @@ async fn hook_accepts_primary_net_event_before_completing_response() { let db = Arc::new(DbWriter::open(&db_path, 64).expect("test db")); let deps = Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), }); @@ -784,6 +786,7 @@ async fn bounded_logger_backpressure_keeps_every_completed_response_event() { let db = Arc::new(DbWriter::open(&db_path, 1).expect("test db")); let hook = TelemetryHook::new(Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), })); diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/headers.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/headers.rs index f427366cf..655c9582f 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/headers.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/headers.rs @@ -21,6 +21,7 @@ async fn an_observed_credential_is_redacted_from_every_stored_header_and_body() let db = Arc::new(DbWriter::open(&db_path, 64).expect("test db")); let hook = TelemetryHook::new(Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), })); diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/ledger.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/ledger.rs index fa20a4f31..2630c3d83 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/ledger.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests/ledger.rs @@ -21,6 +21,7 @@ async fn hook_writes_substitution_event_and_shared_credential_ref() { let db = Arc::new(DbWriter::open(&db_path, 64).expect("test db")); let deps = Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), }); @@ -98,6 +99,7 @@ async fn hook_does_not_repay_capture_ledger_for_repeated_identical_credential() let db = Arc::new(DbWriter::open(&db_path, 64).expect("test db")); let deps = Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), }); @@ -200,6 +202,7 @@ match = 'http.host == "api.anthropic.com" && http.path == "/v1/messages" && tcp. let db = Arc::new(DbWriter::open(&db_path, 64).expect("test db")); let deps = Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), }); @@ -262,6 +265,7 @@ match = 'model.provider == "anthropic" && model.name == "claude-test"' let db = Arc::new(DbWriter::open(&db_path, 64).expect("test db")); let deps = Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), }); @@ -322,6 +326,7 @@ async fn hook_writes_injected_substitution_event_for_broker_ref_replay() { let db = Arc::new(DbWriter::open(&db_path, 64).expect("test db")); let deps = Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), }); @@ -394,6 +399,7 @@ async fn hook_detects_response_body_token_exchange_and_redacts_preview() { let db = Arc::new(DbWriter::open(&db_path, 64).expect("test db")); let deps = Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), }); diff --git a/crates/capsem-core/src/net/mitm_proxy/upgrade.rs b/crates/capsem-core/src/net/mitm_proxy/upgrade.rs index de0c9549b..7970703af 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upgrade.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upgrade.rs @@ -188,12 +188,7 @@ pub(super) async fn handle_upgrade( if let Some(trace_id) = capsem_foundation::telemetry::ambient_capsem_trace_id() { upgrade_event = upgrade_event.with_trace_id(trace_id); } - let rules = Arc::clone(policy_snapshot.security_rules()); - let upgrade_evaluation = crate::security_engine::evaluate_security_boundary( - &rules, - Arc::clone(policy_snapshot.plugins()), - upgrade_event, - ); + let upgrade_evaluation = config.engine.evaluate(policy_snapshot, upgrade_event); let upgrade_decision = match upgrade_evaluation { Ok(evaluation) => evaluation, Err(error) => return Ok(make_ws_error(&error)), diff --git a/crates/capsem-core/src/net/proxy_engine.rs b/crates/capsem-core/src/net/proxy_engine.rs index 02f6bbf37..958df4f7a 100644 --- a/crates/capsem-core/src/net/proxy_engine.rs +++ b/crates/capsem-core/src/net/proxy_engine.rs @@ -4,13 +4,149 @@ //! enforcement, credential handling, and ledger completion. Reload replaces //! the outer `Arc` once, so no request can combine pieces from two active //! policy files. +//! +//! Transport adapters pair this engine with the descriptor-only +//! [`super::mitm_proxy::TcpUpstreamGrants`], [`super::dns::DnsUpstreamGrants`] +//! and [`super::dns::private::PrivateNames`] capabilities. None exposes a +//! filesystem path, process handle, resolver socket, or arbitrary dial. use std::collections::BTreeMap; +use std::future::Future; +use std::pin::Pin; use std::sync::{Arc, RwLock}; +use capsem_logger::{DbWriter, WriteOp}; + +use crate::credential_broker::{BrokeredCredential, BrokeredUpstreamCredentials, CredentialObservation}; +use crate::net::ai_traffic::provider::ProviderKind; +use crate::security_engine::{MaterializedHttpRequest, SecurityActionError, SecurityBoundaryEvaluation, SecurityEvent}; + use super::policy::NetworkMechanics; use super::policy_config::{ModelEndpointRegistry, SecurityPluginConfig, SecurityRuleSet}; +pub type ProxyCapabilityFuture<'a, T> = Pin + Send + 'a>>; + +/// The only ledger authority exposed to a proxy worker. +pub trait ProxyLedger: Send + Sync { + fn write(&self, op: WriteOp) -> ProxyCapabilityFuture<'_, Result<(), String>>; +} + +impl ProxyLedger for DbWriter { + fn write(&self, op: WriteOp) -> ProxyCapabilityFuture<'_, Result<(), String>> { + Box::pin(self.write_checked(op)) + } +} + +/// Credential operations available to a proxy worker, without store paths. +pub trait ProxyCredentials: Send + Sync { + fn capture(&self, observation: &CredentialObservation) -> Result; + + fn substitute_upstream( + &self, + domain: &str, + ai_provider: Option, + headers: &mut http::HeaderMap, + query: Option<&str>, + ) -> Result; + + fn redact_text(&self, text: &str, observations: &[CredentialObservation]) -> String { + crate::credential_broker::redact_observed_credentials_in_text(text, observations) + } + + fn redact_bytes(&self, bytes: &[u8], observations: &[CredentialObservation]) -> Vec { + crate::credential_broker::redact_observed_credentials_in_bytes(bytes, observations) + } +} + +/// In-process adapter used until the proxy runs behind the coordinator IPC. +pub struct LocalProxyCredentials; + +impl ProxyCredentials for LocalProxyCredentials { + fn capture(&self, observation: &CredentialObservation) -> Result { + crate::credential_broker::broker_observed_credential(observation) + } + + fn substitute_upstream( + &self, + domain: &str, + ai_provider: Option, + headers: &mut http::HeaderMap, + query: Option<&str>, + ) -> Result { + crate::credential_broker::substitute_brokered_upstream_credentials(domain, ai_provider, headers, query) + } +} + +/// Transport-independent proxy behavior and its scoped host capabilities. +#[derive(Clone)] +pub struct ProxyEngine { + policy: ProxyPolicyHandle, + ledger: Arc, + credentials: Arc, +} + +impl ProxyEngine { + pub fn new( + policy: ProxyPolicyHandle, + ledger: Arc, + credentials: Arc, + ) -> Self { + Self { + policy, + ledger, + credentials, + } + } + + pub fn local(policy: ProxyPolicyHandle, ledger: Arc) -> Self { + Self::new(policy, ledger, Arc::new(LocalProxyCredentials)) + } + + pub fn policy(&self) -> &ProxyPolicyHandle { + &self.policy + } + + pub async fn write(&self, op: WriteOp) -> Result<(), String> { + self.ledger.write(op).await + } + + pub fn evaluate( + &self, + snapshot: &ProxyPolicySnapshot, + event: SecurityEvent, + ) -> Result { + crate::security_engine::evaluate_security_boundary_with_credentials( + snapshot.security_rules(), + Arc::clone(snapshot.plugins()), + Arc::clone(&self.credentials), + event, + ) + } + + pub fn capture_credential(&self, observation: &CredentialObservation) -> Result { + self.credentials.capture(observation) + } + + pub fn credentials(&self) -> Arc { + Arc::clone(&self.credentials) + } + + pub fn materialize_http_request( + &self, + event: &SecurityEvent, + ) -> Result { + crate::security_engine::materialize_http_request_for_upstream_with_credentials(event, self.credentials.as_ref()) + } + + pub fn redact_text(&self, text: &str, observations: &[CredentialObservation]) -> String { + self.credentials.redact_text(text, observations) + } + + pub fn redact_bytes(&self, bytes: &[u8], observations: &[CredentialObservation]) -> Vec { + self.credentials.redact_bytes(bytes, observations) + } +} + /// One fully compiled active-policy revision. pub struct ProxyPolicySnapshot { digest: String, diff --git a/crates/capsem-core/src/net/proxy_engine/tests.rs b/crates/capsem-core/src/net/proxy_engine/tests.rs index 58d151917..04a780cad 100644 --- a/crates/capsem-core/src/net/proxy_engine/tests.rs +++ b/crates/capsem-core/src/net/proxy_engine/tests.rs @@ -1,11 +1,17 @@ use std::collections::BTreeMap; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; +use std::time::SystemTime; use super::*; +use crate::credential_broker::{BrokeredCredential, BrokeredUpstreamCredentials, CredentialObservation}; use crate::net::policy::NetworkMechanics; use crate::net::policy_config::{ - ModelEndpointRegistry, SecurityPluginConfig, SecurityRuleProfile, SecurityRuleSet, SecurityRuleSource, + DetectionLevel, ModelEndpointRegistry, PolicyActionId, SecurityPluginConfig, SecurityPluginMode, + SecurityRuleProfile, SecurityRuleSet, SecurityRuleSource, }; +use crate::security_engine::{HttpRequestSecurityEvent, RuntimeSecurityEventType, SecurityEvent}; +use capsem_credentials::CredentialProvider; +use capsem_logger::{FileAction, FileEvent, FileKind, WriteOp}; fn revision(marker: usize) -> ProxyPolicySnapshot { let profile = SecurityRuleProfile::parse_toml(&format!( @@ -31,6 +37,22 @@ match = 'http.host == "marker-{marker}.example"' ) } +fn credential_revision() -> ProxyPolicySnapshot { + ProxyPolicySnapshot::new( + "blake3:credentials".to_string(), + NetworkMechanics::new(), + SecurityRuleSet::new(Vec::new()), + BTreeMap::from([( + "credential_broker".to_string(), + SecurityPluginConfig { + mode: SecurityPluginMode::Rewrite, + detection_level: DetectionLevel::Informational, + }, + )]), + ModelEndpointRegistry::default(), + ) +} + #[test] fn policy_handle_replaces_one_immutable_revision_atomically() { let handle = ProxyPolicyHandle::new(revision(0)); @@ -68,3 +90,156 @@ fn request_snapshot_cannot_mix_concurrent_policy_revisions() { } writer.join().unwrap(); } + +#[derive(Default)] +struct RecordingLedger { + writes: Mutex>, +} + +impl ProxyLedger for RecordingLedger { + fn write(&self, op: WriteOp) -> ProxyCapabilityFuture<'_, Result<(), String>> { + Box::pin(async move { + let kind = match op { + WriteOp::FileEvent(_) => "file", + _ => "other", + }; + self.writes.lock().unwrap().push(kind); + Ok(()) + }) + } +} + +#[derive(Default)] +struct RecordingCredentials { + calls: Mutex>, +} + +impl ProxyCredentials for RecordingCredentials { + fn capture(&self, observation: &CredentialObservation) -> Result { + self.calls.lock().unwrap().push("capture"); + Ok(BrokeredCredential { + provider: observation.provider, + credential_ref: observation.credential_ref(), + store_account: "scoped-test-account".to_string(), + newly_captured: true, + }) + } + + fn substitute_upstream( + &self, + _domain: &str, + _ai_provider: Option, + headers: &mut http::HeaderMap, + query: Option<&str>, + ) -> Result { + self.calls.lock().unwrap().push("substitute"); + headers.insert( + http::header::AUTHORIZATION, + http::HeaderValue::from_static("Bearer scoped-secret"), + ); + Ok(BrokeredUpstreamCredentials { + credential_ref: Some("blake3:scoped-reference".to_string()), + query: query.map(str::to_string), + }) + } +} + +#[tokio::test] +async fn engine_exposes_only_typed_ledger_and_credential_capabilities() { + let ledger = Arc::new(RecordingLedger::default()); + let credentials = Arc::new(RecordingCredentials::default()); + let engine = ProxyEngine::new( + ProxyPolicyHandle::new(revision(7)), + Arc::clone(&ledger) as Arc, + Arc::clone(&credentials) as Arc, + ); + + engine + .write(WriteOp::FileEvent(FileEvent { + event_id: None, + timestamp: SystemTime::UNIX_EPOCH, + action: FileAction::Read, + path: "/scoped".to_string(), + size: Some(1), + kind: FileKind::File, + trace_id: None, + credential_ref: None, + })) + .await + .unwrap(); + + let observation = CredentialObservation { + provider: CredentialProvider::Anthropic, + raw_value: "raw-secret-value".to_string(), + source: "test".to_string(), + event_type: None, + trace_id: None, + context_json: None, + }; + let brokered = engine.capture_credential(&observation).unwrap(); + assert_eq!(brokered.credential_ref, observation.credential_ref()); + + let mut event = + SecurityEvent::new(RuntimeSecurityEventType::HttpRequest).with_http_request(HttpRequestSecurityEvent::new( + "api.anthropic.com", + None, + http::HeaderMap::new(), + Some("x=1".to_string()), + )); + event.action_trace.push(PolicyActionId::CredentialBrokerSubstitute); + let materialized = engine.materialize_http_request(&event).unwrap(); + assert_eq!( + materialized.headers.get(http::header::AUTHORIZATION).unwrap(), + "Bearer scoped-secret" + ); + assert_eq!(materialized.credential_ref.as_deref(), Some("blake3:scoped-reference")); + + assert_eq!(*ledger.writes.lock().unwrap(), vec!["file"]); + assert_eq!(*credentials.calls.lock().unwrap(), vec!["capture", "substitute"]); +} + +#[test] +fn engine_redaction_never_returns_observed_secret() { + let ledger = Arc::new(RecordingLedger::default()); + let credentials = Arc::new(RecordingCredentials::default()); + let engine = ProxyEngine::new(ProxyPolicyHandle::new(revision(8)), ledger, credentials); + let observation = CredentialObservation { + provider: CredentialProvider::OpenAi, + raw_value: "raw-secret-value".to_string(), + source: "test".to_string(), + event_type: None, + trace_id: None, + context_json: None, + }; + + let redacted = engine.redact_text("authorization: raw-secret-value", std::slice::from_ref(&observation)); + assert!(!redacted.contains("raw-secret-value")); + assert!(redacted.contains(&observation.credential_ref())); +} + +#[test] +fn engine_evaluation_uses_its_scoped_credential_capability() { + let credentials = Arc::new(RecordingCredentials::default()); + let engine = ProxyEngine::new( + ProxyPolicyHandle::new(credential_revision()), + Arc::new(RecordingLedger::default()), + Arc::clone(&credentials) as Arc, + ); + let snapshot = engine.policy().snapshot(); + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::AUTHORIZATION, + http::HeaderValue::from_static("Bearer sk-scoped-engine-credential"), + ); + let event = SecurityEvent::new(RuntimeSecurityEventType::HttpRequest) + .with_http_request(HttpRequestSecurityEvent::new("api.openai.com", None, headers, None)); + + let evaluated = engine.evaluate(&snapshot, event).unwrap(); + + assert_eq!(*credentials.calls.lock().unwrap(), vec!["capture"]); + assert_eq!(evaluated.event.credential_observations.len(), 1); + assert!(evaluated + .event + .action_trace + .contains(&PolicyActionId::CredentialBrokerCapture)); +} diff --git a/crates/capsem-core/src/security_engine/builtin_actions.rs b/crates/capsem-core/src/security_engine/builtin_actions.rs index 3c72d48f8..0587952b1 100644 --- a/crates/capsem-core/src/security_engine/builtin_actions.rs +++ b/crates/capsem-core/src/security_engine/builtin_actions.rs @@ -17,7 +17,9 @@ pub(super) fn registry() -> &'static SecurityActionRegistry { static BUILTIN: OnceLock = OnceLock::new(); BUILTIN.get_or_init(|| { SecurityActionRegistry::new() - .register_plugin(CredentialBrokerPlugin) + .register_plugin(CredentialBrokerPlugin::new(std::sync::Arc::new( + crate::net::proxy_engine::LocalProxyCredentials, + ))) .and_then(|registry| registry.register_plugin(DummyPreEicarPlugin)) .and_then(|registry| registry.register_plugin(DummyPostAllowPlugin)) .and_then(|registry| registry.register_plugin(LogSanitizerPlugin)) diff --git a/crates/capsem-core/src/security_engine/mod.rs b/crates/capsem-core/src/security_engine/mod.rs index 9430eeb0c..cd139999c 100644 --- a/crates/capsem-core/src/security_engine/mod.rs +++ b/crates/capsem-core/src/security_engine/mod.rs @@ -700,10 +700,36 @@ pub fn emit_security_decision_transition_blocking( pub fn evaluate_security_boundary( rules: &SecurityRuleSet, plugin_policy: impl Into>>, + event: SecurityEvent, +) -> Result { + evaluate_security_boundary_with_registry( + rules, + SecurityActionRegistry::with_builtin_actions().with_plugin_policy(plugin_policy), + event, + ) +} + +pub(crate) fn evaluate_security_boundary_with_credentials( + rules: &SecurityRuleSet, + plugin_policy: impl Into>>, + credentials: Arc, + event: SecurityEvent, +) -> Result { + evaluate_security_boundary_with_registry( + rules, + SecurityActionRegistry::with_builtin_actions() + .with_proxy_credentials(credentials) + .with_plugin_policy(plugin_policy), + event, + ) +} + +fn evaluate_security_boundary_with_registry( + rules: &SecurityRuleSet, + action_registry: SecurityActionRegistry, mut event: SecurityEvent, ) -> Result { event.validate_network(event.event_type)?; - let action_registry = SecurityActionRegistry::with_builtin_actions().with_plugin_policy(plugin_policy); event = action_registry.apply_security_plugins(SecurityPluginStage::Preprocess, event)?; @@ -1673,6 +1699,13 @@ pub struct MaterializedHttpRequest { pub fn materialize_http_request_for_upstream( event: &SecurityEvent, +) -> Result { + materialize_http_request_for_upstream_with_credentials(event, &crate::net::proxy_engine::LocalProxyCredentials) +} + +pub(crate) fn materialize_http_request_for_upstream_with_credentials( + event: &SecurityEvent, + credentials: &dyn crate::net::proxy_engine::ProxyCredentials, ) -> Result { let Some(request) = event.http_request.as_ref() else { return Err(SecurityActionError::new( @@ -1689,8 +1722,8 @@ pub fn materialize_http_request_for_upstream( } let mut headers = request.headers.clone(); - let BrokeredUpstreamCredentials { credential_ref, query } = - crate::credential_broker::substitute_brokered_upstream_credentials( + let BrokeredUpstreamCredentials { credential_ref, query } = credentials + .substitute_upstream( &request.domain, request.ai_provider, &mut headers, @@ -1817,6 +1850,14 @@ impl SecurityActionRegistry { self } + fn with_proxy_credentials(mut self, credentials: Arc) -> Self { + Arc::make_mut(&mut self.plugins).insert( + "credential_broker".to_string(), + Arc::new(CredentialBrokerPlugin::new(credentials)), + ); + self + } + pub fn register_plugin(mut self, plugin: impl SecurityPlugin + 'static) -> Result { let id = plugin.id(); if self.plugins.contains_key(id) { diff --git a/crates/capsem-core/src/security_engine/plugins/pre.rs b/crates/capsem-core/src/security_engine/plugins/pre.rs index b001e276f..edef82bd3 100644 --- a/crates/capsem-core/src/security_engine/plugins/pre.rs +++ b/crates/capsem-core/src/security_engine/plugins/pre.rs @@ -1,13 +1,20 @@ -use crate::credential_broker::{ - broker_observed_credential, detect_brokered_http_references, detect_http_credential_with_provider, -}; +use crate::credential_broker::{detect_brokered_http_references, detect_http_credential_with_provider}; use crate::net::policy_config::{PolicyActionId, SecurityPluginConfig, SecurityPluginMode}; +use crate::net::proxy_engine::ProxyCredentials; use crate::security_engine::{ security_event_contains_text, SecurityActionError, SecurityEvent, SecurityPlugin, SecurityPluginResult, SecurityPluginStage, DUMMY_EICAR_TEST_STRING, }; -pub(in crate::security_engine) struct CredentialBrokerPlugin; +pub(in crate::security_engine) struct CredentialBrokerPlugin { + credentials: std::sync::Arc, +} + +impl CredentialBrokerPlugin { + pub(in crate::security_engine) fn new(credentials: std::sync::Arc) -> Self { + Self { credentials } + } +} impl SecurityPlugin for CredentialBrokerPlugin { fn id(&self) -> &'static str { @@ -58,7 +65,10 @@ impl SecurityPlugin for CredentialBrokerPlugin { } for observation in &event.credential_observations { - let brokered = broker_observed_credential(observation).map_err(SecurityActionError::new)?; + let brokered = self + .credentials + .capture(observation) + .map_err(SecurityActionError::new)?; if event.credential_ref.is_none() { event.credential_ref = Some(brokered.credential_ref); } diff --git a/crates/capsem-core/tests/mitm_integration.rs b/crates/capsem-core/tests/mitm_integration.rs index a60efba57..3934a3550 100644 --- a/crates/capsem-core/tests/mitm_integration.rs +++ b/crates/capsem-core/tests/mitm_integration.rs @@ -190,17 +190,22 @@ fn make_proxy_config_with_mechanics( let dir = tempfile::tempdir().unwrap(); let db = Arc::new(DbWriter::open(&dir.path().join("test.db"), 256).unwrap()); std::mem::forget(dir); // the tempdir lives as long as the test + let engine = Arc::new(capsem_core::net::proxy_engine::ProxyEngine::local( + policy.clone(), + Arc::clone(&db), + )); let telemetry = Arc::new(mitm_proxy::telemetry_hook::TelemetryDeps { db: db.clone(), + credentials: engine.credentials(), pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), trace_state: Arc::new(std::sync::Mutex::new(capsem_core::net::ai_traffic::TraceState::new())), }); let pipeline = mitm_proxy::make_production_pipeline(Arc::clone(&telemetry)); - let upstream_grants = Arc::new(IntegrationGrants::new(policy.clone(), UpstreamResolver::system())); + let upstream_grants = Arc::new(IntegrationGrants::new(policy, UpstreamResolver::system())); let config = Arc::new(MitmProxyConfig { server_tls: mitm_proxy::make_server_tls_config(&ca), ca, - policy, + engine, db: db.clone(), upstream_tls: mitm_proxy::make_upstream_tls_config(), telemetry, diff --git a/crates/capsem-core/tests/mitm_integration/mcp_gateway_route.rs b/crates/capsem-core/tests/mitm_integration/mcp_gateway_route.rs index 249cf822a..7cf91a7a8 100644 --- a/crates/capsem-core/tests/mitm_integration/mcp_gateway_route.rs +++ b/crates/capsem-core/tests/mitm_integration/mcp_gateway_route.rs @@ -60,9 +60,11 @@ fn proxy_with_gateway() -> (Arc, Arc, Called) { config.mcp_endpoint = Some(Arc::new(McpEndpointState::new( aggregator, Arc::new(std::sync::RwLock::new(Arc::clone( - config.policy.snapshot().security_rules(), + config.engine.policy().snapshot().security_rules(), + ))), + Arc::new(std::sync::RwLock::new(Arc::clone( + config.engine.policy().snapshot().plugins(), ))), - Arc::new(std::sync::RwLock::new(Arc::clone(config.policy.snapshot().plugins()))), Arc::new(tokio::sync::Semaphore::new(4)), McpTimeouts::default(), ))); diff --git a/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs b/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs index c071e181c..4b51c22b9 100644 --- a/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs +++ b/crates/capsem-core/tests/mitm_integration/resolved_upstream.rs @@ -178,7 +178,7 @@ fn resolving(config: Arc, name: &str, address: &str) -> Arc Date: Fri, 9 Oct 2026 03:47:55 +0000 Subject: [PATCH 102/464] refactor(process): construct proxy engine --- crates/capsem-process/src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 2adc7bc1b..8005e58aa 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -649,6 +649,10 @@ async fn run_async_main_loop( let security_rules = Arc::new(std::sync::RwLock::new(Arc::new(runtime_config.security_rules.clone()))); let plugin_policy = Arc::new(std::sync::RwLock::new(Arc::new(runtime_config.plugins.clone()))); let proxy_policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new(runtime_config.proxy_policy_snapshot()); + let proxy_engine = Arc::new(capsem_core::net::proxy_engine::ProxyEngine::local( + proxy_policy.clone(), + Arc::clone(&db), + )); let job_store = Arc::new(JobStore { publisher: Arc::new( capsem_core::container::publish::Publisher::for_session( @@ -810,6 +814,7 @@ async fn run_async_main_loop( let telemetry_deps = Arc::new(capsem_core::net::mitm_proxy::telemetry_hook::TelemetryDeps { db: Arc::clone(&db), + credentials: proxy_engine.credentials(), pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), trace_state: Arc::clone(&model_trace_state), }); @@ -817,7 +822,7 @@ async fn run_async_main_loop( let mitm_config = Arc::new(capsem_core::net::mitm_proxy::MitmProxyConfig { ca: Arc::clone(&net_state.ca), server_tls: capsem_core::net::mitm_proxy::make_server_tls_config(&net_state.ca), - policy: proxy_policy.clone(), + engine: proxy_engine, db: Arc::clone(&db), upstream_tls: Arc::clone(&net_state.upstream_tls), telemetry: telemetry_deps, From be1cfee0788fa95c5b4f8c04d03c4e5b42b1dc63 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 03:50:02 +0000 Subject: [PATCH 103/464] security(foundation): add descriptor-only proxy role --- crates/capsem-foundation/src/unix/worker_sandbox.rs | 1 + .../src/unix/worker_sandbox/linux.rs | 4 ++-- .../src/unix/worker_sandbox/seatbelt.rs | 2 +- .../src/unix/worker_sandbox/tests.rs | 13 +++++++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/crates/capsem-foundation/src/unix/worker_sandbox.rs b/crates/capsem-foundation/src/unix/worker_sandbox.rs index 5a0704223..18e17f1d1 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox.rs @@ -8,6 +8,7 @@ use std::path::{Path, PathBuf}; pub enum Role { Gateway, Ledger, + Proxy, VmOwner, } diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs index e4e0417d0..433afa177 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs @@ -253,10 +253,10 @@ fn install_seccomp(role: Role) -> io::Result<()> { libc::SYS_futimesat, libc::SYS_utimensat, ]; - if matches!(role, Role::Gateway | Role::Ledger) { + if matches!(role, Role::Gateway | Role::Ledger | Role::Proxy) { denied.extend([libc::SYS_execve, libc::SYS_execveat]); } - if role == Role::Ledger { + if matches!(role, Role::Ledger | Role::Proxy) { denied.extend([ libc::SYS_socket, libc::SYS_socketpair, diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs b/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs index 4026c0d04..830541e2f 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs @@ -9,7 +9,7 @@ pub(super) struct Compiled { } pub(super) fn compile(policy: &Policy) -> io::Result { - let mut source = if policy.role() == Role::Ledger { + let mut source = if matches!(policy.role(), Role::Ledger | Role::Proxy) { String::from("(version 1)\n(deny default)\n") } else { String::from( diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index 7e3304c72..430ceac0f 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -59,6 +59,19 @@ fn seatbelt_ledger_is_deny_by_default_with_only_its_directory() { assert_eq!(compiled.parameters(), [("PATH_0", "/sessions/one")]); } +#[test] +fn seatbelt_proxy_is_deny_by_default_and_descriptor_only() { + let compiled = super::seatbelt::compile(&Policy::new(Role::Proxy)).unwrap(); + let source = compiled.source().to_str().unwrap(); + assert!(source.starts_with("(version 1)\n(deny default)\n")); + assert!(!source.contains("allow file")); + assert!(!source.contains("allow network")); + assert!(!source.contains("allow process-exec")); + assert!(!source.contains("allow signal")); + assert!(!source.contains("allow mach-lookup")); + assert!(compiled.parameters().is_empty()); +} + #[cfg(target_os = "linux")] #[test] fn linux_policy_preserves_grants_and_denies_ambient_authority() { From f455da5bdcafcdd2369df52f0aa0e2f3c750c7ef Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 03:51:33 +0000 Subject: [PATCH 104/464] feat(proto): frame proxy capability grants --- crates/capsem-proto/src/lib.rs | 1 + crates/capsem-proto/src/proxy_control.rs | 364 ++++++++++++++++++ .../capsem-proto/src/proxy_control/tests.rs | 75 ++++ 3 files changed, 440 insertions(+) create mode 100644 crates/capsem-proto/src/proxy_control.rs create mode 100644 crates/capsem-proto/src/proxy_control/tests.rs diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 0a14816c8..0f7e687d0 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -26,6 +26,7 @@ pub mod mcp_aggregator; pub mod mcp_contracts; pub mod poll; pub mod privatelink; +pub mod proxy_control; pub mod repeated; pub mod router; mod sparse; diff --git a/crates/capsem-proto/src/proxy_control.rs b/crates/capsem-proto/src/proxy_control.rs new file mode 100644 index 000000000..e82d815d1 --- /dev/null +++ b/crates/capsem-proto/src/proxy_control.rs @@ -0,0 +1,364 @@ +//! Fixed coordinator control records for one confined proxy worker. + +use thiserror::Error; + +pub const PROXY_CONTROL_FRAME_SIZE: usize = 32; +pub const PROXY_CONTROL_MAX_FDS: usize = 1; +pub const PROXY_CONTROL_VERSION: u16 = 1; + +const MAGIC: [u8; 2] = *b"PX"; +const VERSION_RANGE: std::ops::Range = 2..4; +const KIND_OFFSET: usize = 4; +const CAPABILITY_OFFSET: usize = 5; +const DETAIL_RANGE: std::ops::Range = 6..8; +const GENERATION_RANGE: std::ops::Range = 8..24; +const GRANT_ID_RANGE: std::ops::Range = 24..32; + +const ATTACH: u8 = 1; +const SHUTDOWN: u8 = 2; +const READY: u8 = 101; +const ADOPTED: u8 = 102; +const REJECTED: u8 = 103; +const STOPPED: u8 = 104; +const CLOSED: u8 = 105; + +/// Fresh identity of one supervised proxy-worker lifetime. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProxyGeneration([u8; 16]); + +impl ProxyGeneration { + pub const fn new(bytes: [u8; 16]) -> Self { + Self(bytes) + } + + pub const fn as_bytes(self) -> [u8; 16] { + self.0 + } + + const fn is_zero(self) -> bool { + let mut index = 0; + while index < self.0.len() { + if self.0[index] != 0 { + return false; + } + index += 1; + } + true + } +} + +/// Connected capabilities the coordinator may grant to a proxy worker. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[repr(u8)] +pub enum ProxyCapability { + Traffic = 1, + Upstream = 2, + Credential = 3, + Ledger = 4, + PrivateNames = 5, + Mcp = 6, + Telemetry = 7, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProxyChannelGrant { + generation: ProxyGeneration, + grant_id: u64, + capability: ProxyCapability, +} + +impl ProxyChannelGrant { + pub fn new( + generation: ProxyGeneration, + grant_id: u64, + capability: ProxyCapability, + ) -> Result { + if generation.is_zero() { + return Err(ProxyControlError::ZeroGeneration); + } + if grant_id == 0 { + return Err(ProxyControlError::ZeroGrantId); + } + Ok(Self { + generation, + grant_id, + capability, + }) + } + + pub const fn generation(self) -> ProxyGeneration { + self.generation + } + + pub const fn grant_id(self) -> u64 { + self.grant_id + } + + pub const fn capability(self) -> ProxyCapability { + self.capability + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProxyControlRequest { + Attach(ProxyChannelGrant), + Shutdown { generation: ProxyGeneration }, +} + +impl ProxyControlRequest { + pub const fn generation(self) -> ProxyGeneration { + match self { + Self::Attach(grant) => grant.generation(), + Self::Shutdown { generation } => generation, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u16)] +pub enum ProxyControlRejection { + Capacity = 1, + DuplicateCapability = 2, + DuplicateGrant = 3, + ShuttingDown = 4, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u16)] +pub enum ProxyChannelCloseReason { + Disconnected = 1, + ProtocolError = 2, + Revoked = 3, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProxyControlEvent { + Ready { + generation: ProxyGeneration, + }, + Adopted { + generation: ProxyGeneration, + grant_id: u64, + }, + Rejected { + generation: ProxyGeneration, + grant_id: u64, + reason: ProxyControlRejection, + }, + Closed { + generation: ProxyGeneration, + grant_id: u64, + reason: ProxyChannelCloseReason, + }, + Stopped { + generation: ProxyGeneration, + }, +} + +impl ProxyControlEvent { + pub const fn generation(self) -> ProxyGeneration { + match self { + Self::Ready { generation } + | Self::Adopted { generation, .. } + | Self::Rejected { generation, .. } + | Self::Closed { generation, .. } + | Self::Stopped { generation } => generation, + } + } +} + +pub fn encode_proxy_control_request(request: ProxyControlRequest) -> [u8; PROXY_CONTROL_FRAME_SIZE] { + match request { + ProxyControlRequest::Attach(grant) => encode( + ATTACH, + grant.capability() as u8, + 0, + grant.generation(), + grant.grant_id(), + ), + ProxyControlRequest::Shutdown { generation } => encode(SHUTDOWN, 0, 0, generation, 0), + } +} + +pub fn decode_proxy_control_request( + frame: &[u8; PROXY_CONTROL_FRAME_SIZE], +) -> Result { + validate_header(frame)?; + let generation = decode_generation(frame); + let grant_id = decode_u64(frame, GRANT_ID_RANGE); + let detail = decode_u16(frame, DETAIL_RANGE); + match frame[KIND_OFFSET] { + ATTACH if detail == 0 => Ok(ProxyControlRequest::Attach(ProxyChannelGrant::new( + generation, + grant_id, + decode_capability(frame[CAPABILITY_OFFSET])?, + )?)), + SHUTDOWN if frame[CAPABILITY_OFFSET] == 0 && detail == 0 && grant_id == 0 => { + validate_generation(generation)?; + Ok(ProxyControlRequest::Shutdown { generation }) + } + kind => Err(ProxyControlError::InvalidRequest(kind)), + } +} + +pub fn encode_proxy_control_event(event: ProxyControlEvent) -> [u8; PROXY_CONTROL_FRAME_SIZE] { + match event { + ProxyControlEvent::Ready { generation } => encode(READY, 0, 0, generation, 0), + ProxyControlEvent::Adopted { generation, grant_id } => encode(ADOPTED, 0, 0, generation, grant_id), + ProxyControlEvent::Rejected { + generation, + grant_id, + reason, + } => encode(REJECTED, 0, reason as u16, generation, grant_id), + ProxyControlEvent::Closed { + generation, + grant_id, + reason, + } => encode(CLOSED, 0, reason as u16, generation, grant_id), + ProxyControlEvent::Stopped { generation } => encode(STOPPED, 0, 0, generation, 0), + } +} + +pub fn decode_proxy_control_event( + frame: &[u8; PROXY_CONTROL_FRAME_SIZE], +) -> Result { + validate_header(frame)?; + if frame[CAPABILITY_OFFSET] != 0 { + return Err(ProxyControlError::ReservedBytes); + } + let generation = decode_generation(frame); + validate_generation(generation)?; + let grant_id = decode_u64(frame, GRANT_ID_RANGE); + let detail = decode_u16(frame, DETAIL_RANGE); + match frame[KIND_OFFSET] { + READY if detail == 0 && grant_id == 0 => Ok(ProxyControlEvent::Ready { generation }), + ADOPTED if detail == 0 && grant_id != 0 => Ok(ProxyControlEvent::Adopted { generation, grant_id }), + REJECTED if grant_id != 0 => Ok(ProxyControlEvent::Rejected { + generation, + grant_id, + reason: decode_rejection(detail)?, + }), + CLOSED if grant_id != 0 => Ok(ProxyControlEvent::Closed { + generation, + grant_id, + reason: decode_close_reason(detail)?, + }), + STOPPED if detail == 0 && grant_id == 0 => Ok(ProxyControlEvent::Stopped { generation }), + kind => Err(ProxyControlError::InvalidEvent(kind)), + } +} + +fn encode( + kind: u8, + capability: u8, + detail: u16, + generation: ProxyGeneration, + grant_id: u64, +) -> [u8; PROXY_CONTROL_FRAME_SIZE] { + let mut frame = [0; PROXY_CONTROL_FRAME_SIZE]; + frame[..2].copy_from_slice(&MAGIC); + frame[VERSION_RANGE].copy_from_slice(&PROXY_CONTROL_VERSION.to_be_bytes()); + frame[KIND_OFFSET] = kind; + frame[CAPABILITY_OFFSET] = capability; + frame[DETAIL_RANGE].copy_from_slice(&detail.to_be_bytes()); + frame[GENERATION_RANGE].copy_from_slice(&generation.as_bytes()); + frame[GRANT_ID_RANGE].copy_from_slice(&grant_id.to_be_bytes()); + frame +} + +fn validate_header(frame: &[u8; PROXY_CONTROL_FRAME_SIZE]) -> Result<(), ProxyControlError> { + if frame[..2] != MAGIC { + return Err(ProxyControlError::Magic); + } + let version = decode_u16(frame, VERSION_RANGE); + if version != PROXY_CONTROL_VERSION { + return Err(ProxyControlError::Version { + expected: PROXY_CONTROL_VERSION, + actual: version, + }); + } + Ok(()) +} + +fn validate_generation(generation: ProxyGeneration) -> Result<(), ProxyControlError> { + if generation.is_zero() { + Err(ProxyControlError::ZeroGeneration) + } else { + Ok(()) + } +} + +fn decode_capability(value: u8) -> Result { + match value { + 1 => Ok(ProxyCapability::Traffic), + 2 => Ok(ProxyCapability::Upstream), + 3 => Ok(ProxyCapability::Credential), + 4 => Ok(ProxyCapability::Ledger), + 5 => Ok(ProxyCapability::PrivateNames), + 6 => Ok(ProxyCapability::Mcp), + 7 => Ok(ProxyCapability::Telemetry), + value => Err(ProxyControlError::InvalidCapability(value)), + } +} + +fn decode_rejection(value: u16) -> Result { + match value { + 1 => Ok(ProxyControlRejection::Capacity), + 2 => Ok(ProxyControlRejection::DuplicateCapability), + 3 => Ok(ProxyControlRejection::DuplicateGrant), + 4 => Ok(ProxyControlRejection::ShuttingDown), + value => Err(ProxyControlError::InvalidRejection(value)), + } +} + +fn decode_close_reason(value: u16) -> Result { + match value { + 1 => Ok(ProxyChannelCloseReason::Disconnected), + 2 => Ok(ProxyChannelCloseReason::ProtocolError), + 3 => Ok(ProxyChannelCloseReason::Revoked), + value => Err(ProxyControlError::InvalidCloseReason(value)), + } +} + +fn decode_u16(frame: &[u8; PROXY_CONTROL_FRAME_SIZE], range: std::ops::Range) -> u16 { + u16::from_be_bytes(frame[range].try_into().expect("fixed proxy control range")) +} + +fn decode_u64(frame: &[u8; PROXY_CONTROL_FRAME_SIZE], range: std::ops::Range) -> u64 { + u64::from_be_bytes(frame[range].try_into().expect("fixed proxy control range")) +} + +fn decode_generation(frame: &[u8; PROXY_CONTROL_FRAME_SIZE]) -> ProxyGeneration { + ProxyGeneration::new( + frame[GENERATION_RANGE] + .try_into() + .expect("fixed proxy generation range"), + ) +} + +#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +pub enum ProxyControlError { + #[error("proxy control magic mismatch")] + Magic, + #[error("proxy control version mismatch: expected {expected}, got {actual}")] + Version { expected: u16, actual: u16 }, + #[error("proxy control request kind {0} is invalid")] + InvalidRequest(u8), + #[error("proxy control event kind {0} is invalid")] + InvalidEvent(u8), + #[error("proxy capability {0} is invalid")] + InvalidCapability(u8), + #[error("proxy control rejection {0} is invalid")] + InvalidRejection(u16), + #[error("proxy channel close reason {0} is invalid")] + InvalidCloseReason(u16), + #[error("proxy control reserved bytes are nonzero")] + ReservedBytes, + #[error("proxy generation must not be zero")] + ZeroGeneration, + #[error("proxy grant id must not be zero")] + ZeroGrantId, +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/proxy_control/tests.rs b/crates/capsem-proto/src/proxy_control/tests.rs new file mode 100644 index 000000000..526a18d29 --- /dev/null +++ b/crates/capsem-proto/src/proxy_control/tests.rs @@ -0,0 +1,75 @@ +use super::*; + +const GENERATION: ProxyGeneration = ProxyGeneration::new([7; 16]); + +#[test] +fn grants_round_trip_without_session_path_or_destination_authority() { + for capability in [ + ProxyCapability::Traffic, + ProxyCapability::Upstream, + ProxyCapability::Credential, + ProxyCapability::Ledger, + ProxyCapability::PrivateNames, + ProxyCapability::Mcp, + ProxyCapability::Telemetry, + ] { + let grant = ProxyChannelGrant::new(GENERATION, 41, capability).unwrap(); + let request = ProxyControlRequest::Attach(grant); + let frame = encode_proxy_control_request(request); + assert_eq!(decode_proxy_control_request(&frame).unwrap(), request); + let visible = String::from_utf8_lossy(&frame); + assert!(!visible.contains("session")); + assert!(!visible.contains("path")); + assert!(!visible.contains("host")); + } + + let shutdown = ProxyControlRequest::Shutdown { generation: GENERATION }; + assert_eq!( + decode_proxy_control_request(&encode_proxy_control_request(shutdown)).unwrap(), + shutdown + ); +} + +#[test] +fn events_round_trip_generation_and_grant_identity() { + for event in [ + ProxyControlEvent::Ready { generation: GENERATION }, + ProxyControlEvent::Adopted { + generation: GENERATION, + grant_id: 42, + }, + ProxyControlEvent::Rejected { + generation: GENERATION, + grant_id: 43, + reason: ProxyControlRejection::DuplicateCapability, + }, + ProxyControlEvent::Closed { + generation: GENERATION, + grant_id: 44, + reason: ProxyChannelCloseReason::Disconnected, + }, + ProxyControlEvent::Stopped { generation: GENERATION }, + ] { + assert_eq!( + decode_proxy_control_event(&encode_proxy_control_event(event)).unwrap(), + event + ); + } +} + +#[test] +fn malformed_or_stale_authority_records_fail_closed() { + let grant = ProxyChannelGrant::new(GENERATION, 41, ProxyCapability::Upstream).unwrap(); + let valid = encode_proxy_control_request(ProxyControlRequest::Attach(grant)); + for (index, value) in [(0, 0), (3, 0), (4, 99), (5, 99)] { + let mut frame = valid; + frame[index] = value; + assert!(decode_proxy_control_request(&frame).is_err()); + } + let mut zero_generation = valid; + zero_generation[GENERATION_RANGE].fill(0); + assert!(decode_proxy_control_request(&zero_generation).is_err()); + let mut zero_grant = valid; + zero_grant[GRANT_ID_RANGE].fill(0); + assert!(decode_proxy_control_request(&zero_grant).is_err()); +} From b16771bf184dc6410d1215a05e26f451cd9521ca Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 03:55:12 +0000 Subject: [PATCH 105/464] security(proxy): confine capability worker --- Cargo.lock | 14 ++ Cargo.toml | 1 + crates/capsem-proxy/Cargo.toml | 25 +++ crates/capsem-proxy/src/main.rs | 215 ++++++++++++++++++++++++ crates/capsem-proxy/src/tests.rs | 19 +++ crates/capsem-proxy/tests/subprocess.rs | 187 +++++++++++++++++++++ 6 files changed, 461 insertions(+) create mode 100644 crates/capsem-proxy/Cargo.toml create mode 100644 crates/capsem-proxy/src/main.rs create mode 100644 crates/capsem-proxy/src/tests.rs create mode 100644 crates/capsem-proxy/tests/subprocess.rs diff --git a/Cargo.lock b/Cargo.lock index 1dca91dee..43c477a63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1202,6 +1202,20 @@ dependencies = [ "tokio", ] +[[package]] +name = "capsem-proxy" +version = "0.7.0" +dependencies = [ + "anyhow", + "capsem-foundation", + "capsem-guard", + "capsem-proto", + "clap", + "tempfile", + "tokio", + "tracing", +] + [[package]] name = "capsem-router" version = "0.7.0" diff --git a/Cargo.toml b/Cargo.toml index 9ec45afb2..d6311661a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ members = [ "crates/capsem-agent", "crates/capsem-logger", "crates/capsem-ledger", + "crates/capsem-proxy", "crates/capsem-process", "crates/capsem-service", "crates/capsem-admin", diff --git a/crates/capsem-proxy/Cargo.toml b/crates/capsem-proxy/Cargo.toml new file mode 100644 index 000000000..731ce1e37 --- /dev/null +++ b/crates/capsem-proxy/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "capsem-proxy" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +description = "Confined per-session Capsem proxy worker." +homepage.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +anyhow.workspace = true +capsem-foundation = { path = "../capsem-foundation" } +capsem-guard = { path = "../capsem-guard" } +capsem-proto = { path = "../capsem-proto" } +clap = { workspace = true, features = ["derive"] } +tokio.workspace = true +tracing.workspace = true + +[dev-dependencies] +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs new file mode 100644 index 000000000..af882744c --- /dev/null +++ b/crates/capsem-proxy/src/main.rs @@ -0,0 +1,215 @@ +use std::collections::{BTreeMap, HashSet}; +use std::io; +use std::os::fd::AsFd; +use std::os::unix::net::UnixStream; +use std::process::Command; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use capsem_foundation::unix::worker_sandbox::{Policy, Role}; +use capsem_foundation::unix::{fd, router_channel}; +use capsem_proto::proxy_control::{ + decode_proxy_control_request, encode_proxy_control_event, ProxyCapability, ProxyControlEvent, + ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, +}; +use clap::Parser; + +const CONTROL_QUEUE_CAPACITY: usize = 16; +const GRANT_LIMIT: usize = 70; +const TRAFFIC_GRANT_LIMIT: usize = 64; +const CONTROL_SEND_TIMEOUT: Duration = Duration::from_secs(5); + +type ControlSender = router_channel::DescriptorSender; +type ControlReceiver = router_channel::DescriptorReceiver; +type ControlFrame = router_channel::DescriptorFrame; + +#[derive(Parser)] +#[command(version, about = "Confined per-session Capsem proxy worker")] +struct Args { + #[arg(long)] + parent_pid: u32, + /// Sixteen-byte worker generation as 32 lowercase hexadecimal digits. + #[arg(long, value_parser = parse_generation)] + generation: ProxyGeneration, +} + +fn main() -> Result<()> { + // SAFETY: process entry, before telemetry, runtimes or capability + // channels can observe environment state or own unrelated descriptors. + unsafe { + capsem_foundation::unix::process::clear_inherited_environment(); + fd::close_inherited_descriptors()?; + } + let _telemetry = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { + service: "capsem-proxy", + sink: capsem_foundation::telemetry::LogSink::Stderr, + default_filter: "capsem_proxy=info,capsem_core=info,capsem_foundation=warn", + })?; + let args = Args::parse(); + capsem_guard::watch_parent_or_exit(Some(args.parent_pid))?; + let stdin = io::stdin(); + let control = UnixStream::from(fd::duplicate(stdin.as_fd())?); + let denied_file = std::env::current_exe().context("resolve proxy executable before confinement")?; + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build()?; + capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Proxy)) + .context("confine proxy worker before readiness")?; + attest_confinement(&denied_file, args.parent_pid)?; + runtime.block_on(run_control(control, args.generation)) +} + +fn attest_confinement(denied_file: &std::path::Path, parent_pid: u32) -> Result<()> { + if std::env::vars_os().next().is_some() { + bail!("proxy worker retained inherited environment state"); + } + require_denied(std::fs::read(denied_file), "read an ambient file")?; + require_denied(UnixStream::pair(), "create a Unix socket")?; + require_denied( + std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)), + "create a TCP listener", + )?; + require_denied( + std::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, 9)), + "dial a TCP address", + )?; + require_denied(Command::new(denied_file).arg("--version").status(), "execute a process")?; + let parent = capsem_foundation::unix::process::ProcessId::try_from(parent_pid)?; + if capsem_foundation::unix::process::probe_signal_authority(parent)? + != capsem_foundation::unix::process::SignalProbe::Denied + { + bail!("proxy confinement allowed it to signal its parent"); + } + Ok(()) +} + +fn require_denied(result: io::Result, operation: &str) -> Result<()> { + if result.is_ok() { + bail!("proxy confinement allowed it to {operation}"); + } + Ok(()) +} + +async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result<()> { + let shutdown = control.try_clone()?; + let sender = ControlSender::new(control.try_clone()?)?; + let receiver = ControlReceiver::new(control)?; + let (frames_sender, mut frames) = tokio::sync::mpsc::channel(CONTROL_QUEUE_CAPACITY); + let reader = tokio::spawn(read_control(receiver, frames_sender)); + + let outcome = async { + send_event(&sender, ProxyControlEvent::Ready { generation }).await?; + let mut grants = BTreeMap::>::new(); + let mut grant_ids = HashSet::new(); + + while let Some(frame) = frames.recv().await { + let request = decode_proxy_control_request(&frame.bytes)?; + if request.generation() != generation { + bail!("proxy control request names a stale generation"); + } + match request { + ProxyControlRequest::Attach(grant) => { + if frame.fds.len() != 1 { + bail!("proxy capability attach requires exactly one descriptor"); + } + let capability = grant.capability(); + let grant_id = grant.grant_id(); + let current = grants.get(&capability).map_or(0, Vec::len); + let rejection = if grant_ids.contains(&grant_id) { + Some(ProxyControlRejection::DuplicateGrant) + } else if grant_ids.len() >= GRANT_LIMIT + || (capability == ProxyCapability::Traffic && current >= TRAFFIC_GRANT_LIMIT) + { + Some(ProxyControlRejection::Capacity) + } else if capability != ProxyCapability::Traffic && current != 0 { + Some(ProxyControlRejection::DuplicateCapability) + } else { + None + }; + if let Some(reason) = rejection { + send_event( + &sender, + ProxyControlEvent::Rejected { + generation, + grant_id, + reason, + }, + ) + .await?; + continue; + } + let descriptor = frame.fds.into_iter().next().expect("one checked descriptor"); + grants + .entry(capability) + .or_default() + .push((grant_id, UnixStream::from(descriptor))); + grant_ids.insert(grant_id); + send_event(&sender, ProxyControlEvent::Adopted { generation, grant_id }).await?; + } + ProxyControlRequest::Shutdown { .. } => { + if !frame.fds.is_empty() { + bail!("proxy shutdown must not carry descriptors"); + } + break; + } + } + } + + drop(grants); + send_event(&sender, ProxyControlEvent::Stopped { generation }).await + } + .await; + + let _ = shutdown.shutdown(std::net::Shutdown::Both); + let reader_outcome = reader.await.context("proxy control reader task failed")?; + outcome?; + reader_outcome?; + Ok(()) +} + +async fn read_control(receiver: ControlReceiver, frames: tokio::sync::mpsc::Sender) -> io::Result<()> { + loop { + match receiver.recv().await { + Ok(frame) => { + if frames.send(frame).await.is_err() { + return Ok(()); + } + } + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(()), + Err(error) => return Err(error), + } + } +} + +async fn send_event(sender: &ControlSender, event: ProxyControlEvent) -> Result<()> { + tokio::time::timeout( + CONTROL_SEND_TIMEOUT, + sender.send(&encode_proxy_control_event(event), &[]), + ) + .await + .context("proxy control event send timed out")??; + Ok(()) +} + +fn parse_generation(value: &str) -> std::result::Result { + if value.len() != 32 + || !value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return Err("generation must be 32 lowercase hexadecimal digits".into()); + } + let mut bytes = [0; 16]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).map_err(|_| "invalid generation")?; + } + let generation = ProxyGeneration::new(bytes); + if generation.as_bytes() == [0; 16] { + return Err("generation must not be zero".into()); + } + Ok(generation) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs new file mode 100644 index 000000000..d625b55ee --- /dev/null +++ b/crates/capsem-proxy/src/tests.rs @@ -0,0 +1,19 @@ +use super::*; + +#[test] +fn generation_parser_requires_canonical_nonzero_hex() { + assert_eq!( + parse_generation("07070707070707070707070707070707").unwrap().as_bytes(), + [7; 16] + ); + for invalid in [ + "", + "00000000000000000000000000000000", + "0707070707070707070707070707070", + "070707070707070707070707070707070", + "0707070707070707070707070707070G", + "0707070707070707070707070707070A", + ] { + assert!(parse_generation(invalid).is_err(), "accepted {invalid}"); + } +} diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs new file mode 100644 index 000000000..b788d7152 --- /dev/null +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -0,0 +1,187 @@ +use std::os::fd::{AsRawFd, OwnedFd}; +use std::os::unix::net::UnixStream; +use std::process::{Child, Command, Stdio}; +use std::time::Duration; + +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::proxy_control::{ + decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyChannelGrant, ProxyControlEvent, + ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, +}; + +type ControlSender = DescriptorSender; +type ControlReceiver = DescriptorReceiver; + +const GENERATION: ProxyGeneration = ProxyGeneration::new([7; 16]); + +struct ChildGuard(Option); + +impl ChildGuard { + fn take(&mut self) -> Child { + self.0.take().expect("child present") + } +} + +impl Drop for ChildGuard { + fn drop(&mut self) { + if let Some(child) = &mut self.0 { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +async fn event(receiver: &ControlReceiver) -> ProxyControlEvent { + let frame = tokio::time::timeout(Duration::from_secs(5), receiver.recv()) + .await + .expect("proxy control event timed out") + .expect("proxy control event"); + assert!(frame.fds.is_empty()); + decode_proxy_control_event(&frame.bytes).expect("valid proxy control event") +} + +fn spawn() -> (ChildGuard, ControlSender, ControlReceiver) { + let (coordinator, child_control) = UnixStream::pair().unwrap(); + let child = Command::new(env!("CARGO_BIN_EXE_capsem-proxy")) + .arg("--parent-pid") + .arg(std::process::id().to_string()) + .arg("--generation") + .arg("07070707070707070707070707070707") + .env("CAPSEM_PROXY_SECRET", "must-not-survive") + .stdin(Stdio::from(OwnedFd::from(child_control))) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let requests = ControlSender::new(coordinator.try_clone().unwrap()).unwrap(); + let events = ControlReceiver::new(coordinator).unwrap(); + (ChildGuard(Some(child)), requests, events) +} + +#[tokio::test] +async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { + let (mut child, requests, events) = spawn(); + assert_eq!( + event(&events).await, + ProxyControlEvent::Ready { generation: GENERATION } + ); + + #[cfg(target_os = "linux")] + { + let environ = std::fs::read(format!("/proc/{}/environ", child.0.as_ref().unwrap().id())).unwrap(); + assert!(!environ + .windows(b"CAPSEM_PROXY_SECRET".len()) + .any(|value| value == b"CAPSEM_PROXY_SECRET")); + } + + let mut peers = Vec::new(); + for (index, capability) in [ + ProxyCapability::Traffic, + ProxyCapability::Upstream, + ProxyCapability::Credential, + ProxyCapability::Ledger, + ProxyCapability::PrivateNames, + ProxyCapability::Mcp, + ProxyCapability::Telemetry, + ] + .into_iter() + .enumerate() + { + let grant_id = index as u64 + 1; + let (peer, granted) = UnixStream::pair().unwrap(); + let grant = ProxyChannelGrant::new(GENERATION, grant_id, capability).unwrap(); + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), + &[granted.as_raw_fd()], + ) + .await + .unwrap(); + drop(granted); + assert_eq!( + event(&events).await, + ProxyControlEvent::Adopted { + generation: GENERATION, + grant_id, + } + ); + peers.push(peer); + } + + let (duplicate_peer, duplicate) = UnixStream::pair().unwrap(); + let duplicate_grant = ProxyChannelGrant::new(GENERATION, 8, ProxyCapability::Ledger).unwrap(); + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Attach(duplicate_grant)), + &[duplicate.as_raw_fd()], + ) + .await + .unwrap(); + drop(duplicate); + assert_eq!( + event(&events).await, + ProxyControlEvent::Rejected { + generation: GENERATION, + grant_id: 8, + reason: ProxyControlRejection::DuplicateCapability, + } + ); + drop(duplicate_peer); + + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Shutdown { generation: GENERATION }), + &[], + ) + .await + .unwrap(); + assert_eq!( + event(&events).await, + ProxyControlEvent::Stopped { generation: GENERATION } + ); + drop(peers); + + let child = child.take(); + let output = tokio::task::spawn_blocking(move || child.wait_with_output()) + .await + .unwrap() + .unwrap(); + assert!( + output.status.success(), + "proxy worker failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[tokio::test] +async fn stale_generation_terminates_worker_and_releases_grants() { + let (mut child, requests, events) = spawn(); + assert_eq!( + event(&events).await, + ProxyControlEvent::Ready { generation: GENERATION } + ); + let stale = ProxyGeneration::new([8; 16]); + let (peer, granted) = UnixStream::pair().unwrap(); + let grant = ProxyChannelGrant::new(stale, 1, ProxyCapability::Upstream).unwrap(); + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), + &[granted.as_raw_fd()], + ) + .await + .unwrap(); + drop(granted); + + let child = child.take(); + let output = tokio::time::timeout( + Duration::from_secs(5), + tokio::task::spawn_blocking(move || child.wait_with_output()), + ) + .await + .expect("stale-generation worker did not exit") + .unwrap() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("stale generation")); + drop(peer); +} From 9a3fced985db978c5987bb57ee2a0245d7df1766 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:04:25 +0000 Subject: [PATCH 106/464] feat(proto): carry proxy policy revisions --- crates/capsem-proto/build.rs | 1 + crates/capsem-proto/src/lib.rs | 1 + crates/capsem-proto/src/proxy_control.rs | 2 + .../capsem-proto/src/proxy_control/tests.rs | 1 + crates/capsem-proto/src/proxy_policy.rs | 88 +++++++++++++++++++ crates/capsem-proto/src/proxy_policy/tests.rs | 36 ++++++++ 6 files changed, 129 insertions(+) create mode 100644 crates/capsem-proto/src/proxy_policy.rs create mode 100644 crates/capsem-proto/src/proxy_policy/tests.rs diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index 7333cbb4f..cd0679e86 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -24,6 +24,7 @@ fn main() { let files = ["lib.rs", "ipc.rs", "handshake.rs", "router.rs", "exec_stream.rs"]; write_hash("schema_hash.txt", &files); write_hash("ledger_schema_hash.txt", &["ledger.rs", "ledger_control.rs"]); + write_hash("proxy_schema_hash.txt", &["proxy_control.rs", "proxy_policy.rs"]); } fn write_hash(output: &str, files: &[&str]) { diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 0f7e687d0..50f92abbe 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -27,6 +27,7 @@ pub mod mcp_contracts; pub mod poll; pub mod privatelink; pub mod proxy_control; +pub mod proxy_policy; pub mod repeated; pub mod router; mod sparse; diff --git a/crates/capsem-proto/src/proxy_control.rs b/crates/capsem-proto/src/proxy_control.rs index e82d815d1..07afa5033 100644 --- a/crates/capsem-proto/src/proxy_control.rs +++ b/crates/capsem-proto/src/proxy_control.rs @@ -58,6 +58,7 @@ pub enum ProxyCapability { PrivateNames = 5, Mcp = 6, Telemetry = 7, + Policy = 8, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -297,6 +298,7 @@ fn decode_capability(value: u8) -> Result { 5 => Ok(ProxyCapability::PrivateNames), 6 => Ok(ProxyCapability::Mcp), 7 => Ok(ProxyCapability::Telemetry), + 8 => Ok(ProxyCapability::Policy), value => Err(ProxyControlError::InvalidCapability(value)), } } diff --git a/crates/capsem-proto/src/proxy_control/tests.rs b/crates/capsem-proto/src/proxy_control/tests.rs index 526a18d29..e27e744f9 100644 --- a/crates/capsem-proto/src/proxy_control/tests.rs +++ b/crates/capsem-proto/src/proxy_control/tests.rs @@ -12,6 +12,7 @@ fn grants_round_trip_without_session_path_or_destination_authority() { ProxyCapability::PrivateNames, ProxyCapability::Mcp, ProxyCapability::Telemetry, + ProxyCapability::Policy, ] { let grant = ProxyChannelGrant::new(GENERATION, 41, capability).unwrap(); let request = ProxyControlRequest::Attach(grant); diff --git a/crates/capsem-proto/src/proxy_policy.rs b/crates/capsem-proto/src/proxy_policy.rs new file mode 100644 index 000000000..c0fc01a1b --- /dev/null +++ b/crates/capsem-proto/src/proxy_policy.rs @@ -0,0 +1,88 @@ +//! Path-free active-policy updates for a confined proxy worker. + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub const PROXY_SCHEMA_HASH: u64 = include!(concat!(env!("OUT_DIR"), "/proxy_schema_hash.txt")); +pub const MAX_ACTIVE_POLICY_BYTES: usize = 1024 * 1024; +pub const MAX_POLICY_ERROR_BYTES: usize = 4 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyPolicyRequest { + Apply { + schema_hash: u64, + request_id: u64, + #[serde(with = "serde_bytes")] + active_policy: Vec, + }, +} + +impl ProxyPolicyRequest { + pub fn validate(&self) -> Result<(), ProxyPolicyError> { + match self { + Self::Apply { + schema_hash, + request_id, + active_policy, + } if *schema_hash == PROXY_SCHEMA_HASH + && *request_id != 0 + && !active_policy.is_empty() + && active_policy.len() <= MAX_ACTIVE_POLICY_BYTES => + { + Ok(()) + } + _ => Err(ProxyPolicyError::InvalidRequest), + } + } + + pub fn apply(request_id: u64, active_policy: Vec) -> Self { + Self::Apply { + schema_hash: PROXY_SCHEMA_HASH, + request_id, + active_policy, + } + } + + pub const fn request_id(&self) -> u64 { + match self { + Self::Apply { request_id, .. } => *request_id, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyPolicyResponse { + Applied { + request_id: u64, + active_policy_digest: String, + }, + Rejected { + request_id: u64, + error: String, + }, +} + +impl ProxyPolicyResponse { + pub fn rejected(request_id: u64, error: impl Into) -> Self { + let mut error = error.into(); + if error.len() > MAX_POLICY_ERROR_BYTES { + let mut end = MAX_POLICY_ERROR_BYTES; + while !error.is_char_boundary(end) { + end -= 1; + } + error.truncate(end); + } + Self::Rejected { request_id, error } + } +} + +#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +pub enum ProxyPolicyError { + #[error("proxy policy request has an incompatible schema, zero id, or invalid active-policy body")] + InvalidRequest, +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/proxy_policy/tests.rs b/crates/capsem-proto/src/proxy_policy/tests.rs new file mode 100644 index 000000000..eb90b4940 --- /dev/null +++ b/crates/capsem-proto/src/proxy_policy/tests.rs @@ -0,0 +1,36 @@ +use super::*; + +#[test] +fn request_is_path_free_and_bounded() { + let request = ProxyPolicyRequest::apply(1, b"network = {}".to_vec()); + request.validate().unwrap(); + let encoded = rmp_serde::to_vec_named(&request).unwrap(); + let visible = String::from_utf8_lossy(&encoded); + assert!(!visible.contains("path")); + assert!(!visible.contains("session")); + assert!(ProxyPolicyRequest::Apply { + schema_hash: PROXY_SCHEMA_HASH, + request_id: 2, + active_policy: vec![0; MAX_ACTIVE_POLICY_BYTES + 1], + } + .validate() + .is_err()); + assert!(ProxyPolicyRequest::Apply { + schema_hash: PROXY_SCHEMA_HASH.wrapping_add(1), + request_id: 3, + active_policy: b"network = {}".to_vec(), + } + .validate() + .is_err()); +} + +#[test] +fn rejection_diagnostic_is_utf8_bounded() { + let response = ProxyPolicyResponse::rejected(3, "é".repeat(MAX_POLICY_ERROR_BYTES)); + let ProxyPolicyResponse::Rejected { request_id, error } = response else { + panic!("expected rejection") + }; + assert_eq!(request_id, 3); + assert!(error.len() <= MAX_POLICY_ERROR_BYTES); + assert!(std::str::from_utf8(error.as_bytes()).is_ok()); +} From be1b66bcf2481ed3ce81a61ffef3ef8742d452fb Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:05:20 +0000 Subject: [PATCH 107/464] feat(proxy): compile path-free runtime policy --- crates/capsem-core/src/net/proxy_engine.rs | 55 ++++++++++++++++++- .../capsem-core/src/net/proxy_engine/tests.rs | 52 ++++++++++++++++++ 2 files changed, 106 insertions(+), 1 deletion(-) diff --git a/crates/capsem-core/src/net/proxy_engine.rs b/crates/capsem-core/src/net/proxy_engine.rs index 958df4f7a..fe9a7a141 100644 --- a/crates/capsem-core/src/net/proxy_engine.rs +++ b/crates/capsem-core/src/net/proxy_engine.rs @@ -12,17 +12,21 @@ use std::collections::BTreeMap; use std::future::Future; +use std::net::SocketAddr; use std::pin::Pin; use std::sync::{Arc, RwLock}; use capsem_logger::{DbWriter, WriteOp}; use crate::credential_broker::{BrokeredCredential, BrokeredUpstreamCredentials, CredentialObservation}; +use crate::mcp::policy::McpConfig; use crate::net::ai_traffic::provider::ProviderKind; use crate::security_engine::{MaterializedHttpRequest, SecurityActionError, SecurityBoundaryEvaluation, SecurityEvent}; use super::policy::NetworkMechanics; -use super::policy_config::{ModelEndpointRegistry, SecurityPluginConfig, SecurityRuleSet}; +use super::policy_config::{ + active_policy_digest, ActivePolicyFile, ModelEndpointRegistry, SecurityPluginConfig, SecurityRuleSet, +}; pub type ProxyCapabilityFuture<'a, T> = Pin + Send + 'a>>; @@ -147,6 +151,55 @@ impl ProxyEngine { } } +/// One path-free active-policy revision compiled for a confined proxy worker. +/// +/// The coordinator sends exact bytes over a capability channel. Compilation +/// cannot consult ambient settings, session directories, or built-in policy, +/// and the digest therefore names precisely the revision put in force. +pub struct ProxyRuntimePolicy { + snapshot: ProxyPolicySnapshot, + dns_upstreams: Vec, + mcp: McpConfig, +} + +impl ProxyRuntimePolicy { + pub fn compile(active_policy: &[u8]) -> Result { + let source = std::str::from_utf8(active_policy).map_err(|error| format!("decode active policy: {error}"))?; + let active = + toml::from_str::(source).map_err(|error| format!("parse active policy: {error}"))?; + let compiled = active + .compile_runtime() + .map_err(|error| format!("compile active policy: {error}"))?; + Ok(Self { + snapshot: ProxyPolicySnapshot::new( + active_policy_digest(active_policy), + compiled.network, + compiled.security_rules, + compiled.plugins, + compiled.model_endpoints, + ), + dns_upstreams: compiled.dns_upstreams, + mcp: compiled.mcp, + }) + } + + pub fn snapshot(&self) -> &ProxyPolicySnapshot { + &self.snapshot + } + + pub fn dns_upstreams(&self) -> &[SocketAddr] { + &self.dns_upstreams + } + + pub fn mcp(&self) -> &McpConfig { + &self.mcp + } + + pub fn into_parts(self) -> (ProxyPolicySnapshot, Vec, McpConfig) { + (self.snapshot, self.dns_upstreams, self.mcp) + } +} + /// One fully compiled active-policy revision. pub struct ProxyPolicySnapshot { digest: String, diff --git a/crates/capsem-core/src/net/proxy_engine/tests.rs b/crates/capsem-core/src/net/proxy_engine/tests.rs index 04a780cad..ef4ab3cc2 100644 --- a/crates/capsem-core/src/net/proxy_engine/tests.rs +++ b/crates/capsem-core/src/net/proxy_engine/tests.rs @@ -53,6 +53,58 @@ fn credential_revision() -> ProxyPolicySnapshot { ) } +const ACTIVE_POLICY: &str = r#" +[network] +[network.dns] +upstreams = ["127.0.0.1:5353"] + +[user_rules.profiles.rules.worker_http] +name = "worker_http" +action = "allow" +priority = 10 +match = 'http.host == "worker.example"' + +[corp_rules] + +[plugins.credential_broker] +mode = "rewrite" +detection_level = "informational" + +[mcp.server_enabled] +local = false +"#; + +#[test] +fn runtime_policy_compiles_exact_bytes_without_a_path() { + let runtime = ProxyRuntimePolicy::compile(ACTIVE_POLICY.as_bytes()).unwrap(); + let snapshot = runtime.snapshot(); + + assert_eq!( + snapshot.digest(), + crate::net::policy_config::active_policy_digest(ACTIVE_POLICY.as_bytes()) + ); + assert!(snapshot + .security_rules() + .rules() + .iter() + .any(|rule| rule.rule_id == "profiles.rules.worker_http")); + assert_eq!( + snapshot.plugins()["credential_broker"].mode, + SecurityPluginMode::Rewrite + ); + assert_eq!(runtime.dns_upstreams(), &["127.0.0.1:5353".parse().unwrap()]); + assert_eq!(runtime.mcp().server_enabled.get("local"), Some(&false)); +} + +#[test] +fn malformed_runtime_policy_is_rejected() { + let error = ProxyRuntimePolicy::compile(b"[network]\nunknown = true") + .err() + .expect("invalid policy must fail"); + + assert!(error.contains("parse active policy"), "{error}"); +} + #[test] fn policy_handle_replaces_one_immutable_revision_atomically() { let handle = ProxyPolicyHandle::new(revision(0)); From ed5251f7a3a8fef739ba0ed5328733d6f5411a0d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:06:07 +0000 Subject: [PATCH 108/464] security(proxy): apply policy inside worker --- Cargo.lock | 3 + crates/capsem-proxy/Cargo.toml | 3 + crates/capsem-proxy/src/main.rs | 272 +++++++++++++++++++----- crates/capsem-proxy/src/tests.rs | 17 ++ crates/capsem-proxy/tests/subprocess.rs | 90 +++++++- 5 files changed, 334 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43c477a63..0876e26c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1207,10 +1207,13 @@ name = "capsem-proxy" version = "0.7.0" dependencies = [ "anyhow", + "capsem-core", "capsem-foundation", "capsem-guard", + "capsem-logger", "capsem-proto", "clap", + "http", "tempfile", "tokio", "tracing", diff --git a/crates/capsem-proxy/Cargo.toml b/crates/capsem-proxy/Cargo.toml index 731ce1e37..c12482da4 100644 --- a/crates/capsem-proxy/Cargo.toml +++ b/crates/capsem-proxy/Cargo.toml @@ -11,10 +11,13 @@ authors.workspace = true [dependencies] anyhow.workspace = true +capsem-core = { path = "../capsem-core" } capsem-foundation = { path = "../capsem-foundation" } capsem-guard = { path = "../capsem-guard" } +capsem-logger = { path = "../capsem-logger" } capsem-proto = { path = "../capsem-proto" } clap = { workspace = true, features = ["derive"] } +http.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index af882744c..82ad5ebd2 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -3,25 +3,37 @@ use std::io; use std::os::fd::AsFd; use std::os::unix::net::UnixStream; use std::process::Command; +use std::sync::{Arc, Mutex}; use std::time::Duration; use anyhow::{bail, Context, Result}; +use capsem_core::credential_broker::{BrokeredCredential, BrokeredUpstreamCredentials, CredentialObservation}; +use capsem_core::net::ai_traffic::provider::ProviderKind; +use capsem_core::net::proxy_engine::{ + ProxyCapabilityFuture, ProxyCredentials, ProxyEngine, ProxyLedger, ProxyPolicyHandle, ProxyRuntimePolicy, +}; +use capsem_foundation::ipc_channel; use capsem_foundation::unix::worker_sandbox::{Policy, Role}; use capsem_foundation::unix::{fd, router_channel}; +use capsem_logger::WriteOp; use capsem_proto::proxy_control::{ - decode_proxy_control_request, encode_proxy_control_event, ProxyCapability, ProxyControlEvent, - ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, + decode_proxy_control_request, encode_proxy_control_event, ProxyCapability, ProxyChannelCloseReason, + ProxyControlEvent, ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, + PROXY_CONTROL_MAX_FDS, }; +use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; use clap::Parser; +use tokio::task::{JoinError, JoinSet}; const CONTROL_QUEUE_CAPACITY: usize = 16; -const GRANT_LIMIT: usize = 70; +const GRANT_LIMIT: usize = 71; const TRAFFIC_GRANT_LIMIT: usize = 64; const CONTROL_SEND_TIMEOUT: Duration = Duration::from_secs(5); type ControlSender = router_channel::DescriptorSender; type ControlReceiver = router_channel::DescriptorReceiver; type ControlFrame = router_channel::DescriptorFrame; +type CapabilityResult = (ProxyCapability, u64, ProxyChannelCloseReason); #[derive(Parser)] #[command(version, about = "Confined per-session Capsem proxy worker")] @@ -100,62 +112,104 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result let outcome = async { send_event(&sender, ProxyControlEvent::Ready { generation }).await?; - let mut grants = BTreeMap::>::new(); + let state = Arc::new(Mutex::new(ProxyRuntimeState::default())); + let mut grants = BTreeMap::>::new(); + let mut descriptors = BTreeMap::::new(); let mut grant_ids = HashSet::new(); + let mut tasks = JoinSet::::new(); - while let Some(frame) = frames.recv().await { - let request = decode_proxy_control_request(&frame.bytes)?; - if request.generation() != generation { - bail!("proxy control request names a stale generation"); - } - match request { - ProxyControlRequest::Attach(grant) => { - if frame.fds.len() != 1 { - bail!("proxy capability attach requires exactly one descriptor"); + loop { + let input = if tasks.is_empty() { + ControlInput::Frame(frames.recv().await.context("proxy control channel closed")?) + } else { + tokio::select! { + frame = frames.recv() => { + ControlInput::Frame(frame.context("proxy control channel closed")?) } - let capability = grant.capability(); - let grant_id = grant.grant_id(); - let current = grants.get(&capability).map_or(0, Vec::len); - let rejection = if grant_ids.contains(&grant_id) { - Some(ProxyControlRejection::DuplicateGrant) - } else if grant_ids.len() >= GRANT_LIMIT - || (capability == ProxyCapability::Traffic && current >= TRAFFIC_GRANT_LIMIT) - { - Some(ProxyControlRejection::Capacity) - } else if capability != ProxyCapability::Traffic && current != 0 { - Some(ProxyControlRejection::DuplicateCapability) - } else { - None - }; - if let Some(reason) = rejection { - send_event( - &sender, - ProxyControlEvent::Rejected { - generation, - grant_id, - reason, - }, - ) - .await?; - continue; + result = tasks.join_next() => { + ControlInput::Capability(result.expect("nonempty proxy capability task set")) } - let descriptor = frame.fds.into_iter().next().expect("one checked descriptor"); - grants - .entry(capability) - .or_default() - .push((grant_id, UnixStream::from(descriptor))); - grant_ids.insert(grant_id); - send_event(&sender, ProxyControlEvent::Adopted { generation, grant_id }).await?; } - ProxyControlRequest::Shutdown { .. } => { - if !frame.fds.is_empty() { - bail!("proxy shutdown must not carry descriptors"); + }; + match input { + ControlInput::Capability(result) => { + let (capability, grant_id, reason) = result.context("proxy capability task failed")?; + remove_grant(&mut grants, &mut grant_ids, capability, grant_id); + send_event( + &sender, + ProxyControlEvent::Closed { + generation, + grant_id, + reason, + }, + ) + .await?; + } + ControlInput::Frame(frame) => { + let request = decode_proxy_control_request(&frame.bytes)?; + if request.generation() != generation { + bail!("proxy control request names a stale generation"); + } + match request { + ProxyControlRequest::Attach(grant) => { + if frame.fds.len() != 1 { + bail!("proxy capability attach requires exactly one descriptor"); + } + let capability = grant.capability(); + let grant_id = grant.grant_id(); + let current = grants.get(&capability).map_or(0, Vec::len); + let rejection = if grant_ids.contains(&grant_id) { + Some(ProxyControlRejection::DuplicateGrant) + } else if grant_ids.len() >= GRANT_LIMIT + || (capability == ProxyCapability::Traffic && current >= TRAFFIC_GRANT_LIMIT) + { + Some(ProxyControlRejection::Capacity) + } else if capability != ProxyCapability::Traffic && current != 0 { + Some(ProxyControlRejection::DuplicateCapability) + } else { + None + }; + if let Some(reason) = rejection { + send_event( + &sender, + ProxyControlEvent::Rejected { + generation, + grant_id, + reason, + }, + ) + .await?; + continue; + } + let descriptor = frame.fds.into_iter().next().expect("one checked descriptor"); + let stream = UnixStream::from(descriptor); + grants.entry(capability).or_default().push(grant_id); + grant_ids.insert(grant_id); + if capability == ProxyCapability::Policy { + let state = Arc::clone(&state); + tasks.spawn(async move { + let reason = serve_policy(stream, state).await; + (capability, grant_id, reason) + }); + } else { + descriptors.insert(grant_id, stream); + } + send_event(&sender, ProxyControlEvent::Adopted { generation, grant_id }).await?; + } + ProxyControlRequest::Shutdown { .. } => { + if !frame.fds.is_empty() { + bail!("proxy shutdown must not carry descriptors"); + } + break; + } } - break; } } } + tasks.abort_all(); + while tasks.join_next().await.is_some() {} + drop(descriptors); drop(grants); send_event(&sender, ProxyControlEvent::Stopped { generation }).await } @@ -168,6 +222,126 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result Ok(()) } +enum ControlInput { + Frame(ControlFrame), + Capability(std::result::Result), +} + +fn remove_grant( + grants: &mut BTreeMap>, + grant_ids: &mut HashSet, + capability: ProxyCapability, + grant_id: u64, +) { + grant_ids.remove(&grant_id); + if let Some(ids) = grants.get_mut(&capability) { + ids.retain(|candidate| *candidate != grant_id); + if ids.is_empty() { + grants.remove(&capability); + } + } +} + +#[derive(Default)] +struct ProxyRuntimeState { + engine: Option, + dns_upstreams: Vec, + mcp: capsem_core::mcp::policy::McpConfig, +} + +impl ProxyRuntimeState { + fn apply(&mut self, active_policy: &[u8]) -> Result { + let runtime = ProxyRuntimePolicy::compile(active_policy).map_err(anyhow::Error::msg)?; + let digest = runtime.snapshot().digest().to_string(); + let (snapshot, dns_upstreams, mcp) = runtime.into_parts(); + if let Some(engine) = &self.engine { + engine.policy().replace(snapshot); + } else { + self.engine = Some(ProxyEngine::new( + ProxyPolicyHandle::new(snapshot), + Arc::new(UnavailableLedger), + Arc::new(UnavailableCredentials), + )); + } + self.dns_upstreams = dns_upstreams; + self.mcp = mcp; + Ok(digest) + } +} + +struct UnavailableLedger; + +impl ProxyLedger for UnavailableLedger { + fn write(&self, _op: WriteOp) -> ProxyCapabilityFuture<'_, std::result::Result<(), String>> { + Box::pin(async { Err("proxy ledger capability is not attached".to_string()) }) + } +} + +struct UnavailableCredentials; + +impl ProxyCredentials for UnavailableCredentials { + fn capture(&self, _observation: &CredentialObservation) -> std::result::Result { + Err("proxy credential capability is not attached".to_string()) + } + + fn substitute_upstream( + &self, + _domain: &str, + _ai_provider: Option, + _headers: &mut http::HeaderMap, + _query: Option<&str>, + ) -> std::result::Result { + Err("proxy credential capability is not attached".to_string()) + } +} + +async fn serve_policy(stream: UnixStream, state: Arc>) -> ProxyChannelCloseReason { + let Ok((sender, receiver)) = ipc_channel::channel_from_std::(stream) + else { + return ProxyChannelCloseReason::ProtocolError; + }; + loop { + let request = match receiver.recv().await { + Ok(request) => request, + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => { + return ProxyChannelCloseReason::Disconnected; + } + Err(error) => { + tracing::warn!(%error, "proxy policy capability received an invalid frame"); + return ProxyChannelCloseReason::ProtocolError; + } + }; + let request_id = request.request_id(); + let response = if let Err(error) = request.validate() { + ProxyPolicyResponse::rejected(request_id, error.to_string()) + } else { + let ProxyPolicyRequest::Apply { active_policy, .. } = request; + match state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .apply(&active_policy) + { + Ok(active_policy_digest) => ProxyPolicyResponse::Applied { + request_id, + active_policy_digest, + }, + Err(error) => ProxyPolicyResponse::rejected(request_id, format!("{error:#}")), + } + }; + match tokio::time::timeout(CONTROL_SEND_TIMEOUT, sender.send(response)).await { + Ok(Ok(())) => {} + Ok(Err(error)) if error.kind() == io::ErrorKind::BrokenPipe => { + return ProxyChannelCloseReason::Disconnected; + } + Ok(Err(error)) => { + tracing::warn!(%error, "proxy policy capability could not send a response"); + return ProxyChannelCloseReason::ProtocolError; + } + Err(_) => return ProxyChannelCloseReason::ProtocolError, + } + } +} + async fn read_control(receiver: ControlReceiver, frames: tokio::sync::mpsc::Sender) -> io::Result<()> { loop { match receiver.recv().await { diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index d625b55ee..f13939423 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -17,3 +17,20 @@ fn generation_parser_requires_canonical_nonzero_hex() { assert!(parse_generation(invalid).is_err(), "accepted {invalid}"); } } + +#[test] +fn failed_policy_update_preserves_the_running_engine_revision() { + let active_policy = br#" +[network] +[user_rules.profiles.rules.worker_http] +name = "worker_http" +action = "allow" +match = 'http.host == "worker.example"' +[corp_rules] +"#; + let mut state = ProxyRuntimeState::default(); + let digest = state.apply(active_policy).unwrap(); + + assert!(state.apply(b"[network]\nunknown = true").is_err()); + assert_eq!(state.engine.as_ref().unwrap().policy().snapshot().digest(), digest); +} diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs index b788d7152..7bfcdb9fa 100644 --- a/crates/capsem-proxy/tests/subprocess.rs +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -8,6 +8,7 @@ use capsem_proto::proxy_control::{ decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyChannelGrant, ProxyControlEvent, ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, }; +use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; type ControlSender = DescriptorSender; type ControlReceiver = DescriptorReceiver; @@ -83,6 +84,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { ProxyCapability::PrivateNames, ProxyCapability::Mcp, ProxyCapability::Telemetry, + ProxyCapability::Policy, ] .into_iter() .enumerate() @@ -109,7 +111,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { } let (duplicate_peer, duplicate) = UnixStream::pair().unwrap(); - let duplicate_grant = ProxyChannelGrant::new(GENERATION, 8, ProxyCapability::Ledger).unwrap(); + let duplicate_grant = ProxyChannelGrant::new(GENERATION, 9, ProxyCapability::Ledger).unwrap(); requests .send( &encode_proxy_control_request(ProxyControlRequest::Attach(duplicate_grant)), @@ -122,7 +124,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { event(&events).await, ProxyControlEvent::Rejected { generation: GENERATION, - grant_id: 8, + grant_id: 9, reason: ProxyControlRejection::DuplicateCapability, } ); @@ -185,3 +187,87 @@ async fn stale_generation_terminates_worker_and_releases_grants() { assert!(String::from_utf8_lossy(&output.stderr).contains("stale generation")); drop(peer); } + +#[tokio::test] +async fn policy_capability_compiles_exact_bytes_and_rejects_bad_revisions() { + let (mut child, requests, events) = spawn(); + assert_eq!( + event(&events).await, + ProxyControlEvent::Ready { generation: GENERATION } + ); + let (peer, granted) = UnixStream::pair().unwrap(); + let grant = ProxyChannelGrant::new(GENERATION, 1, ProxyCapability::Policy).unwrap(); + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), + &[granted.as_raw_fd()], + ) + .await + .unwrap(); + drop(granted); + assert_eq!( + event(&events).await, + ProxyControlEvent::Adopted { + generation: GENERATION, + grant_id: 1, + } + ); + + let (policy_tx, policy_rx) = + capsem_foundation::ipc_channel::channel_from_std::(peer).unwrap(); + let active_policy = br#" +[network] +[network.dns] +upstreams = ["127.0.0.1:5353"] +[user_rules.profiles.rules.worker_http] +name = "worker_http" +action = "allow" +match = 'http.host == "worker.example"' +[corp_rules] +[mcp.server_enabled] +local = false +"#; + policy_tx + .send(ProxyPolicyRequest::apply(1, active_policy.to_vec())) + .await + .unwrap(); + assert_eq!( + policy_rx.recv().await.unwrap(), + ProxyPolicyResponse::Applied { + request_id: 1, + active_policy_digest: capsem_core::net::policy_config::active_policy_digest(active_policy), + } + ); + + policy_tx + .send(ProxyPolicyRequest::apply(2, b"[network]\nunknown = true".to_vec())) + .await + .unwrap(); + let ProxyPolicyResponse::Rejected { request_id, error } = policy_rx.recv().await.unwrap() else { + panic!("invalid active policy was accepted") + }; + assert_eq!(request_id, 2); + assert!(error.contains("parse active policy"), "{error}"); + + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Shutdown { generation: GENERATION }), + &[], + ) + .await + .unwrap(); + assert_eq!( + event(&events).await, + ProxyControlEvent::Stopped { generation: GENERATION } + ); + let child = child.take(); + let output = tokio::task::spawn_blocking(move || child.wait_with_output()) + .await + .unwrap() + .unwrap(); + assert!( + output.status.success(), + "proxy worker failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} From b4eddf7c5675f8d2c1de8a816c3e31e510caa077 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:26:00 +0000 Subject: [PATCH 109/464] security(service): supervise per-session proxy workers --- CHANGELOG.md | 4 + crates/capsem-service/src/active_policy.rs | 24 +- crates/capsem-service/src/instance.rs | 1 + crates/capsem-service/src/instance_reaper.rs | 1 + crates/capsem-service/src/main.rs | 5 + crates/capsem-service/src/owner_connection.rs | 4 + crates/capsem-service/src/proxy_worker.rs | 684 ++++++++++++++++++ .../capsem-service/src/proxy_worker/tests.rs | 218 ++++++ crates/capsem-service/src/service_runtime.rs | 3 + crates/capsem-service/src/tests.rs | 2 + .../capsem-service/src/vm_files/provision.rs | 12 +- crates/capsem-service/src/vm_lifecycle.rs | 4 +- .../src/vm_lifecycle/provision.rs | 17 + .../capsem-service/src/vm_lifecycle/resume.rs | 10 +- .../src/vm_lifecycle/resume_process.rs | 17 + 15 files changed, 993 insertions(+), 13 deletions(-) create mode 100644 crates/capsem-service/src/proxy_worker.rs create mode 100644 crates/capsem-service/src/proxy_worker/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index c30991303..815928900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Start one fresh-generation confined proxy worker with every live VM, apply + exact active-policy bytes before registration, revoke it with the VM owner, + and terminate only the matching owner generation if its proxy dies. + - Apply each proxy policy reload as one digest-keyed immutable revision across HTTP routing, security plugins, provider identity, response telemetry and guest DNS, preventing a request from combining authority from two revisions. diff --git a/crates/capsem-service/src/active_policy.rs b/crates/capsem-service/src/active_policy.rs index d13145aa6..c185ce2f3 100644 --- a/crates/capsem-service/src/active_policy.rs +++ b/crates/capsem-service/src/active_policy.rs @@ -12,6 +12,7 @@ use capsem_core::net::policy_config::ActivePolicyFile; pub(crate) struct PublishedActivePolicy { pub(crate) path: PathBuf, pub(crate) digest: String, + pub(crate) bytes: Vec, runtime: Arc, } @@ -48,6 +49,7 @@ impl ServiceState { Ok(PublishedActivePolicy { path: active_policy_path, digest: capsem_core::net::policy_config::active_policy_digest(serialized.as_bytes()), + bytes: serialized.into_bytes(), runtime, }) } @@ -151,11 +153,23 @@ pub(crate) async fn push_policy_to_running_instances( } }; match publisher { - Some(publisher) => publisher - .publish(published.broker_policy()) - .await - .err() - .map(|error| format!("{id}: {error}")), + Some(publisher) => { + let proxy = match state.proxy_worker(id, *generation) { + Ok(proxy) => proxy, + Err(error) => return Some(format!("{id}: {error}")), + }; + match proxy.apply_policy(published.bytes.clone()).await { + Ok(applied) if applied == *expected => publisher + .publish(published.broker_policy()) + .await + .err() + .map(|error| format!("{id}: {error}")), + Ok(applied) => Some(format!( + "{id}: proxy applied active policy {applied}, expected {expected}" + )), + Err(error) => Some(format!("{id}: proxy reload refused: {error:#}")), + } + } None => None, } } diff --git a/crates/capsem-service/src/instance.rs b/crates/capsem-service/src/instance.rs index 829700844..691eb23a6 100644 --- a/crates/capsem-service/src/instance.rs +++ b/crates/capsem-service/src/instance.rs @@ -122,6 +122,7 @@ impl ServiceState { let removed = instances.remove(id)?; drop(instances); removed.authority.revoke(); + self.remove_proxy_worker(id, generation); Some(removed) } diff --git a/crates/capsem-service/src/instance_reaper.rs b/crates/capsem-service/src/instance_reaper.rs index 79c25d4bb..3e3adca1f 100644 --- a/crates/capsem-service/src/instance_reaper.rs +++ b/crates/capsem-service/src/instance_reaper.rs @@ -228,6 +228,7 @@ pub(super) async fn revoke_exited_generation( drop(instances); if let Some(instance) = removed.as_ref() { instance.authority.revoke(); + state.remove_proxy_worker(id, generation); } ExitedGeneration::Current(removed.map(Box::new)) } diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 0d1939d00..81b91ba53 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -49,6 +49,7 @@ mod mcp_routes; mod plugin_routes; mod private_routes; mod process_control; +mod proxy_worker; mod sandbox_info; mod session_cleanup; mod session_db_handles; @@ -162,6 +163,8 @@ struct Args { #[arg(long)] process_binary: Option, #[arg(long)] + proxy_binary: Option, + #[arg(long)] gateway_binary: Option, #[arg(long)] gateway_port: Option, @@ -205,6 +208,7 @@ pub struct ServiceState { managed: managed_routes::Runtime, connections: connection_routes::Runtime, instances: Mutex>, // instance id to process info + proxy_workers: Mutex>, retirements: instance_reaper::Retirements, /// Logger-owned DB handles keyed by session/VM id. Logged-data routes /// resolve a handle here and call `ready/query`; they do not open SQLite @@ -214,6 +218,7 @@ pub struct ServiceState { /// Named networks as groups of VMs, durable in each network's database. networks: tokio::sync::Mutex, process_binary: PathBuf, + proxy_binary: PathBuf, assets_dir: PathBuf, run_dir: PathBuf, service_socket: PathBuf, // this service's own, where an owner asks on a guest's behalf diff --git a/crates/capsem-service/src/owner_connection.rs b/crates/capsem-service/src/owner_connection.rs index 1c13b04c4..9884a8d1a 100644 --- a/crates/capsem-service/src/owner_connection.rs +++ b/crates/capsem-service/src/owner_connection.rs @@ -38,6 +38,10 @@ impl OwnerConnection { self.identity.uid } + pub(crate) fn generation(&self) -> uuid::Uuid { + self.generation + } + pub(crate) fn authenticate( &self, state: &ServiceState, diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs new file mode 100644 index 000000000..5815baf27 --- /dev/null +++ b/crates/capsem-service/src/proxy_worker.rs @@ -0,0 +1,684 @@ +//! Trusted lifecycle and capability coordinator for one confined proxy worker. + +use std::fmt::Write as _; +use std::io; +use std::os::fd::{AsRawFd as _, OwnedFd}; +use std::os::unix::net::UnixStream; +use std::path::Path; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::{anyhow, bail, Context, Result}; +use capsem_foundation::ipc_channel; +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::proxy_control::{ + decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyControlEvent, ProxyControlRequest, + ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, +}; +use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; +use tokio::process::{Child, Command}; +use tokio::sync::{mpsc, oneshot, watch}; + +use crate::ServiceState; + +#[cfg(not(test))] +const STARTUP_TIMEOUT: Duration = Duration::from_secs(10); +#[cfg(test)] +const STARTUP_TIMEOUT: Duration = Duration::from_secs(2); +#[cfg(not(test))] +const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +const OPERATION_TIMEOUT: Duration = Duration::from_secs(2); +#[cfg(not(test))] +const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +const SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(200); +const COMMAND_QUEUE_CAPACITY: usize = 16; + +type ControlSender = DescriptorSender; +type ControlReceiver = DescriptorReceiver; +type ControlFrame = capsem_foundation::unix::router_channel::DescriptorFrame; + +/// A bounded command handle for one fresh proxy-worker generation. +#[derive(Clone)] +pub(crate) struct ProxyWorker { + generation: ProxyGeneration, + commands: mpsc::Sender, + stopped: watch::Receiver>, +} + +impl ProxyWorker { + /// Spawn the production worker with an empty environment and no inherited + /// descriptors other than its coordinator socket on stdin. + pub(crate) async fn spawn(binary: &Path, active_policy: Vec, stdout: Stdio, stderr: Stdio) -> Result { + let generation = fresh_generation(); + let mut command = Command::new(binary); + command.env_clear().stdout(stdout).stderr(stderr).kill_on_drop(true); + launch(command, generation, active_policy, |command, generation| { + command + .arg("--parent-pid") + .arg(std::process::id().to_string()) + .arg("--generation") + .arg(generation_hex(generation)); + }) + .await + } + + pub(crate) fn generation(&self) -> ProxyGeneration { + self.generation + } + + #[cfg_attr(not(test), expect(dead_code, reason = "S10-003 attaches the traffic capabilities"))] + pub(crate) async fn grant(&self, capability: ProxyCapability, stream: UnixStream) -> Result<()> { + let (completed, result) = oneshot::channel(); + self.commands + .send(CommandRequest::Grant { + capability, + stream, + completed, + }) + .await + .map_err(|_| anyhow!(self.stop_reason("proxy worker supervisor stopped before accepting a grant")))?; + result + .await + .map_err(|_| anyhow!(self.stop_reason("proxy worker supervisor stopped with a grant pending")))? + } + + pub(crate) async fn apply_policy(&self, active_policy: Vec) -> Result { + let (completed, result) = oneshot::channel(); + self.commands + .send(CommandRequest::ApplyPolicy { + active_policy, + completed, + }) + .await + .map_err(|_| anyhow!(self.stop_reason("proxy worker supervisor stopped before accepting policy")))?; + result + .await + .map_err(|_| anyhow!(self.stop_reason("proxy worker supervisor stopped with policy pending")))? + } + + #[cfg_attr( + not(test), + expect(dead_code, reason = "session ownership normally shuts down by dropping the handle") + )] + pub(crate) async fn shutdown(self) -> Result<()> { + let (completed, result) = oneshot::channel(); + self.commands + .send(CommandRequest::Shutdown { completed }) + .await + .map_err(|_| anyhow!(self.stop_reason("proxy worker supervisor already stopped")))?; + result + .await + .map_err(|_| anyhow!(self.stop_reason("proxy worker supervisor stopped during shutdown")))? + } + + pub(crate) fn stop_receiver(&self) -> watch::Receiver> { + self.stopped.clone() + } + + fn stop_reason(&self, fallback: &str) -> String { + self.stopped.borrow().clone().unwrap_or_else(|| fallback.to_string()) + } + + #[cfg(test)] + pub(crate) fn test_stub() -> Self { + let generation = fresh_generation(); + let (commands, mut requests) = mpsc::channel(COMMAND_QUEUE_CAPACITY); + let (stopped_tx, stopped) = watch::channel(None); + tokio::spawn(async move { + while let Some(request) = requests.recv().await { + match request { + CommandRequest::Grant { completed, .. } => { + let _ = completed.send(Ok(())); + } + CommandRequest::Shutdown { completed } => { + let _ = completed.send(Ok(())); + break; + } + CommandRequest::ApplyPolicy { + active_policy, + completed, + } => { + let _ = completed.send(Ok(capsem_core::net::policy_config::active_policy_digest( + &active_policy, + ))); + } + } + } + let _ = stopped_tx.send(Some("test proxy worker stopped".to_string())); + }); + Self { + generation, + commands, + stopped, + } + } +} + +/// Start the proxy from a blocking lifecycle worker and retain its log beside +/// the session. Missing workers fail the session; tests use a scoped stub only +/// when their synthetic service state deliberately has no installed binary. +pub(crate) fn spawn_for_session(binary: &Path, active_policy: Vec, log_path: &Path) -> Result { + #[cfg(test)] + if !binary.exists() { + return Ok(ProxyWorker::test_stub()); + } + let log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("open {}", log_path.display()))?; + tokio::runtime::Handle::current().block_on(ProxyWorker::spawn( + binary, + active_policy, + Stdio::from(log.try_clone()?), + Stdio::from(log), + )) +} + +enum CommandRequest { + Grant { + capability: ProxyCapability, + stream: UnixStream, + completed: oneshot::Sender>, + }, + ApplyPolicy { + active_policy: Vec, + completed: oneshot::Sender>, + }, + Shutdown { + completed: oneshot::Sender>, + }, +} + +struct WorkerProcess { + generation: ProxyGeneration, + child: Child, + control_tx: ControlSender, + control_events: mpsc::Receiver>, + control_reader: tokio::task::JoinHandle<()>, + policy_tx: ipc_channel::Sender, + policy_rx: ipc_channel::Receiver, + next_grant_id: u64, + next_request_id: u64, +} + +enum PolicyApply { + Applied(String), + Rejected(String), +} + +async fn launch( + mut command: Command, + generation: ProxyGeneration, + active_policy: Vec, + configure: impl FnOnce(&mut Command, ProxyGeneration), +) -> Result { + let (coordinator, child_control) = UnixStream::pair().context("create proxy control channel")?; + configure(&mut command, generation); + command.stdin(Stdio::from(OwnedFd::from(child_control))); + let child = command.spawn().context("spawn capsem-proxy")?; + let control_tx = ControlSender::new(coordinator.try_clone()?).context("open proxy control sender")?; + let control_rx = ControlReceiver::new(coordinator).context("open proxy control receiver")?; + let process = WorkerProcess::start(child, control_tx, control_rx, generation, active_policy).await?; + let (commands, requests) = mpsc::channel(COMMAND_QUEUE_CAPACITY); + let (stopped_tx, stopped) = watch::channel(None); + tokio::spawn(supervise(process, requests, stopped_tx)); + Ok(ProxyWorker { + generation, + commands, + stopped, + }) +} + +impl WorkerProcess { + async fn start( + child: Child, + control_tx: ControlSender, + control_rx: ControlReceiver, + generation: ProxyGeneration, + active_policy: Vec, + ) -> Result { + let mut pending = PendingWorker { + generation, + child, + control_tx, + control_rx, + next_grant_id: 1, + }; + let startup = async { + match pending.recv_event().await? { + ProxyControlEvent::Ready { generation: ready } if ready == generation => {} + event => bail!("proxy worker sent {event:?} before readiness"), + } + let (policy_peer, policy_worker) = UnixStream::pair().context("create proxy policy channel")?; + pending.grant(ProxyCapability::Policy, policy_worker).await?; + Ok(ipc_channel::channel_from_std(policy_peer)?) + }; + let (policy_tx, policy_rx) = match tokio::time::timeout(STARTUP_TIMEOUT, startup).await { + Ok(Ok(channel)) => channel, + Ok(Err(error)) => { + pending.kill_reap().await; + return Err(error); + } + Err(_) => { + pending.kill_reap().await; + bail!("proxy worker readiness timed out"); + } + }; + let PendingWorker { + child, + control_tx, + control_rx, + next_grant_id, + .. + } = pending; + let (control_events_tx, control_events) = mpsc::channel(COMMAND_QUEUE_CAPACITY); + let control_reader = tokio::spawn(read_control(control_rx, control_events_tx)); + let mut process = Self { + generation, + child, + control_tx, + control_events, + control_reader, + policy_tx, + policy_rx, + next_grant_id, + next_request_id: 1, + }; + let expected = capsem_core::net::policy_config::active_policy_digest(&active_policy); + let applied = match process.apply_policy(active_policy).await { + Ok(PolicyApply::Applied(digest)) => digest, + Ok(PolicyApply::Rejected(error)) => { + process.kill_reap().await; + bail!("proxy worker rejected initial active policy: {error}"); + } + Err(error) => { + process.kill_reap().await; + return Err(error); + } + }; + if applied != expected { + process.kill_reap().await; + bail!("proxy worker applied active policy {applied}, expected {expected}"); + } + Ok(process) + } + + async fn grant(&mut self, capability: ProxyCapability, stream: UnixStream) -> Result<()> { + let grant_id = self.allocate_grant_id()?; + let grant = capsem_proto::proxy_control::ProxyChannelGrant::new(self.generation, grant_id, capability)?; + self.control_tx + .send( + &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), + &[stream.as_raw_fd()], + ) + .await + .context("send proxy descriptor grant")?; + drop(stream); + match self.recv_event().await? { + ProxyControlEvent::Adopted { + generation, + grant_id: adopted, + } if generation == self.generation && adopted == grant_id => Ok(()), + ProxyControlEvent::Rejected { + generation, + grant_id: rejected, + reason, + } if generation == self.generation && rejected == grant_id => { + bail!("proxy worker rejected {capability:?} grant {grant_id}: {reason:?}") + } + event => bail!("proxy worker sent {event:?} while adopting grant {grant_id}"), + } + } + + async fn apply_policy(&mut self, active_policy: Vec) -> Result { + let request_id = self.allocate_request_id()?; + self.policy_tx + .send(ProxyPolicyRequest::apply(request_id, active_policy)) + .await + .context("send proxy active policy")?; + let response = self.recv_policy().await?; + match response { + ProxyPolicyResponse::Applied { + request_id: applied, + active_policy_digest, + } if applied == request_id => Ok(PolicyApply::Applied(active_policy_digest)), + ProxyPolicyResponse::Rejected { + request_id: rejected, + error, + } if rejected == request_id => Ok(PolicyApply::Rejected(error)), + response => bail!("proxy worker sent mismatched policy response {response:?}"), + } + } + + async fn shutdown(&mut self) -> Result<()> { + self.control_tx + .send( + &encode_proxy_control_request(ProxyControlRequest::Shutdown { + generation: self.generation, + }), + &[], + ) + .await + .context("send proxy shutdown")?; + let stopped = tokio::time::timeout(SHUTDOWN_TIMEOUT, async { + match self.recv_event().await? { + ProxyControlEvent::Stopped { generation } if generation == self.generation => {} + event => bail!("proxy worker sent {event:?} during shutdown"), + } + let status = self.child.wait().await.context("reap proxy worker")?; + if !status.success() { + bail!("proxy worker exited unsuccessfully during shutdown: {status}"); + } + Ok(()) + }) + .await; + match stopped { + Ok(result) => result, + Err(_) => { + let _ = self.child.start_kill(); + let _ = self.child.wait().await; + bail!("proxy worker bounded shutdown timed out") + } + } + } + + async fn recv_event(&mut self) -> Result { + tokio::time::timeout(OPERATION_TIMEOUT, async { + tokio::select! { + status = self.child.wait() => { + let status = status.context("reap proxy worker")?; + bail!("proxy worker exited: {status}") + } + frame = self.control_events.recv() => { + let frame = frame.context("proxy control event pump stopped")??; + if !frame.fds.is_empty() { + bail!("proxy worker returned a descriptor on its control channel"); + } + decode_proxy_control_event(&frame.bytes).map_err(anyhow::Error::from) + } + } + }) + .await + .map_err(|_| anyhow!("proxy worker control operation timed out"))? + } + + async fn recv_policy(&mut self) -> Result { + tokio::time::timeout(OPERATION_TIMEOUT, async { + tokio::select! { + status = self.child.wait() => { + let status = status.context("reap proxy worker")?; + bail!("proxy worker exited with policy pending: {status}") + } + response = self.policy_rx.recv() => response.context("proxy worker stopped with policy pending"), + } + }) + .await + .map_err(|_| anyhow!("proxy worker policy operation timed out"))? + } + + fn allocate_grant_id(&mut self) -> Result { + let id = self.next_grant_id; + self.next_grant_id = id.checked_add(1).context("proxy grant id exhausted")?; + Ok(id) + } + + fn allocate_request_id(&mut self) -> Result { + let id = self.next_request_id; + self.next_request_id = id.checked_add(1).context("proxy policy request id exhausted")?; + Ok(id) + } + + async fn kill_reap(&mut self) { + let _ = self.child.start_kill(); + let _ = self.child.wait().await; + self.control_reader.abort(); + let _ = (&mut self.control_reader).await; + } +} + +struct PendingWorker { + generation: ProxyGeneration, + child: Child, + control_tx: ControlSender, + control_rx: ControlReceiver, + next_grant_id: u64, +} + +impl PendingWorker { + async fn recv_event(&mut self) -> Result { + tokio::select! { + status = self.child.wait() => { + let status = status.context("reap proxy worker during startup")?; + bail!("proxy worker exited before readiness: {status}") + } + frame = self.control_rx.recv() => { + let frame = frame.context("receive proxy startup event")?; + if !frame.fds.is_empty() { + bail!("proxy worker returned a descriptor during startup"); + } + decode_proxy_control_event(&frame.bytes).map_err(anyhow::Error::from) + } + } + } + + async fn grant(&mut self, capability: ProxyCapability, stream: UnixStream) -> Result<()> { + let grant_id = self.next_grant_id; + self.next_grant_id = grant_id.checked_add(1).context("proxy startup grant id exhausted")?; + let grant = capsem_proto::proxy_control::ProxyChannelGrant::new(self.generation, grant_id, capability)?; + self.control_tx + .send( + &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), + &[stream.as_raw_fd()], + ) + .await + .context("send proxy startup grant")?; + drop(stream); + match self.recv_event().await? { + ProxyControlEvent::Adopted { + generation, + grant_id: adopted, + } if generation == self.generation && adopted == grant_id => Ok(()), + event => bail!("proxy worker sent {event:?} while adopting startup grant {grant_id}"), + } + } + + async fn kill_reap(&mut self) { + let _ = self.child.start_kill(); + let _ = self.child.wait().await; + } +} + +async fn supervise( + mut process: WorkerProcess, + mut requests: mpsc::Receiver, + stopped: watch::Sender>, +) { + let reason = loop { + tokio::select! { + status = process.child.wait() => { + break match status { + Ok(status) => format!("proxy worker exited unexpectedly: {status}"), + Err(error) => format!("proxy worker exit wait failed: {error}"), + }; + } + event = process.control_events.recv() => { + break match event { + Some(Ok(frame)) if frame.fds.is_empty() => match decode_proxy_control_event(&frame.bytes) { + Ok(event) => format!("proxy worker sent unsolicited control event {event:?}"), + Err(error) => format!("proxy worker sent invalid control event: {error}"), + }, + Some(Ok(_)) => "proxy worker sent an unsolicited descriptor".to_string(), + Some(Err(error)) => format!("proxy worker control channel failed: {error}"), + None => "proxy worker control event pump stopped".to_string(), + }; + } + request = requests.recv() => { + let Some(request) = request else { + break match process.shutdown().await { + Ok(()) => "proxy worker stopped after its owner released it".to_string(), + Err(error) => format!("proxy worker release failed: {error:#}"), + }; + }; + match request { + CommandRequest::Grant { capability, stream, completed } => { + let result = process.grant(capability, stream).await; + let failed = result.is_err(); + let diagnostic = result.as_ref().err().map(|error| format!("{error:#}")); + let _ = completed.send(result); + if failed { + break diagnostic.expect("failed result has diagnostic"); + } + } + CommandRequest::ApplyPolicy { active_policy, completed } => { + match process.apply_policy(active_policy).await { + Ok(PolicyApply::Applied(digest)) => { + let _ = completed.send(Ok(digest)); + } + Ok(PolicyApply::Rejected(error)) => { + let _ = completed.send(Err(anyhow!("proxy worker rejected active policy: {error}"))); + } + Err(error) => { + let diagnostic = format!("{error:#}"); + let _ = completed.send(Err(error)); + break diagnostic; + } + } + } + CommandRequest::Shutdown { completed } => { + let result = process.shutdown().await; + let reason = result + .as_ref() + .err() + .map_or_else(|| "proxy worker stopped by coordinator".to_string(), |error| format!("{error:#}")); + let _ = completed.send(result); + break reason; + } + } + } + } + }; + process.kill_reap().await; + fail_pending(&mut requests, &reason); + let _ = stopped.send(Some(reason)); +} + +async fn read_control(receiver: ControlReceiver, events: mpsc::Sender>) { + loop { + let event = receiver.recv().await; + let complete = event.is_err(); + if events.send(event).await.is_err() || complete { + return; + } + } +} + +fn fail_pending(requests: &mut mpsc::Receiver, reason: &str) { + while let Ok(request) = requests.try_recv() { + let error = || Err(anyhow!(reason.to_string())); + match request { + CommandRequest::Grant { completed, .. } | CommandRequest::Shutdown { completed } => { + let _ = completed.send(error()); + } + CommandRequest::ApplyPolicy { completed, .. } => { + let _ = completed.send(Err(anyhow!(reason.to_string()))); + } + } + } +} + +fn fresh_generation() -> ProxyGeneration { + ProxyGeneration::new(*uuid::Uuid::new_v4().as_bytes()) +} + +fn generation_hex(generation: ProxyGeneration) -> String { + let mut encoded = String::with_capacity(32); + for byte in generation.as_bytes() { + write!(&mut encoded, "{byte:02x}").expect("write proxy generation to string"); + } + encoded +} + +impl ServiceState { + pub(crate) fn register_proxy_worker( + self: &std::sync::Arc, + session_id: &str, + session_generation: uuid::Uuid, + worker: ProxyWorker, + ) -> Result<()> { + let proxy_generation = generation_hex(worker.generation()); + let mut stopped = worker.stop_receiver(); + { + let mut workers = self.proxy_workers.lock().unwrap(); + if workers.contains_key(session_id) { + bail!("proxy worker slot for {session_id} was already occupied"); + } + workers.insert(session_id.to_string(), (session_generation, worker)); + } + let state = std::sync::Arc::clone(self); + let session_id = session_id.to_string(); + tokio::spawn(async move { + while stopped.borrow().is_none() && stopped.changed().await.is_ok() {} + let reason = stopped + .borrow() + .clone() + .unwrap_or_else(|| "proxy worker supervisor disappeared".to_string()); + let owner_pid = state + .instances + .lock() + .unwrap() + .get(&session_id) + .and_then(|instance| (instance.generation == session_generation).then_some(instance.pid)); + if let Some(pid) = owner_pid { + tracing::error!( + session_id, + %session_generation, + proxy_generation, + pid, + reason, + "proxy worker stopped while its VM owner remained active" + ); + if pid > 0 { + crate::process_control::send_or_log( + pid, + crate::process_control::Signal::Terminate, + "proxy-worker-death", + ); + } + } + }); + Ok(()) + } + + pub(crate) fn remove_proxy_worker(&self, session_id: &str, session_generation: uuid::Uuid) { + let mut workers = self.proxy_workers.lock().unwrap(); + if workers + .get(session_id) + .is_some_and(|(generation, _)| *generation == session_generation) + { + workers.remove(session_id); + } + } + + pub(crate) fn proxy_worker(&self, session_id: &str, session_generation: uuid::Uuid) -> Result { + if let Some((generation, worker)) = self.proxy_workers.lock().unwrap().get(session_id) { + if *generation == session_generation { + return Ok(worker.clone()); + } + return Err("proxy worker belongs to a replaced VM generation".to_string()); + } + #[cfg(test)] + { + Ok(ProxyWorker::test_stub()) + } + #[cfg(not(test))] + { + Err("running VM has no registered proxy worker".to_string()) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/proxy_worker/tests.rs b/crates/capsem-service/src/proxy_worker/tests.rs new file mode 100644 index 000000000..27aaa756a --- /dev/null +++ b/crates/capsem-service/src/proxy_worker/tests.rs @@ -0,0 +1,218 @@ +use std::os::fd::AsFd as _; +use std::sync::Arc; + +use capsem_proto::proxy_control::{decode_proxy_control_request, encode_proxy_control_event}; + +use super::*; + +const GENERATION: ProxyGeneration = ProxyGeneration::new([9; 16]); +const ACTIVE_POLICY: &[u8] = br#" +[network] +[user_rules.profiles.rules.worker_http] +name = "worker_http" +action = "allow" +match = 'http.host == "worker.example"' +[corp_rules] +"#; + +fn decode_generation(value: &str) -> ProxyGeneration { + let mut bytes = [0; 16]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap(); + } + ProxyGeneration::new(bytes) +} + +#[tokio::test] +async fn fake_proxy_worker_child() { + let Ok(mode) = std::env::var("CAPSEM_FAKE_PROXY_WORKER") else { + return; + }; + if mode == "refuse" { + std::process::exit(42); + } + let generation = decode_generation(&std::env::var("CAPSEM_FAKE_PROXY_GENERATION").unwrap()); + let control = UnixStream::from(capsem_foundation::unix::fd::duplicate(std::io::stdin().as_fd()).unwrap()); + let control_tx = Arc::new(ControlSender::new(control.try_clone().unwrap()).unwrap()); + let control_rx = ControlReceiver::new(control).unwrap(); + control_tx + .send( + &encode_proxy_control_event(ProxyControlEvent::Ready { generation }), + &[], + ) + .await + .unwrap(); + + let frame = control_rx.recv().await.unwrap(); + let ProxyControlRequest::Attach(grant) = decode_proxy_control_request(&frame.bytes).unwrap() else { + panic!("expected policy attach") + }; + assert_eq!(grant.generation(), generation); + assert_eq!(grant.capability(), ProxyCapability::Policy); + let policy_stream = UnixStream::from(frame.fds.into_iter().next().unwrap()); + control_tx + .send( + &encode_proxy_control_event(ProxyControlEvent::Adopted { + generation, + grant_id: grant.grant_id(), + }), + &[], + ) + .await + .unwrap(); + let (policy_tx, policy_rx) = + ipc_channel::channel_from_std::(policy_stream).unwrap(); + let (control_events_tx, mut control_events) = mpsc::channel(COMMAND_QUEUE_CAPACITY); + tokio::spawn(read_control(control_rx, control_events_tx)); + let mut policies = 0; + let mut held = Vec::new(); + loop { + tokio::select! { + request = policy_rx.recv() => { + let request = request.unwrap(); + policies += 1; + if mode == "die_with_policy_pending" && policies == 2 { + std::process::exit(43); + } + let ProxyPolicyRequest::Apply { request_id, active_policy, .. } = request; + let response = match capsem_core::net::proxy_engine::ProxyRuntimePolicy::compile(&active_policy) { + Ok(runtime) => ProxyPolicyResponse::Applied { + request_id, + active_policy_digest: runtime.snapshot().digest().to_string(), + }, + Err(error) => ProxyPolicyResponse::rejected(request_id, error), + }; + policy_tx.send(response).await.unwrap(); + } + frame = control_events.recv() => { + let frame = frame.unwrap().unwrap(); + match decode_proxy_control_request(&frame.bytes).unwrap() { + ProxyControlRequest::Attach(grant) => { + held.push(UnixStream::from(frame.fds.into_iter().next().unwrap())); + control_tx.send( + &encode_proxy_control_event(ProxyControlEvent::Adopted { + generation, + grant_id: grant.grant_id(), + }), + &[], + ).await.unwrap(); + } + ProxyControlRequest::Shutdown { .. } if mode == "stall_shutdown" => { + std::future::pending::<()>().await; + } + ProxyControlRequest::Shutdown { .. } => { + control_tx.send( + &encode_proxy_control_event(ProxyControlEvent::Stopped { generation }), + &[], + ).await.unwrap(); + return; + } + } + } + } + } +} + +async fn fake(mode: &'static str) -> Result { + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", "proxy_worker::tests::fake_proxy_worker_child", "--nocapture"]) + .env_clear() + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + launch( + command, + GENERATION, + ACTIVE_POLICY.to_vec(), + move |command, generation| { + command + .env("CAPSEM_FAKE_PROXY_WORKER", mode) + .env("CAPSEM_FAKE_PROXY_GENERATION", generation_hex(generation)); + }, + ) + .await +} + +#[tokio::test] +async fn supervisor_applies_policy_grants_connected_descriptors_and_reaps() { + let worker = fake("normal").await.unwrap(); + assert_eq!(worker.generation(), GENERATION); + let digest = worker.apply_policy(ACTIVE_POLICY.to_vec()).await.unwrap(); + assert_eq!( + digest, + capsem_core::net::policy_config::active_policy_digest(ACTIVE_POLICY) + ); + let (peer, granted) = UnixStream::pair().unwrap(); + worker.grant(ProxyCapability::Telemetry, granted).await.unwrap(); + drop(peer); + worker.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn startup_refusal_never_returns_a_worker_handle() { + let error = fake("refuse").await.err().expect("refused startup must fail"); + assert!(format!("{error:#}").contains("exited before readiness"), "{error:#}"); +} + +#[tokio::test] +async fn death_fails_current_and_queued_policy_requests_explicitly() { + let worker = fake("die_with_policy_pending").await.unwrap(); + let first = worker.clone(); + let second = worker.clone(); + let (first, second) = tokio::join!( + first.apply_policy(ACTIVE_POLICY.to_vec()), + second.apply_policy(ACTIVE_POLICY.to_vec()) + ); + for result in [first, second] { + let error = result.expect_err("worker death must fail policy request"); + let diagnostic = format!("{error:#}"); + assert!( + diagnostic.contains("proxy worker") && (diagnostic.contains("exited") || diagnostic.contains("stopped")), + "{diagnostic}" + ); + } +} + +#[tokio::test] +async fn stalled_shutdown_is_killed_and_reaped_within_the_bound() { + let worker = fake("stall_shutdown").await.unwrap(); + let started = std::time::Instant::now(); + let error = worker.shutdown().await.unwrap_err(); + assert!(format!("{error:#}").contains("bounded shutdown timed out"), "{error:#}"); + assert!(started.elapsed() < Duration::from_secs(2)); +} + +#[tokio::test] +async fn unexpected_proxy_death_terminates_only_its_matching_vm_generation() { + let state = crate::tests::make_test_state(); + let mut owner_a = Command::new("sleep").arg("30").kill_on_drop(true).spawn().unwrap(); + let mut owner_b = Command::new("sleep").arg("30").kill_on_drop(true).spawn().unwrap(); + let session_a = state.run_dir.join("sessions/a"); + let session_b = state.run_dir.join("sessions/b"); + std::fs::create_dir_all(&session_a).unwrap(); + std::fs::create_dir_all(&session_b).unwrap(); + crate::tests::insert_fake_instance_with_session_dir(&state, "a", owner_a.id().unwrap(), session_a); + crate::tests::insert_fake_instance_with_session_dir(&state, "b", owner_b.id().unwrap(), session_b); + let generation = state.instances.lock().unwrap()["a"].generation; + let worker = fake("die_with_policy_pending").await.unwrap(); + let caller = worker.clone(); + state.register_proxy_worker("a", generation, worker).unwrap(); + + assert!(caller.apply_policy(ACTIVE_POLICY.to_vec()).await.is_err()); + let status = tokio::time::timeout(Duration::from_secs(2), owner_a.wait()) + .await + .expect("matching VM owner was not terminated") + .unwrap(); + assert!(!status.success()); + assert!( + owner_b.try_wait().unwrap().is_none(), + "unrelated VM owner was terminated" + ); + + state.evict_instance("a", generation); + let generation_b = state.instances.lock().unwrap()["b"].generation; + state.evict_instance("b", generation_b); + owner_b.start_kill().unwrap(); + owner_b.wait().await.unwrap(); +} diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index 5b479cb82..338c45a56 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -151,6 +151,7 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { let process_binary = args .process_binary .unwrap_or_else(|| PathBuf::from("cache/target/cargo/debug/capsem-process")); + let proxy_binary = args.proxy_binary.unwrap_or_else(|| find_sibling_binary("capsem-proxy")); let assets_base_dir = args .assets_dir .unwrap_or_else(|| run_dir.parent().unwrap().join("assets")); @@ -258,11 +259,13 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { managed, connections, instances: Mutex::new(HashMap::new()), + proxy_workers: Mutex::new(HashMap::new()), retirements: Default::default(), session_db_handles: Mutex::new(HashMap::new()), persistent_registry: SharedRegistry::new(persistent_registry), networks: tokio::sync::Mutex::new(networks), process_binary: process_binary.clone(), + proxy_binary, assets_dir: assets_base_dir, run_dir: run_dir.clone(), service_socket: service_sock.clone(), diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 04920236e..e8aeed825 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -55,6 +55,7 @@ pub(crate) fn test_state( managed, connections, instances: Mutex::new(HashMap::new()), + proxy_workers: Mutex::new(HashMap::new()), retirements: Default::default(), session_db_handles: Mutex::new(HashMap::new()), persistent_registry: SharedRegistry::new( @@ -64,6 +65,7 @@ pub(crate) fn test_state( run_dir.join("networks"), )), process_binary: PathBuf::from("/nonexistent/capsem-process"), + proxy_binary: PathBuf::from("/nonexistent/capsem-proxy"), assets_dir, service_socket: run_dir.join("service.sock"), switches: switches::Switches::in_process(), diff --git a/crates/capsem-service/src/vm_files/provision.rs b/crates/capsem-service/src/vm_files/provision.rs index 627bc3502..8f554537a 100644 --- a/crates/capsem-service/src/vm_files/provision.rs +++ b/crates/capsem-service/src/vm_files/provision.rs @@ -86,7 +86,17 @@ async fn provision(state: Arc, payload: ProvisionRequest) -> Resul let _ = state.persistent_registry.lock().unwrap().unregister(&stale_name); }) .await; - state.instances.lock().unwrap().remove(&id); + let generation = { + state + .instances + .lock() + .unwrap() + .get(&id) + .map(|instance| instance.generation) + }; + if let Some(generation) = generation { + state.evict_instance(&id, generation); + } warn!(id, attempt, "retrying provision after launchd-cleanup transient"); } diff --git a/crates/capsem-service/src/vm_lifecycle.rs b/crates/capsem-service/src/vm_lifecycle.rs index 66bccab28..85294c42f 100644 --- a/crates/capsem-service/src/vm_lifecycle.rs +++ b/crates/capsem-service/src/vm_lifecycle.rs @@ -491,7 +491,7 @@ pub(super) async fn handle_suspend( signal_vm_process(&state, &id, generation, process_control::Signal::Kill).await?; wait_for_process_exit(&state, &id, generation, std::time::Duration::from_millis(500)).await?; tracing::warn!(id, outcome, "handle_suspend removing failed instance"); - state.instances.lock().unwrap().remove(&id); + state.evict_instance(&id, owner.generation()); let _ = std::fs::remove_file(&uds_path); let _ = std::fs::remove_file(uds_path.with_extension("ready")); return Err(AppError(StatusCode::INTERNAL_SERVER_ERROR, error)); @@ -507,7 +507,7 @@ pub(super) async fn handle_suspend( .map_err(|e| AppError(StatusCode::BAD_GATEWAY, e))?; tracing::warn!(id, "handle_suspend (success) removing instance"); - state.instances.lock().unwrap().remove(&id); + state.evict_instance(&id, owner.generation()); state.unregister_session_db_handle(&id); let _ = std::fs::remove_file(&uds_path); let _ = std::fs::remove_file(uds_path.with_extension("ready")); diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index f2e323a7d..8f7245eb3 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -286,6 +286,18 @@ impl ServiceState { let pid = child.id().unwrap_or(0); info!(id, pid, version, asset_version = %resolved.asset_version, "capsem-process spawned"); + let proxy = match crate::proxy_worker::spawn_for_session( + &self.proxy_binary, + active_policy.bytes, + &session_dir.join("proxy.log"), + ) { + Ok(proxy) => proxy, + Err(error) => { + instance_reaper::kill_and_reap(child); + return Err(error.context("start confined proxy worker")); + } + }; + // Provisioning runs on a blocking thread that keeps the runtime // handle, so the event lands before anything can stop the session. let created = capsem_proto::host_session::HostSessionDetail { @@ -359,6 +371,11 @@ impl ServiceState { }, ); drop(instances); + if let Err(error) = self.register_proxy_worker(id, generation, proxy) { + self.evict_instance(id, generation); + instance_reaper::kill_and_reap(child); + return Err(error.context("register proxy worker")); + } let _upstream_broker = upstream_broker.start(upstream_grant); let _reaper = instance_reaper::spawn_exit_reaper( child, diff --git a/crates/capsem-service/src/vm_lifecycle/resume.rs b/crates/capsem-service/src/vm_lifecycle/resume.rs index 401ff70ca..03ca32932 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume.rs @@ -104,18 +104,18 @@ pub(crate) async fn handle_resume( /// Tear down a warm-restore process that failed to reach ready while the /// caller already holds the save/restore locks. -pub(crate) async fn stop_failed_restore_process_under_lock(state: &ServiceState, id: &str) -> Result<(), AppError> { - let Some((uds_path, generation)) = ({ +pub(crate) async fn stop_failed_restore_process_under_lock(state: &ServiceState, id: &str) { + let Some((uds_path, pid, generation)) = ({ let instances = state.instances.lock().unwrap(); - instances.get(id).map(|i| (i.uds_path.clone(), i.generation)) + instances.get(id).map(|i| (i.uds_path.clone(), i.pid, i.generation)) }) else { return Ok(()); }; signal_vm_process(state, id, generation, process_control::Signal::Terminate).await?; - tracing::warn!(id, "removing failed warm restore before cold fallback"); - claim_shutdown_instance(state, id, generation); + tracing::warn!(id, pid, "removing failed warm restore before cold fallback"); + state.evict_instance(id, generation); state.unregister_session_db_handle(id); wait_for_process_exit(state, id, generation, std::time::Duration::from_secs(1)).await?; state diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index b6d603252..43e964c51 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -215,6 +215,18 @@ impl ServiceState { let pid = child.id().unwrap_or(0); info!(name, pid, "capsem-process resumed"); + let proxy = match crate::proxy_worker::spawn_for_session( + &self.proxy_binary, + active_policy.bytes, + &entry.session_dir.join("proxy.log"), + ) { + Ok(proxy) => proxy, + Err(error) => { + instance_reaper::kill_and_reap(child); + return Err(error.context("start confined proxy worker")); + } + }; + if session_db_path_for_session_dir(&entry.session_dir).exists() { if let Err(error) = self.register_session_db_handle(&vm_id, &entry.session_dir) { let _reaper = instance_reaper::kill_and_reap(child, retirement); @@ -255,6 +267,11 @@ impl ServiceState { }, ); drop(instances); + if let Err(error) = self.register_proxy_worker(&vm_id, generation, proxy) { + self.evict_instance(&vm_id, generation); + instance_reaper::kill_and_reap(child); + return Err(error.context("register proxy worker")); + } let _upstream_broker = upstream_broker.start(upstream_grant); let _reaper = instance_reaper::spawn_exit_reaper( child, From 1f0d5e440a27674f8020829f08d5ba7a074bfd53 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:35:51 +0000 Subject: [PATCH 110/464] security(packaging): ship confined workers --- .github/workflows/ci.yaml | 8 +- .github/workflows/release.yaml | 10 +- CHANGELOG.md | 5 + .../tools/check_public_binary_release.py | 2 + .../builder/release/tools/release_cohort.py | 2 + .../release/tools/release_installed_probe.py | 2 + .../packaging/linux/build-linux-package.sh | 4 +- build_system/packaging/linux/deb-postinst.sh | 2 +- build_system/packaging/linux/repack-deb.sh | 4 +- build_system/packaging/macos/build-pkg.sh | 2 +- .../macos/build-test-macos-package.sh | 4 +- .../packaging/macos/macos_tart_guest.sh | 2 + .../packaging/macos/pkg-scripts/postinstall | 19 +- .../macos/prove-macos-package-boot.sh | 2 +- build_system/packaging/shared/retire-cohort | 2 +- .../build/prove-live-public-install.sh | 2 +- build_system/scripts/test/simulate-install.sh | 21 +- build_system/scripts/test/test-linux-rust.sh | 2 + .../tests/gate/test_ci_coverage_contract.py | 6 + .../tests/packaging/test_build_pkg.py | 2 + .../tests/packaging/test_macos_tart_glowup.py | 2 + .../tests/packaging/test_repack_deb.py | 2 + .../tests/packaging/test_router_signing.py | 14 +- .../release/test_release_doctor_contract.py | 10 +- .../tests/scripts/test_rust_affected.py | 2 + codecov.yml | 13 ++ config/dependency-inventory.json | 219 ++++++++++++++---- config/gate.toml | 10 +- 28 files changed, 293 insertions(+), 82 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e0ff45c14..906027e6e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -281,8 +281,8 @@ jobs: - name: Unit tests with coverage run: | set -o pipefail - cargo llvm-cov nextest --no-cfg-coverage --lib --bins --profile ci-unit --codecov --output-path cache/target/coverage/rust/unit.json -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-crypto -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server - cargo llvm-cov report --summary-only -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-crypto -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server 2>&1 | tee cache/target/coverage/rust/summary.txt + cargo llvm-cov nextest --no-cfg-coverage --lib --bins --profile ci-unit --codecov --output-path cache/target/coverage/rust/unit.json -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-ledger -p capsem-proxy -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server + cargo llvm-cov report --summary-only -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-ledger -p capsem-proxy -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server 2>&1 | tee cache/target/coverage/rust/summary.txt # Integration tests (tests/ directory, cross-crate) - name: Integration tests with coverage @@ -358,10 +358,12 @@ jobs: - name: Python integration tests (non-VM suites) run: | bash build_system/scripts/test/prepare-install-test-assets.sh - cargo build -p capsem-process -p capsem-service -p capsem -p capsem-mock-server -p capsem-bench + cargo build -p capsem-process -p capsem-ledger -p capsem-proxy -p capsem-service -p capsem -p capsem-mock-server -p capsem-bench for bin in cache/target/cargo/debug/capsem-process cache/target/cargo/debug/capsem-service cache/target/cargo/debug/capsem cache/target/cargo/debug/capsem-mock-server cache/target/cargo/debug/capsem-bench-rs; do codesign --sign - --entitlements build_system/packaging/macos/entitlements.plist --force "$bin" done + codesign --sign - --force cache/target/cargo/debug/capsem-ledger + codesign --sign - --force cache/target/cargo/debug/capsem-proxy uv run --project build_system --frozen python -m pytest -c build_system/pyproject.toml --rootdir . tests/capsem-bootstrap/ \ tests/capsem-codesign/ \ tests/capsem-rootfs-artifacts/ \ diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ef7abeb66..a08df38b1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -381,6 +381,8 @@ jobs: -p capsem \ -p capsem-service \ -p capsem-process \ + -p capsem-ledger \ + -p capsem-proxy \ -p capsem-tui \ -p capsem-router \ -p capsem-mcp-aggregator \ @@ -396,9 +398,9 @@ jobs: env: APPLE_SIGNING_IDENTITY: ${{ secrets.APPLE_SIGNING_IDENTITY }} run: | - for bin in capsem capsem-service capsem-process capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do + for bin in capsem capsem-service capsem-process capsem-ledger capsem-proxy capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do entitlement_args=() - if [[ "$bin" != capsem-router ]]; then + if [[ "$bin" != capsem-router && "$bin" != capsem-ledger && "$bin" != capsem-proxy ]]; then entitlement_args=(--entitlements build_system/packaging/macos/entitlements.plist) fi codesign --sign "$APPLE_SIGNING_IDENTITY" \ @@ -734,7 +736,7 @@ jobs: pkgutil --pkg-info com.capsem.pkg | grep -F "version: $VERSION" test -d "/Applications/Capsem.app" test -x "/Applications/Capsem.app/Contents/MacOS/capsem-app" - for bin in capsem capsem-admin capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs; do + for bin in capsem capsem-admin capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-ledger capsem-proxy capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs; do test -x "$HOME/.capsem/bin/$bin" "$HOME/.capsem/bin/$bin" --version | grep -F "$VERSION" done @@ -819,7 +821,7 @@ jobs: sudo dpkg -i "$package" test ! -e /var/run/capsem/install-manifest dpkg-query -W -f='${Version}' capsem | grep -Fx "$VERSION" - for bin in capsem capsem-admin capsem-app capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs; do + for bin in capsem capsem-admin capsem-app capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-ledger capsem-proxy capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs; do test -x "/usr/bin/$bin" if [ "$bin" != "capsem-app" ]; then "/usr/bin/$bin" --version | grep -F "$VERSION" diff --git a/CHANGELOG.md b/CHANGELOG.md index 815928900..49942f986 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Ship the confined ledger and proxy workers in every supported package and + sign them without macOS virtualization authority, so clean installs cannot + fall back to privileged in-process execution or grant VM capabilities to + data-plane helpers. + - Start one fresh-generation confined proxy worker with every live VM, apply exact active-policy bytes before registration, revoke it with the VM owner, and terminate only the matching owner generation if its proxy dies. diff --git a/build_system/builder/release/tools/check_public_binary_release.py b/build_system/builder/release/tools/check_public_binary_release.py index 1335f1b22..72fa30ef3 100644 --- a/build_system/builder/release/tools/check_public_binary_release.py +++ b/build_system/builder/release/tools/check_public_binary_release.py @@ -590,6 +590,8 @@ def run_docker_install_smoke( "capsem-mcp-aggregator", "capsem-mcp-builtin", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-service", "capsem-tray", "capsem-tui", diff --git a/build_system/builder/release/tools/release_cohort.py b/build_system/builder/release/tools/release_cohort.py index 74c03a722..fcbadc001 100644 --- a/build_system/builder/release/tools/release_cohort.py +++ b/build_system/builder/release/tools/release_cohort.py @@ -34,6 +34,8 @@ "capsem-mcp-builtin", "capsem-mock-server", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-service", "capsem-tray", "capsem-tui", diff --git a/build_system/builder/release/tools/release_installed_probe.py b/build_system/builder/release/tools/release_installed_probe.py index 8660d5a4e..f70f93582 100644 --- a/build_system/builder/release/tools/release_installed_probe.py +++ b/build_system/builder/release/tools/release_installed_probe.py @@ -14,6 +14,8 @@ "capsem", "capsem-service", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-tui", "capsem-router", "capsem-mcp-aggregator", diff --git a/build_system/packaging/linux/build-linux-package.sh b/build_system/packaging/linux/build-linux-package.sh index c2705253a..b9df2c5f4 100644 --- a/build_system/packaging/linux/build-linux-package.sh +++ b/build_system/packaging/linux/build-linux-package.sh @@ -70,7 +70,7 @@ cp "$RELEASE_DIR/capsem-pty-agent" \ echo "--- Build companion host binaries ---" cargo build --release --locked --offline --target "$RUST_TARGET" \ - -p capsem -p capsem-service -p capsem-process -p capsem-tui \ + -p capsem -p capsem-service -p capsem-process -p capsem-ledger -p capsem-proxy -p capsem-tui \ -p capsem-router -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-gateway \ -p capsem-tray -p capsem-admin -p capsem-mock-server -p capsem-bench bash build_system/scripts/build/check-build-provenance.sh "$RELEASE_DIR/capsem" \ @@ -107,7 +107,7 @@ bash "$SCRIPT_DIR/repack-deb.sh" --manifest "$CAPSEM_INSTALL_MANIFEST_URL" "$DEB echo "--- Validate artifacts ---" dpkg-deb --info "$DEB" -dpkg-deb --contents "$DEB" | grep -E 'usr/bin/(capsem|capsem-service|capsem-process|capsem-tui|capsem-router|capsem-mcp-aggregator|capsem-mcp-builtin|capsem-gateway|capsem-tray|capsem-admin|capsem-mock-server|capsem-bench-rs)$' +dpkg-deb --contents "$DEB" | grep -E 'usr/bin/(capsem|capsem-service|capsem-process|capsem-ledger|capsem-proxy|capsem-tui|capsem-router|capsem-mcp-aggregator|capsem-mcp-builtin|capsem-gateway|capsem-tray|capsem-admin|capsem-mock-server|capsem-bench-rs)$' cp "$DEB" "$OUT/" # Record the exact package this run produced, so a stale cache/target/packages entry diff --git a/build_system/packaging/linux/deb-postinst.sh b/build_system/packaging/linux/deb-postinst.sh index 21150f37a..8c13c2fc9 100755 --- a/build_system/packaging/linux/deb-postinst.sh +++ b/build_system/packaging/linux/deb-postinst.sh @@ -86,7 +86,7 @@ fi # Symlink system binaries into user dir CAPSEM_INSTALL_PHASE="link_binaries" -for bin in capsem capsem-service capsem-process capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do +for bin in capsem capsem-service capsem-process capsem-ledger capsem-proxy capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do if [ ! -f "/usr/bin/$bin" ]; then echo "capsem: packaged binary missing: /usr/bin/$bin" >&2 echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') phase=deb-postinst event=binary_missing bin=$bin src=/usr/bin/$bin" diff --git a/build_system/packaging/linux/repack-deb.sh b/build_system/packaging/linux/repack-deb.sh index c303a699d..f55f4538c 100755 --- a/build_system/packaging/linux/repack-deb.sh +++ b/build_system/packaging/linux/repack-deb.sh @@ -14,6 +14,8 @@ # /usr/bin/capsem # /usr/bin/capsem-service # /usr/bin/capsem-process +# /usr/bin/capsem-ledger +# /usr/bin/capsem-proxy # /usr/bin/capsem-tui # /usr/bin/capsem-gateway # /usr/bin/capsem-tray @@ -227,7 +229,7 @@ done echo "=== Adding companion binaries ===" mkdir -p "$WORK_DIR/deb/usr/bin" -for bin in capsem capsem-service capsem-process capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do +for bin in capsem capsem-service capsem-process capsem-ledger capsem-proxy capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do src="$BIN_DIR/$bin" if [ -f "$src" ]; then cp "$src" "$WORK_DIR/deb/usr/bin/$bin" diff --git a/build_system/packaging/macos/build-pkg.sh b/build_system/packaging/macos/build-pkg.sh index 8c1e5a202..028950fa0 100755 --- a/build_system/packaging/macos/build-pkg.sh +++ b/build_system/packaging/macos/build-pkg.sh @@ -183,7 +183,7 @@ cp -R "$APP_PATH" "$WORK_DIR/payload/Applications/Capsem.app" # Companion binaries SHARE_DIR="$WORK_DIR/payload/usr/local/share/capsem" mkdir -p "$SHARE_DIR/bin" -for bin in capsem capsem-service capsem-process capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do +for bin in capsem capsem-service capsem-process capsem-ledger capsem-proxy capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do src="$BIN_DIR/$bin" if [ -f "$src" ]; then reject_retired_credential_store_markers "$src" diff --git a/build_system/packaging/macos/build-test-macos-package.sh b/build_system/packaging/macos/build-test-macos-package.sh index 21da74fef..bead2a44c 100755 --- a/build_system/packaging/macos/build-test-macos-package.sh +++ b/build_system/packaging/macos/build-test-macos-package.sh @@ -63,6 +63,8 @@ cargo build --release \ -p capsem \ -p capsem-service \ -p capsem-process \ + -p capsem-ledger \ + -p capsem-proxy \ -p capsem-tui \ \ -p capsem-router \ @@ -93,7 +95,7 @@ import sys expected = { "capsem", "capsem-admin", "capsem-app", "capsem-gateway", "capsem-router", "capsem-mcp-aggregator", "capsem-mcp-builtin", - "capsem-process", "capsem-service", "capsem-tray", "capsem-tui", + "capsem-process", "capsem-ledger", "capsem-proxy", "capsem-service", "capsem-tray", "capsem-tui", "capsem-mock-server", "capsem-bench-rs", } document = json.loads(pathlib.Path(sys.argv[1]).read_text()) diff --git a/build_system/packaging/macos/macos_tart_guest.sh b/build_system/packaging/macos/macos_tart_guest.sh index c26216044..8074aa77c 100755 --- a/build_system/packaging/macos/macos_tart_guest.sh +++ b/build_system/packaging/macos/macos_tart_guest.sh @@ -43,6 +43,8 @@ BINARIES=( capsem capsem-service capsem-process + capsem-ledger + capsem-proxy capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin diff --git a/build_system/packaging/macos/pkg-scripts/postinstall b/build_system/packaging/macos/pkg-scripts/postinstall index d05145efd..f09a64070 100755 --- a/build_system/packaging/macos/pkg-scripts/postinstall +++ b/build_system/packaging/macos/pkg-scripts/postinstall @@ -70,7 +70,7 @@ echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') phase=postinstall event=layout_ready caps # Copy companion binaries from pkg payload CAPSEM_INSTALL_PHASE="copy_binaries" -for bin in capsem capsem-service capsem-process capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do +for bin in capsem capsem-service capsem-process capsem-ledger capsem-proxy capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do src="$PKG_SHARE/bin/$bin" if [ ! -f "$src" ]; then echo "capsem: packaged binary missing: $src" >&2 @@ -87,6 +87,8 @@ codesign_identifier_for_bin() { capsem) echo "org.capsem.cli" ;; capsem-service) echo "org.capsem.service" ;; capsem-process) echo "org.capsem.process" ;; + capsem-ledger) echo "org.capsem.ledger" ;; + capsem-proxy) echo "org.capsem.proxy" ;; capsem-tui) echo "org.capsem.tui" ;; capsem-router) echo "org.capsem.router" ;; capsem-mcp-aggregator) echo "org.capsem.mcp.aggregator" ;; @@ -101,7 +103,7 @@ codesign_identifier_for_bin() { } # Preserve production Developer ID signatures. Development packages may carry -# unsigned binaries; the confined port router receives no VM entitlement. +# unsigned binaries; confined workers receive no VM entitlement. CAPSEM_INSTALL_PHASE="codesign_binaries" if [ -f "$PKG_SHARE/entitlements.plist" ]; then for bin in "$CAPSEM_DIR/bin"/capsem*; do @@ -110,11 +112,14 @@ if [ -f "$PKG_SHARE/entitlements.plist" ]; then if codesign --verify --strict "$bin" 2>/dev/null; then echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') phase=postinstall event=binary_signature_preserved bin=$(basename "$bin")" else - if [ "$(basename "$bin")" = capsem-router ]; then - codesign --sign - --identifier "$identifier" --force "$bin" - else - codesign --sign - --identifier "$identifier" --entitlements "$PKG_SHARE/entitlements.plist" --force "$bin" - fi + case "$(basename "$bin")" in + capsem-router|capsem-ledger|capsem-proxy) + codesign --sign - --identifier "$identifier" --force "$bin" + ;; + *) + codesign --sign - --identifier "$identifier" --entitlements "$PKG_SHARE/entitlements.plist" --force "$bin" + ;; + esac echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') phase=postinstall event=binary_signed_ad_hoc bin=$(basename "$bin")" fi fi diff --git a/build_system/packaging/macos/prove-macos-package-boot.sh b/build_system/packaging/macos/prove-macos-package-boot.sh index 7b93276b2..5acdf8ec3 100755 --- a/build_system/packaging/macos/prove-macos-package-boot.sh +++ b/build_system/packaging/macos/prove-macos-package-boot.sh @@ -56,7 +56,7 @@ PERSISTENT_RESUME_EVIDENCE="$WORK_ROOT/persistent-resume.json" stop_isolated_processes() { local name for name in \ - capsem-service capsem-tray capsem-gateway capsem-process \ + capsem-service capsem-tray capsem-gateway capsem-process capsem-ledger capsem-proxy \ capsem-router capsem-mcp-aggregator capsem-mcp-builtin do pkill -9 -f "$CAPSEM_HOME_DIR/bin/$name" 2>/dev/null || true diff --git a/build_system/packaging/shared/retire-cohort b/build_system/packaging/shared/retire-cohort index f39846b03..97e64c3e2 100755 --- a/build_system/packaging/shared/retire-cohort +++ b/build_system/packaging/shared/retire-cohort @@ -54,7 +54,7 @@ capsem_retire_native_cohort() { local killed="" local name pid # Retire the historical router executable when upgrading an older cohort. - for name in capsem-service capsem-gateway capsem-tray capsem-process capsem-router capsem-port-router capsem-mcp-aggregator capsem-mcp-builtin; do + for name in capsem-service capsem-gateway capsem-tray capsem-process capsem-ledger capsem-proxy capsem-router capsem-port-router capsem-mcp-aggregator capsem-mcp-builtin; do for pid in $(pgrep -u "$target_uid" -f "$name" 2>/dev/null || true); do if capsem_process_is_package_owned "$capsem_dir" "$target_uid" "$name" "$pid"; then "$kill_command" -9 "$pid" diff --git a/build_system/scripts/build/prove-live-public-install.sh b/build_system/scripts/build/prove-live-public-install.sh index f76d06f70..0b0f89566 100755 --- a/build_system/scripts/build/prove-live-public-install.sh +++ b/build_system/scripts/build/prove-live-public-install.sh @@ -26,7 +26,7 @@ test -x "$capsem" test -x /usr/bin/capsem-app "$capsem" --version | grep -F "$expected_version" dpkg-query -W -f='${Version}' capsem | grep -Fx "$expected_version" -for bin in capsem capsem-admin capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs; do +for bin in capsem capsem-admin capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-ledger capsem-proxy capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs; do test -x "$capsem_home/bin/$bin" "$capsem_home/bin/$bin" --version | grep -F "$expected_version" done diff --git a/build_system/scripts/test/simulate-install.sh b/build_system/scripts/test/simulate-install.sh index c3614e58f..5f640db5e 100755 --- a/build_system/scripts/test/simulate-install.sh +++ b/build_system/scripts/test/simulate-install.sh @@ -30,7 +30,7 @@ RUN_DIR="${CAPSEM_RUN_DIR:-$CAPSEM_HOME_DIR/run}" # ``cache/target/cargo/debug/capsem-*`` are not caught in the blast. A bare # ``pkill -x capsem-service`` matches every capsem-service on the box, which # poisoned the full test suite whenever any install fixture fired this script. -for name in capsem-service capsem-tray capsem-gateway capsem-process capsem-router capsem-mcp-aggregator capsem-mcp-builtin; do +for name in capsem-service capsem-tray capsem-gateway capsem-process capsem-ledger capsem-proxy capsem-router capsem-mcp-aggregator capsem-mcp-builtin; do pkill -9 -f "$INSTALL_DIR/$name" 2>/dev/null || true done @@ -48,7 +48,7 @@ fi mkdir -p "$ASSETS_DST" # Copy binaries -for bin in capsem capsem-service capsem-process capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do +for bin in capsem capsem-service capsem-process capsem-ledger capsem-proxy capsem-tui capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-gateway capsem-tray capsem-admin capsem-mock-server capsem-bench-rs; do src="$BIN_SRC/$bin" if [[ ! -f "$src" ]]; then echo "ERROR: binary not found: $src" >&2 @@ -63,6 +63,8 @@ codesign_identifier_for_bin() { capsem) echo "org.capsem.cli" ;; capsem-service) echo "org.capsem.service" ;; capsem-process) echo "org.capsem.process" ;; + capsem-ledger) echo "org.capsem.ledger" ;; + capsem-proxy) echo "org.capsem.proxy" ;; capsem-tui) echo "org.capsem.tui" ;; capsem-mcp) echo "org.capsem.mcp" ;; capsem-router) echo "org.capsem.router" ;; @@ -89,12 +91,15 @@ if [[ "$(uname -s)" == "Darwin" ]]; then exit 1 fi identifier="$(codesign_identifier_for_bin "$bin")" - # As the package installs it: the confined router gets no VM entitlement. - if [[ "$(basename "$bin")" == capsem-router ]]; then - codesign --sign - --identifier "$identifier" --force "$bin" - else - codesign --sign - --identifier "$identifier" --entitlements "$ENTITLEMENTS" --force "$bin" - fi + # As the package installs them: confined workers get no VM entitlement. + case "$(basename "$bin")" in + capsem-router|capsem-ledger|capsem-proxy) + codesign --sign - --identifier "$identifier" --force "$bin" + ;; + *) + codesign --sign - --identifier "$identifier" --entitlements "$ENTITLEMENTS" --force "$bin" + ;; + esac fi done fi diff --git a/build_system/scripts/test/test-linux-rust.sh b/build_system/scripts/test/test-linux-rust.sh index 22213fece..925705577 100755 --- a/build_system/scripts/test/test-linux-rust.sh +++ b/build_system/scripts/test/test-linux-rust.sh @@ -26,6 +26,8 @@ packages=( capsem-mcp-aggregator capsem-mcp-builtin capsem-process + capsem-ledger + capsem-proxy capsem-router capsem-bench capsem-mock-server diff --git a/build_system/tests/gate/test_ci_coverage_contract.py b/build_system/tests/gate/test_ci_coverage_contract.py index 022dbd5c9..8c2ca4c76 100644 --- a/build_system/tests/gate/test_ci_coverage_contract.py +++ b/build_system/tests/gate/test_ci_coverage_contract.py @@ -133,6 +133,7 @@ def test_workspace_crates_and_bins_enumerated() -> None: "capsem-bench-rs", "capsem-dns-proxy", "capsem-gateway", + "capsem-ledger", "capsem-mcp-aggregator", "capsem-mcp-builtin", @@ -140,6 +141,7 @@ def test_workspace_crates_and_bins_enumerated() -> None: "capsem-mock-server", "capsem-net-proxy", "capsem-process", + "capsem-proxy", "capsem-pty-agent", "capsem-service", "capsem-sysutil", @@ -374,7 +376,9 @@ def test_release_critical_crates_are_reported() -> None: "crates/capsem-mcp-aggregator/src/**", "crates/capsem-mcp-builtin/src/**", "crates/capsem-mock-server/src/**", + "crates/capsem-ledger/src/**", "crates/capsem-process/src/**", + "crates/capsem-proxy/src/**", "crates/capsem-service/src/**", "crates/capsem-tray/src/**", "crates/capsem-tui/src/**", @@ -428,7 +432,9 @@ def test_release_binaries_and_package_rails_covered() -> None: "capsem-mcp-aggregator", "capsem-mcp-builtin", "capsem-mock-server", + "capsem-ledger", "capsem-process", + "capsem-proxy", "capsem-service", "capsem-tray", "capsem-tui", diff --git a/build_system/tests/packaging/test_build_pkg.py b/build_system/tests/packaging/test_build_pkg.py index 6225c747d..0d81826be 100644 --- a/build_system/tests/packaging/test_build_pkg.py +++ b/build_system/tests/packaging/test_build_pkg.py @@ -20,6 +20,8 @@ "capsem", "capsem-service", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-router", "capsem-tui", diff --git a/build_system/tests/packaging/test_macos_tart_glowup.py b/build_system/tests/packaging/test_macos_tart_glowup.py index e59bb95b7..913ad61dd 100644 --- a/build_system/tests/packaging/test_macos_tart_glowup.py +++ b/build_system/tests/packaging/test_macos_tart_glowup.py @@ -373,6 +373,8 @@ def test_guest_installs_and_verifies_the_exact_shared_package() -> None: "capsem", "capsem-service", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-tui", "capsem-mcp-aggregator", diff --git a/build_system/tests/packaging/test_repack_deb.py b/build_system/tests/packaging/test_repack_deb.py index 693d479d1..7b9d2c6b6 100644 --- a/build_system/tests/packaging/test_repack_deb.py +++ b/build_system/tests/packaging/test_repack_deb.py @@ -35,6 +35,8 @@ "capsem", "capsem-service", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-router", "capsem-tui", diff --git a/build_system/tests/packaging/test_router_signing.py b/build_system/tests/packaging/test_router_signing.py index 954e944a7..ee598ad61 100644 --- a/build_system/tests/packaging/test_router_signing.py +++ b/build_system/tests/packaging/test_router_signing.py @@ -10,7 +10,7 @@ ROOT = Path(__file__).resolve().parents[3] -def test_router_is_signed_without_virtualization_authority(tmp_path): +def test_confined_workers_are_signed_without_virtualization_authority(tmp_path): tools = tmp_path / "tools" tools.mkdir() calls = tmp_path / "calls.jsonl" @@ -40,12 +40,16 @@ def test_router_is_signed_without_virtualization_authority(tmp_path): signed = [json.loads(line) for line in calls.read_text().splitlines()] router = [args for args in signed if args[-1].endswith("/capsem-router")] assert len(router) == 1 and "--entitlements" not in router[0] + proxy = [args for args in signed if args[-1].endswith("/capsem-proxy")] + assert len(proxy) == 1 and "--entitlements" not in proxy[0] + ledger = [args for args in signed if args[-1].endswith("/capsem-ledger")] + assert len(ledger) == 1 and "--entitlements" not in ledger[0] assert any(args[-1].endswith("/capsem-process") and "--entitlements" in args for args in signed) calls.unlink() installed = tmp_path / "installed" (installed / "bin").mkdir(parents=True) - for name in ("capsem-process", "capsem-router"): + for name in ("capsem-process", "capsem-ledger", "capsem-router", "capsem-proxy"): (installed / "bin" / name).touch() entitlements = ROOT / "build_system/packaging/macos/entitlements.plist" (tmp_path / entitlements.name).write_bytes(entitlements.read_bytes()) @@ -67,3 +71,9 @@ def test_router_is_signed_without_virtualization_authority(tmp_path): router = [args for args in signed if args[-1].endswith("/capsem-router")] assert len(router) == 1 and "--entitlements" not in router[0] assert "org.capsem.router" in router[0] + proxy = [args for args in signed if args[-1].endswith("/capsem-proxy")] + assert len(proxy) == 1 and "--entitlements" not in proxy[0] + assert "org.capsem.proxy" in proxy[0] + ledger = [args for args in signed if args[-1].endswith("/capsem-ledger")] + assert len(ledger) == 1 and "--entitlements" not in ledger[0] + assert "org.capsem.ledger" in ledger[0] diff --git a/build_system/tests/release/test_release_doctor_contract.py b/build_system/tests/release/test_release_doctor_contract.py index 05fd31ba3..e102d63d5 100644 --- a/build_system/tests/release/test_release_doctor_contract.py +++ b/build_system/tests/release/test_release_doctor_contract.py @@ -2016,7 +2016,7 @@ def test_binary_release_installs_exact_artifacts_before_publication() -> None: assert 'test -d "/Applications/Capsem.app"' in macos assert 'test -x "/Applications/Capsem.app/Contents/MacOS/capsem-app"' in macos assert ( - "for bin in capsem capsem-admin capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs" + "for bin in capsem capsem-admin capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-ledger capsem-proxy capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs" in macos ) assert 'grep -F "Installed: true" /tmp/capsem-status.txt' in macos @@ -2043,7 +2043,7 @@ def test_binary_release_installs_exact_artifacts_before_publication() -> None: "install-manifest-request.sh write" ) assert ( - "for bin in capsem capsem-admin capsem-app capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs" + "for bin in capsem capsem-admin capsem-app capsem-gateway capsem-router capsem-mcp-aggregator capsem-mcp-builtin capsem-process capsem-ledger capsem-proxy capsem-service capsem-tray capsem-tui capsem-mock-server capsem-bench-rs" in linux ) assert "dpkg-query -W -f='${Version}' capsem | grep -Fx \"$VERSION\"" in linux @@ -4819,6 +4819,8 @@ def test_binary_update_installer_scripts_replace_and_restart_full_helper_cohort( "capsem", "capsem-service", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-tui", "capsem-mcp-aggregator", "capsem-mcp-builtin", @@ -4832,6 +4834,8 @@ def test_binary_update_installer_scripts_replace_and_restart_full_helper_cohort( "capsem-gateway", "capsem-tray", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-mcp-aggregator", "capsem-mcp-builtin", ] @@ -5775,7 +5779,7 @@ def test_pr_ci_non_vm_python_tests_prepare_assets_and_signed_binaries() -> None: asset_pos = block.find("bash build_system/scripts/test/prepare-install-test-assets.sh") build_pos = block.find( - "cargo build -p capsem-process -p capsem-service -p capsem " + "cargo build -p capsem-process -p capsem-ledger -p capsem-proxy -p capsem-service -p capsem " "-p capsem-mock-server -p capsem-bench" ) bench_package_pos = block.find("-p capsem-bench") diff --git a/build_system/tests/scripts/test_rust_affected.py b/build_system/tests/scripts/test_rust_affected.py index 55679b067..03565301b 100644 --- a/build_system/tests/scripts/test_rust_affected.py +++ b/build_system/tests/scripts/test_rust_affected.py @@ -132,6 +132,8 @@ def test_real_workspace_inventory_matches_cargo_members() -> None: "capsem-mock-server", "capsem-network", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-router", "capsem-sdk", "capsem-proto", diff --git a/codecov.yml b/codecov.yml index 933d1cd0f..3ac5b9715 100644 --- a/codecov.yml +++ b/codecov.yml @@ -116,6 +116,7 @@ component_management: - crates/capsem-core/src/net/mitm_proxy/** - crates/capsem-core/src/net/parsers/** - crates/capsem-core/src/net/policy.rs + - crates/capsem-core/src/net/proxy_engine.rs - crates/capsem-core/src/net/address_pool.rs - crates/capsem-core/src/net/network_registry.rs - crates/capsem-core/src/net/network_registry/** @@ -262,6 +263,18 @@ component_management: paths: - crates/capsem-process/src/** + # Per-session network policy enforcement and capability handling. + - component_id: proxy + name: Proxy + paths: + - crates/capsem-proxy/src/** + + # Per-session database ownership and descriptor-only ledger RPC. + - component_id: ledger + name: Ledger + paths: + - crates/capsem-ledger/src/** + # Admin tooling: manifest generation, image build orchestration, and # release asset validation. - component_id: admin diff --git a/config/dependency-inventory.json b/config/dependency-inventory.json index a430ab765..c6ccca4ac 100644 --- a/config/dependency-inventory.json +++ b/config/dependency-inventory.json @@ -525,14 +525,6 @@ "strum" ] }, - "bincode": { - "build_script": false, - "dependencies": [ - "serde", - "serde_bytes", - "serde_derive" - ] - }, "bit-set": { "build_script": false, "dependencies": [ @@ -780,6 +772,7 @@ "build_script": false, "dependencies": [ "capsem-foundation", + "capsem-guard", "capsem-proto", "nix", "serde", @@ -922,7 +915,6 @@ "rmcp", "rusqlite", "rustls", - "rustls-pemfile", "scraper", "serde", "serde_json", @@ -955,7 +947,6 @@ "capsem-foundation": { "build_script": false, "dependencies": [ - "bincode", "blake3", "capsem-proto", "libc", @@ -971,7 +962,7 @@ ] }, "capsem-gateway": { - "build_script": false, + "build_script": true, "dependencies": [ "anyhow", "axum", @@ -989,6 +980,7 @@ "rand", "serde", "serde_json", + "sha2", "tempfile", "tokio", "tokio-tungstenite", @@ -1009,6 +1001,20 @@ "tracing" ] }, + "capsem-ledger": { + "build_script": false, + "dependencies": [ + "anyhow", + "capsem-foundation", + "capsem-guard", + "capsem-logger", + "capsem-proto", + "clap", + "tempfile", + "tokio", + "tracing" + ] + }, "capsem-logger": { "build_script": false, "dependencies": [ @@ -1024,8 +1030,10 @@ "memmap2", "metrics", "metrics-util", + "rmp-serde", "rusqlite", "serde", + "serde_bytes", "serde_json", "sha2", "tempfile", @@ -1112,15 +1120,21 @@ "build_script": false, "dependencies": [ "anyhow", + "async-trait", "axum", + "bytes", "capsem-assets", "capsem-core", "capsem-foundation", + "capsem-guard", "capsem-logger", "capsem-proto", + "capsem-telemetry", "clap", "core-foundation-sys", "futures", + "http", + "opentelemetry-http", "serde", "serde_json", "tempfile", @@ -1145,6 +1159,22 @@ "tokio" ] }, + "capsem-proxy": { + "build_script": false, + "dependencies": [ + "anyhow", + "capsem-core", + "capsem-foundation", + "capsem-guard", + "capsem-logger", + "capsem-proto", + "clap", + "http", + "tempfile", + "tokio", + "tracing" + ] + }, "capsem-router": { "build_script": false, "dependencies": [ @@ -1178,9 +1208,11 @@ "build_script": false, "dependencies": [ "anyhow", + "async-trait", "axum", "base64", "blake3", + "bytes", "capsem-api", "capsem-assets", "capsem-config", @@ -1190,10 +1222,16 @@ "capsem-logger", "capsem-proto", "capsem-router", + "capsem-telemetry", "clap", "filetime", "futures", + "http", "humantime", + "metrics", + "opentelemetry", + "opentelemetry-http", + "opentelemetry_sdk", "rand", "reqwest", "rusqlite", @@ -1203,6 +1241,7 @@ "tokio", "tokio-stream", "tokio-tungstenite", + "tokio-util", "toml", "tower", "tower-http", @@ -1213,14 +1252,23 @@ "capsem-telemetry": { "build_script": false, "dependencies": [ + "async-trait", + "bytes", + "http", "metrics", - "metrics-util" + "metrics-util", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-otlp", + "opentelemetry_sdk", + "reqwest" ] }, "capsem-tray": { "build_script": true, "dependencies": [ "anyhow", + "capsem-api", "capsem-foundation", "capsem-guard", "clap", @@ -2464,14 +2512,6 @@ "thread_local" ] }, - "fxhash": { - "build_script": false, - "dependencies": [ - "byteorder", - "fnv", - "seahash" - ] - }, "gdk": { "build_script": false, "dependencies": [ @@ -3213,9 +3253,6 @@ "crossterm", "dyn-clone", "fuzzy-matcher", - "fxhash", - "newline-converter", - "once_cell", "rstest", "tempfile", "termion", @@ -3896,17 +3933,6 @@ "build_script": false, "dependencies": [] }, - "newline-converter": { - "build_script": false, - "dependencies": [ - "criterion", - "fancy-regex", - "lazy-regex", - "once_cell", - "quickcheck", - "unicode-segmentation" - ] - }, "nibble_vec": { "build_script": false, "dependencies": [ @@ -4345,6 +4371,102 @@ "libc" ] }, + "opentelemetry": { + "build_script": false, + "dependencies": [ + "criterion", + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "rand", + "thiserror", + "tokio", + "tracing" + ] + }, + "opentelemetry-http": { + "build_script": false, + "dependencies": [ + "async-trait", + "bytes", + "futures-executor", + "http", + "http-body-util", + "hyper", + "hyper-util", + "opentelemetry", + "reqwest", + "tokio", + "tower-service" + ] + }, + "opentelemetry-otlp": { + "build_script": false, + "dependencies": [ + "async-trait", + "flate2", + "futures-executor", + "futures-util", + "http", + "httpdate", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "serde_json", + "temp-env", + "thiserror", + "tokio", + "tokio-stream", + "tonic", + "tonic-types", + "zstd" + ] + }, + "opentelemetry-proto": { + "build_script": false, + "dependencies": [ + "base64", + "const-hex", + "opentelemetry", + "opentelemetry_sdk", + "prost", + "schemars", + "serde", + "serde_json", + "tempfile", + "tonic", + "tonic-prost", + "tonic-prost-build" + ] + }, + "opentelemetry_sdk": { + "build_script": false, + "dependencies": [ + "criterion", + "futures-channel", + "futures-executor", + "futures-util", + "http", + "opentelemetry", + "opentelemetry-http", + "percent-encoding", + "portable-atomic", + "pprof", + "rand", + "rstest", + "serde", + "serde_json", + "temp-env", + "thiserror", + "tokio", + "tokio-stream", + "url" + ] + }, "option-ext": { "build_script": false, "dependencies": [] @@ -4803,6 +4925,26 @@ "x86" ] }, + "prost": { + "build_script": false, + "dependencies": [ + "bytes", + "criterion", + "proptest", + "prost-derive", + "rand" + ] + }, + "prost-derive": { + "build_script": false, + "dependencies": [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn" + ] + }, "quanta": { "build_script": false, "dependencies": [ @@ -5373,6 +5515,7 @@ "rand", "reqwest", "rmcp-macros", + "rstest", "schemars", "serde", "serde_json", @@ -5506,13 +5649,6 @@ "zlib-rs" ] }, - "rustls-pemfile": { - "build_script": false, - "dependencies": [ - "bencher", - "rustls-pki-types" - ] - }, "rustls-pki-types": { "build_script": false, "dependencies": [ @@ -7263,7 +7399,6 @@ "unicode-width": { "build_script": false, "dependencies": [ - "compiler_builtins", "rustc-std-workspace-core", "rustc-std-workspace-std" ] diff --git a/config/gate.toml b/config/gate.toml index 9a8680c5b..21555d40e 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -649,6 +649,8 @@ binaries = [ "capsem-mcp-aggregator", "capsem-mcp-builtin", "capsem-process", + "capsem-ledger", + "capsem-proxy", "capsem-service", "capsem-tray", "capsem-tui", @@ -1677,7 +1679,7 @@ binaries = [ # `initrd.guest-agents` cross-compiles. built = [ "capsem", "capsem-admin", "capsem-bench-rs", "capsem-gateway", - "capsem-mcp-aggregator", "capsem-mcp-builtin", "capsem-mock-server", "capsem-process", "capsem-router", + "capsem-ledger", "capsem-mcp-aggregator", "capsem-mcp-builtin", "capsem-mock-server", "capsem-process", "capsem-proxy", "capsem-router", "capsem-service", "capsem-tray", "capsem-tui", ] # Host binaries this list deliberately omits, each built by a step that owns @@ -2418,7 +2420,7 @@ rust_coverage_crate_minimum = 40.0 # Raise to about measured minus 2.5, not minus 3: a floor on the band's edge # is refused by the next run's +0.04 jitter. rust_coverage_ratchet_headroom = 3.0 -rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-config = 89.0, capsem-core = 78.0, capsem-crypto = 93.9, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } +rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-config = 89.0, capsem-core = 78.0, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 40.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 40.0, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } # macOS compiles a different cfg-selected line inventory. Its stronger floors # must not raise Linux thresholds using code that Linux cannot compile or test. # Overrides may only increase the shared floors; no platform gets an exemption. @@ -2727,8 +2729,8 @@ network_reason = "No mid-run fetch. This is the rule the whole profile exists fo # sandboxed cargo. `just doctor` runs the same probe and owns the fetch. cargo_offline_probe = ["cargo", "fetch", "--locked", "--offline"] socket_reason = "`(deny network*)` covers AF_UNIX too, so these come back explicitly." -self_confinement_reason = "Seatbelt cannot be nested. These trusted children replace the inherited gate profile with their stricter router profile before handling data; ordinary test children retain the gate network denial." -self_confined_executables = ["capsem-router", "capsem-router-confinement-test"] +self_confinement_reason = "Seatbelt cannot be nested. These trusted children replace the inherited gate profile with their stricter worker profile before handling data; ordinary test children retain the gate network denial." +self_confined_executables = ["capsem-ledger", "capsem-proxy", "capsem-router", "capsem-router-confinement-test"] # Absolute or `~`-relative. Denying the Docker socket makes a running daemon # look absent -- `permission denied ... docker.sock` -- which is how the first From 4b073f9a00bc108308e26cc1c01eb5f9a4d06a4e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:39:28 +0000 Subject: [PATCH 111/464] security(install): require confined workers --- CHANGELOG.md | 4 ++ crates/capsem/src/client.rs | 2 + crates/capsem/src/client/tests.rs | 11 ++++++ crates/capsem/src/paths.rs | 4 ++ crates/capsem/src/paths/tests.rs | 9 +++++ crates/capsem/src/service_install.rs | 42 ++++++++++++++++---- crates/capsem/src/service_install/tests.rs | 45 ++++++++++++++++++++++ crates/capsem/src/uninstall.rs | 21 +++++++++- 8 files changed, 128 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49942f986..3fb935919 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Pass the packaged proxy worker path explicitly to installed and direct + services, refuse incomplete confined-worker cohorts, and retire ledger and + proxy children during reinstall or uninstall. + - Ship the confined ledger and proxy workers in every supported package and sign them without macOS virtualization authority, so clean installs cannot fall back to privileged in-process execution or grant VM capabilities to diff --git a/crates/capsem/src/client.rs b/crates/capsem/src/client.rs index a555440a2..652471536 100644 --- a/crates/capsem/src/client.rs +++ b/crates/capsem/src/client.rs @@ -303,6 +303,8 @@ fn direct_spawn_service_args( paths.assets_dir.clone().into_os_string(), "--process-binary".into(), paths.process_bin.clone().into_os_string(), + "--proxy-binary".into(), + paths.proxy_bin.clone().into_os_string(), ]; if isolated { args.extend([OsString::from("--gateway-port"), OsString::from("0")]); diff --git a/crates/capsem/src/client/tests.rs b/crates/capsem/src/client/tests.rs index 4f661c276..49e1b32a1 100644 --- a/crates/capsem/src/client/tests.rs +++ b/crates/capsem/src/client/tests.rs @@ -799,6 +799,8 @@ fn isolated_direct_spawn_uses_an_ephemeral_gateway_port() { let paths = paths::CapsemPaths { service_bin: PathBuf::from("/opt/capsem/bin/capsem-service"), process_bin: PathBuf::from("/opt/capsem/bin/capsem-process"), + ledger_bin: PathBuf::from("/opt/capsem/bin/capsem-ledger"), + proxy_bin: PathBuf::from("/opt/capsem/bin/capsem-proxy"), gateway_bin: PathBuf::from("/opt/capsem/bin/capsem-gateway"), tray_bin: PathBuf::from("/opt/capsem/bin/capsem-tray"), assets_dir: PathBuf::from("/opt/capsem/assets"), @@ -810,6 +812,11 @@ fn isolated_direct_spawn_uses_an_ephemeral_gateway_port() { args.windows(2).any(|pair| pair == ["--gateway-port", "0"]), "isolated service must not collide with an installed gateway: {args:?}" ); + assert!( + args.windows(2) + .any(|pair| pair == ["--proxy-binary", "/opt/capsem/bin/capsem-proxy"]), + "direct service must use its sibling confined proxy: {args:?}" + ); } #[test] @@ -817,6 +824,8 @@ fn bounded_direct_spawn_exits_with_its_one_shot_cli_parent() { let paths = paths::CapsemPaths { service_bin: PathBuf::from("/opt/capsem/bin/capsem-service"), process_bin: PathBuf::from("/opt/capsem/bin/capsem-process"), + ledger_bin: PathBuf::from("/opt/capsem/bin/capsem-ledger"), + proxy_bin: PathBuf::from("/opt/capsem/bin/capsem-proxy"), gateway_bin: PathBuf::from("/opt/capsem/bin/capsem-gateway"), tray_bin: PathBuf::from("/opt/capsem/bin/capsem-tray"), assets_dir: PathBuf::from("/opt/capsem/assets"), @@ -868,6 +877,8 @@ fn ordinary_direct_spawn_preserves_the_installed_gateway_port_contract() { let paths = paths::CapsemPaths { service_bin: PathBuf::from("/opt/capsem/bin/capsem-service"), process_bin: PathBuf::from("/opt/capsem/bin/capsem-process"), + ledger_bin: PathBuf::from("/opt/capsem/bin/capsem-ledger"), + proxy_bin: PathBuf::from("/opt/capsem/bin/capsem-proxy"), gateway_bin: PathBuf::from("/opt/capsem/bin/capsem-gateway"), tray_bin: PathBuf::from("/opt/capsem/bin/capsem-tray"), assets_dir: PathBuf::from("/opt/capsem/assets"), diff --git a/crates/capsem/src/paths.rs b/crates/capsem/src/paths.rs index 3c554cc64..9c0d6470f 100644 --- a/crates/capsem/src/paths.rs +++ b/crates/capsem/src/paths.rs @@ -16,6 +16,8 @@ pub fn capsem_home() -> Result { pub struct CapsemPaths { pub service_bin: PathBuf, pub process_bin: PathBuf, + pub ledger_bin: PathBuf, + pub proxy_bin: PathBuf, pub gateway_bin: PathBuf, pub tray_bin: PathBuf, pub assets_dir: PathBuf, @@ -34,6 +36,8 @@ pub fn discover_paths() -> Result { Ok(CapsemPaths { service_bin: bin_dir.join("capsem-service"), process_bin: bin_dir.join("capsem-process"), + ledger_bin: bin_dir.join("capsem-ledger"), + proxy_bin: bin_dir.join("capsem-proxy"), gateway_bin: bin_dir.join("capsem-gateway"), tray_bin: bin_dir.join("capsem-tray"), assets_dir: capsem_foundation::paths::capsem_assets_dir(), diff --git a/crates/capsem/src/paths/tests.rs b/crates/capsem/src/paths/tests.rs index eaa8a6857..d253d3054 100644 --- a/crates/capsem/src/paths/tests.rs +++ b/crates/capsem/src/paths/tests.rs @@ -70,6 +70,8 @@ fn discover_paths_sibling_binaries_use_exe_dir() { let exe_dir = exe.parent().unwrap(); assert_eq!(paths.service_bin.parent().unwrap(), exe_dir); assert_eq!(paths.process_bin.parent().unwrap(), exe_dir); + assert_eq!(paths.ledger_bin.parent().unwrap(), exe_dir); + assert_eq!(paths.proxy_bin.parent().unwrap(), exe_dir); } #[test] @@ -106,6 +108,13 @@ fn discover_paths_process_bin_name() { ); } +#[test] +fn discover_paths_confined_worker_names() { + let paths = discover_paths().unwrap(); + assert_eq!(paths.ledger_bin.file_name().unwrap(), "capsem-ledger"); + assert_eq!(paths.proxy_bin.file_name().unwrap(), "capsem-proxy"); +} + // ----------------------------------------------------------------------- // Installed layout contract: what simulate-install.sh produces // must be what discover_paths + service startup consume. diff --git a/crates/capsem/src/service_install.rs b/crates/capsem/src/service_install.rs index 129dab97c..9e9269962 100644 --- a/crates/capsem/src/service_install.rs +++ b/crates/capsem/src/service_install.rs @@ -57,6 +57,7 @@ pub struct ServiceStatus { pub fn generate_plist( service_bin: &Path, process_bin: &Path, + proxy_bin: &Path, gateway_bin: &Path, tray_bin: &Path, assets_dir: &Path, @@ -65,6 +66,7 @@ pub fn generate_plist( let log_dir = xml_escape(&format!("{}/Library/Logs/capsem", home)); let service_bin = xml_escape(&service_bin.display().to_string()); let process_bin = xml_escape(&process_bin.display().to_string()); + let proxy_bin = xml_escape(&proxy_bin.display().to_string()); let gateway_bin = xml_escape(&gateway_bin.display().to_string()); let tray_bin = xml_escape(&tray_bin.display().to_string()); let assets_dir = xml_escape(&assets_dir.display().to_string()); @@ -89,6 +91,8 @@ pub fn generate_plist( {assets_dir} --process-binary {process_bin} + --proxy-binary + {proxy_bin} --gateway-binary {gateway_bin} --tray-binary @@ -115,12 +119,14 @@ pub fn generate_plist( pub fn generate_systemd_unit( service_bin: &Path, process_bin: &Path, + proxy_bin: &Path, gateway_bin: &Path, tray_bin: &Path, assets_dir: &Path, ) -> String { let service_bin = systemd_escape_path(service_bin); let process_bin = systemd_escape_path(process_bin); + let proxy_bin = systemd_escape_path(proxy_bin); let gateway_bin = systemd_escape_path(gateway_bin); let tray_bin = systemd_escape_path(tray_bin); let assets_dir = systemd_escape_path(assets_dir); @@ -129,7 +135,7 @@ pub fn generate_systemd_unit( Description=Capsem sandbox service [Service] -ExecStart={service_bin} --foreground --assets-dir {assets_dir} --process-binary {process_bin} --gateway-binary {gateway_bin} --tray-binary {tray_bin} +ExecStart={service_bin} --foreground --assets-dir {assets_dir} --process-binary {process_bin} --proxy-binary {proxy_bin} --gateway-binary {gateway_bin} --tray-binary {tray_bin} Restart=always RestartSec=2 @@ -176,6 +182,22 @@ fn reject_test_isolation_env() -> Result<()> { ); } +fn validate_service_binaries(capsem_paths: &paths::CapsemPaths) -> Result<()> { + for (name, path) in [ + ("capsem-service", &capsem_paths.service_bin), + ("capsem-process", &capsem_paths.process_bin), + ("capsem-ledger", &capsem_paths.ledger_bin), + ("capsem-proxy", &capsem_paths.proxy_bin), + ("capsem-gateway", &capsem_paths.gateway_bin), + ("capsem-tray", &capsem_paths.tray_bin), + ] { + if !path.is_file() { + anyhow::bail!("{name} not found at {}", path.display()); + } + } + Ok(()) +} + /// Install the capsem service as a LaunchAgent (macOS) or systemd user unit (Linux). pub async fn install_service() -> Result<()> { reject_test_isolation_env()?; @@ -183,12 +205,7 @@ pub async fn install_service() -> Result<()> { let capsem_paths = paths::discover_paths().context("cannot discover paths for service installation")?; let home = std::env::var("HOME").context("HOME not set")?; - if !capsem_paths.service_bin.exists() { - anyhow::bail!("capsem-service not found at {}", capsem_paths.service_bin.display()); - } - if !capsem_paths.process_bin.exists() { - anyhow::bail!("capsem-process not found at {}", capsem_paths.process_bin.display()); - } + validate_service_binaries(&capsem_paths)?; #[cfg(target_os = "macos")] { @@ -523,7 +540,14 @@ async fn install_launchagent(capsem_paths: &paths::CapsemPaths, home: &str) -> R .map(|d| format!("{}/{name}", d.display())) .unwrap_or_else(|| name.to_string()) }; - let names = ["capsem-service", "capsem-tray", "capsem-gateway", "capsem-process"]; + let names = [ + "capsem-service", + "capsem-tray", + "capsem-gateway", + "capsem-process", + "capsem-ledger", + "capsem-proxy", + ]; for name in names { let pattern = scoped_name(name); let _ = tokio::process::Command::new("pkill") @@ -558,6 +582,7 @@ async fn install_launchagent(capsem_paths: &paths::CapsemPaths, home: &str) -> R let plist_content = generate_plist( &capsem_paths.service_bin, &capsem_paths.process_bin, + &capsem_paths.proxy_bin, &capsem_paths.gateway_bin, &capsem_paths.tray_bin, &capsem_paths.assets_dir, @@ -640,6 +665,7 @@ async fn install_systemd_unit(capsem_paths: &paths::CapsemPaths, home: &str) -> let unit_content = generate_systemd_unit( &capsem_paths.service_bin, &capsem_paths.process_bin, + &capsem_paths.proxy_bin, &capsem_paths.gateway_bin, &capsem_paths.tray_bin, &capsem_paths.assets_dir, diff --git a/crates/capsem/src/service_install/tests.rs b/crates/capsem/src/service_install/tests.rs index 285540e0c..78eab47af 100644 --- a/crates/capsem/src/service_install/tests.rs +++ b/crates/capsem/src/service_install/tests.rs @@ -1,11 +1,45 @@ use super::*; use std::path::Path; +fn service_paths(bin_dir: &Path) -> paths::CapsemPaths { + paths::CapsemPaths { + service_bin: bin_dir.join("capsem-service"), + process_bin: bin_dir.join("capsem-process"), + ledger_bin: bin_dir.join("capsem-ledger"), + proxy_bin: bin_dir.join("capsem-proxy"), + gateway_bin: bin_dir.join("capsem-gateway"), + tray_bin: bin_dir.join("capsem-tray"), + assets_dir: bin_dir.join("assets"), + } +} + +#[test] +fn service_install_refuses_missing_confined_workers() { + for missing in ["capsem-ledger", "capsem-proxy"] { + let dir = tempfile::tempdir().unwrap(); + for name in [ + "capsem-service", + "capsem-process", + "capsem-ledger", + "capsem-proxy", + "capsem-gateway", + "capsem-tray", + ] { + if name != missing { + std::fs::write(dir.path().join(name), b"binary").unwrap(); + } + } + let error = validate_service_binaries(&service_paths(dir.path())).unwrap_err(); + assert!(error.to_string().contains(missing), "{error:#}"); + } +} + #[test] fn test_generate_plist_absolute_paths() { let plist = generate_plist( Path::new("/Users/test/.capsem/bin/capsem-service"), Path::new("/Users/test/.capsem/bin/capsem-process"), + Path::new("/Users/test/.capsem/bin/capsem-proxy"), Path::new("/Users/test/.capsem/bin/capsem-gateway"), Path::new("/Users/test/.capsem/bin/capsem-tray"), Path::new("/Users/test/.capsem/assets"), @@ -14,6 +48,8 @@ fn test_generate_plist_absolute_paths() { // ProgramArguments binary and path args must be absolute assert!(plist.contains("/Users/test/.capsem/bin/capsem-service")); assert!(plist.contains("/Users/test/.capsem/bin/capsem-process")); + assert!(plist.contains("--proxy-binary")); + assert!(plist.contains("/Users/test/.capsem/bin/capsem-proxy")); assert!(plist.contains("/Users/test/.capsem/assets")); // Log path must be absolute assert!(plist.contains("/Users/test/Library/Logs/capsem/service.log")); @@ -26,6 +62,7 @@ fn test_generate_plist_valid_xml() { let plist = generate_plist( Path::new("/usr/local/bin/capsem-service"), Path::new("/usr/local/bin/capsem-process"), + Path::new("/usr/local/bin/capsem-proxy"), Path::new("/usr/local/bin/capsem-gateway"), Path::new("/usr/local/bin/capsem-tray"), Path::new("/home/test/.capsem/assets"), @@ -45,6 +82,7 @@ fn test_generate_plist_has_keep_alive() { let plist = generate_plist( Path::new("/bin/capsem-service"), Path::new("/bin/capsem-process"), + Path::new("/bin/capsem-proxy"), Path::new("/bin/capsem-gateway"), Path::new("/bin/capsem-tray"), Path::new("/assets"), @@ -60,6 +98,7 @@ fn test_generate_plist_pins_file_backed_credential_store() { let plist = generate_plist( Path::new("/Users/test/.capsem/bin/capsem-service"), Path::new("/Users/test/.capsem/bin/capsem-process"), + Path::new("/Users/test/.capsem/bin/capsem-proxy"), Path::new("/Users/test/.capsem/bin/capsem-gateway"), Path::new("/Users/test/.capsem/bin/capsem-tray"), Path::new("/Users/test/.capsem/assets"), @@ -117,6 +156,7 @@ fn test_generate_systemd_unit_absolute_paths() { let unit = generate_systemd_unit( Path::new("/home/test/.capsem/bin/capsem-service"), Path::new("/home/test/.capsem/bin/capsem-process"), + Path::new("/home/test/.capsem/bin/capsem-proxy"), Path::new("/home/test/.capsem/bin/capsem-gateway"), Path::new("/home/test/.capsem/bin/capsem-tray"), Path::new("/home/test/.capsem/assets"), @@ -130,6 +170,7 @@ fn test_generate_systemd_unit_absolute_paths() { ); // --process-binary value should be absolute assert!(exec_line.contains("--process-binary /")); + assert!(exec_line.contains("--proxy-binary /")); // --assets-dir value should be absolute assert!(exec_line.contains("--assets-dir /")); } @@ -139,6 +180,7 @@ fn test_generate_systemd_unit_restart_policy() { let unit = generate_systemd_unit( Path::new("/bin/svc"), Path::new("/bin/proc"), + Path::new("/bin/proxy"), Path::new("/bin/gw"), Path::new("/bin/tray"), Path::new("/assets"), @@ -152,6 +194,7 @@ fn test_generate_systemd_unit_wanted_by() { let unit = generate_systemd_unit( Path::new("/bin/svc"), Path::new("/bin/proc"), + Path::new("/bin/proxy"), Path::new("/bin/gw"), Path::new("/bin/tray"), Path::new("/assets"), @@ -182,6 +225,7 @@ fn test_plist_with_special_chars_in_path() { let plist = generate_plist( Path::new("/Users/AT&T Corp/.capsem/bin/capsem-service"), Path::new("/Users/AT&T Corp/.capsem/bin/capsem-process"), + Path::new("/Users/AT&T Corp/.capsem/bin/capsem-proxy"), Path::new("/Users/AT&T Corp/.capsem/bin/capsem-gateway"), Path::new("/Users/AT&T Corp/.capsem/bin/capsem-tray"), Path::new("/Users/AT&T Corp/.capsem/assets"), @@ -215,6 +259,7 @@ fn test_systemd_unit_with_spaces_in_path() { let unit = generate_systemd_unit( Path::new("/home/John Doe/.capsem/bin/capsem-service"), Path::new("/home/John Doe/.capsem/bin/capsem-process"), + Path::new("/home/John Doe/.capsem/bin/capsem-proxy"), Path::new("/home/John Doe/.capsem/bin/capsem-gateway"), Path::new("/home/John Doe/.capsem/bin/capsem-tray"), Path::new("/home/John Doe/.capsem/assets"), diff --git a/crates/capsem/src/uninstall.rs b/crates/capsem/src/uninstall.rs index f5eb88344..94141b3bb 100644 --- a/crates/capsem/src/uninstall.rs +++ b/crates/capsem/src/uninstall.rs @@ -45,7 +45,14 @@ pub async fn run_uninstall(yes: bool) -> Result<()> { let install_dir = std::env::current_exe() .ok() .and_then(|p| p.parent().map(|p| p.to_path_buf())); - for name in ["capsem-service", "capsem-process", "capsem-gateway", "capsem-tray"] { + for name in [ + "capsem-service", + "capsem-process", + "capsem-ledger", + "capsem-proxy", + "capsem-gateway", + "capsem-tray", + ] { let pattern = match install_dir.as_ref() { Some(dir) => format!("{}/{name}", dir.display()), None => name.to_string(), @@ -62,10 +69,20 @@ pub async fn run_uninstall(yes: bool) -> Result<()> { // Remove binaries from the detected install location const CAPSEM_BINARIES: &[&str] = &[ "capsem", + "capsem-admin", + "capsem-app", + "capsem-bench-rs", + "capsem-gateway", + "capsem-ledger", + "capsem-mcp-aggregator", + "capsem-mcp-builtin", + "capsem-mock-server", "capsem-service", "capsem-process", - "capsem-gateway", + "capsem-proxy", + "capsem-router", "capsem-tray", + "capsem-tui", ]; if let Some(bin_dir) = platform::install_bin_dir() { if bin_dir.exists() { From af6eedc121b9d9ac12f2254dfc11a1590561c898 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:49:01 +0000 Subject: [PATCH 112/464] security(service): supervise ledger worker protocol --- crates/capsem-service/src/ledger_worker.rs | 447 ++++++++++++++++++ .../capsem-service/src/ledger_worker/tests.rs | 194 ++++++++ crates/capsem-service/src/main.rs | 3 +- 3 files changed, 642 insertions(+), 2 deletions(-) create mode 100644 crates/capsem-service/src/ledger_worker.rs create mode 100644 crates/capsem-service/src/ledger_worker/tests.rs diff --git a/crates/capsem-service/src/ledger_worker.rs b/crates/capsem-service/src/ledger_worker.rs new file mode 100644 index 000000000..4286a77e7 --- /dev/null +++ b/crates/capsem-service/src/ledger_worker.rs @@ -0,0 +1,447 @@ +//! Trusted lifecycle and channel-grant coordinator for one session ledger. + +#![cfg_attr( + not(test), + expect( + dead_code, + reason = "S09-004 wires producer and reader clients to the supervised ledger" + ) +)] + +use std::fmt::Write as _; +use std::io; +use std::os::fd::{AsRawFd as _, OwnedFd}; +use std::os::unix::net::UnixStream; +use std::path::Path; +use std::process::Stdio; +use std::time::Duration; + +use anyhow::{anyhow, bail, Context, Result}; +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; +use capsem_proto::ledger_control::{ + decode_ledger_control_event, encode_ledger_control_request, LedgerControlEvent, LedgerControlRequest, + LEDGER_CONTROL_FRAME_SIZE, LEDGER_CONTROL_MAX_FDS, +}; +use tokio::process::{Child, Command}; +use tokio::sync::{mpsc, oneshot, watch}; + +#[cfg(not(test))] +const STARTUP_TIMEOUT: Duration = Duration::from_secs(10); +#[cfg(test)] +const STARTUP_TIMEOUT: Duration = Duration::from_secs(2); +#[cfg(not(test))] +const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +const OPERATION_TIMEOUT: Duration = Duration::from_secs(2); +#[cfg(not(test))] +const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +const SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(200); +const COMMAND_QUEUE_CAPACITY: usize = 64; + +type ControlSender = DescriptorSender; +type ControlReceiver = DescriptorReceiver; +type ControlFrame = capsem_foundation::unix::router_channel::DescriptorFrame; + +#[derive(Debug)] +pub(crate) struct LedgerClient { + stream: UnixStream, + grant: LedgerChannelGrant, +} + +impl LedgerClient { + pub(crate) const fn grant(&self) -> LedgerChannelGrant { + self.grant + } + + pub(crate) fn into_parts(self) -> (UnixStream, LedgerChannelGrant) { + (self.stream, self.grant) + } +} + +#[derive(Clone)] +pub(crate) struct LedgerWorker { + generation: LedgerGeneration, + commands: mpsc::Sender, + stopped: watch::Receiver>, +} + +impl LedgerWorker { + #[cfg_attr( + test, + expect(dead_code, reason = "production launch is covered by capsem-ledger subprocess tests") + )] + pub(crate) async fn spawn(binary: &Path, database: &Path, stdout: Stdio, stderr: Stdio) -> Result { + let generation = fresh_generation(); + let mut command = Command::new(binary); + command.env_clear().stdout(stdout).stderr(stderr).kill_on_drop(true); + launch(command, database, generation, |command, generation, database| { + command + .arg("--parent-pid") + .arg(std::process::id().to_string()) + .arg("--database") + .arg(database) + .arg("--generation") + .arg(generation_hex(generation)); + }) + .await + } + + pub(crate) const fn generation(&self) -> LedgerGeneration { + self.generation + } + + pub(crate) async fn connect(&self, role: LedgerClientRole) -> Result { + let (completed, result) = oneshot::channel(); + self.commands + .send(CommandRequest::Connect { role, completed }) + .await + .map_err(|_| anyhow!(self.stop_reason("ledger worker supervisor stopped before accepting a client")))?; + result + .await + .map_err(|_| anyhow!(self.stop_reason("ledger worker supervisor stopped with a client pending")))? + } + + pub(crate) async fn shutdown(self) -> Result<()> { + let (completed, result) = oneshot::channel(); + self.commands + .send(CommandRequest::Shutdown { completed }) + .await + .map_err(|_| anyhow!(self.stop_reason("ledger worker supervisor already stopped")))?; + result + .await + .map_err(|_| anyhow!(self.stop_reason("ledger worker supervisor stopped during shutdown")))? + } + + pub(crate) fn stop_receiver(&self) -> watch::Receiver> { + self.stopped.clone() + } + + fn stop_reason(&self, fallback: &str) -> String { + self.stopped.borrow().clone().unwrap_or_else(|| fallback.to_string()) + } +} + +enum CommandRequest { + Connect { + role: LedgerClientRole, + completed: oneshot::Sender>, + }, + Shutdown { + completed: oneshot::Sender>, + }, +} + +struct WorkerProcess { + generation: LedgerGeneration, + child: Child, + control_tx: ControlSender, + control_events: mpsc::Receiver>, + control_reader: tokio::task::JoinHandle<()>, + next_client_id: u64, +} + +async fn launch( + mut command: Command, + database: &Path, + generation: LedgerGeneration, + configure: impl FnOnce(&mut Command, LedgerGeneration, &Path), +) -> Result { + if !database.is_absolute() || database.file_name().and_then(|name| name.to_str()) != Some("session.db") { + bail!("ledger database must be an absolute session.db path"); + } + let (coordinator, child_control) = UnixStream::pair().context("create ledger control channel")?; + configure(&mut command, generation, database); + command.stdin(Stdio::from(OwnedFd::from(child_control))); + let child = command.spawn().context("spawn capsem-ledger")?; + let control_tx = ControlSender::new(coordinator.try_clone()?).context("open ledger control sender")?; + let control_rx = ControlReceiver::new(coordinator).context("open ledger control receiver")?; + let process = WorkerProcess::start(child, control_tx, control_rx, generation).await?; + let (commands, requests) = mpsc::channel(COMMAND_QUEUE_CAPACITY); + let (stopped_tx, stopped) = watch::channel(None); + tokio::spawn(supervise(process, requests, stopped_tx)); + Ok(LedgerWorker { + generation, + commands, + stopped, + }) +} + +impl WorkerProcess { + async fn start( + child: Child, + control_tx: ControlSender, + control_rx: ControlReceiver, + generation: LedgerGeneration, + ) -> Result { + let mut pending = PendingWorker { + child, + control_tx, + control_rx, + }; + let ready = tokio::time::timeout(STARTUP_TIMEOUT, pending.recv_event()).await; + match ready { + Ok(Ok(LedgerControlEvent::Ready { generation: announced })) if announced == generation => {} + Ok(Ok(LedgerControlEvent::Ready { .. })) => { + pending.kill_reap().await; + bail!("ledger worker announced a stale generation before readiness"); + } + Ok(Ok(event)) => { + pending.kill_reap().await; + bail!("ledger worker sent {event:?} before readiness"); + } + Ok(Err(error)) => { + pending.kill_reap().await; + return Err(error); + } + Err(_) => { + pending.kill_reap().await; + bail!("ledger worker readiness timed out"); + } + } + let PendingWorker { + child, + control_tx, + control_rx, + } = pending; + let (events_tx, control_events) = mpsc::channel(COMMAND_QUEUE_CAPACITY); + let control_reader = tokio::spawn(read_control(control_rx, events_tx)); + Ok(Self { + generation, + child, + control_tx, + control_events, + control_reader, + next_client_id: 1, + }) + } + + async fn connect(&mut self, role: LedgerClientRole) -> Result { + let client_id = self.next_client_id; + self.next_client_id = client_id.checked_add(1).context("ledger client id exhausted")?; + let grant = LedgerChannelGrant::new(self.generation, client_id, role)?; + let (client, worker) = UnixStream::pair().context("create ledger client channel")?; + self.control_tx + .send( + &encode_ledger_control_request(LedgerControlRequest::Attach(grant)), + &[worker.as_raw_fd()], + ) + .await + .context("send ledger descriptor grant")?; + drop(worker); + loop { + match self.recv_event().await? { + LedgerControlEvent::Adopted { + generation, + client_id: adopted, + } if generation == self.generation && adopted == client_id => { + return Ok(LedgerClient { stream: client, grant }); + } + LedgerControlEvent::Rejected { + generation, + client_id: rejected, + reason, + } if generation == self.generation && rejected == client_id => { + bail!("ledger worker rejected {role:?} client {client_id}: {reason:?}"); + } + LedgerControlEvent::Closed { generation, .. } if generation == self.generation => {} + event => bail!("ledger worker sent {event:?} while adopting client {client_id}"), + } + } + } + + async fn shutdown(&mut self) -> Result<()> { + self.control_tx + .send( + &encode_ledger_control_request(LedgerControlRequest::Shutdown { + generation: self.generation, + }), + &[], + ) + .await + .context("send ledger shutdown")?; + let stopped = tokio::time::timeout(SHUTDOWN_TIMEOUT, async { + loop { + match self.recv_event().await? { + LedgerControlEvent::Stopped { generation } if generation == self.generation => break, + LedgerControlEvent::Closed { generation, .. } if generation == self.generation => {} + event => bail!("ledger worker sent {event:?} during shutdown"), + } + } + let status = self.child.wait().await.context("reap ledger worker")?; + if !status.success() { + bail!("ledger worker exited unsuccessfully during shutdown: {status}"); + } + Ok(()) + }) + .await; + match stopped { + Ok(result) => result, + Err(_) => { + let _ = self.child.start_kill(); + let _ = self.child.wait().await; + bail!("ledger worker bounded shutdown timed out") + } + } + } + + async fn recv_event(&mut self) -> Result { + tokio::time::timeout(OPERATION_TIMEOUT, async { + tokio::select! { + status = self.child.wait() => { + let status = status.context("reap ledger worker")?; + bail!("ledger worker exited: {status}") + } + frame = self.control_events.recv() => { + let frame = frame + .context("ledger control event pump stopped")? + .context("ledger worker control channel failed")?; + if !frame.fds.is_empty() { + bail!("ledger worker returned a descriptor on its control channel"); + } + decode_ledger_control_event(&frame.bytes).map_err(anyhow::Error::from) + } + } + }) + .await + .map_err(|_| anyhow!("ledger worker control operation timed out"))? + } + + async fn kill_reap(&mut self) { + let _ = self.child.start_kill(); + let _ = self.child.wait().await; + self.control_reader.abort(); + let _ = (&mut self.control_reader).await; + } +} + +struct PendingWorker { + child: Child, + control_tx: ControlSender, + control_rx: ControlReceiver, +} + +impl PendingWorker { + async fn recv_event(&mut self) -> Result { + tokio::select! { + status = self.child.wait() => { + let status = status.context("reap ledger worker during startup")?; + bail!("ledger worker exited before readiness: {status}") + } + frame = self.control_rx.recv() => { + let frame = frame.context("receive ledger startup event")?; + if !frame.fds.is_empty() { + bail!("ledger worker returned a descriptor during startup"); + } + decode_ledger_control_event(&frame.bytes).map_err(anyhow::Error::from) + } + } + } + + async fn kill_reap(&mut self) { + let _ = self.child.start_kill(); + let _ = self.child.wait().await; + } +} + +async fn supervise( + mut process: WorkerProcess, + mut requests: mpsc::Receiver, + stopped: watch::Sender>, +) { + let reason = loop { + tokio::select! { + status = process.child.wait() => { + break match status { + Ok(status) => format!("ledger worker exited unexpectedly: {status}"), + Err(error) => format!("ledger worker exit wait failed: {error}"), + }; + } + event = process.control_events.recv() => { + match event { + Some(Ok(frame)) if frame.fds.is_empty() => match decode_ledger_control_event(&frame.bytes) { + Ok(LedgerControlEvent::Closed { generation, client_id, reason }) + if generation == process.generation => { + tracing::debug!(client_id, ?reason, "ledger client channel closed"); + } + Ok(event) => break format!("ledger worker sent unsolicited control event {event:?}"), + Err(error) => break format!("ledger worker sent invalid control event: {error}"), + }, + Some(Ok(_)) => break "ledger worker sent an unsolicited descriptor".to_string(), + Some(Err(error)) => break format!("ledger worker control channel failed: {error}"), + None => break "ledger worker control event pump stopped".to_string(), + } + } + request = requests.recv() => { + let Some(request) = request else { + break match process.shutdown().await { + Ok(()) => "ledger worker stopped after its owner released it".to_string(), + Err(error) => format!("ledger worker release failed: {error:#}"), + }; + }; + match request { + CommandRequest::Connect { role, completed } => { + let result = process.connect(role).await; + let failed = result.is_err(); + let diagnostic = result.as_ref().err().map(|error| format!("{error:#}")); + let _ = completed.send(result); + if failed { + break diagnostic.expect("failed result has diagnostic"); + } + } + CommandRequest::Shutdown { completed } => { + let result = process.shutdown().await; + let reason = result + .as_ref() + .err() + .map_or_else(|| "ledger worker stopped by coordinator".to_string(), |error| format!("{error:#}")); + let _ = completed.send(result); + break reason; + } + } + } + } + }; + process.kill_reap().await; + fail_pending(&mut requests, &reason); + let _ = stopped.send(Some(reason)); +} + +async fn read_control(receiver: ControlReceiver, events: mpsc::Sender>) { + loop { + let event = receiver.recv().await; + let complete = event.is_err(); + if events.send(event).await.is_err() || complete { + return; + } + } +} + +fn fail_pending(requests: &mut mpsc::Receiver, reason: &str) { + while let Ok(request) = requests.try_recv() { + match request { + CommandRequest::Connect { completed, .. } => { + let _ = completed.send(Err(anyhow!(reason.to_string()))); + } + CommandRequest::Shutdown { completed } => { + let _ = completed.send(Err(anyhow!(reason.to_string()))); + } + } + } +} + +fn fresh_generation() -> LedgerGeneration { + LedgerGeneration::new(*uuid::Uuid::new_v4().as_bytes()) +} + +fn generation_hex(generation: LedgerGeneration) -> String { + let mut encoded = String::with_capacity(32); + for byte in generation.as_bytes() { + write!(&mut encoded, "{byte:02x}").expect("write ledger generation to string"); + } + encoded +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/ledger_worker/tests.rs b/crates/capsem-service/src/ledger_worker/tests.rs new file mode 100644 index 000000000..1c442c80e --- /dev/null +++ b/crates/capsem-service/src/ledger_worker/tests.rs @@ -0,0 +1,194 @@ +use std::os::fd::AsFd as _; +use std::sync::Arc; + +use capsem_proto::ledger::LedgerClientRole; +use capsem_proto::ledger_control::{ + decode_ledger_control_request, encode_ledger_control_event, LedgerClientCloseReason, +}; + +use super::*; + +const GENERATION: LedgerGeneration = LedgerGeneration::new([7; 16]); + +fn decode_generation(value: &str) -> LedgerGeneration { + let mut bytes = [0; 16]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16).unwrap(); + } + LedgerGeneration::new(bytes) +} + +#[tokio::test] +async fn fake_ledger_worker_child() { + let Ok(mode) = std::env::var("CAPSEM_FAKE_LEDGER_WORKER") else { + return; + }; + if mode == "refuse" { + std::process::exit(42); + } + let generation = decode_generation(&std::env::var("CAPSEM_FAKE_LEDGER_GENERATION").unwrap()); + let announced = if mode == "stale_ready" { + LedgerGeneration::new([8; 16]) + } else { + generation + }; + let control = UnixStream::from(capsem_foundation::unix::fd::duplicate(std::io::stdin().as_fd()).unwrap()); + let control_tx = Arc::new(ControlSender::new(control.try_clone().unwrap()).unwrap()); + let control_rx = ControlReceiver::new(control).unwrap(); + control_tx + .send( + &encode_ledger_control_event(LedgerControlEvent::Ready { generation: announced }), + &[], + ) + .await + .unwrap(); + if mode == "stale_ready" { + std::future::pending::<()>().await; + } + + let mut held = Vec::new(); + loop { + let frame = control_rx.recv().await.unwrap(); + match decode_ledger_control_request(&frame.bytes).unwrap() { + LedgerControlRequest::Attach(grant) => { + if mode == "die_with_attach_pending" { + std::process::exit(43); + } + let stream = UnixStream::from(frame.fds.into_iter().next().unwrap()); + control_tx + .send( + &encode_ledger_control_event(LedgerControlEvent::Adopted { + generation, + client_id: grant.client_id(), + }), + &[], + ) + .await + .unwrap(); + if mode == "close_clients" { + drop(stream); + control_tx + .send( + &encode_ledger_control_event(LedgerControlEvent::Closed { + generation, + client_id: grant.client_id(), + reason: LedgerClientCloseReason::Disconnected, + }), + &[], + ) + .await + .unwrap(); + } else { + held.push(stream); + } + } + LedgerControlRequest::Shutdown { .. } if mode == "stall_shutdown" => { + std::future::pending::<()>().await; + } + LedgerControlRequest::Shutdown { .. } => { + control_tx + .send( + &encode_ledger_control_event(LedgerControlEvent::Stopped { generation }), + &[], + ) + .await + .unwrap(); + return; + } + } + } +} + +async fn fake(mode: &'static str) -> Result { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("session.db"); + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "ledger_worker::tests::fake_ledger_worker_child", + "--nocapture", + ]) + .env_clear() + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + launch(command, &database, GENERATION, move |command, generation, _database| { + command + .env("CAPSEM_FAKE_LEDGER_WORKER", mode) + .env("CAPSEM_FAKE_LEDGER_GENERATION", generation_hex(generation)); + }) + .await +} + +#[tokio::test] +async fn supervisor_mints_unique_role_bound_channels_and_reaps() { + let worker = fake("normal").await.unwrap(); + assert_eq!(worker.generation(), GENERATION); + let mut stopped = worker.stop_receiver(); + let reader = worker.connect(LedgerClientRole::Reader).await.unwrap(); + let producer = worker.connect(LedgerClientRole::VmOwner).await.unwrap(); + assert_eq!(reader.grant().generation(), GENERATION); + assert_eq!(reader.grant().role(), LedgerClientRole::Reader); + assert_eq!(producer.grant().role(), LedgerClientRole::VmOwner); + assert_ne!(reader.grant().client_id(), producer.grant().client_id()); + let (reader_stream, reader_grant) = reader.into_parts(); + assert_eq!(reader_grant.role(), LedgerClientRole::Reader); + drop(reader_stream); + drop(producer); + worker.shutdown().await.unwrap(); + stopped.changed().await.unwrap(); + assert!(stopped.borrow().as_deref().unwrap().contains("coordinator")); +} + +#[test] +fn generated_worker_lifetimes_are_fresh_and_nonzero() { + assert_ne!(fresh_generation(), fresh_generation()); +} + +#[tokio::test] +async fn client_close_events_do_not_kill_the_worker_supervisor() { + let worker = fake("close_clients").await.unwrap(); + drop(worker.connect(LedgerClientRole::Reader).await.unwrap()); + tokio::task::yield_now().await; + let next = worker.connect(LedgerClientRole::Maintainer).await.unwrap(); + assert_eq!(next.grant().client_id(), 2); + worker.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn startup_refusal_and_stale_readiness_never_return_authority() { + for mode in ["refuse", "stale_ready"] { + let error = fake(mode).await.err().expect("invalid startup must fail"); + let diagnostic = format!("{error:#}"); + assert!( + diagnostic.contains("before readiness") || diagnostic.contains("stale generation"), + "{diagnostic}" + ); + } +} + +#[tokio::test] +async fn death_fails_current_and_queued_channel_grants_explicitly() { + let worker = fake("die_with_attach_pending").await.unwrap(); + let first = worker.clone(); + let second = worker.clone(); + let (first, second) = tokio::join!( + first.connect(LedgerClientRole::Reader), + second.connect(LedgerClientRole::Maintainer) + ); + for result in [first, second] { + let error = result.expect_err("worker death must fail every pending grant"); + let diagnostic = format!("{error:#}"); + assert!(diagnostic.contains("ledger worker"), "{diagnostic}"); + } +} + +#[tokio::test] +async fn stalled_shutdown_is_killed_and_reaped_within_the_bound() { + let worker = fake("stall_shutdown").await.unwrap(); + let started = std::time::Instant::now(); + let error = worker.shutdown().await.unwrap_err(); + assert!(format!("{error:#}").contains("bounded shutdown timed out"), "{error:#}"); + assert!(started.elapsed() < Duration::from_secs(2)); +} diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 81b91ba53..50cac2c7e 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -43,8 +43,7 @@ mod owner_connection; mod owner_handoff; mod policy_mutation; use policy_mutation::{apply_policy_mutation, MutationRoute, PolicyMutation}; -mod connection_routes; -mod credential_routes; +mod ledger_worker; mod mcp_routes; mod plugin_routes; mod private_routes; From 5f28073aa8f514b9cc041e870561c9bcc0d7f294 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:56:33 +0000 Subject: [PATCH 113/464] security(service): serialize ledger worker lifetimes --- crates/capsem-service/src/ledger_worker.rs | 141 ++++++++++++++++-- .../capsem-service/src/ledger_worker/tests.rs | 132 ++++++++++++++++ crates/capsem-service/src/main.rs | 7 + crates/capsem-service/src/service_runtime.rs | 4 + crates/capsem-service/src/tests.rs | 3 + 5 files changed, 278 insertions(+), 9 deletions(-) diff --git a/crates/capsem-service/src/ledger_worker.rs b/crates/capsem-service/src/ledger_worker.rs index 4286a77e7..81c225946 100644 --- a/crates/capsem-service/src/ledger_worker.rs +++ b/crates/capsem-service/src/ledger_worker.rs @@ -12,8 +12,9 @@ use std::fmt::Write as _; use std::io; use std::os::fd::{AsRawFd as _, OwnedFd}; use std::os::unix::net::UnixStream; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Stdio; +use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; @@ -44,6 +45,126 @@ type ControlSender = DescriptorSender; type ControlFrame = capsem_foundation::unix::router_channel::DescriptorFrame; +struct LedgerSlot { + database: PathBuf, + worker: LedgerWorker, +} + +/// Service-owned table that serializes each session's ledger lifetime. +pub(crate) struct LedgerWorkers { + binary: PathBuf, + slots: tokio::sync::Mutex>, +} + +impl LedgerWorkers { + pub(crate) fn new(binary: PathBuf) -> Self { + Self { + binary, + slots: tokio::sync::Mutex::new(std::collections::HashMap::new()), + } + } + + /// Acquire a role-bound channel from the one worker for `session_id`. + /// The slot lock deliberately spans readiness: no replacement generation + /// can start before a stopped generation has been fully reaped. + pub(crate) async fn acquire( + self: &Arc, + session_id: &str, + database: &Path, + log_path: &Path, + role: LedgerClientRole, + ) -> Result { + let worker = { + let mut slots = self.slots.lock().await; + let mut watcher = None; + if slots + .get(session_id) + .is_some_and(|slot| slot.worker.stop_receiver().borrow().is_some() && slot.database == database) + { + slots.remove(session_id); + } + let worker = if let Some(slot) = slots.get(session_id) { + if slot.database != database { + bail!( + "ledger worker slot for {session_id} owns {}, not {}", + slot.database.display(), + database.display() + ); + } + slot.worker.clone() + } else { + let worker = self.spawn(database, log_path).await?; + slots.insert( + session_id.to_string(), + LedgerSlot { + database: database.to_path_buf(), + worker: worker.clone(), + }, + ); + watcher = Some((worker.generation(), worker.stop_receiver())); + worker + }; + drop(slots); + if let Some((generation, mut stopped)) = watcher { + let registry = Arc::clone(self); + let watched_session_id = session_id.to_string(); + tokio::spawn(async move { + if stopped.borrow().is_none() { + let _ = stopped.changed().await; + } + let mut slots = registry.slots.lock().await; + if slots + .get(&watched_session_id) + .is_some_and(|slot| slot.worker.generation() == generation) + { + slots.remove(&watched_session_id); + } + }); + } + worker + }; + worker.connect(role).await + } + + /// Stop and reap the current generation before vacating its slot. + pub(crate) async fn shutdown(&self, session_id: &str) -> Result<()> { + let Some(worker) = self.slots.lock().await.get(session_id).map(|slot| slot.worker.clone()) else { + return Ok(()); + }; + let generation = worker.generation(); + let result = worker.shutdown().await; + let mut slots = self.slots.lock().await; + if slots + .get(session_id) + .is_some_and(|slot| slot.worker.generation() == generation) + { + slots.remove(session_id); + } + result + } + + async fn spawn(&self, database: &Path, log_path: &Path) -> Result { + let log = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .await + .with_context(|| format!("open {}", log_path.display()))? + .into_std() + .await; + LedgerWorker::spawn(&self.binary, database, Stdio::from(log.try_clone()?), Stdio::from(log)).await + } + + #[cfg(test)] + async fn generation(&self, session_id: &str) -> Option { + self.slots + .lock() + .await + .get(session_id) + .map(|slot| slot.worker.generation()) + } +} + #[derive(Debug)] pub(crate) struct LedgerClient { stream: UnixStream, @@ -68,10 +189,6 @@ pub(crate) struct LedgerWorker { } impl LedgerWorker { - #[cfg_attr( - test, - expect(dead_code, reason = "production launch is covered by capsem-ledger subprocess tests") - )] pub(crate) async fn spawn(binary: &Path, database: &Path, stdout: Stdio, stderr: Stdio) -> Result { let generation = fresh_generation(); let mut command = Command::new(binary); @@ -138,7 +255,7 @@ struct WorkerProcess { child: Child, control_tx: ControlSender, control_events: mpsc::Receiver>, - control_reader: tokio::task::JoinHandle<()>, + control_reader: Option>, next_client_id: u64, } @@ -212,7 +329,7 @@ impl WorkerProcess { child, control_tx, control_events, - control_reader, + control_reader: Some(control_reader), next_client_id: 1, }) } @@ -311,8 +428,10 @@ impl WorkerProcess { async fn kill_reap(&mut self) { let _ = self.child.start_kill(); let _ = self.child.wait().await; - self.control_reader.abort(); - let _ = (&mut self.control_reader).await; + if let Some(mut control_reader) = self.control_reader.take() { + control_reader.abort(); + let _ = (&mut control_reader).await; + } } } @@ -392,6 +511,10 @@ async fn supervise( } CommandRequest::Shutdown { completed } => { let result = process.shutdown().await; + // Completion is the replacement barrier: even a send + // or protocol failure is reaped before the coordinator + // is allowed to vacate this generation's slot. + process.kill_reap().await; let reason = result .as_ref() .err() diff --git a/crates/capsem-service/src/ledger_worker/tests.rs b/crates/capsem-service/src/ledger_worker/tests.rs index 1c442c80e..b1a16f47d 100644 --- a/crates/capsem-service/src/ledger_worker/tests.rs +++ b/crates/capsem-service/src/ledger_worker/tests.rs @@ -1,4 +1,5 @@ use std::os::fd::AsFd as _; +use std::os::unix::fs::PermissionsExt as _; use std::sync::Arc; use capsem_proto::ledger::LedgerClientRole; @@ -121,6 +122,37 @@ async fn fake(mode: &'static str) -> Result { .await } +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\"'\"'")) +} + +fn fake_binary(directory: &Path, mode: &str) -> PathBuf { + let binary = directory.join(format!("capsem-ledger-{mode}")); + write_fake_binary(&binary, mode); + binary +} + +fn write_fake_binary(binary: &Path, mode: &str) { + let test_executable = std::env::current_exe().unwrap(); + let generations = binary.with_extension("generations"); + let script = format!( + "#!/bin/sh\ngeneration=\nwhile [ \"$#\" -gt 0 ]; do\n if [ \"$1\" = \"--generation\" ]; then\n shift\n generation=$1\n fi\n shift\ndone\nprintf '%s\\n' \"$generation\" >> {}\nCAPSEM_FAKE_LEDGER_WORKER={} CAPSEM_FAKE_LEDGER_GENERATION=\"$generation\" exec {} --exact ledger_worker::tests::fake_ledger_worker_child --nocapture\n", + shell_quote(&generations.to_string_lossy()), + shell_quote(mode), + shell_quote(&test_executable.to_string_lossy()), + ); + std::fs::write(binary, script).unwrap(); + let mut permissions = std::fs::metadata(binary).unwrap().permissions(); + permissions.set_mode(0o755); + std::fs::set_permissions(binary, permissions).unwrap(); +} + +fn session_paths(root: &Path, name: &str) -> (PathBuf, PathBuf) { + let directory = root.join(name); + std::fs::create_dir(&directory).unwrap(); + (directory.join("session.db"), directory.join("ledger.log")) +} + #[tokio::test] async fn supervisor_mints_unique_role_bound_channels_and_reaps() { let worker = fake("normal").await.unwrap(); @@ -192,3 +224,103 @@ async fn stalled_shutdown_is_killed_and_reaped_within_the_bound() { assert!(format!("{error:#}").contains("bounded shutdown timed out"), "{error:#}"); assert!(started.elapsed() < Duration::from_secs(2)); } + +#[tokio::test] +async fn lifecycle_slot_serializes_concurrent_leases_and_replacement() { + let directory = tempfile::tempdir().unwrap(); + let binary = fake_binary(directory.path(), "normal"); + let workers = Arc::new(LedgerWorkers::new(binary)); + let (database, log) = session_paths(directory.path(), "session-a"); + + let (reader, maintainer) = tokio::join!( + workers.acquire("a", &database, &log, LedgerClientRole::Reader), + workers.acquire("a", &database, &log, LedgerClientRole::Maintainer), + ); + let reader = reader.unwrap(); + let maintainer = maintainer.unwrap(); + assert_eq!(reader.grant().generation(), maintainer.grant().generation()); + assert_ne!(reader.grant().client_id(), maintainer.grant().client_id()); + let first_generation = reader.grant().generation(); + + let (other_database, other_log) = session_paths(directory.path(), "rebound"); + let error = workers + .acquire("a", &other_database, &other_log, LedgerClientRole::Reader) + .await + .unwrap_err(); + assert!(format!("{error:#}").contains("owns"), "{error:#}"); + + drop(reader); + drop(maintainer); + workers.shutdown("a").await.unwrap(); + assert_eq!(workers.generation("a").await, None); + let replacement = workers + .acquire("a", &database, &log, LedgerClientRole::Reader) + .await + .unwrap(); + assert_ne!(replacement.grant().generation(), first_generation); + drop(replacement); + workers.shutdown("a").await.unwrap(); +} + +#[tokio::test] +async fn stopping_one_session_leaves_an_unrelated_session_generation_alive() { + let directory = tempfile::tempdir().unwrap(); + let binary = fake_binary(directory.path(), "normal"); + let workers = Arc::new(LedgerWorkers::new(binary)); + let (database_a, log_a) = session_paths(directory.path(), "session-a"); + let (database_b, log_b) = session_paths(directory.path(), "session-b"); + let client_a = workers + .acquire("a", &database_a, &log_a, LedgerClientRole::Reader) + .await + .unwrap(); + let client_b = workers + .acquire("b", &database_b, &log_b, LedgerClientRole::Reader) + .await + .unwrap(); + let generation_b = client_b.grant().generation(); + + drop(client_a); + workers.shutdown("a").await.unwrap(); + let second_b = workers + .acquire("b", &database_b, &log_b, LedgerClientRole::Maintainer) + .await + .unwrap(); + assert_eq!(second_b.grant().generation(), generation_b); + + drop(client_b); + drop(second_b); + workers.shutdown("b").await.unwrap(); +} + +#[tokio::test] +async fn crashed_slot_restarts_only_after_reap_with_a_fresh_generation() { + let directory = tempfile::tempdir().unwrap(); + let binary = fake_binary(directory.path(), "die_with_attach_pending"); + let workers = Arc::new(LedgerWorkers::new(binary.clone())); + let (database, log) = session_paths(directory.path(), "stopped-session"); + + let error = workers + .acquire("stopped", &database, &log, LedgerClientRole::Reader) + .await + .unwrap_err(); + assert!(format!("{error:#}").contains("ledger worker"), "{error:#}"); + tokio::time::timeout(Duration::from_secs(2), async { + while workers.generation("stopped").await.is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("crashed generation was not reaped and removed"); + + write_fake_binary(&binary, "normal"); + let reader = workers + .acquire("stopped", &database, &log, LedgerClientRole::Reader) + .await + .unwrap(); + let generations = std::fs::read_to_string(binary.with_extension("generations")).unwrap(); + let generations: Vec<_> = generations.lines().collect(); + assert_eq!(generations.len(), 2); + assert_ne!(generations[0], generations[1]); + drop(reader); + workers.shutdown("stopped").await.unwrap(); +} diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 50cac2c7e..fdd872c50 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -164,6 +164,8 @@ struct Args { #[arg(long)] proxy_binary: Option, #[arg(long)] + ledger_binary: Option, + #[arg(long)] gateway_binary: Option, #[arg(long)] gateway_port: Option, @@ -208,6 +210,11 @@ pub struct ServiceState { connections: connection_routes::Runtime, instances: Mutex>, // instance id to process info proxy_workers: Mutex>, + #[expect( + dead_code, + reason = "S09-004 connects the VM and proxy producers through this lifecycle table" + )] + ledger_workers: Arc, retirements: instance_reaper::Retirements, /// Logger-owned DB handles keyed by session/VM id. Logged-data routes /// resolve a handle here and call `ready/query`; they do not open SQLite diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index 338c45a56..7df344141 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -152,6 +152,9 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { .process_binary .unwrap_or_else(|| PathBuf::from("cache/target/cargo/debug/capsem-process")); let proxy_binary = args.proxy_binary.unwrap_or_else(|| find_sibling_binary("capsem-proxy")); + let ledger_binary = args + .ledger_binary + .unwrap_or_else(|| find_sibling_binary("capsem-ledger")); let assets_base_dir = args .assets_dir .unwrap_or_else(|| run_dir.parent().unwrap().join("assets")); @@ -260,6 +263,7 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { connections, instances: Mutex::new(HashMap::new()), proxy_workers: Mutex::new(HashMap::new()), + ledger_workers: Arc::new(ledger_worker::LedgerWorkers::new(ledger_binary)), retirements: Default::default(), session_db_handles: Mutex::new(HashMap::new()), persistent_registry: SharedRegistry::new(persistent_registry), diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index e8aeed825..662ffd0fb 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -56,6 +56,9 @@ pub(crate) fn test_state( connections, instances: Mutex::new(HashMap::new()), proxy_workers: Mutex::new(HashMap::new()), + ledger_workers: Arc::new(ledger_worker::LedgerWorkers::new(PathBuf::from( + "/nonexistent/capsem-ledger", + ))), retirements: Default::default(), session_db_handles: Mutex::new(HashMap::new()), persistent_registry: SharedRegistry::new( From e068dcfd2ec12382aebc8f6bca203ab61a41aaee Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 04:59:44 +0000 Subject: [PATCH 114/464] security(install): pin ledger worker path --- CHANGELOG.md | 6 +- crates/capsem/src/client.rs | 2 + crates/capsem/src/client/tests.rs | 5 + crates/capsem/src/service_install.rs | 101 ++++++++++------- crates/capsem/src/service_install/tests.rs | 120 +++++++++++++-------- 5 files changed, 148 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fb935919..6576b6635 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security -- Pass the packaged proxy worker path explicitly to installed and direct - services, refuse incomplete confined-worker cohorts, and retire ledger and - proxy children during reinstall or uninstall. +- Pass the packaged ledger and proxy worker paths explicitly to installed and + direct services, refuse incomplete confined-worker cohorts, and retire both + children during reinstall or uninstall. - Ship the confined ledger and proxy workers in every supported package and sign them without macOS virtualization authority, so clean installs cannot diff --git a/crates/capsem/src/client.rs b/crates/capsem/src/client.rs index 652471536..5176d13ab 100644 --- a/crates/capsem/src/client.rs +++ b/crates/capsem/src/client.rs @@ -303,6 +303,8 @@ fn direct_spawn_service_args( paths.assets_dir.clone().into_os_string(), "--process-binary".into(), paths.process_bin.clone().into_os_string(), + "--ledger-binary".into(), + paths.ledger_bin.clone().into_os_string(), "--proxy-binary".into(), paths.proxy_bin.clone().into_os_string(), ]; diff --git a/crates/capsem/src/client/tests.rs b/crates/capsem/src/client/tests.rs index 49e1b32a1..adf2b0aa6 100644 --- a/crates/capsem/src/client/tests.rs +++ b/crates/capsem/src/client/tests.rs @@ -817,6 +817,11 @@ fn isolated_direct_spawn_uses_an_ephemeral_gateway_port() { .any(|pair| pair == ["--proxy-binary", "/opt/capsem/bin/capsem-proxy"]), "direct service must use its sibling confined proxy: {args:?}" ); + assert!( + args.windows(2) + .any(|pair| pair == ["--ledger-binary", "/opt/capsem/bin/capsem-ledger"]), + "direct service must use its sibling confined ledger: {args:?}" + ); } #[test] diff --git a/crates/capsem/src/service_install.rs b/crates/capsem/src/service_install.rs index 9e9269962..c557818f7 100644 --- a/crates/capsem/src/service_install.rs +++ b/crates/capsem/src/service_install.rs @@ -50,25 +50,48 @@ pub struct ServiceStatus { pub unit_path: Option, } +#[derive(Clone, Copy)] +pub struct ServiceBinaries<'a> { + service: &'a Path, + process: &'a Path, + ledger: &'a Path, + proxy: &'a Path, + gateway: &'a Path, + tray: &'a Path, +} + +impl<'a> ServiceBinaries<'a> { + pub const fn new( + service: &'a Path, + process: &'a Path, + ledger: &'a Path, + proxy: &'a Path, + gateway: &'a Path, + tray: &'a Path, + ) -> Self { + Self { + service, + process, + ledger, + proxy, + gateway, + tray, + } + } +} + /// Generate a macOS LaunchAgent plist for capsem-service. /// /// All paths are absolute and XML-escaped for safe embedding. #[cfg_attr(not(target_os = "macos"), allow(dead_code))] -pub fn generate_plist( - service_bin: &Path, - process_bin: &Path, - proxy_bin: &Path, - gateway_bin: &Path, - tray_bin: &Path, - assets_dir: &Path, - home: &str, -) -> String { +pub fn generate_plist(binaries: ServiceBinaries<'_>, assets_dir: &Path, home: &str) -> String { let log_dir = xml_escape(&format!("{}/Library/Logs/capsem", home)); - let service_bin = xml_escape(&service_bin.display().to_string()); - let process_bin = xml_escape(&process_bin.display().to_string()); - let proxy_bin = xml_escape(&proxy_bin.display().to_string()); - let gateway_bin = xml_escape(&gateway_bin.display().to_string()); - let tray_bin = xml_escape(&tray_bin.display().to_string()); + let service_bin = xml_escape(&binaries.service.display().to_string()); + let process_bin = xml_escape(&binaries.process.display().to_string()); + let ledger_bin = xml_escape(&binaries.ledger.display().to_string()); + let proxy_bin = xml_escape(&binaries.proxy.display().to_string()); + let gateway_bin = xml_escape(&binaries.gateway.display().to_string()); + let tray_bin = xml_escape(&binaries.tray.display().to_string()); let assets_dir = xml_escape(&assets_dir.display().to_string()); let credential_store_path = xml_escape(&format!("{}/.capsem/credentials/credential-store.json", home)); format!( @@ -91,6 +114,8 @@ pub fn generate_plist( {assets_dir} --process-binary {process_bin} + --ledger-binary + {ledger_bin} --proxy-binary {proxy_bin} --gateway-binary @@ -116,26 +141,20 @@ pub fn generate_plist( /// /// All paths are absolute. Spaces are escaped with `\x20` per systemd syntax. #[cfg_attr(not(target_os = "linux"), allow(dead_code))] -pub fn generate_systemd_unit( - service_bin: &Path, - process_bin: &Path, - proxy_bin: &Path, - gateway_bin: &Path, - tray_bin: &Path, - assets_dir: &Path, -) -> String { - let service_bin = systemd_escape_path(service_bin); - let process_bin = systemd_escape_path(process_bin); - let proxy_bin = systemd_escape_path(proxy_bin); - let gateway_bin = systemd_escape_path(gateway_bin); - let tray_bin = systemd_escape_path(tray_bin); +pub fn generate_systemd_unit(binaries: ServiceBinaries<'_>, assets_dir: &Path) -> String { + let service_bin = systemd_escape_path(binaries.service); + let process_bin = systemd_escape_path(binaries.process); + let ledger_bin = systemd_escape_path(binaries.ledger); + let proxy_bin = systemd_escape_path(binaries.proxy); + let gateway_bin = systemd_escape_path(binaries.gateway); + let tray_bin = systemd_escape_path(binaries.tray); let assets_dir = systemd_escape_path(assets_dir); format!( r#"[Unit] Description=Capsem sandbox service [Service] -ExecStart={service_bin} --foreground --assets-dir {assets_dir} --process-binary {process_bin} --proxy-binary {proxy_bin} --gateway-binary {gateway_bin} --tray-binary {tray_bin} +ExecStart={service_bin} --foreground --assets-dir {assets_dir} --process-binary {process_bin} --ledger-binary {ledger_bin} --proxy-binary {proxy_bin} --gateway-binary {gateway_bin} --tray-binary {tray_bin} Restart=always RestartSec=2 @@ -580,11 +599,14 @@ async fn install_launchagent(capsem_paths: &paths::CapsemPaths, home: &str) -> R // Install service plist let plist_content = generate_plist( - &capsem_paths.service_bin, - &capsem_paths.process_bin, - &capsem_paths.proxy_bin, - &capsem_paths.gateway_bin, - &capsem_paths.tray_bin, + ServiceBinaries::new( + &capsem_paths.service_bin, + &capsem_paths.process_bin, + &capsem_paths.ledger_bin, + &capsem_paths.proxy_bin, + &capsem_paths.gateway_bin, + &capsem_paths.tray_bin, + ), &capsem_paths.assets_dir, home, ); @@ -663,11 +685,14 @@ async fn install_systemd_unit(capsem_paths: &paths::CapsemPaths, home: &str) -> std::fs::create_dir_all(&unit_dir).context("cannot create systemd user unit directory")?; let unit_content = generate_systemd_unit( - &capsem_paths.service_bin, - &capsem_paths.process_bin, - &capsem_paths.proxy_bin, - &capsem_paths.gateway_bin, - &capsem_paths.tray_bin, + ServiceBinaries::new( + &capsem_paths.service_bin, + &capsem_paths.process_bin, + &capsem_paths.ledger_bin, + &capsem_paths.proxy_bin, + &capsem_paths.gateway_bin, + &capsem_paths.tray_bin, + ), &capsem_paths.assets_dir, ); diff --git a/crates/capsem/src/service_install/tests.rs b/crates/capsem/src/service_install/tests.rs index 78eab47af..796ea25ba 100644 --- a/crates/capsem/src/service_install/tests.rs +++ b/crates/capsem/src/service_install/tests.rs @@ -37,17 +37,22 @@ fn service_install_refuses_missing_confined_workers() { #[test] fn test_generate_plist_absolute_paths() { let plist = generate_plist( - Path::new("/Users/test/.capsem/bin/capsem-service"), - Path::new("/Users/test/.capsem/bin/capsem-process"), - Path::new("/Users/test/.capsem/bin/capsem-proxy"), - Path::new("/Users/test/.capsem/bin/capsem-gateway"), - Path::new("/Users/test/.capsem/bin/capsem-tray"), + ServiceBinaries::new( + Path::new("/Users/test/.capsem/bin/capsem-service"), + Path::new("/Users/test/.capsem/bin/capsem-process"), + Path::new("/Users/test/.capsem/bin/capsem-ledger"), + Path::new("/Users/test/.capsem/bin/capsem-proxy"), + Path::new("/Users/test/.capsem/bin/capsem-gateway"), + Path::new("/Users/test/.capsem/bin/capsem-tray"), + ), Path::new("/Users/test/.capsem/assets"), "/Users/test", ); // ProgramArguments binary and path args must be absolute assert!(plist.contains("/Users/test/.capsem/bin/capsem-service")); assert!(plist.contains("/Users/test/.capsem/bin/capsem-process")); + assert!(plist.contains("--ledger-binary")); + assert!(plist.contains("/Users/test/.capsem/bin/capsem-ledger")); assert!(plist.contains("--proxy-binary")); assert!(plist.contains("/Users/test/.capsem/bin/capsem-proxy")); assert!(plist.contains("/Users/test/.capsem/assets")); @@ -60,11 +65,14 @@ fn test_generate_plist_absolute_paths() { #[test] fn test_generate_plist_valid_xml() { let plist = generate_plist( - Path::new("/usr/local/bin/capsem-service"), - Path::new("/usr/local/bin/capsem-process"), - Path::new("/usr/local/bin/capsem-proxy"), - Path::new("/usr/local/bin/capsem-gateway"), - Path::new("/usr/local/bin/capsem-tray"), + ServiceBinaries::new( + Path::new("/usr/local/bin/capsem-service"), + Path::new("/usr/local/bin/capsem-process"), + Path::new("/usr/local/bin/capsem-ledger"), + Path::new("/usr/local/bin/capsem-proxy"), + Path::new("/usr/local/bin/capsem-gateway"), + Path::new("/usr/local/bin/capsem-tray"), + ), Path::new("/home/test/.capsem/assets"), "/home/test", ); @@ -80,11 +88,14 @@ fn test_generate_plist_valid_xml() { #[test] fn test_generate_plist_has_keep_alive() { let plist = generate_plist( - Path::new("/bin/capsem-service"), - Path::new("/bin/capsem-process"), - Path::new("/bin/capsem-proxy"), - Path::new("/bin/capsem-gateway"), - Path::new("/bin/capsem-tray"), + ServiceBinaries::new( + Path::new("/bin/capsem-service"), + Path::new("/bin/capsem-process"), + Path::new("/bin/capsem-ledger"), + Path::new("/bin/capsem-proxy"), + Path::new("/bin/capsem-gateway"), + Path::new("/bin/capsem-tray"), + ), Path::new("/assets"), "/home", ); @@ -96,11 +107,14 @@ fn test_generate_plist_has_keep_alive() { #[test] fn test_generate_plist_pins_file_backed_credential_store() { let plist = generate_plist( - Path::new("/Users/test/.capsem/bin/capsem-service"), - Path::new("/Users/test/.capsem/bin/capsem-process"), - Path::new("/Users/test/.capsem/bin/capsem-proxy"), - Path::new("/Users/test/.capsem/bin/capsem-gateway"), - Path::new("/Users/test/.capsem/bin/capsem-tray"), + ServiceBinaries::new( + Path::new("/Users/test/.capsem/bin/capsem-service"), + Path::new("/Users/test/.capsem/bin/capsem-process"), + Path::new("/Users/test/.capsem/bin/capsem-ledger"), + Path::new("/Users/test/.capsem/bin/capsem-proxy"), + Path::new("/Users/test/.capsem/bin/capsem-gateway"), + Path::new("/Users/test/.capsem/bin/capsem-tray"), + ), Path::new("/Users/test/.capsem/assets"), "/Users/test", ); @@ -154,11 +168,14 @@ fn macos_stop_uses_bootout_so_keepalive_does_not_restart_service() { #[test] fn test_generate_systemd_unit_absolute_paths() { let unit = generate_systemd_unit( - Path::new("/home/test/.capsem/bin/capsem-service"), - Path::new("/home/test/.capsem/bin/capsem-process"), - Path::new("/home/test/.capsem/bin/capsem-proxy"), - Path::new("/home/test/.capsem/bin/capsem-gateway"), - Path::new("/home/test/.capsem/bin/capsem-tray"), + ServiceBinaries::new( + Path::new("/home/test/.capsem/bin/capsem-service"), + Path::new("/home/test/.capsem/bin/capsem-process"), + Path::new("/home/test/.capsem/bin/capsem-ledger"), + Path::new("/home/test/.capsem/bin/capsem-proxy"), + Path::new("/home/test/.capsem/bin/capsem-gateway"), + Path::new("/home/test/.capsem/bin/capsem-tray"), + ), Path::new("/home/test/.capsem/assets"), ); // ExecStart line should have absolute path @@ -170,6 +187,7 @@ fn test_generate_systemd_unit_absolute_paths() { ); // --process-binary value should be absolute assert!(exec_line.contains("--process-binary /")); + assert!(exec_line.contains("--ledger-binary /")); assert!(exec_line.contains("--proxy-binary /")); // --assets-dir value should be absolute assert!(exec_line.contains("--assets-dir /")); @@ -178,11 +196,14 @@ fn test_generate_systemd_unit_absolute_paths() { #[test] fn test_generate_systemd_unit_restart_policy() { let unit = generate_systemd_unit( - Path::new("/bin/svc"), - Path::new("/bin/proc"), - Path::new("/bin/proxy"), - Path::new("/bin/gw"), - Path::new("/bin/tray"), + ServiceBinaries::new( + Path::new("/bin/svc"), + Path::new("/bin/proc"), + Path::new("/bin/ledger"), + Path::new("/bin/proxy"), + Path::new("/bin/gw"), + Path::new("/bin/tray"), + ), Path::new("/assets"), ); assert!(unit.contains("Restart=always")); @@ -192,11 +213,14 @@ fn test_generate_systemd_unit_restart_policy() { #[test] fn test_generate_systemd_unit_wanted_by() { let unit = generate_systemd_unit( - Path::new("/bin/svc"), - Path::new("/bin/proc"), - Path::new("/bin/proxy"), - Path::new("/bin/gw"), - Path::new("/bin/tray"), + ServiceBinaries::new( + Path::new("/bin/svc"), + Path::new("/bin/proc"), + Path::new("/bin/ledger"), + Path::new("/bin/proxy"), + Path::new("/bin/gw"), + Path::new("/bin/tray"), + ), Path::new("/assets"), ); assert!(unit.contains("[Install]")); @@ -223,11 +247,14 @@ fn test_xml_escape_angle_brackets() { #[test] fn test_plist_with_special_chars_in_path() { let plist = generate_plist( - Path::new("/Users/AT&T Corp/.capsem/bin/capsem-service"), - Path::new("/Users/AT&T Corp/.capsem/bin/capsem-process"), - Path::new("/Users/AT&T Corp/.capsem/bin/capsem-proxy"), - Path::new("/Users/AT&T Corp/.capsem/bin/capsem-gateway"), - Path::new("/Users/AT&T Corp/.capsem/bin/capsem-tray"), + ServiceBinaries::new( + Path::new("/Users/AT&T Corp/.capsem/bin/capsem-service"), + Path::new("/Users/AT&T Corp/.capsem/bin/capsem-process"), + Path::new("/Users/AT&T Corp/.capsem/bin/capsem-ledger"), + Path::new("/Users/AT&T Corp/.capsem/bin/capsem-proxy"), + Path::new("/Users/AT&T Corp/.capsem/bin/capsem-gateway"), + Path::new("/Users/AT&T Corp/.capsem/bin/capsem-tray"), + ), Path::new("/Users/AT&T Corp/.capsem/assets"), "/Users/AT&T Corp", ); @@ -257,11 +284,14 @@ fn test_systemd_escape_path_with_spaces() { #[test] fn test_systemd_unit_with_spaces_in_path() { let unit = generate_systemd_unit( - Path::new("/home/John Doe/.capsem/bin/capsem-service"), - Path::new("/home/John Doe/.capsem/bin/capsem-process"), - Path::new("/home/John Doe/.capsem/bin/capsem-proxy"), - Path::new("/home/John Doe/.capsem/bin/capsem-gateway"), - Path::new("/home/John Doe/.capsem/bin/capsem-tray"), + ServiceBinaries::new( + Path::new("/home/John Doe/.capsem/bin/capsem-service"), + Path::new("/home/John Doe/.capsem/bin/capsem-process"), + Path::new("/home/John Doe/.capsem/bin/capsem-ledger"), + Path::new("/home/John Doe/.capsem/bin/capsem-proxy"), + Path::new("/home/John Doe/.capsem/bin/capsem-gateway"), + Path::new("/home/John Doe/.capsem/bin/capsem-tray"), + ), Path::new("/home/John Doe/.capsem/assets"), ); let exec_line = unit.lines().find(|l| l.starts_with("ExecStart=")).unwrap(); From 24f804b1fee26b7530017309ef5bf6ec948fa25b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:06:45 +0000 Subject: [PATCH 115/464] feat(logger): write through ledger descriptors --- crates/capsem-logger/src/writer.rs | 38 ++ crates/capsem-logger/src/writer/producer.rs | 15 + crates/capsem-logger/src/writer/remote.rs | 342 ++++++++++++++++++ .../capsem-logger/src/writer/remote/tests.rs | 180 +++++++++ 4 files changed, 575 insertions(+) create mode 100644 crates/capsem-logger/src/writer/remote.rs create mode 100644 crates/capsem-logger/src/writer/remote/tests.rs diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 9470a4624..2b76dac2d 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -291,6 +291,7 @@ mod barriers; mod batch; mod operation; mod recording; +mod remote; mod retention; mod retention_faults; mod writer_lock; @@ -326,6 +327,7 @@ pub struct DbWriter { /// Published by the writer thread after every batch so a test can prove /// the bound without a second view of the thread's state. pending_body_bytes: Arc, + remote: Option, } impl DbWriter { @@ -428,6 +430,7 @@ impl DbWriter { join_handle: std::sync::Mutex::new(Some(join_handle)), db_path, pending_body_bytes, + remote: None, }) } @@ -464,6 +467,25 @@ impl DbWriter { join_handle: std::sync::Mutex::new(Some(join_handle)), db_path: PathBuf::from(":memory:"), pending_body_bytes, + remote: None, + }) + } + + /// Construct the familiar producer facade over a coordinator-minted + /// channel. This process retains no SQLite connection or writer lock; the + /// dedicated ledger worker owns both. + pub fn from_ledger_channel( + stream: std::os::unix::net::UnixStream, + grant: capsem_proto::ledger::LedgerChannelGrant, + logical_path: &Path, + capacity: usize, + ) -> std::io::Result { + Ok(Self { + tx: std::sync::Mutex::new(None), + join_handle: std::sync::Mutex::new(None), + db_path: logical_path.to_path_buf(), + pending_body_bytes: Arc::new(AtomicU64::new(0)), + remote: Some(remote::RemoteWriter::start(stream, grant, capacity)?), }) } @@ -480,6 +502,9 @@ impl DbWriter { /// Every reader reads the file, so an `Err` means no reader will see the /// rows yet, and the caller must not claim otherwise. pub async fn flush_checked(&self) -> Result<(), String> { + if let Some(remote) = &self.remote { + return remote.flush().await; + } let Some(tx) = self.clone_sender() else { return Ok(()); }; @@ -502,6 +527,9 @@ impl DbWriter { /// halves of the ledger. A caller that holds no writer holds no right to /// rewrite either one. pub async fn retain_bodies_since(&self, cutoff: &str) -> Result { + if let Some(remote) = &self.remote { + return remote.retain(cutoff).await; + } let Some(tx) = self.clone_sender() else { return Err("db writer is shut down; nothing was retained".to_string()); }; @@ -537,6 +565,10 @@ impl DbWriter { /// and runs the final `PRAGMA wal_checkpoint(TRUNCATE)`. Call from a /// blocking thread (e.g. via `tokio::task::spawn_blocking`). pub fn shutdown_blocking(&self) { + if let Some(remote) = &self.remote { + remote.shutdown(); + return; + } let _ = self.tx.lock().unwrap().take(); let handle = self.join_handle.lock().unwrap().take(); if let Some(handle) = handle { @@ -548,6 +580,9 @@ impl DbWriter { /// It sees what the writer has flushed, not what it has only accepted. /// Returns Err for in-memory writers (no file to share between connections). pub fn reader(&self) -> rusqlite::Result { + if self.remote.is_some() { + return Err(rusqlite::Error::InvalidPath(self.db_path.clone())); + } if self.db_path.to_str() == Some(":memory:") { return Err(rusqlite::Error::InvalidPath(self.db_path.clone())); } @@ -563,6 +598,9 @@ impl DbWriter { /// block but not yet written. They reach `session.bodies` at the next /// disk flush or when the block closes, so this is the backlog a crash would lose. pub fn pending_body_bytes(&self) -> u64 { + if self.remote.is_some() { + return 0; + } self.pending_body_bytes.load(Ordering::Acquire) } } diff --git a/crates/capsem-logger/src/writer/producer.rs b/crates/capsem-logger/src/writer/producer.rs index 8022da240..92065ca3a 100644 --- a/crates/capsem-logger/src/writer/producer.rs +++ b/crates/capsem-logger/src/writer/producer.rs @@ -24,6 +24,11 @@ impl DbWriter { queue_result = tracing::field::Empty, ); let started = Instant::now(); + if let Some(remote) = &self.remote { + let result = remote.admit(op).await; + record_enqueue(started, if result.is_ok() { "queued" } else { "closed" }, &span); + return result; + } let Some(tx) = self.clone_sender() else { record_enqueue(started, "missing_sender", &span); return Err("db writer sender missing".to_string()); @@ -46,6 +51,11 @@ impl DbWriter { queue_result = tracing::field::Empty, ); let started = Instant::now(); + if let Some(remote) = &self.remote { + let accepted = remote.try_admit(op); + record_enqueue(started, if accepted { "queued" } else { "full" }, &span); + return accepted; + } let queue_result = match self.clone_sender() { Some(tx) => match tx.try_send(WriterMessage::write(op)) { Ok(()) => "queued", @@ -78,6 +88,11 @@ impl DbWriter { queue_result = tracing::field::Empty, ); let started = Instant::now(); + if let Some(remote) = &self.remote { + let result = remote.admit_blocking(op); + record_enqueue(started, if result.is_ok() { "queued" } else { "closed" }, &span); + return result; + } let Some(tx) = self.clone_sender() else { record_enqueue(started, "missing_sender", &span); return Err("db writer sender missing".to_string()); diff --git a/crates/capsem-logger/src/writer/remote.rs b/crates/capsem-logger/src/writer/remote.rs new file mode 100644 index 000000000..0dc0b6f2a --- /dev/null +++ b/crates/capsem-logger/src/writer/remote.rs @@ -0,0 +1,342 @@ +//! Descriptor-backed writer client for the dedicated ledger owner. + +use std::io; +use std::os::unix::net::UnixStream; +use std::sync::{mpsc, Mutex}; +use std::time::Duration; + +use capsem_foundation::ipc_channel; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerHello, LedgerOutcome, LedgerRequest}; +use tokio::sync::oneshot; + +use crate::ledger_protocol::{LedgerClientMessage, LedgerCommand, LedgerReply, LedgerServerMessage}; +use crate::{RetainOutcome, WriteOp}; + +const STARTUP_TIMEOUT: Duration = Duration::from_secs(10); +#[cfg(not(test))] +const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +const OPERATION_TIMEOUT: Duration = Duration::from_millis(200); +const BACKPRESSURE_MAX_WAIT: Duration = Duration::from_millis(5); + +type ClientSender = ipc_channel::Sender; +type ClientReceiver = ipc_channel::Receiver; + +enum AdmitReply { + Async(oneshot::Sender>), + Blocking(mpsc::SyncSender>), +} + +impl AdmitReply { + fn send(self, result: Result<(), String>) { + match self { + Self::Async(reply) => { + let _ = reply.send(result); + } + Self::Blocking(reply) => { + let _ = reply.send(result); + } + } + } +} + +enum RemoteMessage { + Admit { + event: Box, + reply: Option, + }, + Flush { + reply: oneshot::Sender>, + }, + Retain { + cutoff: String, + reply: oneshot::Sender>, + }, +} + +pub(super) struct RemoteWriter { + tx: Mutex>>, + join: Mutex>>, +} + +impl RemoteWriter { + pub(super) fn start(stream: UnixStream, grant: LedgerChannelGrant, capacity: usize) -> io::Result { + let abort = stream.try_clone()?; + let (tx, rx) = mpsc::sync_channel(capacity.max(1)); + let (startup_tx, startup_rx) = mpsc::sync_channel(1); + let join = std::thread::Builder::new() + .name("capsem-ledger-client".into()) + .spawn(move || run(stream, grant, rx, startup_tx))?; + match startup_rx.recv_timeout(STARTUP_TIMEOUT) { + Ok(Ok(())) => Ok(Self { + tx: Mutex::new(Some(tx)), + join: Mutex::new(Some(join)), + }), + Ok(Err(error)) => { + let _ = join.join(); + Err(io::Error::other(error)) + } + Err(mpsc::RecvTimeoutError::Timeout) => { + let _ = abort.shutdown(std::net::Shutdown::Both); + let _ = join.join(); + Err(io::Error::new( + io::ErrorKind::TimedOut, + "ledger client handshake timed out", + )) + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + let _ = join.join(); + Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "ledger client stopped during handshake", + )) + } + } + } + + pub(super) async fn admit(&self, event: WriteOp) -> Result<(), String> { + let (reply, result) = oneshot::channel(); + self.send_async(RemoteMessage::Admit { + event: Box::new(event), + reply: Some(AdmitReply::Async(reply)), + }) + .await?; + result + .await + .map_err(|_| "ledger client stopped with admission pending".to_string())? + } + + pub(super) fn try_admit(&self, event: WriteOp) -> bool { + let Some(tx) = self.sender() else { + return false; + }; + tx.try_send(RemoteMessage::Admit { + event: Box::new(event), + reply: None, + }) + .is_ok() + } + + pub(super) fn admit_blocking(&self, event: WriteOp) -> Result<(), String> { + let Some(tx) = self.sender() else { + return Err("ledger client is shut down".to_string()); + }; + let (reply, result) = mpsc::sync_channel(1); + tx.send(RemoteMessage::Admit { + event: Box::new(event), + reply: Some(AdmitReply::Blocking(reply)), + }) + .map_err(|_| "ledger client channel closed".to_string())?; + result + .recv() + .map_err(|_| "ledger client stopped with blocking admission pending".to_string())? + } + + pub(super) async fn flush(&self) -> Result<(), String> { + let (reply, result) = oneshot::channel(); + self.send_async(RemoteMessage::Flush { reply }).await?; + result + .await + .map_err(|_| "ledger client stopped with flush pending".to_string())? + } + + pub(super) async fn retain(&self, cutoff: &str) -> Result { + let (reply, result) = oneshot::channel(); + self.send_async(RemoteMessage::Retain { + cutoff: cutoff.to_string(), + reply, + }) + .await?; + result + .await + .map_err(|_| "ledger client stopped with retention pending".to_string())? + } + + pub(super) fn shutdown(&self) { + let _ = self.tx.lock().unwrap().take(); + if let Some(join) = self.join.lock().unwrap().take() { + let _ = join.join(); + } + } + + fn sender(&self) -> Option> { + self.tx.lock().unwrap().clone() + } + + async fn send_async(&self, mut message: RemoteMessage) -> Result<(), String> { + let Some(tx) = self.sender() else { + return Err("ledger client is shut down".to_string()); + }; + let mut wait = Duration::from_micros(50); + let mut attempts = 0_u32; + loop { + match tx.try_send(message) { + Ok(()) => return Ok(()), + Err(mpsc::TrySendError::Full(returned)) => { + message = returned; + if attempts == 0 { + tokio::task::yield_now().await; + } else { + tokio::time::sleep(wait).await; + wait = (wait * 2).min(BACKPRESSURE_MAX_WAIT); + } + attempts += 1; + } + Err(mpsc::TrySendError::Disconnected(_)) => { + return Err("ledger client channel closed".to_string()); + } + } + } + } +} + +fn run( + stream: UnixStream, + grant: LedgerChannelGrant, + messages: mpsc::Receiver, + startup: mpsc::SyncSender>, +) { + let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() { + Ok(runtime) => runtime, + Err(error) => { + let _ = startup.send(Err(format!("build ledger client runtime: {error}"))); + return; + } + }; + let channel = { + let _entered = runtime.enter(); + ipc_channel::channel_from_std::(stream) + }; + let (sender, receiver) = match channel { + Ok(channel) => channel, + Err(error) => { + let _ = startup.send(Err(format!("open ledger client channel: {error}"))); + return; + } + }; + let handshake = runtime.block_on(async { + tokio::time::timeout(OPERATION_TIMEOUT, handshake(&sender, &receiver, grant)) + .await + .map_err(|_| "ledger client handshake timed out".to_string())? + }); + if let Err(error) = handshake { + let _ = startup.send(Err(error)); + return; + } + if startup.send(Ok(())).is_err() { + return; + } + + let mut request_id = 1_u64; + while let Ok(message) = messages.recv() { + let result = runtime.block_on(async { + tokio::time::timeout(OPERATION_TIMEOUT, dispatch(&sender, &receiver, request_id, &message)) + .await + .map_err(|_| "ledger client operation timed out".to_string())? + }); + finish(message, result.clone()); + if let Err(error) = result { + tracing::error!(%error, "ledger client operation failed"); + fail_pending(&messages, &error); + return; + } + let Some(next) = request_id.checked_add(1) else { + fail_pending(&messages, "ledger request id exhausted"); + return; + }; + request_id = next; + } +} + +async fn handshake(sender: &ClientSender, receiver: &ClientReceiver, grant: LedgerChannelGrant) -> Result<(), String> { + sender + .send(LedgerClientMessage::Hello { + hello: LedgerHello::for_grant(&grant), + }) + .await + .map_err(|error| format!("send ledger hello: {error}"))?; + let message = receiver + .recv() + .await + .map_err(|error| format!("receive ledger welcome: {error}"))?; + let LedgerServerMessage::Welcome { welcome } = message else { + return Err("ledger worker sent a response before welcome".to_string()); + }; + grant.validate_welcome(&welcome).map_err(|error| error.to_string()) +} + +async fn dispatch( + sender: &ClientSender, + receiver: &ClientReceiver, + request_id: u64, + message: &RemoteMessage, +) -> Result { + let command = match message { + RemoteMessage::Admit { event, .. } => LedgerCommand::Admit { event: event.clone() }, + RemoteMessage::Flush { .. } => LedgerCommand::Flush, + RemoteMessage::Retain { cutoff, .. } => LedgerCommand::Retain { cutoff: cutoff.clone() }, + }; + sender + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(request_id, command).map_err(|error| error.to_string())?, + }) + .await + .map_err(|error| format!("send ledger request: {error}"))?; + let message = receiver + .recv() + .await + .map_err(|error| format!("receive ledger response: {error}"))?; + let LedgerServerMessage::Response { response } = message else { + return Err("ledger worker repeated welcome".to_string()); + }; + if response.request_id() != request_id { + return Err(format!( + "ledger response id {} does not match {request_id}", + response.request_id() + )); + } + match response.outcome() { + LedgerOutcome::Success { reply } => { + reply.validate().map_err(|error| error.to_string())?; + Ok(reply.clone()) + } + LedgerOutcome::Failure { failure } => Err(format!("ledger {:?}: {}", failure.code(), failure.message())), + } +} + +fn finish(message: RemoteMessage, result: Result) { + match message { + RemoteMessage::Admit { reply, .. } => { + let result = result.and_then(|reply| match reply { + LedgerReply::Accepted { .. } => Ok(()), + reply => Err(format!("ledger returned {reply:?} for admission")), + }); + if let Some(reply) = reply { + reply.send(result); + } + } + RemoteMessage::Flush { reply } => { + let result = result.and_then(|response| match response { + LedgerReply::Durable { .. } => Ok(()), + response => Err(format!("ledger returned {response:?} for flush")), + }); + let _ = reply.send(result); + } + RemoteMessage::Retain { reply, .. } => { + let result = result.and_then(|response| match response { + LedgerReply::Retained { outcome } => Ok(outcome), + response => Err(format!("ledger returned {response:?} for retention")), + }); + let _ = reply.send(result); + } + } +} + +fn fail_pending(messages: &mpsc::Receiver, reason: &str) { + while let Ok(message) = messages.try_recv() { + finish(message, Err(reason.to_string())); + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-logger/src/writer/remote/tests.rs b/crates/capsem-logger/src/writer/remote/tests.rs new file mode 100644 index 000000000..4d644af06 --- /dev/null +++ b/crates/capsem-logger/src/writer/remote/tests.rs @@ -0,0 +1,180 @@ +use std::os::unix::net::UnixStream; +use std::sync::Arc; +use std::time::SystemTime; + +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; + +use crate::ledger_protocol::{LedgerClientMessage, LedgerServerMessage}; +use crate::ledger_server::LedgerServer; +use crate::{Decision, NetEvent, WriteOp}; + +fn event(domain: &str) -> WriteOp { + event_with_id(domain, "0123456789ab") +} + +fn event_with_id(domain: &str, event_id: &str) -> WriteOp { + WriteOp::NetEvent(NetEvent { + event_id: Some(event_id.into()), + timestamp: SystemTime::UNIX_EPOCH, + domain: domain.into(), + port: 443, + decision: Decision::Allowed, + process_name: None, + pid: None, + method: Some("GET".into()), + path: Some("/v1".into()), + query: None, + status_code: Some(200), + bytes_sent: 0, + bytes_received: 0, + duration_ms: 1, + matched_rule: None, + request_headers: None, + response_headers: None, + request_body: None, + response_body: None, + conn_type: Some("https".into()), + policy_mode: None, + policy_action: None, + policy_rule: None, + policy_reason: None, + trace_id: None, + credential_ref: None, + }) +} + +async fn remote_writer( + role: LedgerClientRole, +) -> ( + tempfile::TempDir, + crate::DbWriter, + tokio::task::JoinHandle>, +) { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("session.db"); + let server = Arc::new(LedgerServer::open(&path).unwrap()); + let generation = LedgerGeneration::new([5; 16]); + let grant = LedgerChannelGrant::new(generation, 9, role).unwrap(); + let (client, worker) = UnixStream::pair().unwrap(); + let task = tokio::spawn(async move { server.serve_client(worker, grant).await }); + let writer = tokio::task::spawn_blocking({ + let path = path.clone(); + move || crate::DbWriter::from_ledger_channel(client, grant, &path, 8) + }) + .await + .unwrap() + .unwrap(); + (directory, writer, task) +} + +#[tokio::test] +async fn descriptor_writer_waits_for_admission_and_flushes_through_the_owner() { + let (directory, writer, task) = remote_writer(LedgerClientRole::VmOwner).await; + writer.write_checked(event("remote.example")).await.unwrap(); + writer.flush_checked().await.unwrap(); + let path = directory.path().join("session.db"); + let rows: i64 = rusqlite::Connection::open(path) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM net_events WHERE domain = 'remote.example'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(rows, 1); + tokio::task::spawn_blocking(move || writer.shutdown_blocking()) + .await + .unwrap(); + assert_eq!( + task.await.unwrap().unwrap(), + crate::ledger_server::LedgerClientExit::Disconnected + ); +} + +#[tokio::test] +async fn descriptor_writer_reports_role_refusal_to_the_producer() { + let (_directory, writer, task) = remote_writer(LedgerClientRole::Reader).await; + let error = writer.write_checked(event("denied.example")).await.unwrap_err(); + assert!( + error.contains("cannot perform") || error.contains("unauthorized"), + "{error}" + ); + tokio::task::spawn_blocking(move || writer.shutdown_blocking()) + .await + .unwrap(); + assert_eq!( + task.await.unwrap().unwrap(), + crate::ledger_server::LedgerClientExit::Disconnected + ); +} + +#[tokio::test] +async fn descriptor_writer_preserves_blocking_and_try_admission_order() { + let (directory, writer, task) = remote_writer(LedgerClientRole::VmOwner).await; + let writer = Arc::new(writer); + let blocking = Arc::clone(&writer); + tokio::task::spawn_blocking(move || { + blocking + .write_blocking_checked(event_with_id("blocking.example", "0123456789ac")) + .unwrap(); + }) + .await + .unwrap(); + assert!(writer.try_write(event_with_id("try.example", "0123456789ad"))); + writer.flush_checked().await.unwrap(); + let path = directory.path().join("session.db"); + let rows: i64 = rusqlite::Connection::open(path) + .unwrap() + .query_row( + "SELECT COUNT(*) FROM net_events WHERE domain IN ('blocking.example', 'try.example')", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(rows, 2); + let writer = Arc::try_unwrap(writer).ok().unwrap(); + tokio::task::spawn_blocking(move || writer.shutdown_blocking()) + .await + .unwrap(); + assert_eq!( + task.await.unwrap().unwrap(), + crate::ledger_server::LedgerClientExit::Disconnected + ); +} + +#[tokio::test] +async fn a_stalled_owner_fails_the_pending_operation_and_bounded_shutdown() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("session.db"); + let grant = LedgerChannelGrant::new(LedgerGeneration::new([6; 16]), 10, LedgerClientRole::VmOwner).unwrap(); + let (client, worker) = UnixStream::pair().unwrap(); + let server = tokio::spawn(async move { + let (sender, receiver) = + capsem_foundation::ipc_channel::channel_from_std::(worker) + .unwrap(); + let LedgerClientMessage::Hello { hello } = receiver.recv().await.unwrap() else { + panic!("expected hello") + }; + grant.validate_hello(&hello).unwrap(); + sender + .send(LedgerServerMessage::Welcome { + welcome: capsem_proto::ledger::LedgerWelcome::for_grant(&grant), + }) + .await + .unwrap(); + let _request = receiver.recv().await.unwrap(); + std::future::pending::<()>().await; + }); + let writer = tokio::task::spawn_blocking(move || crate::DbWriter::from_ledger_channel(client, grant, &path, 2)) + .await + .unwrap() + .unwrap(); + let started = std::time::Instant::now(); + let error = writer.write_checked(event("stalled.example")).await.unwrap_err(); + assert!(error.contains("timed out"), "{error}"); + tokio::task::spawn_blocking(move || writer.shutdown_blocking()) + .await + .unwrap(); + assert!(started.elapsed() < std::time::Duration::from_secs(2)); + server.abort(); +} From 39608ed7c8cbfb372aad89c3f928df96141a8b63 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:10:04 +0000 Subject: [PATCH 116/464] feat(proto): grant ledger channels to workers --- crates/capsem-proto/src/upstream_grant.rs | 108 +++++++++++++++++- .../capsem-proto/src/upstream_grant/tests.rs | 72 ++++++++++++ 2 files changed, 175 insertions(+), 5 deletions(-) diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index c6a5f29d3..a3c8a3157 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -10,9 +10,11 @@ use std::net::IpAddr; use anyhow::{bail, Result}; +use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; + /// Exact record size carried by the bounded SCM_RIGHTS channel. pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 4480; -/// A response grants at most one connected TCP or UDP descriptor. +/// A response grants at most one connected upstream or ledger descriptor. pub const UPSTREAM_GRANT_MAX_FDS: usize = 1; /// DNS names are at most 253 wire-text bytes without a root dot. pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; @@ -20,7 +22,7 @@ pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; pub const MAX_GUEST_SHARE_PATH_BYTES: usize = 4096; const MAGIC: [u8; 2] = *b"UG"; -const VERSION: u8 = 1; +const VERSION: u8 = 2; const MAGIC_RANGE: std::ops::Range = 0..2; const VERSION_OFFSET: usize = 2; const KIND_OFFSET: usize = 3; @@ -42,10 +44,12 @@ const OPEN_DNS: u8 = 3; const ADOPTED: u8 = 4; const RELEASE: u8 = 5; const SET_GUEST_MODE: u8 = 6; +const OPEN_LEDGER: u8 = 7; const TCP_RESOLVED: u8 = 101; const DESCRIPTOR_GRANTED: u8 = 102; const DENIED: u8 = 103; const GUEST_MODE_SET: u8 = 104; +const LEDGER_GRANTED: u8 = 105; /// Application protocol spoken over a granted TCP stream. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -163,9 +167,12 @@ pub enum UpstreamGrantRequest { relative_path: Vec, mode: u16, }, + OpenLedger { + request_id: u64, + }, } -/// Coordinator-to-worker record. Only `DescriptorGranted` carries one fd. +/// Coordinator-to-worker record. Successful descriptor grants carry one fd. #[derive(Clone, Debug, Eq, PartialEq)] pub enum UpstreamGrantResponse { TcpResolved { @@ -188,6 +195,10 @@ pub enum UpstreamGrantResponse { GuestModeSet { request_id: u64, }, + LedgerGranted { + request_id: u64, + grant: LedgerChannelGrant, + }, } impl UpstreamGrantRequest { @@ -198,10 +209,10 @@ impl UpstreamGrantRequest { } impl UpstreamGrantResponse { - /// Only a successful connected-socket grant carries a descriptor. + /// Successful connected-socket and ledger grants carry one descriptor. pub const fn expected_descriptor_count(&self) -> usize { match self { - Self::DescriptorGranted { .. } => 1, + Self::DescriptorGranted { .. } | Self::LedgerGranted { .. } => 1, Self::TcpResolved { .. } | Self::Denied { .. } | Self::GuestModeSet { .. } => 0, } } @@ -272,6 +283,11 @@ pub fn encode_upstream_grant_request(request: &UpstreamGrantRequest) -> Result<[ put_u16(&mut frame, DETAIL_RANGE, *mode); put_path(&mut frame, relative_path)?; } + UpstreamGrantRequest::OpenLedger { request_id } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = OPEN_LEDGER; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + } } Ok(frame) } @@ -353,6 +369,15 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> mode: detail, }) } + OPEN_LEDGER => { + require_nonzero("request id", request_id)?; + if resource_id != 0 { + bail!("ledger request cannot carry a resource"); + } + require_empty_fields(port, detail, &name)?; + require_empty_path(&relative_path)?; + Ok(UpstreamGrantRequest::OpenLedger { request_id }) + } kind => bail!("invalid upstream request kind {kind}"), } } @@ -404,6 +429,14 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul frame[KIND_OFFSET] = GUEST_MODE_SET; put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); } + UpstreamGrantResponse::LedgerGranted { request_id, grant } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = LEDGER_GRANTED; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u64(&mut frame, RESOURCE_ID_RANGE, grant.client_id()); + put_u16(&mut frame, DETAIL_RANGE, ledger_role_code(grant.role())); + put_name(&mut frame, &encode_ledger_generation(grant.generation()))?; + } } Ok(frame) } @@ -474,6 +507,15 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - } Ok(UpstreamGrantResponse::GuestModeSet { request_id }) } + LEDGER_GRANTED => { + if policy_digest.is_some() { + bail!("ledger grant cannot carry a policy digest"); + } + let generation = decode_ledger_generation(&name)?; + let role = decode_ledger_role(detail)?; + let grant = LedgerChannelGrant::new(generation, resource_id, role)?; + Ok(UpstreamGrantResponse::LedgerGranted { request_id, grant }) + } kind => bail!("invalid upstream response kind {kind}"), } } @@ -618,6 +660,62 @@ fn require_empty_policy_digest(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Resul Ok(()) } +fn ledger_role_code(role: LedgerClientRole) -> u16 { + match role { + LedgerClientRole::VmOwner => 1, + LedgerClientRole::Proxy => 2, + LedgerClientRole::Coordinator => 3, + LedgerClientRole::Reader => 4, + LedgerClientRole::Maintainer => 5, + LedgerClientRole::Supervisor => 6, + } +} + +fn decode_ledger_role(code: u16) -> Result { + match code { + 1 => Ok(LedgerClientRole::VmOwner), + 2 => Ok(LedgerClientRole::Proxy), + 3 => Ok(LedgerClientRole::Coordinator), + 4 => Ok(LedgerClientRole::Reader), + 5 => Ok(LedgerClientRole::Maintainer), + 6 => Ok(LedgerClientRole::Supervisor), + _ => bail!("invalid ledger client role {code}"), + } +} + +fn encode_ledger_generation(generation: LedgerGeneration) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(32); + for byte in generation.as_bytes() { + encoded.push(char::from(HEX[usize::from(byte >> 4)])); + encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + encoded +} + +fn decode_ledger_generation(encoded: &str) -> Result { + if encoded.len() != 32 + || !encoded + .bytes() + .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f')) + { + bail!("ledger generation must contain 32 lowercase hexadecimal bytes"); + } + let mut bytes = [0_u8; 16]; + for (index, pair) in encoded.as_bytes().chunks_exact(2).enumerate() { + bytes[index] = (decode_hex_nibble(pair[0])? << 4) | decode_hex_nibble(pair[1])?; + } + Ok(LedgerGeneration::new(bytes)) +} + +fn decode_hex_nibble(byte: u8) -> Result { + match byte { + b'0'..=b'9' => Ok(byte - b'0'), + b'a'..=b'f' => Ok(byte - b'a' + 10), + _ => bail!("invalid lowercase hexadecimal byte"), + } +} + fn validate_policy_digest(digest: &str) -> Result<()> { let Some(hash) = digest.strip_prefix("blake3:") else { bail!("policy digest must use blake3"); diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index ebf517949..9f75a39fb 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; const POLICY_DIGEST: &str = "blake3:0000000000000000000000000000000000000000000000000000000000000001"; @@ -35,6 +36,7 @@ fn every_request_round_trips_exactly() { request_id: 4, upstream_index: 0, }, + UpstreamGrantRequest::OpenLedger { request_id: 5 }, UpstreamGrantRequest::Adopted { grant_id: 51 }, UpstreamGrantRequest::Release { resource_id: 61 }, UpstreamGrantRequest::SetGuestMode { @@ -88,6 +90,10 @@ fn every_response_round_trips_exactly() { reason: UpstreamGrantDenial::Revoked, }, UpstreamGrantResponse::GuestModeSet { request_id: 7 }, + UpstreamGrantResponse::LedgerGranted { + request_id: 8, + grant: LedgerChannelGrant::new(LedgerGeneration::new([0x5a; 16]), 16, LedgerClientRole::VmOwner).unwrap(), + }, ] { response_roundtrip(response); } @@ -137,6 +143,14 @@ fn descriptor_expectations_are_explicit() { upstream_index: 0, }; assert_eq!(request.expected_descriptor_count(), 0); + assert_eq!( + UpstreamGrantResponse::LedgerGranted { + request_id: 2, + grant: LedgerChannelGrant::new(LedgerGeneration::new([1; 16]), 3, LedgerClientRole::VmOwner).unwrap(), + } + .expected_descriptor_count(), + 1 + ); assert_eq!( UpstreamGrantResponse::DescriptorGranted { request_id: 1, @@ -157,6 +171,64 @@ fn descriptor_expectations_are_explicit() { ); } +#[test] +fn ledger_grants_preserve_every_authorized_role() { + for (index, role) in [ + LedgerClientRole::VmOwner, + LedgerClientRole::Proxy, + LedgerClientRole::Coordinator, + LedgerClientRole::Reader, + LedgerClientRole::Maintainer, + LedgerClientRole::Supervisor, + ] + .into_iter() + .enumerate() + { + response_roundtrip(UpstreamGrantResponse::LedgerGranted { + request_id: 10 + index as u64, + grant: LedgerChannelGrant::new( + LedgerGeneration::new([u8::try_from(index + 1).unwrap(); 16]), + 20 + index as u64, + role, + ) + .unwrap(), + }); + } +} + +#[test] +fn malformed_ledger_grant_authority_is_rejected() { + let response = UpstreamGrantResponse::LedgerGranted { + request_id: 1, + grant: LedgerChannelGrant::new(LedgerGeneration::new([0xab; 16]), 2, LedgerClientRole::VmOwner).unwrap(), + }; + let valid = encode_upstream_grant_response(&response).unwrap(); + + let mut zero_generation = valid; + put_name(&mut zero_generation, "00000000000000000000000000000000").unwrap(); + assert!(decode_upstream_grant_response(&zero_generation).is_err()); + + let mut uppercase_generation = valid; + put_name(&mut uppercase_generation, "ABABABABABABABABABABABABABABABAB").unwrap(); + assert!(decode_upstream_grant_response(&uppercase_generation).is_err()); + + let mut short_generation = valid; + put_name(&mut short_generation, "abab").unwrap(); + assert!(decode_upstream_grant_response(&short_generation).is_err()); + + let mut unknown_role = valid; + put_u16(&mut unknown_role, DETAIL_RANGE, 99); + assert!(decode_upstream_grant_response(&unknown_role).is_err()); + + let mut unrelated_policy = valid; + put_policy_digest(&mut unrelated_policy, POLICY_DIGEST).unwrap(); + assert!(decode_upstream_grant_response(&unrelated_policy).is_err()); + + let mut generation_padding = valid; + generation_padding[NAME_RANGE.start + 32] = 1; + assert!(decode_upstream_grant_response(&generation_padding).is_err()); +} + #[test] fn request_encoding_rejects_noncanonical_or_oversized_hosts() { for host in [ From 549e585cfb4b4c79078f6c36442fda7d8017e13b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:21:02 +0000 Subject: [PATCH 117/464] test(service): use authenticated readiness stamps --- crates/capsem-service/src/container_setup/tests.rs | 12 ++++++------ crates/capsem-service/src/tests/settings_files.rs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/capsem-service/src/container_setup/tests.rs b/crates/capsem-service/src/container_setup/tests.rs index be16d323a..9b7a9f7ff 100644 --- a/crates/capsem-service/src/container_setup/tests.rs +++ b/crates/capsem-service/src/container_setup/tests.rs @@ -139,7 +139,7 @@ fn fixture(images: FixtureImages) -> Fixture { std::fs::create_dir_all(&workspace).unwrap(); insert_fake_instance_with_session_dir(&state, "box", std::process::id(), session_dir); let uds_path = state.instances.lock().unwrap()["box"].uds_path.clone(); - std::fs::write(uds_path.with_extension("ready"), b"1\n").unwrap(); + std::fs::write(uds_path.with_extension("ready"), b"ready\n").unwrap(); Fixture { state, workspace, @@ -182,7 +182,7 @@ async fn pull_admission_waits_for_the_vm_owner_launch_barrier() { "the registry was contacted before owner readiness" ); - std::fs::write(fx.uds_path.with_extension("launched"), b"1\n").unwrap(); + std::fs::write(fx.uds_path.with_extension("launched"), b"launched\n").unwrap(); let status = wait_for(&fx.state, "box", |s| s.state == ContainerState::Failed).await; owner.await.unwrap(); assert!(status.error.as_deref().unwrap().contains("blocked after readiness")); @@ -230,7 +230,7 @@ async fn admitted_pull_progresses_while_the_guest_is_still_booting() { }) }); std::fs::remove_file(fx.uds_path.with_extension("ready")).unwrap(); - std::fs::write(fx.uds_path.with_extension("launched"), b"1\n").unwrap(); + std::fs::write(fx.uds_path.with_extension("launched"), b"launched\n").unwrap(); start(&fx.state, "box".into(), spec(None)); tokio::time::timeout(std::time::Duration::from_secs(1), async { while access.lock().unwrap().is_none() { @@ -251,7 +251,7 @@ async fn staged_workload_waits_for_guest_readiness_before_launching() { let owner = owner_accepting_stage_and_launch(&fx.uds_path, 4); let ready = fx.uds_path.with_extension("ready"); std::fs::remove_file(&ready).unwrap(); - std::fs::write(fx.uds_path.with_extension("launched"), b"1\n").unwrap(); + std::fs::write(fx.uds_path.with_extension("launched"), b"launched\n").unwrap(); start(&fx.state, "box".into(), spec(None)); tokio::time::timeout(std::time::Duration::from_secs(2), async { while !fx.workspace.join(".capsem-image/launch.py").exists() { @@ -266,7 +266,7 @@ async fn staged_workload_waits_for_guest_readiness_before_launching() { fx.state.containers.status("box").unwrap().state, ContainerState::Staging ); - std::fs::write(ready, b"1\n").unwrap(); + std::fs::write(ready, b"ready\n").unwrap(); owner.await.unwrap(); wait_for(&fx.state, "box", |status| status.state == ContainerState::Starting).await; } @@ -292,7 +292,7 @@ async fn an_import_refused_during_boot_writes_no_staged_control_bytes() { Box::pin(async move { Some(reply) }) }); std::fs::remove_file(fx.uds_path.with_extension("ready")).unwrap(); - std::fs::write(fx.uds_path.with_extension("launched"), b"1\n").unwrap(); + std::fs::write(fx.uds_path.with_extension("launched"), b"launched\n").unwrap(); start(&fx.state, "box".into(), spec(None)); let status = wait_for(&fx.state, "box", |status| status.state == ContainerState::Failed).await; owner.await.unwrap(); diff --git a/crates/capsem-service/src/tests/settings_files.rs b/crates/capsem-service/src/tests/settings_files.rs index 1bbef41d7..3576934e4 100644 --- a/crates/capsem-service/src/tests/settings_files.rs +++ b/crates/capsem-service/src/tests/settings_files.rs @@ -518,7 +518,7 @@ async fn wait_for_vm_ready_uses_tight_poll_contract_and_detects_ready() { let ready_clone = ready_path.clone(); let creator = std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(200)); - std::fs::write(&ready_clone, b"").unwrap(); + std::fs::write(&ready_clone, b"ready\n").unwrap(); }); wait_for_vm_ready(&uds_path, 30, None, None) From cb1fc43d0c50b62dfc08d7af43e57f7384af9ab5 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:22:26 +0000 Subject: [PATCH 118/464] security(service): grant VM ledger authority --- crates/capsem-service/src/ledger_worker.rs | 6 + crates/capsem-service/src/main.rs | 4 - crates/capsem-service/src/upstream_broker.rs | 126 +++++++++++++++++- .../src/upstream_broker/tests.rs | 119 +++++++++++++++++ .../src/vm_lifecycle/provision.rs | 1 + .../src/vm_lifecycle/resume_process.rs | 2 + 6 files changed, 247 insertions(+), 11 deletions(-) diff --git a/crates/capsem-service/src/ledger_worker.rs b/crates/capsem-service/src/ledger_worker.rs index 81c225946..f8709f56e 100644 --- a/crates/capsem-service/src/ledger_worker.rs +++ b/crates/capsem-service/src/ledger_worker.rs @@ -179,6 +179,12 @@ impl LedgerClient { pub(crate) fn into_parts(self) -> (UnixStream, LedgerChannelGrant) { (self.stream, self.grant) } + + #[cfg(test)] + pub(crate) fn test_pair(grant: LedgerChannelGrant) -> io::Result<(Self, UnixStream)> { + let (stream, worker) = UnixStream::pair()?; + Ok((Self { stream, grant }, worker)) + } } #[derive(Clone)] diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index fdd872c50..62e3a25e8 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -210,10 +210,6 @@ pub struct ServiceState { connections: connection_routes::Runtime, instances: Mutex>, // instance id to process info proxy_workers: Mutex>, - #[expect( - dead_code, - reason = "S09-004 connects the VM and proxy producers through this lifecycle table" - )] ledger_workers: Arc, retirements: instance_reaper::Retirements, /// Logger-owned DB handles keyed by session/VM id. Logged-data routes diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 3a071be2c..7cb0fbf55 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -21,6 +21,7 @@ use capsem_core::net::policy_config::CompiledActivePolicy; use capsem_core::net::upstream_address::UpstreamResolver; use capsem_foundation::unix::fd::{self, SocketShutdown}; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::upstream_grant::{ decode_upstream_grant_request, encode_upstream_grant_response, UpstreamDescriptorKind, UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, @@ -29,6 +30,7 @@ use tokio::sync::{mpsc, oneshot}; use tracing::{debug, warn}; use crate::instance::WorkerGrant; +use crate::ledger_worker::{LedgerClient, LedgerWorkers}; const MAX_SELECTIONS: usize = 64; const MAX_ACTIVE_GRANTS: usize = 256; @@ -53,6 +55,7 @@ pub(crate) struct PendingBroker { initial_policy: Arc, updates: mpsc::Receiver, session_dir: Option, + ledger: Option, } struct PolicyUpdate { @@ -71,6 +74,54 @@ struct ActiveGrant { adopted: bool, } +enum LedgerSource { + #[cfg(test)] + Ready(LedgerClient), + VmOwner { + workers: Arc, + session_id: String, + database: PathBuf, + log_path: PathBuf, + }, +} + +impl LedgerSource { + async fn acquire(self) -> Result { + match self { + #[cfg(test)] + Self::Ready(client) => Ok(client), + Self::VmOwner { + workers, + session_id, + database, + log_path, + } => workers + .acquire( + &session_id, + &database, + &log_path, + capsem_proto::ledger::LedgerClientRole::VmOwner, + ) + .await + .map_err(|error| format!("acquire VM ledger channel: {error:#}")), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AwaitingAdoption { + Upstream(u64), + Ledger(u64), +} + +impl AwaitingAdoption { + const fn grant_id(self) -> u64 { + match self { + Self::Upstream(grant_id) | Self::Ledger(grant_id) => grant_id, + } + } +} + impl BrokerPolicy { pub(crate) fn new(digest: String, runtime: Arc) -> Arc { Arc::new(Self { digest, runtime }) @@ -104,6 +155,7 @@ impl PendingBroker { initial_policy, updates, session_dir: None, + ledger: None, }, PolicyPublisher { updates: Some(updates_tx), @@ -111,6 +163,22 @@ impl PendingBroker { )) } + pub(crate) fn with_vm_ledger(mut self, workers: Arc, session_id: &str, session_dir: &Path) -> Self { + self.ledger = Some(LedgerSource::VmOwner { + workers, + session_id: session_id.to_string(), + database: session_dir.join("session.db"), + log_path: session_dir.join("ledger.log"), + }); + self + } + + #[cfg(test)] + fn with_ledger(mut self, ledger: LedgerClient) -> Self { + self.ledger = Some(LedgerSource::Ready(ledger)); + self + } + pub(crate) fn pair_for_session( initial_policy: Arc, session_dir: PathBuf, @@ -133,6 +201,7 @@ impl PendingBroker { self.updates, authority, self.session_dir, + self.ledger, ) .await { @@ -148,6 +217,7 @@ async fn run( mut updates: mpsc::Receiver, authority: WorkerGrant, session_dir: Option, + mut ledger: Option, ) -> Result<(), String> { let requests = WireReceiver::new(socket.try_clone().map_err(|error| error.to_string())?).map_err(|error| error.to_string())?; @@ -166,6 +236,8 @@ async fn run( let mut active = HashMap::new(); let mut next_resource_id = 1_u64; let mut awaiting_adoption = None; + let ledger_configured = ledger.is_some(); + let mut pending_ledger: Option<(LedgerChannelGrant, OwnedFd)> = None; let result = async { 'requests: loop { let frame = tokio::select! { @@ -191,9 +263,14 @@ async fn run( let request = decode_upstream_grant_request(&frame.bytes) .map_err(|error| format!("decode upstream grant request: {error:#}"))?; if let Some(expected) = awaiting_adoption { - if request != (UpstreamGrantRequest::Adopted { grant_id: expected }) { + if request + != (UpstreamGrantRequest::Adopted { + grant_id: expected.grant_id(), + }) + { break Err(format!( - "descriptor grant {expected} was not adopted before the next request" + "descriptor grant {} was not adopted before the next request", + expected.grant_id() )); } } @@ -333,7 +410,7 @@ async fn run( ¤t_digest, ) .await?; - awaiting_adoption = Some(grant_id); + awaiting_adoption = Some(AwaitingAdoption::Upstream(grant_id)); } UpstreamGrantRequest::OpenDns { request_id, @@ -407,13 +484,24 @@ async fn run( ¤t.digest, ) .await?; - awaiting_adoption = Some(grant_id); + awaiting_adoption = Some(AwaitingAdoption::Upstream(grant_id)); } UpstreamGrantRequest::Adopted { grant_id } => { + if awaiting_adoption == Some(AwaitingAdoption::Ledger(grant_id)) { + let Some((grant, descriptor)) = pending_ledger.take() else { + break Err(format!("ledger grant {grant_id} disappeared before adoption")); + }; + if grant.client_id() != grant_id { + break Err(format!("ledger grant {grant_id} changed before adoption")); + } + drop(descriptor); + awaiting_adoption = None; + continue; + } let Some(grant) = active.get_mut(&grant_id) else { break Err(format!("adopted unknown descriptor grant {grant_id}")); }; - if grant.adopted || awaiting_adoption != Some(grant_id) { + if grant.adopted || awaiting_adoption != Some(AwaitingAdoption::Upstream(grant_id)) { break Err(format!("invalid descriptor adoption {grant_id}")); } grant.adopted = true; @@ -450,6 +538,28 @@ async fn run( } } } + UpstreamGrantRequest::OpenLedger { request_id } => { + let Some(source) = ledger.take() else { + let reason = if ledger_configured { + UpstreamGrantDenial::InvalidResource + } else { + UpstreamGrantDenial::NotConfigured + }; + send_denied(&responses, request_id, reason).await?; + continue; + }; + let client = source.acquire().await?; + let (stream, grant) = client.into_parts(); + let descriptor = OwnedFd::from(stream); + send_response( + &responses, + &UpstreamGrantResponse::LedgerGranted { request_id, grant }, + Some(&descriptor), + ) + .await?; + pending_ledger = Some((grant, descriptor)); + awaiting_adoption = Some(AwaitingAdoption::Ledger(grant.client_id())); + } } } } @@ -526,12 +636,14 @@ fn apply_policy_update( current: &mut Arc, selections: &mut HashMap, active: &mut HashMap, - awaiting_adoption: &mut Option, + awaiting_adoption: &mut Option, ) -> Result<(), String> { *current = update.policy; selections.clear(); let result = revoke_active(std::mem::take(active)); - *awaiting_adoption = None; + if matches!(awaiting_adoption, Some(AwaitingAdoption::Upstream(_))) { + *awaiting_adoption = None; + } let _ = update.applied.send(result.clone()); result } diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index 43feccd26..56e9cb132 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use capsem_core::net::policy::{UpstreamOverride, UpstreamOverrideProtocol}; use capsem_core::net::policy_config::{ActivePolicyFile, SettingsFile}; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; use capsem_proto::upstream_grant::{ decode_upstream_grant_response, encode_upstream_grant_request, UpstreamGrantResponse, }; @@ -14,6 +15,7 @@ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use super::*; use crate::instance::WorkerAuthority; +use crate::ledger_worker::LedgerClient; struct TestClient { requests: WireSender, @@ -79,6 +81,31 @@ fn start_broker( pending.updates, authority.grant(), pending.session_dir, + pending.ledger, + )); + (client, publisher, authority, task) +} + +fn start_broker_with_ledger( + policy: Arc, + ledger: LedgerClient, +) -> ( + TestClient, + PolicyPublisher, + WorkerAuthority, + tokio::task::JoinHandle>, +) { + let (pending, publisher) = PendingBroker::pair(policy).unwrap(); + let pending = pending.with_ledger(ledger); + let client = TestClient::new(pending.worker); + let authority = WorkerAuthority::default(); + let task = tokio::spawn(run( + pending.coordinator, + pending.initial_policy, + pending.updates, + authority.grant(), + pending.session_dir, + pending.ledger, )); (client, publisher, authority, task) } @@ -101,10 +128,102 @@ fn start_broker_for_session( pending.updates, authority.grant(), pending.session_dir, + pending.ledger, )); (client, publisher, authority, task) } +#[tokio::test] +async fn ledger_grant_is_exact_single_use_and_releases_the_broker_copy_after_adoption() { + let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x4d; 16]), 77, LedgerClientRole::VmOwner).unwrap(); + let (ledger, worker_peer) = LedgerClient::test_pair(grant).unwrap(); + let (client, _publisher, authority, task) = start_broker_with_ledger(test_policy("policy-a", None, vec![]), ledger); + + let (response, mut fds) = client + .request(&UpstreamGrantRequest::OpenLedger { request_id: 1 }) + .await; + assert_eq!(response, UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }); + let granted = UnixStream::from(fds.pop().unwrap()); + client + .send(&UpstreamGrantRequest::Adopted { + grant_id: grant.client_id(), + }) + .await; + + let (response, fds) = client + .request(&UpstreamGrantRequest::OpenLedger { request_id: 2 }) + .await; + assert!(fds.is_empty()); + assert_eq!( + response, + UpstreamGrantResponse::Denied { + request_id: 2, + reason: UpstreamGrantDenial::InvalidResource, + } + ); + + drop(granted); + worker_peer.set_nonblocking(true).unwrap(); + let mut worker_peer = tokio::net::UnixStream::from_std(worker_peer).unwrap(); + let mut byte = [0_u8; 1]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), worker_peer.read(&mut byte)) + .await + .expect("ledger peer observes descriptor close") + .unwrap(), + 0 + ); + stop_broker(authority, task).await; +} + +#[tokio::test] +async fn ledger_request_without_a_coordinator_grant_is_denied() { + let (client, _publisher, authority, task) = start_broker(test_policy("policy-a", None, vec![])); + let (response, fds) = client + .request(&UpstreamGrantRequest::OpenLedger { request_id: 1 }) + .await; + assert!(fds.is_empty()); + assert_eq!( + response, + UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::NotConfigured, + } + ); + stop_broker(authority, task).await; +} + +#[tokio::test] +async fn policy_updates_do_not_revoke_a_pending_ledger_grant() { + let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x37; 16]), 91, LedgerClientRole::VmOwner).unwrap(); + let (ledger, worker_peer) = LedgerClient::test_pair(grant).unwrap(); + let (client, publisher, authority, task) = start_broker_with_ledger(test_policy("policy-a", None, vec![]), ledger); + let (response, mut fds) = client + .request(&UpstreamGrantRequest::OpenLedger { request_id: 1 }) + .await; + assert_eq!(response, UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }); + + publisher.publish(test_policy("policy-b", None, vec![])).await.unwrap(); + client + .send(&UpstreamGrantRequest::Adopted { + grant_id: grant.client_id(), + }) + .await; + + drop(fds.pop().unwrap()); + worker_peer.set_nonblocking(true).unwrap(); + let mut worker_peer = tokio::net::UnixStream::from_std(worker_peer).unwrap(); + let mut byte = [0_u8; 1]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), worker_peer.read(&mut byte)) + .await + .expect("ledger peer observes descriptor close after policy update") + .unwrap(), + 0 + ); + stop_broker(authority, task).await; +} + #[tokio::test] async fn guest_mode_requests_are_confined_to_the_session_share() { let session = tempfile::tempdir().unwrap(); diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index 8f7245eb3..caf820988 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -160,6 +160,7 @@ impl ServiceState { active_policy.broker_policy(), session_dir.clone(), )?; + let upstream_broker = upstream_broker.with_vm_ledger(Arc::clone(&self.ledger_workers), id, &session_dir); let upstream_stdio = upstream_broker.worker_stdio()?; info!(process_binary = %self.process_binary.display(), exists = self.process_binary.exists(), "checking process_binary"); diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 43e964c51..3adbe4173 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -71,6 +71,8 @@ impl ServiceState { active_policy.broker_policy(), entry.session_dir.clone(), )?; + let upstream_broker = + upstream_broker.with_vm_ledger(Arc::clone(&self.ledger_workers), &vm_id, &entry.session_dir); let upstream_stdio = upstream_broker.worker_stdio()?; let scratch_disk_size_gb = session_rootfs_size_gb(&entry)?; let resolved = self.resolve_pinned_asset_paths(&entry.asset_pins)?; From eb3dbdd5e8ba3aee894195aca3d3f88e59ef3951 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:26:35 +0000 Subject: [PATCH 119/464] security(process): write through supervised ledger --- CHANGELOG.md | 4 + crates/capsem-process/src/main.rs | 15 +++- crates/capsem-process/src/tests.rs | 7 ++ crates/capsem-process/src/upstream_grant.rs | 53 ++++++++++++ .../src/upstream_grant/tests.rs | 81 +++++++++++++++++++ 5 files changed, 159 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6576b6635..60c732a07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Give each VM owner one coordinator-minted, generation- and role-bound + ledger channel and route its existing event writer through the supervised + session owner, eliminating the VM process's direct SQLite open. + - Pass the packaged ledger and proxy worker paths explicitly to installed and direct services, refuse incomplete confined-worker cohorts, and retire both children during reinstall or uninstall. diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 8005e58aa..d0327fadc 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -622,10 +622,23 @@ async fn run_async_main_loop( ) -> Result<()> { let terminal_output = Arc::new(capsem_core::TerminalOutputQueue::new()); + let (ledger_stream, ledger_grant) = upstream_grants + .open_ledger() + .await + .context("acquire supervised session ledger")?; + let ledger_path = session_dir.join("session.db"); // 1024 queued events: a guest resolving and fetching in parallel enqueues // several rows per request, and a full queue makes every producer sleep // in 5 ms steps on its reply path (see `DbWriter::send_with_backpressure`). - let db = Arc::new(capsem_logger::DbWriter::open(&session_dir.join("session.db"), 1024)?); + // The logical path is diagnostic identity only; the descriptor selects + // the worker that exclusively owns and opens the database. + let db = Arc::new( + tokio::task::spawn_blocking(move || { + capsem_logger::DbWriter::from_ledger_channel(ledger_stream, ledger_grant, &ledger_path, 1024) + }) + .await + .context("join supervised session ledger handshake")??, + ); // Register the DbWriter with the SIGTERM handler BEFORE any work that // produces writes. If the signal fires before the workspace monitor // starts, we still want a clean checkpoint. diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index 5ba0e1a50..d712e3553 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -13,6 +13,13 @@ fn entry_closes_ambient_descriptors_before_runtime_initialization() { assert!(close < telemetry && close < arguments); } +#[test] +fn production_vm_owner_never_opens_the_session_database() { + let source = include_str!("main.rs"); + assert!(!source.contains("DbWriter::open(")); + assert!(source.contains("DbWriter::from_ledger_channel(")); +} + #[test] fn platform_confinement_and_attestation_precede_owner_readiness() { let source = include_str!("main.rs"); diff --git a/crates/capsem-process/src/upstream_grant.rs b/crates/capsem-process/src/upstream_grant.rs index b5a5c5da9..a6da1e011 100644 --- a/crates/capsem-process/src/upstream_grant.rs +++ b/crates/capsem-process/src/upstream_grant.rs @@ -13,6 +13,7 @@ use capsem_core::net::mitm_proxy::{ }; use capsem_foundation::unix::fd::{self, SocketShutdown}; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::upstream_grant::{ decode_upstream_grant_response, encode_upstream_grant_request, UpstreamDescriptorKind, UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, @@ -28,6 +29,9 @@ type WireSender = DescriptorSender; enum Command { + OpenLedger { + reply: oneshot::Sender>, + }, SetGuestMode { relative_path: Vec, mode: u16, @@ -75,6 +79,18 @@ impl UpstreamGrantClient { }); Ok(Self { commands: commands_tx }) } + + pub(crate) async fn open_ledger(&self) -> Result<(UnixStream, LedgerChannelGrant)> { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::OpenLedger { reply }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + } } impl capsem_core::GuestMetadataAuthority for UpstreamGrantClient { @@ -198,6 +214,7 @@ async fn dispatch( command: Command, ) -> Result<(), String> { match command { + Command::OpenLedger { reply } => finish(reply, open_ledger(sender, receiver, request_id).await), Command::SetGuestMode { relative_path, mode, @@ -245,6 +262,42 @@ async fn dispatch( } } +async fn open_ledger( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, +) -> Result, String> { + send_request(sender, &UpstreamGrantRequest::OpenLedger { request_id }).await?; + let (response, mut fds) = receive_response(receiver).await?; + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("session ledger grant denied: {reason:?}"))), + UpstreamGrantResponse::LedgerGranted { + request_id: response_id, + grant, + } if response_id == request_id => { + let descriptor = fds.pop().ok_or("ledger grant omitted its descriptor")?; + send_request( + sender, + &UpstreamGrantRequest::Adopted { + grant_id: grant.client_id(), + }, + ) + .await?; + let stream = UnixStream::from(descriptor); + stream + .peer_addr() + .map_err(|error| format!("adopt granted ledger channel: {error}"))?; + Ok(Ok((stream, grant))) + } + response => Err(format!( + "unexpected session ledger response for request {request_id}: {response:?}" + )), + } +} + async fn set_guest_mode( sender: &WireSender, receiver: &WireReceiver, diff --git a/crates/capsem-process/src/upstream_grant/tests.rs b/crates/capsem-process/src/upstream_grant/tests.rs index c3d7d77bb..e80b1e84a 100644 --- a/crates/capsem-process/src/upstream_grant/tests.rs +++ b/crates/capsem-process/src/upstream_grant/tests.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use capsem_core::net::dns::{DnsResolver, DnsUpstreamGrants}; use capsem_core::net::mitm_proxy::{TcpUpstreamGrants, UpstreamTarget}; use capsem_core::GuestMetadataAuthority as _; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; use capsem_proto::upstream_grant::{ decode_upstream_grant_request, encode_upstream_grant_response, UpstreamDescriptorKind, UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, @@ -16,6 +17,86 @@ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; const POLICY_A: &str = "blake3:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const POLICY_B: &str = "blake3:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +#[tokio::test] +async fn ledger_channel_preserves_grant_and_is_adopted_before_use() { + let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x6c; 16]), 44, LedgerClientRole::VmOwner).unwrap(); + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let (ledger, mut ledger_peer) = UnixStream::pair().unwrap(); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenLedger { request_id: 1 } + ); + send_response( + &responses, + &UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }, + Some(ledger.as_raw_fd()), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { + grant_id: grant.client_id(), + } + ); + }); + + let (mut stream, received) = client.open_ledger().await.unwrap(); + assert_eq!(received, grant); + stream.write_all(b"hello").unwrap(); + let mut message = [0_u8; 5]; + ledger_peer.read_exact(&mut message).unwrap(); + assert_eq!(&message, b"hello"); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn ledger_denial_and_missing_descriptor_fail_closed() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let denied = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenLedger { request_id: 1 } + ); + send_response( + &responses, + &UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::NotConfigured, + }, + None, + ) + .await; + }); + let error = client.open_ledger().await.unwrap_err().to_string(); + assert!(error.contains("NotConfigured"), "{error}"); + denied.await.unwrap(); + + let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x7d; 16]), 45, LedgerClientRole::VmOwner).unwrap(); + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let malformed = tokio::spawn(async move { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenLedger { request_id: 1 } + )); + send_response( + &responses, + &UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }, + None, + ) + .await; + }); + let error = client.open_ledger().await.unwrap_err().to_string(); + assert!(error.contains("carried 0 descriptors"), "{error}"); + malformed.await.unwrap(); +} + async fn open_error(client: &UpstreamGrantClient) -> String { match client.open(0, POLICY_A).await { Ok(_) => panic!("unexpected DNS grant"), From 4033f14d4a12146a09b01eedad54f5c7c3508681 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:35:09 +0000 Subject: [PATCH 120/464] feat(logger): add authenticated ledger client --- crates/capsem-logger/src/ledger_client.rs | 503 ++++++++++++++++++ .../capsem-logger/src/ledger_client/tests.rs | 169 ++++++ crates/capsem-logger/src/ledger_server.rs | 7 +- crates/capsem-logger/src/lib.rs | 1 + 4 files changed, 679 insertions(+), 1 deletion(-) create mode 100644 crates/capsem-logger/src/ledger_client.rs create mode 100644 crates/capsem-logger/src/ledger_client/tests.rs diff --git a/crates/capsem-logger/src/ledger_client.rs b/crates/capsem-logger/src/ledger_client.rs new file mode 100644 index 000000000..b7b277753 --- /dev/null +++ b/crates/capsem-logger/src/ledger_client.rs @@ -0,0 +1,503 @@ +//! Cancellation-safe client for one authenticated session-ledger channel. + +use std::collections::BTreeMap; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use capsem_foundation::ipc_channel; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerHello, LedgerOutcome, LedgerRequest}; +use capsem_proto::ledger_counters::LedgerCounters; +use tokio::sync::{mpsc, oneshot}; + +use crate::ledger_protocol::{ + LedgerBodyMetadata, LedgerClientMessage, LedgerCommand, LedgerExportSummary, LedgerQuery, LedgerReply, LedgerRows, + LedgerServerMessage, +}; +use crate::{RetainOutcome, StoredBody}; + +const COMMAND_CAPACITY: usize = 16; +const EXPORT_CHUNK_CAPACITY: usize = 4; +#[cfg(not(test))] +const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); +#[cfg(test)] +const OPERATION_TIMEOUT: Duration = Duration::from_secs(1); +const STREAM_TIMEOUT: Duration = Duration::from_secs(30); + +type ClientSender = ipc_channel::Sender; +type ClientReceiver = ipc_channel::Receiver; + +enum Command { + Query { + query: LedgerQuery, + reply: oneshot::Sender, String>>, + }, + Counters { + reply: oneshot::Sender, String>>, + }, + ReadBodies { + event_id: String, + reply: oneshot::Sender, String>>, + }, + Retain { + cutoff: String, + reply: oneshot::Sender>, + }, + ExportWarc { + reply: oneshot::Sender>, + }, +} + +/// One authenticated, serial connection to a dedicated session-ledger owner. +/// +/// The background actor owns all framing. Dropping a caller future only drops +/// its reply channel; the actor still consumes the complete worker response. +#[derive(Clone)] +pub struct LedgerClient { + commands: mpsc::Sender, + path: Arc, + role: LedgerClientRole, + read_cache_epoch: Arc, +} + +impl LedgerClient { + pub async fn connect(stream: UnixStream, grant: LedgerChannelGrant, logical_path: PathBuf) -> Result { + let (sender, receiver) = ipc_channel::channel_from_std::(stream) + .map_err(|error| format!("open ledger client channel: {error}"))?; + tokio::time::timeout(OPERATION_TIMEOUT, handshake(&sender, &receiver, grant)) + .await + .map_err(|_| "ledger client handshake timed out".to_string())??; + + let (commands, inbox) = mpsc::channel(COMMAND_CAPACITY); + let read_cache_epoch = Arc::new(AtomicU64::new(0)); + tokio::spawn(run_actor(sender, receiver, inbox, Arc::clone(&read_cache_epoch))); + Ok(Self { + commands, + path: Arc::new(logical_path), + role: grant.role(), + read_cache_epoch, + }) + } + + #[must_use] + pub fn path(&self) -> &Path { + self.path.as_path() + } + + #[must_use] + pub const fn role(&self) -> LedgerClientRole { + self.role + } + + #[must_use] + pub fn read_cache_epoch(&self) -> u64 { + self.read_cache_epoch.load(Ordering::Acquire) + } + + pub async fn query(&self, query: LedgerQuery) -> Result, String> { + let (reply, result) = oneshot::channel(); + self.send(Command::Query { query, reply }).await?; + receive_result(result, "query").await + } + + pub async fn counters(&self) -> Result, String> { + let (reply, result) = oneshot::channel(); + self.send(Command::Counters { reply }).await?; + receive_result(result, "counters").await + } + + pub async fn read_bodies(&self, event_id: &str) -> Result, String> { + let (reply, result) = oneshot::channel(); + self.send(Command::ReadBodies { + event_id: event_id.to_string(), + reply, + }) + .await?; + receive_result(result, "body read").await + } + + pub async fn retain_bodies_since(&self, cutoff: &str) -> Result { + let (reply, result) = oneshot::channel(); + self.send(Command::Retain { + cutoff: cutoff.to_string(), + reply, + }) + .await?; + receive_result(result, "retention").await + } + + pub async fn export_warc(&self) -> Result { + let (reply, result) = oneshot::channel(); + self.send(Command::ExportWarc { reply }).await?; + receive_result(result, "WARC export").await + } + + async fn send(&self, command: Command) -> Result<(), String> { + self.commands + .send(command) + .await + .map_err(|_| "ledger client channel closed".to_string()) + } +} + +/// A bounded WARC byte stream followed by its worker-produced summary. +pub struct LedgerWarcExport { + chunks: mpsc::Receiver, String>>, + completion: oneshot::Receiver>, +} + +impl LedgerWarcExport { + pub async fn next_chunk(&mut self) -> Option, String>> { + self.chunks.recv().await + } + + pub async fn finish(mut self) -> Result { + while self.chunks.recv().await.is_some() {} + self.completion + .await + .map_err(|_| "ledger client stopped with WARC export pending".to_string())? + } +} + +async fn receive_result(result: oneshot::Receiver>, operation: &str) -> Result { + result + .await + .map_err(|_| format!("ledger client stopped with {operation} pending"))? +} + +async fn handshake(sender: &ClientSender, receiver: &ClientReceiver, grant: LedgerChannelGrant) -> Result<(), String> { + sender + .send(LedgerClientMessage::Hello { + hello: LedgerHello::for_grant(&grant), + }) + .await + .map_err(|error| format!("send ledger hello: {error}"))?; + let message = receiver + .recv() + .await + .map_err(|error| format!("receive ledger welcome: {error}"))?; + let LedgerServerMessage::Welcome { welcome } = message else { + return Err("ledger worker sent a response before welcome".to_string()); + }; + grant.validate_welcome(&welcome).map_err(|error| error.to_string()) +} + +async fn run_actor( + sender: ClientSender, + receiver: ClientReceiver, + mut commands: mpsc::Receiver, + read_cache_epoch: Arc, +) { + let mut request_id = 1_u64; + let mut counters = None; + while let Some(command) = commands.recv().await { + let result = dispatch( + &sender, + &receiver, + request_id, + command, + &mut counters, + &read_cache_epoch, + ) + .await; + if let Err(error) = result { + tracing::error!(%error, "ledger client operation failed"); + fail_pending(&mut commands, &error); + return; + } + let Some(next) = request_id.checked_add(1) else { + fail_pending(&mut commands, "ledger request id exhausted"); + return; + }; + request_id = next; + } +} + +async fn dispatch( + sender: &ClientSender, + receiver: &ClientReceiver, + request_id: u64, + command: Command, + previous_counters: &mut Option>, + read_cache_epoch: &AtomicU64, +) -> Result<(), String> { + match command { + Command::Query { query, reply } => { + let result = one_reply(sender, receiver, request_id, LedgerCommand::Query { query }) + .await + .and_then(|response| match response { + LedgerReply::Query { sets } => Ok(sets), + response => Err(unexpected("query", response)), + }); + finish(reply, result) + } + Command::Counters { reply } => { + let result = one_reply(sender, receiver, request_id, LedgerCommand::Counters) + .await + .and_then(|response| match response { + LedgerReply::Counters { counters } => Ok(Arc::new(*counters)), + response => Err(unexpected("counters", response)), + }); + if let Ok(current) = &result { + if previous_counters.as_deref() != Some(current.as_ref()) { + *previous_counters = Some(Arc::clone(current)); + read_cache_epoch.fetch_add(1, Ordering::AcqRel); + } + } + finish(reply, result) + } + Command::ReadBodies { event_id, reply } => { + let result = read_bodies(sender, receiver, request_id, &event_id).await; + finish(reply, result) + } + Command::Retain { cutoff, reply } => { + let result = one_reply(sender, receiver, request_id, LedgerCommand::Retain { cutoff }) + .await + .and_then(|response| match response { + LedgerReply::Retained { outcome } => Ok(outcome), + response => Err(unexpected("retention", response)), + }); + finish(reply, result) + } + Command::ExportWarc { reply } => export_warc(sender, receiver, request_id, reply).await, + } +} + +fn finish(reply: oneshot::Sender>, result: Result) -> Result<(), String> { + let failure = result.as_ref().err().cloned(); + let _ = reply.send(result); + match failure { + Some(error) => Err(error), + None => Ok(()), + } +} + +async fn send_request(sender: &ClientSender, request_id: u64, command: LedgerCommand) -> Result<(), String> { + sender + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(request_id, command).map_err(|error| error.to_string())?, + }) + .await + .map_err(|error| format!("send ledger request: {error}")) +} + +async fn one_reply( + sender: &ClientSender, + receiver: &ClientReceiver, + request_id: u64, + command: LedgerCommand, +) -> Result { + send_request(sender, request_id, command).await?; + tokio::time::timeout(OPERATION_TIMEOUT, receive_reply(receiver, request_id)) + .await + .map_err(|_| "ledger client operation timed out".to_string())? +} + +async fn receive_reply(receiver: &ClientReceiver, request_id: u64) -> Result { + let message = receiver + .recv() + .await + .map_err(|error| format!("receive ledger response: {error}"))?; + let LedgerServerMessage::Response { response } = message else { + return Err("ledger worker repeated welcome".to_string()); + }; + if response.request_id() != request_id { + return Err(format!( + "ledger response id {} does not match {request_id}", + response.request_id() + )); + } + match response.outcome() { + LedgerOutcome::Success { reply } => { + reply.validate().map_err(|error| error.to_string())?; + Ok(reply.clone()) + } + LedgerOutcome::Failure { failure } => Err(format!("ledger {:?}: {}", failure.code(), failure.message())), + } +} + +struct BodyAssembly { + metadata: LedgerBodyMetadata, + bytes: Vec, +} + +async fn read_bodies( + sender: &ClientSender, + receiver: &ClientReceiver, + request_id: u64, + event_id: &str, +) -> Result, String> { + send_request( + sender, + request_id, + LedgerCommand::ReadBodies { + event_id: event_id.to_string(), + }, + ) + .await?; + let mut bodies = BTreeMap::::new(); + loop { + let reply = tokio::time::timeout(OPERATION_TIMEOUT, receive_reply(receiver, request_id)) + .await + .map_err(|_| "ledger body stream timed out".to_string())??; + match reply { + LedgerReply::BodyStart { body_id, metadata } => { + if metadata.event_id != event_id { + return Err("ledger body event id does not match the request".to_string()); + } + let capacity = usize::try_from(metadata.stored_bytes) + .map_err(|_| "ledger body length does not fit memory".to_string())?; + if bodies + .insert( + body_id, + BodyAssembly { + metadata, + bytes: Vec::with_capacity(capacity), + }, + ) + .is_some() + { + return Err("ledger repeated a body id".to_string()); + } + } + LedgerReply::BodyChunk { body_id, offset, bytes } => { + let body = bodies + .get_mut(&body_id) + .ok_or_else(|| "ledger sent a body chunk before its metadata".to_string())?; + if offset != body.bytes.len() as u64 { + return Err("ledger body chunk offset is not contiguous".to_string()); + } + let new_len = body + .bytes + .len() + .checked_add(bytes.len()) + .ok_or_else(|| "ledger body length overflowed".to_string())?; + if new_len as u64 > body.metadata.stored_bytes { + return Err("ledger body exceeded its declared length".to_string()); + } + body.bytes.extend_from_slice(&bytes); + } + LedgerReply::BodiesComplete => return finish_bodies(bodies), + response => return Err(unexpected("body read", response)), + } + } +} + +fn finish_bodies(bodies: BTreeMap) -> Result, String> { + bodies + .into_values() + .map(|body| { + if body.bytes.len() as u64 != body.metadata.stored_bytes { + return Err("ledger body ended before its declared length".to_string()); + } + let hash = format!("blake3:{}", blake3::hash(&body.bytes).to_hex()); + if hash != body.metadata.body_hash { + return Err("ledger body hash does not match its bytes".to_string()); + } + Ok(StoredBody { + event_id: body.metadata.event_id, + source_table: body.metadata.source_table, + direction: body.metadata.direction, + content_type: body.metadata.content_type, + original_bytes: body.metadata.original_bytes, + truncated: body.metadata.truncated, + body_hash: body.metadata.body_hash, + bytes: body.bytes, + }) + }) + .collect() +} + +async fn export_warc( + sender: &ClientSender, + receiver: &ClientReceiver, + request_id: u64, + reply: oneshot::Sender>, +) -> Result<(), String> { + send_request(sender, request_id, LedgerCommand::ExportWarc).await?; + let (chunks, chunk_rx) = mpsc::channel(EXPORT_CHUNK_CAPACITY); + let (completion, completion_rx) = oneshot::channel(); + let export = LedgerWarcExport { + chunks: chunk_rx, + completion: completion_rx, + }; + let mut consumer_open = reply.send(Ok(export)).is_ok(); + let mut offset = 0_u64; + loop { + let response = tokio::time::timeout(STREAM_TIMEOUT, receive_reply(receiver, request_id)) + .await + .map_err(|_| "ledger WARC stream timed out".to_string())?; + let response = match response { + Ok(response) => response, + Err(error) => { + if consumer_open { + let _ = chunks.send(Err(error.clone())).await; + } + let _ = completion.send(Err(error.clone())); + return Err(error); + } + }; + match response { + LedgerReply::WarcChunk { offset: actual, bytes } => { + if actual != offset { + let error = "ledger WARC chunk offset is not contiguous".to_string(); + if consumer_open { + let _ = chunks.send(Err(error.clone())).await; + } + let _ = completion.send(Err(error.clone())); + return Err(error); + } + offset += bytes.len() as u64; + if consumer_open && chunks.send(Ok(bytes)).await.is_err() { + consumer_open = false; + } + } + LedgerReply::WarcComplete { summary } => { + if summary.bytes_written != offset { + let error = "ledger WARC summary byte count does not match the stream".to_string(); + let _ = completion.send(Err(error.clone())); + return Err(error); + } + drop(chunks); + let _ = completion.send(Ok(summary)); + return Ok(()); + } + response => { + let error = unexpected("WARC export", response); + let _ = completion.send(Err(error.clone())); + return Err(error); + } + } + } +} + +fn unexpected(operation: &str, response: LedgerReply) -> String { + format!("ledger returned {response:?} for {operation}") +} + +fn fail_pending(commands: &mut mpsc::Receiver, reason: &str) { + while let Ok(command) = commands.try_recv() { + match command { + Command::Query { reply, .. } => { + let _ = reply.send(Err(reason.to_string())); + } + Command::Counters { reply } => { + let _ = reply.send(Err(reason.to_string())); + } + Command::ReadBodies { reply, .. } => { + let _ = reply.send(Err(reason.to_string())); + } + Command::Retain { reply, .. } => { + let _ = reply.send(Err(reason.to_string())); + } + Command::ExportWarc { reply } => { + let _ = reply.send(Err(reason.to_string())); + } + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-logger/src/ledger_client/tests.rs b/crates/capsem-logger/src/ledger_client/tests.rs new file mode 100644 index 000000000..984e90f2d --- /dev/null +++ b/crates/capsem-logger/src/ledger_client/tests.rs @@ -0,0 +1,169 @@ +use std::sync::Arc; +use std::time::SystemTime; + +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; + +use super::*; +use crate::events::{Decision, NetEvent}; +use crate::ledger_protocol::LedgerQuery; +use crate::{ledger_server::LedgerServer, WriteOp}; + +const GENERATION: LedgerGeneration = LedgerGeneration::new([0x51; 16]); +const EVENT_ID: &str = "0123456789ab"; + +struct Fixture { + _dir: tempfile::TempDir, + client: LedgerClient, + writer: crate::DbWriter, + producer_task: tokio::task::JoinHandle>, + reader_task: tokio::task::JoinHandle>, +} + +impl Fixture { + async fn start(role: LedgerClientRole) -> Self { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("session.db"); + let server = Arc::new(LedgerServer::open(&path).unwrap()); + + let producer_grant = LedgerChannelGrant::new(GENERATION, 1, LedgerClientRole::VmOwner).unwrap(); + let (producer, producer_server) = std::os::unix::net::UnixStream::pair().unwrap(); + let producer_owner = Arc::clone(&server); + let producer_task = + tokio::spawn(async move { producer_owner.serve_client(producer_server, producer_grant).await }); + let writer = tokio::task::spawn_blocking(move || { + crate::DbWriter::from_ledger_channel(producer, producer_grant, &path, 16) + }) + .await + .unwrap() + .unwrap(); + + let reader_grant = LedgerChannelGrant::new(GENERATION, 2, role).unwrap(); + let (reader, reader_server) = std::os::unix::net::UnixStream::pair().unwrap(); + let reader_owner = Arc::clone(&server); + let reader_task = tokio::spawn(async move { reader_owner.serve_client(reader_server, reader_grant).await }); + let client = LedgerClient::connect(reader, reader_grant, dir.path().join("session.db")) + .await + .unwrap(); + Self { + _dir: dir, + client, + writer, + producer_task, + reader_task, + } + } + + async fn write_network_body(&self) { + self.writer + .write_checked(WriteOp::NetEvent(NetEvent { + event_id: Some(EVENT_ID.into()), + timestamp: SystemTime::now(), + domain: "example.test".into(), + port: 443, + decision: Decision::Allowed, + process_name: Some("curl".into()), + pid: Some(7), + method: Some("POST".into()), + path: Some("/v1".into()), + query: None, + status_code: Some(200), + bytes_sent: 7, + bytes_received: 8, + duration_ms: 9, + matched_rule: None, + request_headers: None, + response_headers: None, + request_body: Some(b"request".to_vec()), + response_body: Some(b"response".to_vec()), + conn_type: Some("tls".into()), + policy_mode: None, + policy_action: None, + policy_rule: None, + policy_reason: None, + trace_id: None, + credential_ref: None, + })) + .await + .unwrap(); + self.writer.flush_checked().await.unwrap(); + } + + async fn stop(self) { + let Fixture { + client, + writer, + producer_task, + reader_task, + .. + } = self; + drop(client); + tokio::task::spawn_blocking(move || writer.shutdown_blocking()) + .await + .unwrap(); + assert_eq!( + producer_task.await.unwrap().unwrap(), + crate::ledger_server::LedgerClientExit::Disconnected + ); + assert_eq!( + reader_task.await.unwrap().unwrap(), + crate::ledger_server::LedgerClientExit::Disconnected + ); + } +} + +#[tokio::test] +async fn reader_queries_counters_bodies_and_warc_through_one_serial_channel() { + let fixture = Fixture::start(LedgerClientRole::Reader).await; + assert_eq!(fixture.client.path(), fixture._dir.path().join("session.db")); + assert_eq!(fixture.client.role(), LedgerClientRole::Reader); + fixture.write_network_body().await; + + let counters = fixture.client.counters().await.unwrap(); + assert_eq!(counters.net.total, 1); + assert_eq!(fixture.client.read_cache_epoch(), 1); + assert_eq!(fixture.client.counters().await.unwrap().net.total, 1); + assert_eq!(fixture.client.read_cache_epoch(), 1); + + let sets = fixture.client.query(LedgerQuery::StatsDetail).await.unwrap(); + assert_eq!(sets.len(), 11); + let bodies = fixture.client.read_bodies(EVENT_ID).await.unwrap(); + assert_eq!(bodies.len(), 2); + assert_eq!(bodies[0].bytes, b"request"); + assert_eq!(bodies[1].bytes, b"response"); + + let mut export = fixture.client.export_warc().await.unwrap(); + let mut warc = Vec::new(); + while let Some(chunk) = export.next_chunk().await { + warc.extend_from_slice(&chunk.unwrap()); + } + let summary = export.finish().await.unwrap(); + assert_eq!(summary.records, 2); + assert_eq!(summary.bytes_written, warc.len() as u64); + assert!(!warc.is_empty()); + fixture.stop().await; +} + +#[tokio::test] +async fn counters_advance_the_cache_epoch_only_when_the_snapshot_changes() { + let fixture = Fixture::start(LedgerClientRole::Reader).await; + assert_eq!(fixture.client.counters().await.unwrap().net.total, 0); + assert_eq!(fixture.client.read_cache_epoch(), 1); + fixture.write_network_body().await; + assert_eq!(fixture.client.counters().await.unwrap().net.total, 1); + assert_eq!(fixture.client.read_cache_epoch(), 2); + fixture.stop().await; +} + +#[tokio::test] +async fn server_refusal_is_reported_and_closes_the_failed_client_actor() { + let fixture = Fixture::start(LedgerClientRole::Reader).await; + let error = fixture + .client + .retain_bodies_since("2026-01-01T00:00:00.000000Z") + .await + .unwrap_err(); + assert!(error.contains("UnauthorizedOperation"), "{error}"); + let stopped = fixture.client.counters().await.unwrap_err(); + assert!(stopped.contains("stopped") || stopped.contains("closed"), "{stopped}"); + fixture.stop().await; +} diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index 0a39943df..dbf54d24e 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -139,7 +139,7 @@ impl LedgerServer { Ok(bodies) => self.send_bodies(sender, request_id, bodies).await, Err(error) => Err(storage(error)), }, - LedgerCommand::Counters => match self.db.ledger_counters().await { + LedgerCommand::Counters => match self.current_counters().await { Ok(counters) => { send_success( sender, @@ -168,6 +168,11 @@ impl LedgerServer { Ok(false) } + async fn current_counters(&self) -> Result, String> { + self.db.ready().await?; + self.db.ledger_counters().await + } + async fn send_bodies(&self, sender: &ServerSender, request_id: u64, bodies: Vec) -> Result<(), String> { for (index, body) in bodies.into_iter().enumerate() { let body_id = u32::try_from(index + 1).map_err(|_| "too many bodies in one ledger reply")?; diff --git a/crates/capsem-logger/src/lib.rs b/crates/capsem-logger/src/lib.rs index 48a532e52..4d72e3de0 100644 --- a/crates/capsem-logger/src/lib.rs +++ b/crates/capsem-logger/src/lib.rs @@ -1,6 +1,7 @@ pub mod counters; pub mod db; pub mod events; +pub mod ledger_client; pub mod ledger_protocol; pub mod ledger_server; pub mod network_db; From b4da8d613335de7eb12c9e5357b976cb68acc768 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:36:31 +0000 Subject: [PATCH 121/464] refactor(logger): share named query dispatch --- crates/capsem-logger/src/ledger_server.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index dbf54d24e..c6cc83241 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -244,6 +244,16 @@ impl LedgerServer { } } +/// Execute the same bounded named intent as a descriptor client against an +/// already-owned handle. Embedded tests and the worker dispatch share this +/// rail so neither can grow a second SQL definition. +pub async fn execute_named_query( + db: &DbHandle, + query: crate::ledger_protocol::LedgerQuery, +) -> Result, String> { + queries::execute(db, query).await +} + struct WarcChunkWriter { chunks: tokio::sync::mpsc::Sender>, } From c2491791a397614391b86ed7da07d6903ebddab0 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:45:08 +0000 Subject: [PATCH 122/464] fix(logger): bound history offsets for sqlite --- .../capsem-logger/src/ledger_client/tests.rs | 20 ++++++++++++++++++- .../src/ledger_server/queries.rs | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/capsem-logger/src/ledger_client/tests.rs b/crates/capsem-logger/src/ledger_client/tests.rs index 984e90f2d..521398541 100644 --- a/crates/capsem-logger/src/ledger_client/tests.rs +++ b/crates/capsem-logger/src/ledger_client/tests.rs @@ -5,7 +5,7 @@ use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneratio use super::*; use crate::events::{Decision, NetEvent}; -use crate::ledger_protocol::LedgerQuery; +use crate::ledger_protocol::{LedgerHistoryLayer, LedgerQuery}; use crate::{ledger_server::LedgerServer, WriteOp}; const GENERATION: LedgerGeneration = LedgerGeneration::new([0x51; 16]); @@ -167,3 +167,21 @@ async fn server_refusal_is_reported_and_closes_the_failed_client_actor() { assert!(stopped.contains("stopped") || stopped.contains("closed"), "{stopped}"); fixture.stop().await; } + +#[tokio::test] +async fn history_offset_beyond_sqlite_integer_range_is_an_empty_page() { + let fixture = Fixture::start(LedgerClientRole::Reader).await; + let sets = fixture + .client + .query(LedgerQuery::History { + layers: vec![LedgerHistoryLayer::Exec], + search: None, + limit: 1, + offset: u64::MAX, + }) + .await + .unwrap(); + assert_eq!(sets.len(), 1); + assert!(sets[0].rows.is_empty()); + fixture.stop().await; +} diff --git a/crates/capsem-logger/src/ledger_server/queries.rs b/crates/capsem-logger/src/ledger_server/queries.rs index 1255a16ef..36b7350b8 100644 --- a/crates/capsem-logger/src/ledger_server/queries.rs +++ b/crates/capsem-logger/src/ledger_server/queries.rs @@ -132,9 +132,9 @@ fn history_statements( .join(" UNION ALL "); let params = vec![ search.clone().map_or(Value::Null, Value::String), - json!(offset.saturating_add(u64::from(limit))), + json!(i64::try_from(offset.saturating_add(u64::from(limit))).unwrap_or(i64::MAX)), json!(limit), - json!(offset), + json!(i64::try_from(offset).unwrap_or(i64::MAX)), ]; let mut statements = vec![(format!("{page} ORDER BY timestamp DESC LIMIT ?3 OFFSET ?4"), params)]; if let Some(search) = search { From 90f6343d3036fac1c5706ec3b0eadf2bc737e90a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:49:21 +0000 Subject: [PATCH 123/464] security(service): read sessions through ledger grants --- CHANGELOG.md | 5 + crates/capsem-service/src/instance/tests.rs | 20 +- crates/capsem-service/src/ledger_routes.rs | 6 +- .../src/ledger_routes/bodies.rs | 88 ++----- .../src/ledger_routes/history.rs | 75 +++--- .../capsem-service/src/ledger_routes/rows.rs | 104 +++------ .../src/ledger_routes/security.rs | 35 +-- .../src/ledger_routes/stats_detail.rs | 15 +- .../src/ledger_routes/timeline.rs | 41 ++-- crates/capsem-service/src/main.rs | 7 +- crates/capsem-service/src/service_runtime.rs | 2 +- .../capsem-service/src/session_db_handles.rs | 218 +++++++++++++++--- crates/capsem-service/src/tests.rs | 6 + .../src/tests/db_handle_ownership.rs | 18 +- crates/capsem-service/src/tests/lifecycle.rs | 2 +- .../src/tests/route_query_plans.rs | 5 +- .../src/tests/session_identity.rs | 4 +- crates/capsem-service/src/tests/vm_info.rs | 2 +- .../src/vm_files/diagnostics.rs | 59 +++-- crates/capsem-service/src/vm_lifecycle.rs | 13 ++ .../src/vm_lifecycle/resume_process.rs | 19 +- .../src/vm_lifecycle/session_dirs.rs | 12 +- 22 files changed, 451 insertions(+), 305 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 60c732a07..8671ebedf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Serve live and stopped-session inspection through coordinator-granted, + role-bound ledger channels and bounded named operations, eliminating the + service's per-session SQLite readers and closing ledger workers before a + session is deleted. + - Give each VM owner one coordinator-minted, generation- and role-bound ledger channel and route its existing event writer through the supervised session owner, eliminating the VM process's direct SQLite open. diff --git a/crates/capsem-service/src/instance/tests.rs b/crates/capsem-service/src/instance/tests.rs index 8d48ddb86..ea764ad13 100644 --- a/crates/capsem-service/src/instance/tests.rs +++ b/crates/capsem-service/src/instance/tests.rs @@ -14,19 +14,13 @@ async fn stale_shutdown_binding_leaves_replacement_socket_and_ledger_intact() { instance.uds_path = socket.clone(); let generation = instance.generation; state.instances.lock().unwrap().insert("replacement".into(), instance); - let ledger = Arc::clone(&state.host_ledger); - state - .session_db_handles - .lock() - .unwrap() - .insert("replacement".into(), Arc::clone(&ledger)); - let result = shutdown_vm_process( - &state, - "replacement", - ShutdownMode::Discard, - Some(capsem_crypto::uuid_v4()), - ) - .await; + let ledger_dir = tempfile::tempdir().unwrap(); + let writer = capsem_logger::DbWriter::open(&ledger_dir.path().join("session.db"), 16).unwrap(); + writer.shutdown_blocking(); + let ledger = state + .register_session_db_handle("replacement", ledger_dir.path()) + .unwrap(); + let result = shutdown_vm_process(&state, "replacement", ShutdownMode::Discard, Some(uuid::Uuid::new_v4())).await; assert_eq!(result.unwrap_err().0, StatusCode::CONFLICT); assert_eq!( state.instances.lock().unwrap().get("replacement").unwrap().generation, diff --git a/crates/capsem-service/src/ledger_routes.rs b/crates/capsem-service/src/ledger_routes.rs index 9f269c053..0887957c7 100644 --- a/crates/capsem-service/src/ledger_routes.rs +++ b/crates/capsem-service/src/ledger_routes.rs @@ -5,7 +5,7 @@ pub(super) use bodies::{handle_bodies_warc_export, handle_event_bodies}; mod response_cache; mod rows; pub(crate) use response_cache::{forget_session_responses, session_response_cache_lookup, SessionResponseCache}; -use rows::{query_route_objects, query_route_typed_rows, route_query_objects}; +use rows::{ledger_rows_to_objects, query_route_typed_rows}; pub(crate) mod stats_detail; pub(super) use stats_detail::read_stats_detail_payload_from_session_db; pub(crate) mod history; @@ -149,7 +149,7 @@ pub(super) async fn open_ready_session_db( vm_id: &str, ledger: &str, db_path: &StdPath, -) -> Result, AppError> { +) -> Result, AppError> { let db = session_db(state, vm_id, ledger, db_path).await?; db.ready() .await @@ -166,7 +166,7 @@ pub(super) async fn session_db( vm_id: &str, ledger: &str, db_path: &StdPath, -) -> Result, AppError> { +) -> Result, AppError> { if !db_path.exists() { error!( vm_id, diff --git a/crates/capsem-service/src/ledger_routes/bodies.rs b/crates/capsem-service/src/ledger_routes/bodies.rs index 731f067d9..bbfbfb418 100644 --- a/crates/capsem-service/src/ledger_routes/bodies.rs +++ b/crates/capsem-service/src/ledger_routes/bodies.rs @@ -248,23 +248,9 @@ pub(crate) async fn handle_event_bodies( })) } -/// Chunks in flight between the blocking export and the HTTP response. -/// -/// Small on purpose: the channel is backpressure, not a buffer. A slow client -/// stalls the export thread instead of letting a whole session's bodies pile -/// up in the service's memory. +/// Chunks in flight between the ledger worker and the HTTP response. const EXPORT_CHANNEL_CHUNKS: usize = 4; -/// The export's writer: each `write` hands a chunk to the response stream and -/// waits with a deadline while the client is behind. `export_warc` moves this -/// writer onto a blocking task, so entering the runtime here cannot block one -/// of its worker threads. -struct ExportChannelWriter { - chunks: tokio::sync::mpsc::Sender>, - runtime: tokio::runtime::Handle, - deadline: std::time::Instant, -} - /// What a dropped receiver is reported as. /// /// The export flattens its writer's `io::Error` into a message on the way out, @@ -272,37 +258,7 @@ struct ExportChannelWriter { /// all that is left to tell a cancelled download from a broken one. It is a /// constant rather than two literals precisely because a match on a message /// written twice is a match that drifts. -const EXPORT_CLIENT_GONE: &str = "the export client went away"; const EXPORT_BLOCKED_WRITE: std::time::Duration = std::time::Duration::from_secs(30); -const EXPORT_TOTAL_DEADLINE: std::time::Duration = std::time::Duration::from_secs(15 * 60); - -impl std::io::Write for ExportChannelWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let remaining = self - .deadline - .checked_duration_since(std::time::Instant::now()) - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::TimedOut, "WARC export exceeded 15 minutes"))?; - let wait = remaining.min(EXPORT_BLOCKED_WRITE); - match self.runtime.block_on(tokio::time::timeout( - wait, - self.chunks.send(Ok(Bytes::copy_from_slice(buf))), - )) { - Ok(Ok(())) => {} - Ok(Err(_)) => return Err(std::io::Error::new(std::io::ErrorKind::BrokenPipe, EXPORT_CLIENT_GONE)), - Err(_) => { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "WARC export response was blocked for 30 seconds", - )) - } - } - Ok(buf.len()) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} /// `GET /vms/{id}/bodies/export.warc.gz` -- the whole session's archived /// bodies as a WARC 1.1 file any web-archive tool can read. @@ -341,16 +297,30 @@ pub(crate) async fn handle_bodies_warc_export( let (chunks, receiver) = tokio::sync::mpsc::channel(EXPORT_CHANNEL_CHUNKS); let vm_id = id.clone(); - let runtime = tokio::runtime::Handle::current(); tokio::spawn(async move { - match db - .export_warc(ExportChannelWriter { - chunks, - runtime, - deadline: std::time::Instant::now() + EXPORT_TOTAL_DEADLINE, - }) - .await - { + let mut export = match db.export_warc().await { + Ok(export) => export, + Err(error) => { + error!(vm_id = vm_id.as_str(), error = %error, "session body WARC export failed before streaming"); + let _ = chunks.send(Err(std::io::Error::other(error))).await; + return; + } + }; + while let Some(chunk) = export.next_chunk().await { + let chunk = chunk.map(Bytes::from).map_err(std::io::Error::other); + match tokio::time::timeout(EXPORT_BLOCKED_WRITE, chunks.send(chunk)).await { + Ok(Ok(())) => {} + Ok(Err(_)) => return, + Err(_) => { + error!( + vm_id = vm_id.as_str(), + "WARC export response was blocked for 30 seconds" + ); + return; + } + } + } + match export.finish().await { Ok(summary) => info!( route = "/vms/{id}/bodies/export.warc.gz", vm_id = vm_id.as_str(), @@ -359,16 +329,6 @@ pub(crate) async fn handle_bodies_warc_export( skipped = summary.skipped_count, "bodies_warc_export" ), - // A client that closes the connection mid-download is the normal - // way this ends -- a reviewer who saw enough, a page navigated - // away from -- and logging it at error level would fill the log - // with the one outcome nobody needs to investigate, beside the one - // they do. - Err(error) if error.contains(EXPORT_CLIENT_GONE) => info!( - route = "/vms/{id}/bodies/export.warc.gz", - vm_id = vm_id.as_str(), - "session body WARC export was cancelled by the client" - ), Err(error) => error!( route = "/vms/{id}/bodies/export.warc.gz", vm_id = vm_id.as_str(), diff --git a/crates/capsem-service/src/ledger_routes/history.rs b/crates/capsem-service/src/ledger_routes/history.rs index b7ffb976b..682d7fb91 100644 --- a/crates/capsem-service/src/ledger_routes/history.rs +++ b/crates/capsem-service/src/ledger_routes/history.rs @@ -18,6 +18,7 @@ use super::*; +#[cfg(test)] const EXEC_ROWS: &str = r#" SELECT timestamp, 'exec' AS layer, command, exit_code, duration_ms, stdout_preview, stderr_preview, @@ -30,6 +31,7 @@ SELECT timestamp, 'exec' AS layer, command, exit_code, duration_ms, FROM exec_events "#; +#[cfg(test)] const AUDIT_ROWS: &str = r#" SELECT timestamp, 'audit' AS layer, argv AS command, exit_code, NULL AS duration_ms, NULL AS stdout_preview, NULL AS stderr_preview, @@ -49,15 +51,18 @@ FROM audit_events "#; /// `?1` is the search text in both statements. +#[cfg(test)] const SEARCH: &str = " WHERE instr(command, ?1) > 0 OR instr(stdout_preview, ?1) > 0 \ OR instr(stderr_preview, ?1) > 0 OR instr(details, ?1) > 0"; /// The one aggregate here: how many rows a search matches. +#[cfg(test)] const SEARCH_TOTAL: &str = "SELECT COUNT(*) AS total FROM ({arms})"; /// The maximum rows the reader returns for one page, as the API caps it. pub(crate) const PAGE_CAP: usize = 2000; +#[cfg(test)] fn arms(layer: api::HistoryLayerFilter) -> Vec<&'static str> { [ (api::HistoryLayer::Exec, EXEC_ROWS), @@ -69,6 +74,7 @@ fn arms(layer: api::HistoryLayerFilter) -> Vec<&'static str> { .collect() } +#[cfg(test)] fn filtered(rows: &str, search: bool) -> String { let search = if search { SEARCH } else { "" }; format!("SELECT * FROM ({rows}){search}") @@ -76,6 +82,7 @@ fn filtered(rows: &str, search: bool) -> String { /// One page: `?1` search (NULL and unused without one), `?2` rows per arm /// (`offset + limit`), `?3` limit, `?4` offset. +#[cfg(test)] pub(crate) fn page_sql(layer: api::HistoryLayerFilter, search: bool) -> String { let arms = arms(layer) .into_iter() @@ -92,6 +99,7 @@ pub(crate) fn page_sql(layer: api::HistoryLayerFilter, search: bool) -> String { ) } +#[cfg(test)] pub(crate) fn search_total_sql(layer: api::HistoryLayerFilter) -> String { let arms = arms(layer) .into_iter() @@ -100,10 +108,6 @@ pub(crate) fn search_total_sql(layer: api::HistoryLayerFilter) -> String { SEARCH_TOTAL.replace("{arms}", &arms.join(" UNION ALL ")) } -fn sql_integer(value: usize) -> serde_json::Value { - json!(i64::try_from(value).unwrap_or(i64::MAX)) -} - pub(crate) async fn history_page( state: &ServiceState, vm_id: &str, @@ -114,6 +118,39 @@ pub(crate) async fn history_page( let db = open_ready_session_db(state, vm_id, "history", &db_path).await?; let limit = params.limit.min(PAGE_CAP); let search = params.search.as_deref(); + let query = capsem_logger::ledger_protocol::LedgerQuery::History { + layers: [ + ( + api::HistoryLayer::Exec, + capsem_logger::ledger_protocol::LedgerHistoryLayer::Exec, + ), + ( + api::HistoryLayer::Audit, + capsem_logger::ledger_protocol::LedgerHistoryLayer::Audit, + ), + ] + .into_iter() + .filter(|(layer, _)| params.layer.includes(*layer)) + .map(|(_, layer)| layer) + .collect(), + search: params.search.clone(), + limit: u16::try_from(limit.max(1)).unwrap_or(2000), + offset: u64::try_from(params.offset).unwrap_or(u64::MAX), + }; + let mut sets = db + .query(query) + .await + .map_err(|error| ledger_route_error(vm_id, "history", "query", &db_path, error))?; + if sets.is_empty() { + return Err(ledger_route_error( + vm_id, + "history", + "query", + &db_path, + "history query returned no page", + )); + } + let page = ledger_rows_to_objects(sets.remove(0)); let total = match search { None => { let counters = db @@ -129,17 +166,8 @@ pub(crate) async fn history_page( .map(|(_, count)| count) .sum() } - Some(search) => { - let rows = query_route_objects( - vm_id, - "history", - "search total", - &db_path, - &db, - &search_total_sql(params.layer), - &[json!(search)], - ) - .await?; + Some(_) => { + let rows = sets.into_iter().next().map(ledger_rows_to_objects).unwrap_or_default(); rows.first() .and_then(|row| row.get("total")) .and_then(serde_json::Value::as_u64) @@ -149,22 +177,7 @@ pub(crate) async fn history_page( let commands = if limit == 0 || u64::try_from(params.offset).map_or(true, |offset| offset >= total) { Vec::new() } else { - let rows = query_route_objects( - vm_id, - "history", - "entries", - &db_path, - &db, - &page_sql(params.layer, search.is_some()), - &[ - search.map_or(serde_json::Value::Null, |search| json!(search)), - sql_integer(params.offset.saturating_add(limit)), - sql_integer(limit), - sql_integer(params.offset), - ], - ) - .await?; - rows.into_iter() + page.into_iter() .map(|row| decode_entry(vm_id, &db_path, row)) .collect::, AppError>>()? }; diff --git a/crates/capsem-service/src/ledger_routes/rows.rs b/crates/capsem-service/src/ledger_routes/rows.rs index 81d8361f6..3021b5aa8 100644 --- a/crates/capsem-service/src/ledger_routes/rows.rs +++ b/crates/capsem-service/src/ledger_routes/rows.rs @@ -1,61 +1,34 @@ -//! Ledger query results as route rows. -//! -//! The DB handle answers a query as `{"columns":[...],"rows":[[...]]}` JSON; -//! these turn that into the objects and typed rows routes return. +//! Typed ledger-worker results mapped into route JSON objects. use super::*; -/// A ledger query's `{"columns":[...],"rows":[[...]]}` result, as parsed. -#[derive(Deserialize)] -struct QueryRows { - columns: Vec, - rows: Vec>, -} - -/// One JSON object per row of a ledger query result, keyed by column. -/// -/// Values are moved out of the parsed result, never cloned: a polled route's -/// rows used to be parsed into a tree, deep-copied out of it, then copied -/// again into objects. -pub(super) fn query_json_to_objects(raw: &str) -> serde_json::Result> { - let QueryRows { columns, rows } = serde_json::from_str(raw)?; - Ok(rows +pub(super) fn ledger_rows_to_objects(rows: capsem_logger::ledger_protocol::LedgerRows) -> Vec { + rows.rows .into_iter() .map(|values| { let mut values = values.into_iter(); serde_json::Value::Object( - columns + rows.columns .iter() - .map(|column| (column.clone(), values.next().unwrap_or(serde_json::Value::Null))) + .map(|column| { + let value = values.next().map_or(serde_json::Value::Null, ledger_value_to_json); + (column.clone(), value) + }) .collect(), ) }) - .collect()) + .collect() } -/// [`query_json_to_objects`], naming the route and query when the result is -/// not the shape the DB handle promises. -pub(super) fn route_query_objects( - vm_id: &str, - ledger: &str, - query_name: &str, - db_path: &StdPath, - raw: &str, -) -> Result, AppError> { - query_json_to_objects(raw).map_err(|error| { - error!( - vm_id, - ledger, - operation = "parse query json", - query_name, - db_path = %db_path.display(), - error = %error, - "session ledger route DB query returned invalid JSON" - ); - AppError( - StatusCode::INTERNAL_SERVER_ERROR, - format!("{ledger} ledger query {query_name} returned invalid json for {vm_id}: {error}"), - ) - }) +fn ledger_value_to_json(value: capsem_logger::ledger_protocol::LedgerValue) -> serde_json::Value { + match value { + capsem_logger::ledger_protocol::LedgerValue::Null => serde_json::Value::Null, + capsem_logger::ledger_protocol::LedgerValue::Integer(value) => value.into(), + capsem_logger::ledger_protocol::LedgerValue::Real(value) => serde_json::json!(value), + capsem_logger::ledger_protocol::LedgerValue::Text(value) => value.into(), + capsem_logger::ledger_protocol::LedgerValue::Blob(value) => { + serde_json::Value::Array(value.into_iter().map(serde_json::Value::from).collect()) + } + } } pub(super) async fn query_route_objects( @@ -63,23 +36,11 @@ pub(super) async fn query_route_objects( ledger: &str, query_name: &str, db_path: &StdPath, - db: &capsem_logger::DbHandle, - sql: &str, - params: &[serde_json::Value], + db: &session_db_handles::SessionLedger, + query: capsem_logger::ledger_protocol::LedgerQuery, + set: usize, ) -> Result, AppError> { - // Down the batch rail, which carries its own readiness check and answers - // a repeat fetch of a ledger that has not moved from the handle's cache - // instead of executing the statement again. - let raw = db - .query_many(vec![(sql.to_string(), params.to_vec())]) - .await - .and_then(|results| { - let [raw]: [String; 1] = results - .try_into() - .map_err(|results: Vec| format!("query returned {} results, expected 1", results.len()))?; - Ok(raw) - }); - let raw = raw.map_err(|error| { + let mut sets = db.query(query).await.map_err(|error| { error!( vm_id, ledger, @@ -94,7 +55,16 @@ pub(super) async fn query_route_objects( format!("{ledger} ledger query {query_name} failed for {vm_id}: {error}"), ) })?; - route_query_objects(vm_id, ledger, query_name, db_path, &raw) + if set >= sets.len() { + return Err(ledger_route_error( + vm_id, + ledger, + query_name, + db_path, + format!("query returned {} result sets, missing set {set}", sets.len()), + )); + } + Ok(ledger_rows_to_objects(sets.swap_remove(set))) } pub(super) async fn query_route_typed_rows( @@ -102,14 +72,14 @@ pub(super) async fn query_route_typed_rows( ledger: &str, query_name: &str, db_path: &StdPath, - db: &capsem_logger::DbHandle, - sql: &str, - params: &[serde_json::Value], + db: &session_db_handles::SessionLedger, + query: capsem_logger::ledger_protocol::LedgerQuery, + set: usize, ) -> Result, AppError> where T: DeserializeOwned, { - let objects = query_route_objects(vm_id, ledger, query_name, db_path, db, sql, params).await?; + let objects = query_route_objects(vm_id, ledger, query_name, db_path, db, query, set).await?; objects .into_iter() .map(|object| { diff --git a/crates/capsem-service/src/ledger_routes/security.rs b/crates/capsem-service/src/ledger_routes/security.rs index 8716387d5..221c3ec88 100644 --- a/crates/capsem-service/src/ledger_routes/security.rs +++ b/crates/capsem-service/src/ledger_routes/security.rs @@ -20,6 +20,7 @@ pub(crate) struct SecuritySessionLedger { // The matched event's payload is archive-backed: it is read by event id with // `BodyDirection::Payload`, and the index metadata travels with the other // bodies in the stats-detail payload. +#[cfg(test)] pub(crate) const SECURITY_LATEST_SQL: &str = r#" SELECT event.timestamp_unix_ms, event.event_id, event.event_type, event.rule_id, event.rule_action, event.detection_level, @@ -54,8 +55,11 @@ async fn read_security_ledger( "latest", db_path, &db, - SECURITY_LATEST_SQL, - &[json!(SECURITY_LATEST_LIMIT)], + capsem_logger::ledger_protocol::LedgerQuery::SecurityLatest { + limit: SECURITY_LATEST_LIMIT as u16, + detection_only: false, + }, + 0, ) .await?; Ok(Some(SecuritySessionLedger { latest })) @@ -68,16 +72,21 @@ pub(crate) async fn security_latest_for_vm( detection_only: bool, ) -> Result, AppError> { let session_dir = resolve_session_dir(state, vm_id)?; - let Some(session) = read_security_session_ledger(state, vm_id, &session_dir.join("session.db")).await? else { - return Ok(Vec::new()); - }; - Ok(session - .latest - .iter() - .filter(|event| !detection_only || is_detection_rule_event(event)) - .take(limit) - .cloned() - .collect()) + let db_path = session_dir.join("session.db"); + let db = session_db(state, vm_id, "security", &db_path).await?; + query_route_typed_rows( + vm_id, + "security", + "latest", + &db_path, + &db, + capsem_logger::ledger_protocol::LedgerQuery::SecurityLatest { + limit: u16::try_from(limit).unwrap_or(2000), + detection_only, + }, + 0, + ) + .await } /// The security ledger's aggregates alone. @@ -110,7 +119,7 @@ pub(crate) async fn security_stats_for_session( async fn security_stats( vm_id: &str, db_path: &StdPath, - db: &capsem_logger::DbHandle, + db: &session_db_handles::SessionLedger, ) -> Result { let counters = db .ledger_counters() diff --git a/crates/capsem-service/src/ledger_routes/stats_detail.rs b/crates/capsem-service/src/ledger_routes/stats_detail.rs index c47092009..efdbb8b50 100644 --- a/crates/capsem-service/src/ledger_routes/stats_detail.rs +++ b/crates/capsem-service/src/ledger_routes/stats_detail.rs @@ -137,9 +137,9 @@ pub(super) fn decode_rows( vm_id: &str, db_path: &StdPath, query_name: &str, - raw: &str, + rows: capsem_logger::ledger_protocol::LedgerRows, ) -> Result, AppError> { - route_query_objects(vm_id, "stats_detail", query_name, db_path, raw)? + ledger_rows_to_objects(rows) .into_iter() .map(|mut row| { for name in BOOLEAN_FLAGS { @@ -205,15 +205,10 @@ pub(crate) async fn read_stats_detail_payload_from_session_db( let db = session_db(state, vm_id, "stats_detail", db_path).await?; let statements = stats_detail_statements(); let raw = db - .query_many( - statements - .iter() - .map(|(_, sql, params)| (sql.to_string(), params.clone())) - .collect(), - ) + .query(capsem_logger::ledger_protocol::LedgerQuery::StatsDetail) .await .map_err(|error| ledger_route_error(vm_id, "stats_detail", "query", db_path, error))?; - let raw = <[String; 11]>::try_from(raw).map_err(|raw| { + let raw = <[capsem_logger::ledger_protocol::LedgerRows; 11]>::try_from(raw).map_err(|raw| { ledger_route_error( vm_id, "stats_detail", @@ -222,7 +217,7 @@ pub(crate) async fn read_stats_detail_payload_from_session_db( format!("batch returned {} results, expected {}", raw.len(), statements.len()), ) })?; - let [body_blobs, models, tools, model_events, tool_events, http, dns, files, processes, audit, credentials] = &raw; + let [body_blobs, models, tools, model_events, tool_events, http, dns, files, processes, audit, credentials] = raw; let name = |index: usize| statements[index].0; let bodies: Vec = decode_rows(vm_id, db_path, name(0), body_blobs)?; let mut body_blobs: BTreeMap> = BTreeMap::new(); diff --git a/crates/capsem-service/src/ledger_routes/timeline.rs b/crates/capsem-service/src/ledger_routes/timeline.rs index fa04d68d9..efe21369a 100644 --- a/crates/capsem-service/src/ledger_routes/timeline.rs +++ b/crates/capsem-service/src/ledger_routes/timeline.rs @@ -35,13 +35,6 @@ pub(crate) async fn handle_timeline( }) .transpose()? .unwrap_or_default(); - // A tool call that never recorded its time reads as the epoch, and the - // filter has always compared that displayed time: an epoch cutoff keeps it. - let tool_cutoff = if cutoff.as_str() <= UNDATED_TOOL_CALL { - String::new() - } else { - cutoff.clone() - }; let limit = params.limit.unwrap_or(200).min(2000); let session_dir = resolve_session_dir(&state, &id)?; let db_path = session_dir.join("session.db"); @@ -52,15 +45,22 @@ pub(crate) async fn handle_timeline( "events", &db_path, &db, - &timeline_sql(&layers), - &[ - json!(limit), - json!(cutoff), - json!(tool_cutoff), - params - .trace_id - .map_or(serde_json::Value::Null, serde_json::Value::String), - ], + capsem_logger::ledger_protocol::LedgerQuery::Timeline { + layers: layers + .iter() + .map(|layer| match layer { + TimelineLayer::Exec => capsem_logger::ledger_protocol::LedgerTimelineLayer::Exec, + TimelineLayer::Tool => capsem_logger::ledger_protocol::LedgerTimelineLayer::Tool, + TimelineLayer::Net => capsem_logger::ledger_protocol::LedgerTimelineLayer::Net, + TimelineLayer::Fs => capsem_logger::ledger_protocol::LedgerTimelineLayer::File, + TimelineLayer::Model => capsem_logger::ledger_protocol::LedgerTimelineLayer::Model, + }) + .collect(), + cutoff, + trace_id: params.trace_id, + limit: u16::try_from(limit).unwrap_or(2000), + }, + 0, ) .await?; Ok(Json(api::TimelineResponse { events })) @@ -74,11 +74,9 @@ const LAYERS: [TimelineLayer; 5] = [ TimelineLayer::Model, ]; -/// How a tool call without a recorded time is displayed and ordered. -const UNDATED_TOOL_CALL: &str = "1970-01-01T00:00:00Z"; - /// Rows a layer keeps: at or after the cutoff, and in the requested trace or /// in none. The cutoff is `''` without `since`, which every timestamp passes. +#[cfg(test)] const EXEC_WINDOW: &str = "SELECT timestamp, 'exec' AS layer, exec_id AS ref, command AS summary, \ exit_code AS status, duration_ms, trace_id FROM exec_events \ WHERE timestamp >= ?2 AND (?4 IS NULL OR trace_id = ?4 OR trace_id IS NULL) \ @@ -88,6 +86,7 @@ const EXEC_WINDOW: &str = "SELECT timestamp, 'exec' AS layer, exec_id AS ref, co /// serves; an empty one sorts first, where the epoch it displays as would. /// `+tc.origin` keeps the planner on the timestamp index: nearly every call /// has one of these origins, so the origin index would only add a sort. +#[cfg(test)] const TOOL_WINDOW: &str = "SELECT COALESCE(NULLIF(tc.timestamp, ''), '1970-01-01T00:00:00Z') AS timestamp, \ 'tool' AS layer, tc.event_id AS ref, \ COALESCE(tc.server_name, tc.origin) || '/' || tc.tool_name || COALESCE(' (call_id=' || tc.call_id || ')', '') AS summary, \ @@ -97,17 +96,20 @@ const TOOL_WINDOW: &str = "SELECT COALESCE(NULLIF(tc.timestamp, ''), '1970-01-01 AND tc.timestamp >= ?3 AND (?4 IS NULL OR tc.trace_id = ?4 OR tc.trace_id IS NULL) \ ORDER BY tc.timestamp ASC LIMIT ?1"; +#[cfg(test)] const NET_WINDOW: &str = "SELECT timestamp, 'net' AS layer, id AS ref, \ COALESCE(method, 'GET') || ' ' || domain || COALESCE(path, '') AS summary, \ status_code AS status, duration_ms, trace_id FROM net_events \ WHERE timestamp >= ?2 AND (?4 IS NULL OR trace_id = ?4 OR trace_id IS NULL) \ ORDER BY timestamp ASC LIMIT ?1"; +#[cfg(test)] const FS_WINDOW: &str = "SELECT timestamp, 'fs' AS layer, id AS ref, action || ' ' || path AS summary, \ NULL AS status, NULL AS duration_ms, trace_id FROM fs_events \ WHERE timestamp >= ?2 AND (?4 IS NULL OR trace_id = ?4 OR trace_id IS NULL) \ ORDER BY timestamp ASC LIMIT ?1"; +#[cfg(test)] const MODEL_WINDOW: &str = "SELECT timestamp, 'model' AS layer, id AS ref, \ provider || '/' || COALESCE(model, '?') AS summary, \ status_code AS status, duration_ms, trace_id FROM model_calls \ @@ -117,6 +119,7 @@ const MODEL_WINDOW: &str = "SELECT timestamp, 'model' AS layer, id AS ref, \ /// The requested layers' windows, merged oldest first. Parameters: `?1` /// limit, `?2` cutoff, `?3` tool cutoff, `?4` trace id or NULL. Every window /// names `?4`, so the statement always takes all four. +#[cfg(test)] pub(crate) fn timeline_sql(layers: &[TimelineLayer]) -> String { let windows = layers .iter() diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 62e3a25e8..954fb01a6 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -212,10 +212,9 @@ pub struct ServiceState { proxy_workers: Mutex>, ledger_workers: Arc, retirements: instance_reaper::Retirements, - /// Logger-owned DB handles keyed by session/VM id. Logged-data routes - /// resolve a handle here and call `ready/query`; they do not open SQLite - /// readers or create per-route projection caches. - session_db_handles: Mutex>>, + /// Authenticated ledger channels keyed by session/VM id. Logged-data + /// routes use named operations; only the supervised worker opens SQLite. + session_db_handles: Mutex>>, persistent_registry: SharedRegistry, /// Named networks as groups of VMs, durable in each network's database. networks: tokio::sync::Mutex, diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index 7df344141..bb209eba9 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -300,7 +300,7 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { _test_tempdir: None, }); - state.hydrate_session_db_handles(); + state.hydrate_session_db_handles().await; state.hydrate_host_stats().await?; state .record_service_event(capsem_logger::HostEventKind::ServiceStarted) diff --git a/crates/capsem-service/src/session_db_handles.rs b/crates/capsem-service/src/session_db_handles.rs index cc914ffaa..16ac6b8b7 100644 --- a/crates/capsem-service/src/session_db_handles.rs +++ b/crates/capsem-service/src/session_db_handles.rs @@ -1,57 +1,219 @@ -//! The per-session logger DB handles the service holds for its routes: -//! registration, replacement on a rebound path, and startup hydration. +//! Authenticated per-session ledger channels held by service routes. use super::*; +use capsem_logger::ledger_protocol::{LedgerQuery, LedgerRows}; +#[cfg(not(test))] +use capsem_proto::ledger::LedgerClientRole; +use capsem_proto::ledger_counters::LedgerCounters; pub(super) fn session_db_path_for_session_dir(session_dir: &StdPath) -> PathBuf { session_dir.join("session.db") } -/// The one place the service opens a per-session external reader. Blocking: -/// callers on the runtime reach it through `spawn_blocking`. -fn open_session_db_reader(db_path: &StdPath) -> Result { - capsem_logger::DbHandle::open_external_reader(db_path).map_err(|error| error.to_string()) +pub(crate) struct SessionLedger { + path: PathBuf, + client: Option, + #[cfg(test)] + embedded: Option, +} + +impl SessionLedger { + #[cfg(not(test))] + fn remote(path: PathBuf, client: capsem_logger::ledger_client::LedgerClient) -> Self { + Self { + path, + client: Some(client), + #[cfg(test)] + embedded: None, + } + } + + #[cfg(test)] + pub(crate) fn embedded(path: PathBuf, db: capsem_logger::DbHandle) -> Self { + Self { + path, + client: None, + embedded: Some(db), + } + } + + pub(crate) fn path(&self) -> &StdPath { + &self.path + } + + pub(crate) async fn ready(&self) -> Result<(), String> { + #[cfg(test)] + if let Some(db) = &self.embedded { + return db.ready().await; + } + self.client().counters().await.map(|_| ()) + } + + pub(crate) async fn query(&self, query: LedgerQuery) -> Result, String> { + #[cfg(test)] + if let Some(db) = &self.embedded { + return capsem_logger::ledger_server::execute_named_query(db, query).await; + } + self.client().query(query).await + } + + pub(crate) async fn ledger_counters(&self) -> Result, String> { + #[cfg(test)] + if let Some(db) = &self.embedded { + return db.ledger_counters().await; + } + self.client().counters().await + } + + pub(crate) fn read_cache_epoch(&self, _domain: capsem_logger::ReadCacheDomain) -> u64 { + #[cfg(test)] + if let Some(db) = &self.embedded { + return db.read_cache_epoch(capsem_logger::ReadCacheDomain::All); + } + self.client().read_cache_epoch() + } + + #[cfg(test)] + pub(crate) fn reader_requests(&self) -> u64 { + self.embedded + .as_ref() + .map_or(0, capsem_logger::DbHandle::reader_requests) + } + + pub(crate) async fn read_bodies(&self, event_id: &str) -> Result, String> { + #[cfg(test)] + if let Some(db) = &self.embedded { + return db.read_bodies(event_id).await; + } + self.client().read_bodies(event_id).await + } + + pub(crate) async fn export_warc(&self) -> Result { + #[cfg(test)] + if let Some(db) = &self.embedded { + let bytes = Arc::new(Mutex::new(Vec::new())); + let writer = TestWarcWriter(Arc::clone(&bytes)); + let summary = db.export_warc(writer).await?; + let bytes = std::mem::take(&mut *bytes.lock().unwrap()); + return Ok(SessionWarcExport::Embedded { + bytes: Some(bytes), + summary: capsem_logger::ledger_protocol::LedgerExportSummary { + records: summary.records, + bytes_written: summary.bytes_written, + skipped_count: summary.skipped_count, + skipped_by_reason: summary + .counts_by_reason() + .into_iter() + .map(|(reason, count)| (reason.to_string(), count)) + .collect(), + }, + }); + } + Ok(SessionWarcExport::Remote(self.client().export_warc().await?)) + } + + fn client(&self) -> &capsem_logger::ledger_client::LedgerClient { + self.client.as_ref().expect("production session ledger has a client") + } +} + +#[cfg(test)] +struct TestWarcWriter(Arc>>); + +#[cfg(test)] +impl std::io::Write for TestWarcWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +pub(crate) enum SessionWarcExport { + Remote(capsem_logger::ledger_client::LedgerWarcExport), + #[cfg(test)] + Embedded { + bytes: Option>, + summary: capsem_logger::ledger_protocol::LedgerExportSummary, + }, +} + +impl SessionWarcExport { + pub(crate) async fn next_chunk(&mut self) -> Option, String>> { + match self { + Self::Remote(export) => export.next_chunk().await, + #[cfg(test)] + Self::Embedded { bytes, .. } => bytes.take().map(Ok), + } + } + + pub(crate) async fn finish(self) -> Result { + match self { + Self::Remote(export) => export.finish().await, + #[cfg(test)] + Self::Embedded { summary, .. } => Ok(summary), + } + } } impl ServiceState { - /// Register `vm_id`'s ledger reader, opening it on this thread. - /// - /// Opening a reader is a blocking SQLite open and schema check, so this is - /// for code already off the runtime: startup hydration and `resume_sandbox`. - /// Async routes call [`Self::register_session_db_handle_async`]. + #[cfg(test)] pub(crate) fn register_session_db_handle( &self, vm_id: &str, session_dir: &StdPath, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let db_path = session_db_path_for_session_dir(session_dir); let started = std::time::Instant::now(); if let Some(handle) = self.registered_session_db_handle(vm_id, &db_path, started) { return Ok(handle); } - let opened = open_session_db_reader(&db_path); + let opened = crate::tests::test_session_ledger(db_path.clone()); self.install_session_db_handle(vm_id, db_path, opened, started) } - /// Register `vm_id`'s ledger reader without blocking a tokio worker. - /// - /// Same answer as [`Self::register_session_db_handle`]; the open runs on - /// the blocking pool, where a slow disk or a large schema check stalls one - /// blocking thread instead of every request sharing the worker. pub(crate) async fn register_session_db_handle_async( &self, vm_id: &str, session_dir: &StdPath, - ) -> anyhow::Result> { + ) -> anyhow::Result> { + #[cfg(test)] + return self.register_session_db_handle(vm_id, session_dir); + + #[cfg(not(test))] let db_path = session_db_path_for_session_dir(session_dir); + #[cfg(not(test))] let started = std::time::Instant::now(); + #[cfg(not(test))] if let Some(handle) = self.registered_session_db_handle(vm_id, &db_path, started) { return Ok(handle); } - let open_path = db_path.clone(); - let opened = tokio::task::spawn_blocking(move || open_session_db_reader(&open_path)) + #[cfg(not(test))] + let opened = self + .ledger_workers + .acquire( + vm_id, + &db_path, + &session_dir.join("ledger.log"), + LedgerClientRole::Reader, + ) .await - .unwrap_or_else(|error| Err(format!("session DB open task failed: {error}"))); + .map_err(|error| error.to_string()) + .map(|channel| { + let (stream, grant) = channel.into_parts(); + (stream, grant) + }); + #[cfg(not(test))] + let opened = match opened { + Ok((stream, grant)) => capsem_logger::ledger_client::LedgerClient::connect(stream, grant, db_path.clone()) + .await + .map(|client| SessionLedger::remote(db_path.clone(), client)), + Err(error) => Err(error), + }; + #[cfg(not(test))] self.install_session_db_handle(vm_id, db_path, opened, started) } @@ -62,7 +224,7 @@ impl ServiceState { vm_id: &str, db_path: &StdPath, started: std::time::Instant, - ) -> Option> { + ) -> Option> { let handle = self.session_db_handles.lock().unwrap().get(vm_id).cloned()?; if handle.path() == db_path { tracing::debug!( @@ -88,9 +250,9 @@ impl ServiceState { &self, vm_id: &str, db_path: PathBuf, - opened: Result, + opened: Result, started: std::time::Instant, - ) -> anyhow::Result> { + ) -> anyhow::Result> { let handle = match opened { Ok(handle) => Arc::new(handle), Err(error) => { @@ -149,11 +311,11 @@ impl ServiceState { } } - pub(crate) fn session_db_handle(&self, vm_id: &str) -> Option> { + pub(crate) fn session_db_handle(&self, vm_id: &str) -> Option> { self.session_db_handles.lock().unwrap().get(vm_id).cloned() } - pub(crate) fn hydrate_session_db_handles(&self) { + pub(crate) async fn hydrate_session_db_handles(&self) { let mut candidates: Vec<(String, PathBuf)> = { let instances = self.instances.lock().unwrap(); instances @@ -184,7 +346,7 @@ impl ServiceState { ); continue; } - match self.register_session_db_handle(&vm_id, &session_dir) { + match self.register_session_db_handle_async(&vm_id, &session_dir).await { Ok(_) => hydrated += 1, Err(error) => { warn!( diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 662ffd0fb..fa0115414 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -39,6 +39,12 @@ fn test_host_ledger(run_dir: &StdPath) -> Arc { host_ledger::open_host_ledger(&run_dir.parent().unwrap().join("sessions")).unwrap() } +pub(crate) fn test_session_ledger(db_path: PathBuf) -> Result { + capsem_logger::DbHandle::open_external_reader(&db_path) + .map(|db| session_db_handles::SessionLedger::embedded(db_path, db)) + .map_err(|error| error.to_string()) +} + /// The one test `ServiceState` constructor. The manifest is whatever /// `assets_dir` holds, as at service startup. pub(crate) fn test_state( diff --git a/crates/capsem-service/src/tests/db_handle_ownership.rs b/crates/capsem-service/src/tests/db_handle_ownership.rs index afdc75ea6..a5a714a9d 100644 --- a/crates/capsem-service/src/tests/db_handle_ownership.rs +++ b/crates/capsem-service/src/tests/db_handle_ownership.rs @@ -1,5 +1,5 @@ -//! Source contract: exactly one owner opens the main ledger and exactly one -//! opens per-session external readers. +//! Source contract: the service owns only its host ledger; session SQLite is +//! opened by the supervised ledger worker and reached through a granted FD. #[test] fn service_db_handle_open_is_owned_by_explicit_service_state_owners() { @@ -16,18 +16,16 @@ fn service_db_handle_open_is_owned_by_explicit_service_state_owners() { assert_eq!( opens, 1, "DbHandle::open must live only in the service main-ledger owner. Per-session \ - route ledgers are external readers because capsem-process owns writes. Routes and helpers \ - resolve registered handles and call ready/query/write; they do not create a second \ - DB lifecycle." + ledgers belong to capsem-ledger." ); assert_eq!( - external_reader_opens, 1, - "DbHandle::open_external_reader must live only in register_session_db_handle for \ - per-session ledgers written by capsem-process." + external_reader_opens, 0, + "service production code must not open per-session SQLite readers" ); assert!( - source.contains("fn register_session_db_handle(") && source.contains("DbHandle::open_external_reader("), - "the session-state registration method must own the external DB reader lifecycle" + source.contains("LedgerClientRole::Reader") + && source.contains("capsem_logger::ledger_client::LedgerClient::connect"), + "session registration must request reader authority and adopt the granted channel" ); assert!( source.contains("fn open_host_ledger("), diff --git a/crates/capsem-service/src/tests/lifecycle.rs b/crates/capsem-service/src/tests/lifecycle.rs index 2e4c80ab8..0657f04a9 100644 --- a/crates/capsem-service/src/tests/lifecycle.rs +++ b/crates/capsem-service/src/tests/lifecycle.rs @@ -1267,7 +1267,7 @@ async fn service_rehydrates_session_db_handles() { state.session_db_handle(&vm_id).is_none(), "test must prove startup hydration installs the handle" ); - state.hydrate_session_db_handles(); + state.hydrate_session_db_handles().await; let handle = state .session_db_handle(&vm_id) diff --git a/crates/capsem-service/src/tests/route_query_plans.rs b/crates/capsem-service/src/tests/route_query_plans.rs index 69c09f075..c1e626457 100644 --- a/crates/capsem-service/src/tests/route_query_plans.rs +++ b/crates/capsem-service/src/tests/route_query_plans.rs @@ -478,7 +478,10 @@ async fn stats_detail_lists_the_newest_tool_calls_with_their_joins() { async fn triage_lists_the_newest_tool_errors_first() { let dir = tempfile::tempdir().unwrap(); let db_path = seed_ledger(&dir.path().join("triage"), SMALL_LEDGER_ROWS); - let db = capsem_logger::DbHandle::open_external_reader(&db_path).unwrap(); + let state = make_test_state(); + let db = state + .register_session_db_handle("plans", db_path.parent().unwrap()) + .unwrap(); let triage = session_db_triage("plans", &db, &db_path, 5).await.unwrap(); let rows = triage["tool_errors"]["rows"].as_array().unwrap(); let messages: Vec<&str> = rows.iter().map(|row| row[8].as_str().unwrap()).collect(); diff --git a/crates/capsem-service/src/tests/session_identity.rs b/crates/capsem-service/src/tests/session_identity.rs index 62268c381..f4f5bc328 100644 --- a/crates/capsem-service/src/tests/session_identity.rs +++ b/crates/capsem-service/src/tests/session_identity.rs @@ -141,7 +141,7 @@ async fn startup_hydration_keys_persistent_db_handles_by_session_id() { register_persistent_entry(&state, &session_dir); write_one_security_event(session_dir.join("session.db")).await; - state.hydrate_session_db_handles(); + state.hydrate_session_db_handles().await; assert_eq!(session_db_handle_keys(&state), vec![VM_ID.to_string()]); assert!( state.session_db_handle(DISPLAY_NAME).is_none(), @@ -150,7 +150,7 @@ async fn startup_hydration_keys_persistent_db_handles_by_session_id() { // A running persistent VM must not gain a second handle on rehydration. insert_running_persistent_instance(&state, &session_dir); - state.hydrate_session_db_handles(); + state.hydrate_session_db_handles().await; assert_eq!(session_db_handle_keys(&state), vec![VM_ID.to_string()]); } diff --git a/crates/capsem-service/src/tests/vm_info.rs b/crates/capsem-service/src/tests/vm_info.rs index 089a95a24..0625753b5 100644 --- a/crates/capsem-service/src/tests/vm_info.rs +++ b/crates/capsem-service/src/tests/vm_info.rs @@ -233,7 +233,7 @@ async fn broken_session_db_schema_is_explicit_error_for_session_status() { .data .vms .insert("status-broken-db-vm".to_string(), entry); - state.hydrate_session_db_handles(); + state.hydrate_session_db_handles().await; let handle = state .session_db_handle(&vm_id) .expect("startup hydration installs the handle so routes surface the schema error explicitly"); diff --git a/crates/capsem-service/src/vm_files/diagnostics.rs b/crates/capsem-service/src/vm_files/diagnostics.rs index 87a587f45..a0b574690 100644 --- a/crates/capsem-service/src/vm_files/diagnostics.rs +++ b/crates/capsem-service/src/vm_files/diagnostics.rs @@ -202,6 +202,7 @@ pub(crate) async fn handle_triage( /// `idx_tool_calls_origin`, which for an `IN` list returns rows out of id order /// and sorts every counted tool call, and insertion order is newest first where /// `timestamp` is not (a call recorded without one sorted last). +#[cfg(test)] pub(crate) fn session_triage_statements(limit: usize) -> [(&'static str, String); 3] { let origins = capsem_logger::counters::counted_tool_origins_sql(); [ @@ -237,40 +238,64 @@ pub(crate) fn session_triage_statements(limit: usize) -> [(&'static str, String) pub(crate) async fn session_db_triage( vm_id: &str, - db: &capsem_logger::DbHandle, + db: &crate::session_db_handles::SessionLedger, db_path: &std::path::Path, limit: usize, ) -> anyhow::Result { db.ready() .await .map_err(|error| anyhow!("session triage ledger is not ready for {vm_id}: {error}"))?; + let sets = db + .query(capsem_logger::ledger_protocol::LedgerQuery::Triage { + limit: u16::try_from(limit.clamp(1, 2000)).unwrap_or(2000), + }) + .await + .map_err(|error| anyhow!("session triage query failed: {error}"))?; + let names = ["denied_net", "tool_errors", "exec_failures"]; + if sets.len() != names.len() { + return Err(anyhow!( + "session triage query returned {} result sets, expected {}", + sets.len(), + names.len() + )); + } let mut answers = serde_json::Map::new(); - for (query_name, sql) in session_triage_statements(limit) { - let raw = db.query(&sql, &[]).await.map_err(|error| { - error!( - vm_id, - query_name, - db_path = %db_path.display(), - error = %error, - "session triage ledger query failed" - ); - anyhow!("session triage query {query_name} failed: {error}") - })?; - let value = serde_json::from_str(&raw).map_err(|error| { + for (query_name, rows) in names.into_iter().zip(sets) { + let value = ledger_rows_columnar(rows); + if value.get("rows").is_none() { error!( vm_id, query_name, db_path = %db_path.display(), - error = %error, - "session triage ledger query returned invalid JSON" + "session triage ledger query returned invalid rows" ); - anyhow!("session triage query {query_name} returned invalid JSON: {error}") - })?; + } answers.insert(query_name.to_string(), value); } Ok(serde_json::Value::Object(answers)) } +fn ledger_rows_columnar(rows: capsem_logger::ledger_protocol::LedgerRows) -> serde_json::Value { + let values = rows + .rows + .into_iter() + .map(|row| row.into_iter().map(ledger_value_json).collect::>()) + .collect::>(); + json!({ "columns": rows.columns, "rows": values }) +} + +fn ledger_value_json(value: capsem_logger::ledger_protocol::LedgerValue) -> serde_json::Value { + match value { + capsem_logger::ledger_protocol::LedgerValue::Null => serde_json::Value::Null, + capsem_logger::ledger_protocol::LedgerValue::Integer(value) => value.into(), + capsem_logger::ledger_protocol::LedgerValue::Real(value) => json!(value), + capsem_logger::ledger_protocol::LedgerValue::Text(value) => value.into(), + capsem_logger::ledger_protocol::LedgerValue::Blob(value) => { + serde_json::Value::Array(value.into_iter().map(serde_json::Value::from).collect()) + } + } +} + pub(crate) fn limit_columnar_query_json(value: &serde_json::Value, limit: usize) -> serde_json::Value { let columns = value.get("columns").cloned().unwrap_or_else(|| json!([])); let rows = value diff --git a/crates/capsem-service/src/vm_lifecycle.rs b/crates/capsem-service/src/vm_lifecycle.rs index 85294c42f..6d47cb94f 100644 --- a/crates/capsem-service/src/vm_lifecycle.rs +++ b/crates/capsem-service/src/vm_lifecycle.rs @@ -376,6 +376,12 @@ pub(super) async fn shutdown_vm_process( "VM was killed before it finished flushing; writes made just before stop may be lost" ); } + state.ledger_workers.shutdown(id).await.map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to stop session ledger worker for {id}: {error}"), + ) + })?; let _ = std::fs::remove_file(&uds_path); let _ = std::fs::remove_file(uds_path.with_extension("ready")); state @@ -588,6 +594,13 @@ pub(super) async fn handle_delete( // success until the directory is actually gone. Failed VM exits use the // separate `preserve_failed_session_dir` path; a clean delete must never // be relabelled as a failure. + state.unregister_session_db_handle(&id); + state.ledger_workers.shutdown(&id).await.map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to stop session ledger worker for {id}: {error}"), + ) + })?; let state_clone = Arc::clone(&state); tokio::task::spawn_blocking(move || state_clone.delete_session_dir(&session_dir)) .await diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 3adbe4173..ca42bca8c 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -229,19 +229,12 @@ impl ServiceState { } }; - if session_db_path_for_session_dir(&entry.session_dir).exists() { - if let Err(error) = self.register_session_db_handle(&vm_id, &entry.session_dir) { - let _reaper = instance_reaper::kill_and_reap(child, retirement); - return Err(error); - } - } else { - info!( - vm_id = vm_id, - operation = "defer_session_db_handle_registration", - session_dir = %entry.session_dir.display(), - "session DB not present yet; route will register the external reader lazily" - ); - } + info!( + vm_id, + operation = "defer_session_db_handle_registration", + session_dir = %entry.session_dir.display(), + "session ledger reader will be granted lazily to the first route" + ); let authority = crate::instance::WorkerAuthority::default(); let upstream_grant = authority.grant(); diff --git a/crates/capsem-service/src/vm_lifecycle/session_dirs.rs b/crates/capsem-service/src/vm_lifecycle/session_dirs.rs index fdd56404d..095db1d92 100644 --- a/crates/capsem-service/src/vm_lifecycle/session_dirs.rs +++ b/crates/capsem-service/src/vm_lifecycle/session_dirs.rs @@ -79,13 +79,11 @@ pub(crate) fn settle_persistent_session_dir(state: &ServiceState, name: &str, ex /// reports is data, not authority, and used to go straight to /// `remove_dir_all`. pub(super) async fn remove_purged_session_dir(state: &Arc, vm_id: &str, session_dir: PathBuf) -> bool { - let _deletion_guard = match connection_routes::lifecycle::deletion(state, vm_id).await { - Ok(guard) => guard, - Err(error) => { - warn!(vm_id, error = %error.1, "purge credential retirement remains pending"); - return false; - } - }; + state.unregister_session_db_handle(vm_id); + if let Err(error) = state.ledger_workers.shutdown(vm_id).await { + warn!(vm_id, error = %error, "purge could not stop the session ledger worker"); + return false; + } let delete_state = Arc::clone(state); let dir = session_dir.clone(); let outcome = tokio::task::spawn_blocking(move || delete_state.delete_session_dir(&dir)).await; From 0e4ac804ac655667a554403dfd3bf6d11fc40e3c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:52:44 +0000 Subject: [PATCH 124/464] security(process): stop delegating ledger paths --- crates/capsem-process/src/main.rs | 2 -- crates/capsem-process/src/tests.rs | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index d0327fadc..61934ac7b 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -755,8 +755,6 @@ async fn run_async_main_loop( .and_then(|p| p.parent().map(|d| d.join("capsem-mcp-builtin"))); let mut builtin_env = std::collections::HashMap::new(); builtin_env.insert("CAPSEM_SESSION_DIR".into(), session_dir.to_string_lossy().to_string()); - let db_path = session_dir.join("session.db"); - builtin_env.insert("CAPSEM_SESSION_DB".into(), db_path.to_string_lossy().to_string()); builtin_env.insert( "CAPSEM_ACTIVE_POLICY".into(), runtime_config.active_policy_path.to_string_lossy().to_string(), diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index d712e3553..a270e246d 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -18,6 +18,10 @@ fn production_vm_owner_never_opens_the_session_database() { let source = include_str!("main.rs"); assert!(!source.contains("DbWriter::open(")); assert!(source.contains("DbWriter::from_ledger_channel(")); + assert!( + !source.contains("CAPSEM_SESSION_DB"), + "VM owner must not delegate the ledger path to built-in MCP servers" + ); } #[test] From 3acd0ef6e5832fee39a66ef1c534cf0e5270c7a5 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 05:58:47 +0000 Subject: [PATCH 125/464] fix(service): reconnect failed ledger readers --- .../capsem-service/src/session_db_handles.rs | 132 ++++++++++++++++-- .../src/tests/db_handle_ownership.rs | 4 + 2 files changed, 121 insertions(+), 15 deletions(-) diff --git a/crates/capsem-service/src/session_db_handles.rs b/crates/capsem-service/src/session_db_handles.rs index 16ac6b8b7..9464312a2 100644 --- a/crates/capsem-service/src/session_db_handles.rs +++ b/crates/capsem-service/src/session_db_handles.rs @@ -2,7 +2,6 @@ use super::*; use capsem_logger::ledger_protocol::{LedgerQuery, LedgerRows}; -#[cfg(not(test))] use capsem_proto::ledger::LedgerClientRole; use capsem_proto::ledger_counters::LedgerCounters; @@ -12,17 +11,38 @@ pub(super) fn session_db_path_for_session_dir(session_dir: &StdPath) -> PathBuf pub(crate) struct SessionLedger { path: PathBuf, - client: Option, + remote: Option, #[cfg(test)] embedded: Option, } +struct RemoteLedger { + vm_id: String, + session_dir: PathBuf, + workers: Arc, + client: tokio::sync::Mutex, + counters: Mutex>>, + read_cache_epoch: AtomicU64, +} + impl SessionLedger { #[cfg(not(test))] - fn remote(path: PathBuf, client: capsem_logger::ledger_client::LedgerClient) -> Self { + fn remote( + vm_id: String, + session_dir: PathBuf, + workers: Arc, + client: capsem_logger::ledger_client::LedgerClient, + ) -> Self { Self { - path, - client: Some(client), + path: session_db_path_for_session_dir(&session_dir), + remote: Some(RemoteLedger { + vm_id, + session_dir, + workers, + client: tokio::sync::Mutex::new(client), + counters: Mutex::new(None), + read_cache_epoch: AtomicU64::new(0), + }), #[cfg(test)] embedded: None, } @@ -32,7 +52,7 @@ impl SessionLedger { pub(crate) fn embedded(path: PathBuf, db: capsem_logger::DbHandle) -> Self { Self { path, - client: None, + remote: None, embedded: Some(db), } } @@ -46,7 +66,10 @@ impl SessionLedger { if let Some(db) = &self.embedded { return db.ready().await; } - self.client().counters().await.map(|_| ()) + self.remote_state() + .call(|client| async move { client.counters().await }) + .await + .map(|_| ()) } pub(crate) async fn query(&self, query: LedgerQuery) -> Result, String> { @@ -54,7 +77,12 @@ impl SessionLedger { if let Some(db) = &self.embedded { return capsem_logger::ledger_server::execute_named_query(db, query).await; } - self.client().query(query).await + self.remote_state() + .call(|client| { + let query = query.clone(); + async move { client.query(query).await } + }) + .await } pub(crate) async fn ledger_counters(&self) -> Result, String> { @@ -62,7 +90,17 @@ impl SessionLedger { if let Some(db) = &self.embedded { return db.ledger_counters().await; } - self.client().counters().await + let current = self + .remote_state() + .call(|client| async move { client.counters().await }) + .await?; + let mut previous = self.remote_state().counters.lock().unwrap(); + if previous.as_deref() != Some(current.as_ref()) { + *previous = Some(Arc::clone(¤t)); + self.remote_state().read_cache_epoch.fetch_add(1, Ordering::AcqRel); + } + drop(previous); + Ok(current) } pub(crate) fn read_cache_epoch(&self, _domain: capsem_logger::ReadCacheDomain) -> u64 { @@ -70,7 +108,7 @@ impl SessionLedger { if let Some(db) = &self.embedded { return db.read_cache_epoch(capsem_logger::ReadCacheDomain::All); } - self.client().read_cache_epoch() + self.remote_state().read_cache_epoch.load(Ordering::Acquire) } #[cfg(test)] @@ -85,7 +123,13 @@ impl SessionLedger { if let Some(db) = &self.embedded { return db.read_bodies(event_id).await; } - self.client().read_bodies(event_id).await + let event_id = event_id.to_string(); + self.remote_state() + .call(|client| { + let event_id = event_id.clone(); + async move { client.read_bodies(&event_id).await } + }) + .await } pub(crate) async fn export_warc(&self) -> Result { @@ -109,11 +153,62 @@ impl SessionLedger { }, }); } - Ok(SessionWarcExport::Remote(self.client().export_warc().await?)) + Ok(SessionWarcExport::Remote( + self.remote_state() + .call(|client| async move { client.export_warc().await }) + .await?, + )) + } + + fn remote_state(&self) -> &RemoteLedger { + self.remote + .as_ref() + .expect("production session ledger has a remote client") + } +} + +impl RemoteLedger { + async fn call(&self, operation: F) -> Result + where + F: Fn(capsem_logger::ledger_client::LedgerClient) -> Fut, + Fut: std::future::Future>, + { + let mut client = self.client.lock().await; + let first = operation(client.clone()).await; + let Err(first_error) = first else { + drop(client); + return first; + }; + let replacement = self + .reconnect() + .await + .map_err(|error| format!("ledger operation failed ({first_error}); reconnect failed: {error}"))?; + *client = replacement.clone(); + drop(client); + operation(replacement).await } - fn client(&self) -> &capsem_logger::ledger_client::LedgerClient { - self.client.as_ref().expect("production session ledger has a client") + async fn reconnect(&self) -> Result { + let database = session_db_path_for_session_dir(&self.session_dir); + let log = self.session_dir.join("ledger.log"); + let mut last_error = String::new(); + for _ in 0..2 { + match self + .workers + .acquire(&self.vm_id, &database, &log, LedgerClientRole::Reader) + .await + { + Ok(channel) => { + let (stream, grant) = channel.into_parts(); + return capsem_logger::ledger_client::LedgerClient::connect(stream, grant, database).await; + } + Err(error) => { + last_error = format!("{error:#}"); + tokio::task::yield_now().await; + } + } + } + Err(last_error) } } @@ -210,7 +305,14 @@ impl ServiceState { let opened = match opened { Ok((stream, grant)) => capsem_logger::ledger_client::LedgerClient::connect(stream, grant, db_path.clone()) .await - .map(|client| SessionLedger::remote(db_path.clone(), client)), + .map(|client| { + SessionLedger::remote( + vm_id.to_string(), + session_dir.to_path_buf(), + Arc::clone(&self.ledger_workers), + client, + ) + }), Err(error) => Err(error), }; #[cfg(not(test))] diff --git a/crates/capsem-service/src/tests/db_handle_ownership.rs b/crates/capsem-service/src/tests/db_handle_ownership.rs index a5a714a9d..a6245e0bb 100644 --- a/crates/capsem-service/src/tests/db_handle_ownership.rs +++ b/crates/capsem-service/src/tests/db_handle_ownership.rs @@ -27,6 +27,10 @@ fn service_db_handle_open_is_owned_by_explicit_service_state_owners() { && source.contains("capsem_logger::ledger_client::LedgerClient::connect"), "session registration must request reader authority and adopt the granted channel" ); + assert!( + source.contains("async fn reconnect(&self)") && source.contains("operation(replacement).await"), + "a cached reader must replace a failed generation and retry through a fresh grant" + ); assert!( source.contains("fn open_host_ledger("), "one DbHandle::open owner must be the host ledger" From 68135dd2e93e5ca163a39b5376b8b12214700fd3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:01:20 +0000 Subject: [PATCH 126/464] fix(logger): bound unread export delivery --- crates/capsem-logger/src/ledger_client.rs | 16 ++++++++--- .../capsem-logger/src/ledger_client/tests.rs | 27 +++++++++++++++++-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/crates/capsem-logger/src/ledger_client.rs b/crates/capsem-logger/src/ledger_client.rs index b7b277753..bad59f120 100644 --- a/crates/capsem-logger/src/ledger_client.rs +++ b/crates/capsem-logger/src/ledger_client.rs @@ -24,7 +24,10 @@ const EXPORT_CHUNK_CAPACITY: usize = 4; const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); #[cfg(test)] const OPERATION_TIMEOUT: Duration = Duration::from_secs(1); +#[cfg(not(test))] const STREAM_TIMEOUT: Duration = Duration::from_secs(30); +#[cfg(test)] +const STREAM_TIMEOUT: Duration = Duration::from_millis(250); type ClientSender = ipc_channel::Sender; type ClientReceiver = ipc_channel::Receiver; @@ -433,7 +436,7 @@ async fn export_warc( Ok(response) => response, Err(error) => { if consumer_open { - let _ = chunks.send(Err(error.clone())).await; + let _ = forward_export_chunk(&chunks, Err(error.clone())).await; } let _ = completion.send(Err(error.clone())); return Err(error); @@ -444,13 +447,13 @@ async fn export_warc( if actual != offset { let error = "ledger WARC chunk offset is not contiguous".to_string(); if consumer_open { - let _ = chunks.send(Err(error.clone())).await; + let _ = forward_export_chunk(&chunks, Err(error.clone())).await; } let _ = completion.send(Err(error.clone())); return Err(error); } offset += bytes.len() as u64; - if consumer_open && chunks.send(Ok(bytes)).await.is_err() { + if consumer_open && !forward_export_chunk(&chunks, Ok(bytes)).await { consumer_open = false; } } @@ -473,6 +476,13 @@ async fn export_warc( } } +async fn forward_export_chunk(chunks: &mpsc::Sender, String>>, chunk: Result, String>) -> bool { + matches!( + tokio::time::timeout(STREAM_TIMEOUT, chunks.send(chunk)).await, + Ok(Ok(())) + ) +} + fn unexpected(operation: &str, response: LedgerReply) -> String { format!("ledger returned {response:?} for {operation}") } diff --git a/crates/capsem-logger/src/ledger_client/tests.rs b/crates/capsem-logger/src/ledger_client/tests.rs index 521398541..b2c83bb29 100644 --- a/crates/capsem-logger/src/ledger_client/tests.rs +++ b/crates/capsem-logger/src/ledger_client/tests.rs @@ -54,9 +54,13 @@ impl Fixture { } async fn write_network_body(&self) { + self.write_network_body_named(EVENT_ID, b"request".to_vec()).await; + } + + async fn write_network_body_named(&self, event_id: &str, request_body: Vec) { self.writer .write_checked(WriteOp::NetEvent(NetEvent { - event_id: Some(EVENT_ID.into()), + event_id: Some(event_id.into()), timestamp: SystemTime::now(), domain: "example.test".into(), port: 443, @@ -73,7 +77,7 @@ impl Fixture { matched_rule: None, request_headers: None, response_headers: None, - request_body: Some(b"request".to_vec()), + request_body: Some(request_body), response_body: Some(b"response".to_vec()), conn_type: Some("tls".into()), policy_mode: None, @@ -111,6 +115,25 @@ impl Fixture { } } +#[tokio::test] +async fn an_unread_warc_export_cannot_block_later_operations_forever() { + let fixture = Fixture::start(LedgerClientRole::Reader).await; + for index in 0..16 { + fixture + .write_network_body_named(&format!("{index:012x}"), vec![index as u8; 1024 * 1024]) + .await; + } + + let export = fixture.client.export_warc().await.unwrap(); + let counters = tokio::time::timeout(Duration::from_secs(2), fixture.client.counters()) + .await + .expect("an unread local export wedged the ledger client") + .unwrap(); + assert_eq!(counters.net.total, 16); + drop(export); + fixture.stop().await; +} + #[tokio::test] async fn reader_queries_counters_bodies_and_warc_through_one_serial_channel() { let fixture = Fixture::start(LedgerClientRole::Reader).await; From b2af8624db487b1719d7e677c4bd5a8d3cee8d68 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:04:24 +0000 Subject: [PATCH 127/464] feat(proto): define proxy traffic handoff --- crates/capsem-proto/src/upstream_grant.rs | 73 ++++++++++++++++++- .../capsem-proto/src/upstream_grant/tests.rs | 21 ++++++ 2 files changed, 91 insertions(+), 3 deletions(-) diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index a3c8a3157..8287cc02f 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -22,7 +22,7 @@ pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; pub const MAX_GUEST_SHARE_PATH_BYTES: usize = 4096; const MAGIC: [u8; 2] = *b"UG"; -const VERSION: u8 = 2; +const VERSION: u8 = 3; const MAGIC_RANGE: std::ops::Range = 0..2; const VERSION_OFFSET: usize = 2; const KIND_OFFSET: usize = 3; @@ -45,11 +45,13 @@ const ADOPTED: u8 = 4; const RELEASE: u8 = 5; const SET_GUEST_MODE: u8 = 6; const OPEN_LEDGER: u8 = 7; +const ATTACH_PROXY_TRAFFIC: u8 = 8; const TCP_RESOLVED: u8 = 101; const DESCRIPTOR_GRANTED: u8 = 102; const DENIED: u8 = 103; const GUEST_MODE_SET: u8 = 104; const LEDGER_GRANTED: u8 = 105; +const PROXY_TRAFFIC_ADOPTED: u8 = 106; /// Application protocol spoken over a granted TCP stream. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -82,6 +84,30 @@ pub enum UpstreamDescriptorKind { DnsUdp, } +/// Guest-facing service selected by a surrendered traffic descriptor. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ProxyTrafficService { + Http, + Dns, +} + +impl ProxyTrafficService { + const fn code(self) -> u16 { + match self { + Self::Http => 1, + Self::Dns => 2, + } + } + + fn decode(code: u16) -> Result { + match code { + 1 => Ok(Self::Http), + 2 => Ok(Self::Dns), + _ => bail!("invalid proxy traffic service {code}"), + } + } +} + impl UpstreamDescriptorKind { const fn code(self) -> u16 { match self { @@ -170,6 +196,10 @@ pub enum UpstreamGrantRequest { OpenLedger { request_id: u64, }, + AttachProxyTraffic { + request_id: u64, + service: ProxyTrafficService, + }, } /// Coordinator-to-worker record. Successful descriptor grants carry one fd. @@ -199,12 +229,18 @@ pub enum UpstreamGrantResponse { request_id: u64, grant: LedgerChannelGrant, }, + ProxyTrafficAdopted { + request_id: u64, + }, } impl UpstreamGrantRequest { /// Requests never carry descriptors from the untrusted worker. pub const fn expected_descriptor_count(&self) -> usize { - 0 + match self { + Self::AttachProxyTraffic { .. } => 1, + _ => 0, + } } } @@ -213,7 +249,10 @@ impl UpstreamGrantResponse { pub const fn expected_descriptor_count(&self) -> usize { match self { Self::DescriptorGranted { .. } | Self::LedgerGranted { .. } => 1, - Self::TcpResolved { .. } | Self::Denied { .. } | Self::GuestModeSet { .. } => 0, + Self::TcpResolved { .. } + | Self::Denied { .. } + | Self::GuestModeSet { .. } + | Self::ProxyTrafficAdopted { .. } => 0, } } } @@ -288,6 +327,12 @@ pub fn encode_upstream_grant_request(request: &UpstreamGrantRequest) -> Result<[ frame[KIND_OFFSET] = OPEN_LEDGER; put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); } + UpstreamGrantRequest::AttachProxyTraffic { request_id, service } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = ATTACH_PROXY_TRAFFIC; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u16(&mut frame, DETAIL_RANGE, service.code()); + } } Ok(frame) } @@ -378,6 +423,17 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> require_empty_path(&relative_path)?; Ok(UpstreamGrantRequest::OpenLedger { request_id }) } + ATTACH_PROXY_TRAFFIC => { + require_nonzero("request id", request_id)?; + if resource_id != 0 || port != 0 || !name.is_empty() { + bail!("proxy traffic request carries unrelated fields"); + } + require_empty_path(&relative_path)?; + Ok(UpstreamGrantRequest::AttachProxyTraffic { + request_id, + service: ProxyTrafficService::decode(detail)?, + }) + } kind => bail!("invalid upstream request kind {kind}"), } } @@ -437,6 +493,11 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul put_u16(&mut frame, DETAIL_RANGE, ledger_role_code(grant.role())); put_name(&mut frame, &encode_ledger_generation(grant.generation()))?; } + UpstreamGrantResponse::ProxyTrafficAdopted { request_id } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = PROXY_TRAFFIC_ADOPTED; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + } } Ok(frame) } @@ -516,6 +577,12 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - let grant = LedgerChannelGrant::new(generation, resource_id, role)?; Ok(UpstreamGrantResponse::LedgerGranted { request_id, grant }) } + PROXY_TRAFFIC_ADOPTED => { + if resource_id != 0 || detail != 0 || !name.is_empty() || policy_digest.is_some() { + bail!("proxy traffic adoption carries unrelated fields"); + } + Ok(UpstreamGrantResponse::ProxyTrafficAdopted { request_id }) + } kind => bail!("invalid upstream response kind {kind}"), } } diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index 9f75a39fb..9d2adb87d 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -37,6 +37,14 @@ fn every_request_round_trips_exactly() { upstream_index: 0, }, UpstreamGrantRequest::OpenLedger { request_id: 5 }, + UpstreamGrantRequest::AttachProxyTraffic { + request_id: 6, + service: ProxyTrafficService::Http, + }, + UpstreamGrantRequest::AttachProxyTraffic { + request_id: 7, + service: ProxyTrafficService::Dns, + }, UpstreamGrantRequest::Adopted { grant_id: 51 }, UpstreamGrantRequest::Release { resource_id: 61 }, UpstreamGrantRequest::SetGuestMode { @@ -94,6 +102,7 @@ fn every_response_round_trips_exactly() { request_id: 8, grant: LedgerChannelGrant::new(LedgerGeneration::new([0x5a; 16]), 16, LedgerClientRole::VmOwner).unwrap(), }, + UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 9 }, ] { response_roundtrip(response); } @@ -143,6 +152,18 @@ fn descriptor_expectations_are_explicit() { upstream_index: 0, }; assert_eq!(request.expected_descriptor_count(), 0); + assert_eq!( + UpstreamGrantRequest::AttachProxyTraffic { + request_id: 9, + service: ProxyTrafficService::Dns, + } + .expected_descriptor_count(), + 1 + ); + assert_eq!( + UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 9 }.expected_descriptor_count(), + 0 + ); assert_eq!( UpstreamGrantResponse::LedgerGranted { request_id: 2, From 63efd998e2e49a609665a02cb04a8f3716a90a09 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:08:17 +0000 Subject: [PATCH 128/464] feat(proto): type proxy traffic grants --- crates/capsem-proto/src/proxy_control.rs | 36 ++++++++++--------- .../capsem-proto/src/proxy_control/tests.rs | 3 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/crates/capsem-proto/src/proxy_control.rs b/crates/capsem-proto/src/proxy_control.rs index 07afa5033..54f8754a3 100644 --- a/crates/capsem-proto/src/proxy_control.rs +++ b/crates/capsem-proto/src/proxy_control.rs @@ -4,7 +4,7 @@ use thiserror::Error; pub const PROXY_CONTROL_FRAME_SIZE: usize = 32; pub const PROXY_CONTROL_MAX_FDS: usize = 1; -pub const PROXY_CONTROL_VERSION: u16 = 1; +pub const PROXY_CONTROL_VERSION: u16 = 2; const MAGIC: [u8; 2] = *b"PX"; const VERSION_RANGE: std::ops::Range = 2..4; @@ -51,14 +51,15 @@ impl ProxyGeneration { #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] #[repr(u8)] pub enum ProxyCapability { - Traffic = 1, - Upstream = 2, - Credential = 3, - Ledger = 4, - PrivateNames = 5, - Mcp = 6, - Telemetry = 7, - Policy = 8, + HttpTraffic = 1, + DnsTraffic = 2, + Upstream = 3, + Credential = 4, + Ledger = 5, + PrivateNames = 6, + Mcp = 7, + Telemetry = 8, + Policy = 9, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -291,14 +292,15 @@ fn validate_generation(generation: ProxyGeneration) -> Result<(), ProxyControlEr fn decode_capability(value: u8) -> Result { match value { - 1 => Ok(ProxyCapability::Traffic), - 2 => Ok(ProxyCapability::Upstream), - 3 => Ok(ProxyCapability::Credential), - 4 => Ok(ProxyCapability::Ledger), - 5 => Ok(ProxyCapability::PrivateNames), - 6 => Ok(ProxyCapability::Mcp), - 7 => Ok(ProxyCapability::Telemetry), - 8 => Ok(ProxyCapability::Policy), + 1 => Ok(ProxyCapability::HttpTraffic), + 2 => Ok(ProxyCapability::DnsTraffic), + 3 => Ok(ProxyCapability::Upstream), + 4 => Ok(ProxyCapability::Credential), + 5 => Ok(ProxyCapability::Ledger), + 6 => Ok(ProxyCapability::PrivateNames), + 7 => Ok(ProxyCapability::Mcp), + 8 => Ok(ProxyCapability::Telemetry), + 9 => Ok(ProxyCapability::Policy), value => Err(ProxyControlError::InvalidCapability(value)), } } diff --git a/crates/capsem-proto/src/proxy_control/tests.rs b/crates/capsem-proto/src/proxy_control/tests.rs index e27e744f9..357499cd2 100644 --- a/crates/capsem-proto/src/proxy_control/tests.rs +++ b/crates/capsem-proto/src/proxy_control/tests.rs @@ -5,7 +5,8 @@ const GENERATION: ProxyGeneration = ProxyGeneration::new([7; 16]); #[test] fn grants_round_trip_without_session_path_or_destination_authority() { for capability in [ - ProxyCapability::Traffic, + ProxyCapability::HttpTraffic, + ProxyCapability::DnsTraffic, ProxyCapability::Upstream, ProxyCapability::Credential, ProxyCapability::Ledger, From 42ac0d7c40d5c7850699292d1290c21bfe0c5c10 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:10:11 +0000 Subject: [PATCH 129/464] refactor(proxy): distinguish traffic grants --- crates/capsem-proxy/src/main.rs | 8 ++++++-- crates/capsem-proxy/tests/subprocess.rs | 7 ++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 82ad5ebd2..0bc14932a 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -161,10 +161,10 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result let rejection = if grant_ids.contains(&grant_id) { Some(ProxyControlRejection::DuplicateGrant) } else if grant_ids.len() >= GRANT_LIMIT - || (capability == ProxyCapability::Traffic && current >= TRAFFIC_GRANT_LIMIT) + || (is_traffic(capability) && current >= TRAFFIC_GRANT_LIMIT) { Some(ProxyControlRejection::Capacity) - } else if capability != ProxyCapability::Traffic && current != 0 { + } else if !is_traffic(capability) && current != 0 { Some(ProxyControlRejection::DuplicateCapability) } else { None @@ -222,6 +222,10 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result Ok(()) } +const fn is_traffic(capability: ProxyCapability) -> bool { + matches!(capability, ProxyCapability::HttpTraffic | ProxyCapability::DnsTraffic) +} + enum ControlInput { Frame(ControlFrame), Capability(std::result::Result), diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs index 7bfcdb9fa..e37da8b4a 100644 --- a/crates/capsem-proxy/tests/subprocess.rs +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -77,7 +77,8 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { let mut peers = Vec::new(); for (index, capability) in [ - ProxyCapability::Traffic, + ProxyCapability::HttpTraffic, + ProxyCapability::DnsTraffic, ProxyCapability::Upstream, ProxyCapability::Credential, ProxyCapability::Ledger, @@ -111,7 +112,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { } let (duplicate_peer, duplicate) = UnixStream::pair().unwrap(); - let duplicate_grant = ProxyChannelGrant::new(GENERATION, 9, ProxyCapability::Ledger).unwrap(); + let duplicate_grant = ProxyChannelGrant::new(GENERATION, 10, ProxyCapability::Ledger).unwrap(); requests .send( &encode_proxy_control_request(ProxyControlRequest::Attach(duplicate_grant)), @@ -124,7 +125,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { event(&events).await, ProxyControlEvent::Rejected { generation: GENERATION, - grant_id: 9, + grant_id: 10, reason: ProxyControlRejection::DuplicateCapability, } ); From e33d4ad736e9836e434987fef2db32ea3392c83b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:14:11 +0000 Subject: [PATCH 130/464] feat(service): broker guest traffic to proxy workers --- crates/capsem-service/src/proxy_worker.rs | 1 - crates/capsem-service/src/upstream_broker.rs | 49 ++++++++++++++++-- .../src/upstream_broker/tests.rs | 50 ++++++++++++++++++- .../src/vm_lifecycle/provision.rs | 1 + .../src/vm_lifecycle/resume_process.rs | 1 + 5 files changed, 95 insertions(+), 7 deletions(-) diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index 5815baf27..197bbfa91 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -68,7 +68,6 @@ impl ProxyWorker { self.generation } - #[cfg_attr(not(test), expect(dead_code, reason = "S10-003 attaches the traffic capabilities"))] pub(crate) async fn grant(&self, capability: ProxyCapability, stream: UnixStream) -> Result<()> { let (completed, result) = oneshot::channel(); self.commands diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 7cb0fbf55..fa8600bff 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -23,8 +23,9 @@ use capsem_foundation::unix::fd::{self, SocketShutdown}; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::upstream_grant::{ - decode_upstream_grant_request, encode_upstream_grant_response, UpstreamDescriptorKind, UpstreamGrantDenial, - UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, + decode_upstream_grant_request, encode_upstream_grant_response, ProxyTrafficService, UpstreamDescriptorKind, + UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, + UPSTREAM_GRANT_MAX_FDS, }; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, warn}; @@ -56,6 +57,7 @@ pub(crate) struct PendingBroker { updates: mpsc::Receiver, session_dir: Option, ledger: Option, + proxy: Option, } struct PolicyUpdate { @@ -156,6 +158,7 @@ impl PendingBroker { updates, session_dir: None, ledger: None, + proxy: None, }, PolicyPublisher { updates: Some(updates_tx), @@ -173,6 +176,11 @@ impl PendingBroker { self } + pub(crate) fn with_proxy(mut self, proxy: crate::proxy_worker::ProxyWorker) -> Self { + self.proxy = Some(proxy); + self + } + #[cfg(test)] fn with_ledger(mut self, ledger: LedgerClient) -> Self { self.ledger = Some(LedgerSource::Ready(ledger)); @@ -202,6 +210,7 @@ impl PendingBroker { authority, self.session_dir, self.ledger, + self.proxy, ) .await { @@ -218,6 +227,7 @@ async fn run( authority: WorkerGrant, session_dir: Option, mut ledger: Option, + proxy: Option, ) -> Result<(), String> { let requests = WireReceiver::new(socket.try_clone().map_err(|error| error.to_string())?).map_err(|error| error.to_string())?; @@ -257,11 +267,15 @@ async fn run( .ok_or_else(|| "upstream grant request pump stopped".to_string())? .map_err(|error| format!("receive upstream grant request: {error}"))?, }; - if !frame.fds.is_empty() { - break Err("worker sent an upstream descriptor".into()); - } let request = decode_upstream_grant_request(&frame.bytes) .map_err(|error| format!("decode upstream grant request: {error:#}"))?; + if frame.fds.len() != request.expected_descriptor_count() { + break Err(format!( + "worker upstream request carried {} descriptors, expected {}", + frame.fds.len(), + request.expected_descriptor_count() + )); + } if let Some(expected) = awaiting_adoption { if request != (UpstreamGrantRequest::Adopted { @@ -560,6 +574,31 @@ async fn run( pending_ledger = Some((grant, descriptor)); awaiting_adoption = Some(AwaitingAdoption::Ledger(grant.client_id())); } + UpstreamGrantRequest::AttachProxyTraffic { request_id, service } => { + let Some(proxy) = proxy.as_ref() else { + send_denied(&responses, request_id, UpstreamGrantDenial::NotConfigured).await?; + continue; + }; + let descriptor = frame.fds.into_iter().next().expect("one validated traffic descriptor"); + let capability = match service { + ProxyTrafficService::Http => capsem_proto::proxy_control::ProxyCapability::HttpTraffic, + ProxyTrafficService::Dns => capsem_proto::proxy_control::ProxyCapability::DnsTraffic, + }; + match proxy.grant(capability, UnixStream::from(descriptor)).await { + Ok(()) => { + send_response( + &responses, + &UpstreamGrantResponse::ProxyTrafficAdopted { request_id }, + None, + ) + .await?; + } + Err(error) => { + warn!(%error, ?service, "proxy worker refused traffic descriptor"); + send_denied(&responses, request_id, UpstreamGrantDenial::Revoked).await?; + } + } + } } } } diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index 56e9cb132..71d100a6a 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -9,7 +9,7 @@ use capsem_core::net::policy::{UpstreamOverride, UpstreamOverrideProtocol}; use capsem_core::net::policy_config::{ActivePolicyFile, SettingsFile}; use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; use capsem_proto::upstream_grant::{ - decode_upstream_grant_response, encode_upstream_grant_request, UpstreamGrantResponse, + decode_upstream_grant_response, encode_upstream_grant_request, ProxyTrafficService, UpstreamGrantResponse, }; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; @@ -35,6 +35,18 @@ impl TestClient { self.requests.send(&bytes, &[]).await.unwrap(); } + async fn request_with_descriptor( + &self, + request: &UpstreamGrantRequest, + descriptor: &impl std::os::fd::AsRawFd, + ) -> UpstreamGrantResponse { + let bytes = encode_upstream_grant_request(request).unwrap(); + self.requests.send(&bytes, &[descriptor.as_raw_fd()]).await.unwrap(); + let frame = self.responses.recv().await.unwrap(); + assert!(frame.fds.is_empty()); + decode_upstream_grant_response(&frame.bytes).unwrap() + } + async fn request(&self, request: &UpstreamGrantRequest) -> (UpstreamGrantResponse, Vec) { self.send(request).await; let frame = self.responses.recv().await.unwrap(); @@ -44,6 +56,39 @@ impl TestClient { } } +#[tokio::test] +async fn typed_guest_traffic_is_adopted_by_the_registered_proxy_generation() { + let policy = test_policy("policy-a", None, vec![]); + let (pending, _publisher) = PendingBroker::pair(policy).unwrap(); + let pending = pending.with_proxy(crate::proxy_worker::ProxyWorker::test_stub()); + let client = TestClient::new(pending.worker); + let authority = WorkerAuthority::default(); + let task = tokio::spawn(run( + pending.coordinator, + pending.initial_policy, + pending.updates, + authority.grant(), + pending.session_dir, + pending.ledger, + pending.proxy, + )); + let (traffic, _peer) = UnixStream::pair().unwrap(); + + let response = client + .request_with_descriptor( + &UpstreamGrantRequest::AttachProxyTraffic { + request_id: 1, + service: ProxyTrafficService::Http, + }, + &traffic, + ) + .await; + assert_eq!(response, UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 1 }); + + authority.revoke(); + assert!(task.await.unwrap().unwrap_err().contains("revoked")); +} + fn test_policy(digest: &str, tcp_override: Option, dns_upstreams: Vec) -> Arc { let active = ActivePolicyFile::from_settings_and_corp(&SettingsFile::default(), &SettingsFile::default()).unwrap(); let mut runtime = active.compile_runtime().unwrap(); @@ -82,6 +127,7 @@ fn start_broker( authority.grant(), pending.session_dir, pending.ledger, + pending.proxy, )); (client, publisher, authority, task) } @@ -106,6 +152,7 @@ fn start_broker_with_ledger( authority.grant(), pending.session_dir, pending.ledger, + pending.proxy, )); (client, publisher, authority, task) } @@ -129,6 +176,7 @@ fn start_broker_for_session( authority.grant(), pending.session_dir, pending.ledger, + pending.proxy, )); (client, publisher, authority, task) } diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index caf820988..0185b282b 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -298,6 +298,7 @@ impl ServiceState { return Err(error.context("start confined proxy worker")); } }; + let upstream_broker = upstream_broker.with_proxy(proxy.clone()); // Provisioning runs on a blocking thread that keeps the runtime // handle, so the event lands before anything can stop the session. diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index ca42bca8c..b99f1abc2 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -228,6 +228,7 @@ impl ServiceState { return Err(error.context("start confined proxy worker")); } }; + let upstream_broker = upstream_broker.with_proxy(proxy.clone()); info!( vm_id, From b8454da4aa3aec9095e8052fb7ceffd7c15badea Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:16:39 +0000 Subject: [PATCH 131/464] feat(process): surrender proxy traffic descriptors --- crates/capsem-process/src/upstream_grant.rs | 74 ++++++++++++++++++- .../src/upstream_grant/tests.rs | 40 +++++++++- 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/crates/capsem-process/src/upstream_grant.rs b/crates/capsem-process/src/upstream_grant.rs index a6da1e011..91dcd6153 100644 --- a/crates/capsem-process/src/upstream_grant.rs +++ b/crates/capsem-process/src/upstream_grant.rs @@ -1,7 +1,7 @@ //! Process-side client for coordinator-minted upstream descriptors. use std::io; -use std::os::fd::{AsFd as _, BorrowedFd, OwnedFd}; +use std::os::fd::{AsFd as _, AsRawFd as _, BorrowedFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::time::Duration; @@ -15,8 +15,8 @@ use capsem_foundation::unix::fd::{self, SocketShutdown}; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::upstream_grant::{ - decode_upstream_grant_response, encode_upstream_grant_request, UpstreamDescriptorKind, UpstreamGrantRequest, - UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, + decode_upstream_grant_response, encode_upstream_grant_request, ProxyTrafficService, UpstreamDescriptorKind, + UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, }; use tokio::sync::{mpsc, oneshot}; @@ -29,6 +29,15 @@ type WireSender = DescriptorSender; enum Command { + #[cfg_attr( + not(test), + expect(dead_code, reason = "guest dispatch uses this after proxy traffic handlers land") + )] + AttachProxyTraffic { + service: ProxyTrafficService, + descriptor: OwnedFd, + reply: oneshot::Sender>, + }, OpenLedger { reply: oneshot::Sender>, }, @@ -91,6 +100,26 @@ impl UpstreamGrantClient { .map_err(|_| anyhow!("upstream grant channel stopped"))? .map_err(anyhow::Error::msg) } + + #[cfg_attr( + not(test), + expect(dead_code, reason = "guest dispatch uses this after proxy traffic handlers land") + )] + pub(crate) async fn attach_proxy_traffic(&self, service: ProxyTrafficService, descriptor: OwnedFd) -> Result<()> { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::AttachProxyTraffic { + service, + descriptor, + reply, + }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + } } impl capsem_core::GuestMetadataAuthority for UpstreamGrantClient { @@ -214,6 +243,14 @@ async fn dispatch( command: Command, ) -> Result<(), String> { match command { + Command::AttachProxyTraffic { + service, + descriptor, + reply, + } => finish( + reply, + attach_proxy_traffic(sender, receiver, request_id, service, descriptor).await, + ), Command::OpenLedger { reply } => finish(reply, open_ledger(sender, receiver, request_id).await), Command::SetGuestMode { relative_path, @@ -262,6 +299,37 @@ async fn dispatch( } } +async fn attach_proxy_traffic( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, + service: ProxyTrafficService, + descriptor: OwnedFd, +) -> Result, String> { + let request = UpstreamGrantRequest::AttachProxyTraffic { request_id, service }; + let bytes = + encode_upstream_grant_request(&request).map_err(|error| format!("encode proxy traffic handoff: {error:#}"))?; + tokio::time::timeout(WIRE_TIMEOUT, sender.send(&bytes, &[descriptor.as_raw_fd()])) + .await + .map_err(|_| "send proxy traffic handoff timed out".to_string())? + .map_err(|error| format!("send proxy traffic handoff: {error}"))?; + drop(descriptor); + let (response, fds) = receive_response(receiver).await?; + debug_assert!(fds.is_empty()); + match response { + UpstreamGrantResponse::ProxyTrafficAdopted { + request_id: response_id, + } if response_id == request_id => Ok(Ok(())), + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("proxy traffic handoff denied: {reason:?}"))), + response => Err(format!( + "unexpected proxy traffic response for request {request_id}: {response:?}" + )), + } +} + async fn open_ledger( sender: &WireSender, receiver: &WireReceiver, diff --git a/crates/capsem-process/src/upstream_grant/tests.rs b/crates/capsem-process/src/upstream_grant/tests.rs index e80b1e84a..ae17e19bf 100644 --- a/crates/capsem-process/src/upstream_grant/tests.rs +++ b/crates/capsem-process/src/upstream_grant/tests.rs @@ -7,8 +7,8 @@ use capsem_core::net::mitm_proxy::{TcpUpstreamGrants, UpstreamTarget}; use capsem_core::GuestMetadataAuthority as _; use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; use capsem_proto::upstream_grant::{ - decode_upstream_grant_request, encode_upstream_grant_response, UpstreamDescriptorKind, UpstreamGrantDenial, - UpstreamGrantRequest, UpstreamGrantResponse, + decode_upstream_grant_request, encode_upstream_grant_response, ProxyTrafficService, UpstreamDescriptorKind, + UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, }; use super::*; @@ -52,6 +52,42 @@ async fn ledger_channel_preserves_grant_and_is_adopted_before_use() { broker_task.await.unwrap(); } +#[tokio::test] +async fn proxy_traffic_descriptor_is_surrendered_before_adoption_returns() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let (traffic, mut guest) = UnixStream::pair().unwrap(); + let broker_task = tokio::spawn(async move { + let frame = requests.recv().await.unwrap(); + assert_eq!(frame.fds.len(), 1); + assert_eq!( + decode_upstream_grant_request(&frame.bytes).unwrap(), + UpstreamGrantRequest::AttachProxyTraffic { + request_id: 1, + service: ProxyTrafficService::Dns, + } + ); + let mut adopted = UnixStream::from(frame.fds.into_iter().next().unwrap()); + adopted.write_all(b"owned").unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 1 }, + None, + ) + .await; + }); + + client + .attach_proxy_traffic(ProxyTrafficService::Dns, traffic.into()) + .await + .unwrap(); + let mut proof = [0; 5]; + guest.read_exact(&mut proof).unwrap(); + assert_eq!(&proof, b"owned"); + broker_task.await.unwrap(); +} + #[tokio::test] async fn ledger_denial_and_missing_descriptor_fail_closed() { let (broker, worker) = UnixStream::pair().unwrap(); From c6ee33d53c5f34c7074eaac2276015671ae7c5e1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:21:09 +0000 Subject: [PATCH 132/464] feat(proto): bind proxy ledger grants --- crates/capsem-proto/src/proxy_control.rs | 129 ++++++++++++++++-- .../capsem-proto/src/proxy_control/tests.rs | 22 ++- 2 files changed, 135 insertions(+), 16 deletions(-) diff --git a/crates/capsem-proto/src/proxy_control.rs b/crates/capsem-proto/src/proxy_control.rs index 54f8754a3..17a70e5bb 100644 --- a/crates/capsem-proto/src/proxy_control.rs +++ b/crates/capsem-proto/src/proxy_control.rs @@ -2,9 +2,11 @@ use thiserror::Error; -pub const PROXY_CONTROL_FRAME_SIZE: usize = 32; +use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerProtocolError}; + +pub const PROXY_CONTROL_FRAME_SIZE: usize = 56; pub const PROXY_CONTROL_MAX_FDS: usize = 1; -pub const PROXY_CONTROL_VERSION: u16 = 2; +pub const PROXY_CONTROL_VERSION: u16 = 3; const MAGIC: [u8; 2] = *b"PX"; const VERSION_RANGE: std::ops::Range = 2..4; @@ -13,6 +15,8 @@ const CAPABILITY_OFFSET: usize = 5; const DETAIL_RANGE: std::ops::Range = 6..8; const GENERATION_RANGE: std::ops::Range = 8..24; const GRANT_ID_RANGE: std::ops::Range = 24..32; +const LEDGER_GENERATION_RANGE: std::ops::Range = 32..48; +const LEDGER_CLIENT_ID_RANGE: std::ops::Range = 48..56; const ATTACH: u8 = 1; const SHUTDOWN: u8 = 2; @@ -67,6 +71,7 @@ pub struct ProxyChannelGrant { generation: ProxyGeneration, grant_id: u64, capability: ProxyCapability, + ledger_grant: Option, } impl ProxyChannelGrant { @@ -81,10 +86,36 @@ impl ProxyChannelGrant { if grant_id == 0 { return Err(ProxyControlError::ZeroGrantId); } + if capability == ProxyCapability::Ledger { + return Err(ProxyControlError::MissingLedgerAuthority); + } Ok(Self { generation, grant_id, capability, + ledger_grant: None, + }) + } + + pub fn with_ledger( + generation: ProxyGeneration, + grant_id: u64, + ledger_grant: LedgerChannelGrant, + ) -> Result { + if ledger_grant.role() != LedgerClientRole::Proxy { + return Err(ProxyControlError::InvalidLedgerRole(ledger_grant.role())); + } + if generation.is_zero() { + return Err(ProxyControlError::ZeroGeneration); + } + if grant_id == 0 { + return Err(ProxyControlError::ZeroGrantId); + } + Ok(Self { + generation, + grant_id, + capability: ProxyCapability::Ledger, + ledger_grant: Some(ledger_grant), }) } @@ -99,6 +130,10 @@ impl ProxyChannelGrant { pub const fn capability(self) -> ProxyCapability { self.capability } + + pub const fn ledger_grant(self) -> Option { + self.ledger_grant + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -171,13 +206,20 @@ impl ProxyControlEvent { pub fn encode_proxy_control_request(request: ProxyControlRequest) -> [u8; PROXY_CONTROL_FRAME_SIZE] { match request { - ProxyControlRequest::Attach(grant) => encode( - ATTACH, - grant.capability() as u8, - 0, - grant.generation(), - grant.grant_id(), - ), + ProxyControlRequest::Attach(grant) => { + let mut frame = encode( + ATTACH, + grant.capability() as u8, + grant.ledger_grant().map_or(0, |ledger| ledger_role_code(ledger.role())), + grant.generation(), + grant.grant_id(), + ); + if let Some(ledger) = grant.ledger_grant() { + frame[LEDGER_GENERATION_RANGE].copy_from_slice(&ledger.generation().as_bytes()); + frame[LEDGER_CLIENT_ID_RANGE].copy_from_slice(&ledger.client_id().to_be_bytes()); + } + frame + } ProxyControlRequest::Shutdown { generation } => encode(SHUTDOWN, 0, 0, generation, 0), } } @@ -190,12 +232,35 @@ pub fn decode_proxy_control_request( let grant_id = decode_u64(frame, GRANT_ID_RANGE); let detail = decode_u16(frame, DETAIL_RANGE); match frame[KIND_OFFSET] { - ATTACH if detail == 0 => Ok(ProxyControlRequest::Attach(ProxyChannelGrant::new( - generation, - grant_id, - decode_capability(frame[CAPABILITY_OFFSET])?, - )?)), - SHUTDOWN if frame[CAPABILITY_OFFSET] == 0 && detail == 0 && grant_id == 0 => { + ATTACH => { + let capability = decode_capability(frame[CAPABILITY_OFFSET])?; + let grant = if capability == ProxyCapability::Ledger { + let ledger_generation = LedgerGeneration::new( + frame[LEDGER_GENERATION_RANGE] + .try_into() + .expect("fixed ledger generation range"), + ); + let ledger_client_id = decode_u64(frame, LEDGER_CLIENT_ID_RANGE); + let ledger = LedgerChannelGrant::new(ledger_generation, ledger_client_id, decode_ledger_role(detail)?)?; + ProxyChannelGrant::with_ledger(generation, grant_id, ledger)? + } else { + if detail != 0 + || frame[LEDGER_GENERATION_RANGE].iter().any(|byte| *byte != 0) + || decode_u64(frame, LEDGER_CLIENT_ID_RANGE) != 0 + { + return Err(ProxyControlError::ReservedBytes); + } + ProxyChannelGrant::new(generation, grant_id, capability)? + }; + Ok(ProxyControlRequest::Attach(grant)) + } + SHUTDOWN + if frame[CAPABILITY_OFFSET] == 0 + && detail == 0 + && grant_id == 0 + && frame[LEDGER_GENERATION_RANGE].iter().all(|byte| *byte == 0) + && decode_u64(frame, LEDGER_CLIENT_ID_RANGE) == 0 => + { validate_generation(generation)?; Ok(ProxyControlRequest::Shutdown { generation }) } @@ -228,6 +293,9 @@ pub fn decode_proxy_control_event( if frame[CAPABILITY_OFFSET] != 0 { return Err(ProxyControlError::ReservedBytes); } + if frame[LEDGER_GENERATION_RANGE].iter().any(|byte| *byte != 0) || decode_u64(frame, LEDGER_CLIENT_ID_RANGE) != 0 { + return Err(ProxyControlError::ReservedBytes); + } let generation = decode_generation(frame); validate_generation(generation)?; let grant_id = decode_u64(frame, GRANT_ID_RANGE); @@ -305,6 +373,29 @@ fn decode_capability(value: u8) -> Result { } } +const fn ledger_role_code(role: LedgerClientRole) -> u16 { + match role { + LedgerClientRole::VmOwner => 1, + LedgerClientRole::Proxy => 2, + LedgerClientRole::Coordinator => 3, + LedgerClientRole::Reader => 4, + LedgerClientRole::Maintainer => 5, + LedgerClientRole::Supervisor => 6, + } +} + +fn decode_ledger_role(value: u16) -> Result { + match value { + 1 => Ok(LedgerClientRole::VmOwner), + 2 => Ok(LedgerClientRole::Proxy), + 3 => Ok(LedgerClientRole::Coordinator), + 4 => Ok(LedgerClientRole::Reader), + 5 => Ok(LedgerClientRole::Maintainer), + 6 => Ok(LedgerClientRole::Supervisor), + value => Err(ProxyControlError::InvalidLedgerRoleCode(value)), + } +} + fn decode_rejection(value: u16) -> Result { match value { 1 => Ok(ProxyControlRejection::Capacity), @@ -352,6 +443,14 @@ pub enum ProxyControlError { InvalidEvent(u8), #[error("proxy capability {0} is invalid")] InvalidCapability(u8), + #[error("proxy ledger capability is missing its exact ledger authority")] + MissingLedgerAuthority, + #[error("proxy ledger capability must carry the proxy role, got {0:?}")] + InvalidLedgerRole(LedgerClientRole), + #[error("proxy ledger role code {0} is invalid")] + InvalidLedgerRoleCode(u16), + #[error("proxy ledger authority is invalid: {0}")] + LedgerAuthority(#[from] LedgerProtocolError), #[error("proxy control rejection {0} is invalid")] InvalidRejection(u16), #[error("proxy channel close reason {0} is invalid")] diff --git a/crates/capsem-proto/src/proxy_control/tests.rs b/crates/capsem-proto/src/proxy_control/tests.rs index 357499cd2..b76509104 100644 --- a/crates/capsem-proto/src/proxy_control/tests.rs +++ b/crates/capsem-proto/src/proxy_control/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; const GENERATION: ProxyGeneration = ProxyGeneration::new([7; 16]); @@ -9,7 +10,6 @@ fn grants_round_trip_without_session_path_or_destination_authority() { ProxyCapability::DnsTraffic, ProxyCapability::Upstream, ProxyCapability::Credential, - ProxyCapability::Ledger, ProxyCapability::PrivateNames, ProxyCapability::Mcp, ProxyCapability::Telemetry, @@ -32,6 +32,26 @@ fn grants_round_trip_without_session_path_or_destination_authority() { ); } +#[test] +fn ledger_grants_round_trip_exact_worker_authority() { + let ledger = LedgerChannelGrant::new(LedgerGeneration::new([9; 16]), 73, LedgerClientRole::Proxy).unwrap(); + let grant = ProxyChannelGrant::with_ledger(GENERATION, 41, ledger).unwrap(); + let request = ProxyControlRequest::Attach(grant); + assert_eq!( + decode_proxy_control_request(&encode_proxy_control_request(request)).unwrap(), + request + ); + assert_eq!(grant.ledger_grant(), Some(ledger)); +} + +#[test] +fn ledger_capability_without_exact_authority_is_rejected() { + assert_eq!( + ProxyChannelGrant::new(GENERATION, 41, ProxyCapability::Ledger).unwrap_err(), + ProxyControlError::MissingLedgerAuthority + ); +} + #[test] fn events_round_trip_generation_and_grant_identity() { for event in [ From fab5bc46ff77a96d948362f6d4ac86b531a11dd1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:27:21 +0000 Subject: [PATCH 133/464] feat(proxy): authenticate ledger capability --- crates/capsem-proxy/src/main.rs | 50 +++++++++++++- crates/capsem-proxy/tests/subprocess.rs | 90 ++++++++++++++++++++++++- 2 files changed, 135 insertions(+), 5 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 0bc14932a..938b1bf8a 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, HashSet}; use std::io; use std::os::fd::AsFd; use std::os::unix::net::UnixStream; +use std::path::Path; use std::process::Command; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -191,6 +192,25 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result let reason = serve_policy(stream, state).await; (capability, grant_id, reason) }); + } else if capability == ProxyCapability::Ledger { + let ledger_grant = grant + .ledger_grant() + .expect("ledger capability was decoded with exact authority"); + let writer = tokio::task::spawn_blocking(move || { + capsem_logger::DbWriter::from_ledger_channel( + stream, + ledger_grant, + Path::new("capability:session-ledger"), + 1024, + ) + }) + .await + .map_err(|error| anyhow::anyhow!("join proxy ledger handshake: {error}"))? + .context("authenticate proxy ledger capability")?; + state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .attach_ledger(Arc::new(writer)); } else { descriptors.insert(grant_id, stream); } @@ -246,13 +266,26 @@ fn remove_grant( } } -#[derive(Default)] struct ProxyRuntimeState { engine: Option, + ledger: Arc, + credentials: Arc, dns_upstreams: Vec, mcp: capsem_core::mcp::policy::McpConfig, } +impl Default for ProxyRuntimeState { + fn default() -> Self { + Self { + engine: None, + ledger: Arc::new(UnavailableLedger), + credentials: Arc::new(UnavailableCredentials), + dns_upstreams: Vec::new(), + mcp: capsem_core::mcp::policy::McpConfig::default(), + } + } +} + impl ProxyRuntimeState { fn apply(&mut self, active_policy: &[u8]) -> Result { let runtime = ProxyRuntimePolicy::compile(active_policy).map_err(anyhow::Error::msg)?; @@ -263,14 +296,25 @@ impl ProxyRuntimeState { } else { self.engine = Some(ProxyEngine::new( ProxyPolicyHandle::new(snapshot), - Arc::new(UnavailableLedger), - Arc::new(UnavailableCredentials), + Arc::clone(&self.ledger), + Arc::clone(&self.credentials), )); } self.dns_upstreams = dns_upstreams; self.mcp = mcp; Ok(digest) } + + fn attach_ledger(&mut self, ledger: Arc) { + self.ledger = ledger; + if let Some(engine) = self.engine.take() { + self.engine = Some(ProxyEngine::new( + engine.policy().clone(), + Arc::clone(&self.ledger), + Arc::clone(&self.credentials), + )); + } + } } struct UnavailableLedger; diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs index e37da8b4a..17b0bfa40 100644 --- a/crates/capsem-proxy/tests/subprocess.rs +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -4,6 +4,7 @@ use std::process::{Child, Command, Stdio}; use std::time::Duration; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; use capsem_proto::proxy_control::{ decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyChannelGrant, ProxyControlEvent, ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, @@ -81,7 +82,6 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { ProxyCapability::DnsTraffic, ProxyCapability::Upstream, ProxyCapability::Credential, - ProxyCapability::Ledger, ProxyCapability::PrivateNames, ProxyCapability::Mcp, ProxyCapability::Telemetry, @@ -112,7 +112,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { } let (duplicate_peer, duplicate) = UnixStream::pair().unwrap(); - let duplicate_grant = ProxyChannelGrant::new(GENERATION, 10, ProxyCapability::Ledger).unwrap(); + let duplicate_grant = ProxyChannelGrant::new(GENERATION, 10, ProxyCapability::Telemetry).unwrap(); requests .send( &encode_proxy_control_request(ProxyControlRequest::Attach(duplicate_grant)), @@ -156,6 +156,92 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { ); } +#[tokio::test] +async fn ledger_grant_is_authenticated_before_adoption() { + let (mut child, requests, events) = spawn(); + assert_eq!( + event(&events).await, + ProxyControlEvent::Ready { generation: GENERATION } + ); + let (event_tx, mut event_rx) = tokio::sync::mpsc::channel(4); + let event_pump = tokio::spawn(async move { + while let Ok(frame) = events.recv().await { + let event = decode_proxy_control_event(&frame.bytes).unwrap(); + if event_tx.send(event).await.is_err() { + return; + } + } + }); + let (server_stream, granted) = UnixStream::pair().unwrap(); + let ledger_grant = LedgerChannelGrant::new(LedgerGeneration::new([4; 16]), 17, LedgerClientRole::Proxy).unwrap(); + let grant = ProxyChannelGrant::with_ledger(GENERATION, 1, ledger_grant).unwrap(); + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), + &[granted.as_raw_fd()], + ) + .await + .unwrap(); + drop(granted); + + assert!(tokio::time::timeout(Duration::from_millis(100), event_rx.recv()) + .await + .is_err()); + let ledger_dir = tempfile::tempdir().unwrap(); + let ledger = std::sync::Arc::new( + capsem_logger::ledger_server::LedgerServer::open(&ledger_dir.path().join("session.db")).unwrap(), + ); + let serving = tokio::spawn(async move { ledger.serve_client(server_stream, ledger_grant).await }); + let adopted = match tokio::time::timeout(Duration::from_secs(5), event_rx.recv()).await { + Ok(Some(event)) => event, + _ => { + let child = child.take(); + let output = tokio::task::spawn_blocking(move || child.wait_with_output()) + .await + .unwrap() + .unwrap(); + panic!( + "proxy ledger adoption failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + }; + assert_eq!( + adopted, + ProxyControlEvent::Adopted { + generation: GENERATION, + grant_id: 1, + } + ); + + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Shutdown { generation: GENERATION }), + &[], + ) + .await + .unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), event_rx.recv()) + .await + .unwrap() + .unwrap(), + ProxyControlEvent::Stopped { generation: GENERATION } + ); + event_pump.await.unwrap(); + assert!(serving.await.unwrap().is_ok()); + let child = child.take(); + let output = tokio::task::spawn_blocking(move || child.wait_with_output()) + .await + .unwrap() + .unwrap(); + assert!( + output.status.success(), + "proxy worker failed: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + #[tokio::test] async fn stale_generation_terminates_worker_and_releases_grants() { let (mut child, requests, events) = spawn(); From 9ef64b2eaed1bcdf96ba93cab815ae13be301ec9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:30:00 +0000 Subject: [PATCH 134/464] feat(service): grant proxy ledger authority --- crates/capsem-service/src/proxy_worker.rs | 74 ++++++++++++++++++- .../capsem-service/src/proxy_worker/tests.rs | 18 +++++ .../src/vm_lifecycle/provision.rs | 6 ++ .../src/vm_lifecycle/resume_process.rs | 6 ++ 4 files changed, 100 insertions(+), 4 deletions(-) diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index 197bbfa91..f5864f236 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -11,6 +11,7 @@ use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; use capsem_foundation::ipc_channel; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::proxy_control::{ decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyControlEvent, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, @@ -69,6 +70,15 @@ impl ProxyWorker { } pub(crate) async fn grant(&self, capability: ProxyCapability, stream: UnixStream) -> Result<()> { + self.grant_capability(GrantedCapability::Ordinary(capability), stream) + .await + } + + pub(crate) async fn grant_ledger(&self, stream: UnixStream, grant: LedgerChannelGrant) -> Result<()> { + self.grant_capability(GrantedCapability::Ledger(grant), stream).await + } + + async fn grant_capability(&self, capability: GrantedCapability, stream: UnixStream) -> Result<()> { let (completed, result) = oneshot::channel(); self.commands .send(CommandRequest::Grant { @@ -178,7 +188,7 @@ pub(crate) fn spawn_for_session(binary: &Path, active_policy: Vec, log_path: enum CommandRequest { Grant { - capability: ProxyCapability, + capability: GrantedCapability, stream: UnixStream, completed: oneshot::Sender>, }, @@ -191,6 +201,36 @@ enum CommandRequest { }, } +#[derive(Clone, Copy)] +enum GrantedCapability { + Ordinary(ProxyCapability), + Ledger(LedgerChannelGrant), +} + +impl GrantedCapability { + const fn kind(self) -> ProxyCapability { + match self { + Self::Ordinary(capability) => capability, + Self::Ledger(_) => ProxyCapability::Ledger, + } + } + + fn channel_grant( + self, + generation: ProxyGeneration, + grant_id: u64, + ) -> Result { + match self { + Self::Ordinary(capability) => Ok(capsem_proto::proxy_control::ProxyChannelGrant::new( + generation, grant_id, capability, + )?), + Self::Ledger(ledger) => Ok(capsem_proto::proxy_control::ProxyChannelGrant::with_ledger( + generation, grant_id, ledger, + )?), + } + } +} + struct WorkerProcess { generation: ProxyGeneration, child: Child, @@ -305,9 +345,9 @@ impl WorkerProcess { Ok(process) } - async fn grant(&mut self, capability: ProxyCapability, stream: UnixStream) -> Result<()> { + async fn grant(&mut self, capability: GrantedCapability, stream: UnixStream) -> Result<()> { let grant_id = self.allocate_grant_id()?; - let grant = capsem_proto::proxy_control::ProxyChannelGrant::new(self.generation, grant_id, capability)?; + let grant = capability.channel_grant(self.generation, grant_id)?; self.control_tx .send( &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), @@ -326,7 +366,10 @@ impl WorkerProcess { grant_id: rejected, reason, } if generation == self.generation && rejected == grant_id => { - bail!("proxy worker rejected {capability:?} grant {grant_id}: {reason:?}") + bail!( + "proxy worker rejected {:?} grant {grant_id}: {reason:?}", + capability.kind() + ) } event => bail!("proxy worker sent {event:?} while adopting grant {grant_id}"), } @@ -601,6 +644,29 @@ fn generation_hex(generation: ProxyGeneration) -> String { } impl ServiceState { + pub(crate) async fn grant_proxy_ledger( + &self, + session_id: &str, + session_dir: &Path, + worker: &ProxyWorker, + ) -> Result<()> { + let client = self + .ledger_workers + .acquire( + session_id, + &session_dir.join("session.db"), + &session_dir.join("ledger.log"), + capsem_proto::ledger::LedgerClientRole::Proxy, + ) + .await + .context("acquire proxy ledger channel")?; + let (stream, grant) = client.into_parts(); + worker + .grant_ledger(stream, grant) + .await + .context("grant proxy ledger channel") + } + pub(crate) fn register_proxy_worker( self: &std::sync::Arc, session_id: &str, diff --git a/crates/capsem-service/src/proxy_worker/tests.rs b/crates/capsem-service/src/proxy_worker/tests.rs index 27aaa756a..16014beae 100644 --- a/crates/capsem-service/src/proxy_worker/tests.rs +++ b/crates/capsem-service/src/proxy_worker/tests.rs @@ -1,6 +1,7 @@ use std::os::fd::AsFd as _; use std::sync::Arc; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; use capsem_proto::proxy_control::{decode_proxy_control_request, encode_proxy_control_event}; use super::*; @@ -88,6 +89,10 @@ async fn fake_proxy_worker_child() { let frame = frame.unwrap().unwrap(); match decode_proxy_control_request(&frame.bytes).unwrap() { ProxyControlRequest::Attach(grant) => { + if mode == "ledger" { + assert_eq!(grant.capability(), ProxyCapability::Ledger); + assert_eq!(grant.ledger_grant(), Some(proxy_ledger_grant())); + } held.push(UnixStream::from(frame.fds.into_iter().next().unwrap())); control_tx.send( &encode_proxy_control_event(ProxyControlEvent::Adopted { @@ -113,6 +118,10 @@ async fn fake_proxy_worker_child() { } } +fn proxy_ledger_grant() -> LedgerChannelGrant { + LedgerChannelGrant::new(LedgerGeneration::new([3; 16]), 29, LedgerClientRole::Proxy).unwrap() +} + async fn fake(mode: &'static str) -> Result { let mut command = Command::new(std::env::current_exe().unwrap()); command @@ -149,6 +158,15 @@ async fn supervisor_applies_policy_grants_connected_descriptors_and_reaps() { worker.shutdown().await.unwrap(); } +#[tokio::test] +async fn ledger_descriptor_carries_its_exact_proxy_authority() { + let worker = fake("ledger").await.unwrap(); + let (peer, granted) = UnixStream::pair().unwrap(); + worker.grant_ledger(granted, proxy_ledger_grant()).await.unwrap(); + drop(peer); + worker.shutdown().await.unwrap(); +} + #[tokio::test] async fn startup_refusal_never_returns_a_worker_handle() { let error = fake("refuse").await.err().expect("refused startup must fail"); diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index 0185b282b..e9dcbe3ae 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -298,6 +298,12 @@ impl ServiceState { return Err(error.context("start confined proxy worker")); } }; + if let Err(error) = + tokio::runtime::Handle::current().block_on(self.grant_proxy_ledger(id, &session_dir, &proxy)) + { + instance_reaper::kill_and_reap(child); + return Err(error); + } let upstream_broker = upstream_broker.with_proxy(proxy.clone()); // Provisioning runs on a blocking thread that keeps the runtime diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index b99f1abc2..6ed8bd2ce 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -228,6 +228,12 @@ impl ServiceState { return Err(error.context("start confined proxy worker")); } }; + if let Err(error) = + tokio::runtime::Handle::current().block_on(self.grant_proxy_ledger(&vm_id, &entry.session_dir, &proxy)) + { + instance_reaper::kill_and_reap(child); + return Err(error); + } let upstream_broker = upstream_broker.with_proxy(proxy.clone()); info!( From 9ef5d1f6b8bd20c9302f1ad45784aac7fad300f6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:32:09 +0000 Subject: [PATCH 135/464] feat(proto): define proxy credential capability --- crates/capsem-proto/src/lib.rs | 1 + crates/capsem-proto/src/proxy_credentials.rs | 231 +++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 crates/capsem-proto/src/proxy_credentials.rs diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 50f92abbe..8b2b87bfa 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -27,6 +27,7 @@ pub mod mcp_contracts; pub mod poll; pub mod privatelink; pub mod proxy_control; +pub mod proxy_credentials; pub mod proxy_policy; pub mod repeated; pub mod router; diff --git a/crates/capsem-proto/src/proxy_credentials.rs b/crates/capsem-proto/src/proxy_credentials.rs new file mode 100644 index 000000000..7fd5de76e --- /dev/null +++ b/crates/capsem-proto/src/proxy_credentials.rs @@ -0,0 +1,231 @@ +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub const MAX_CREDENTIAL_VALUE_BYTES: usize = 64 * 1024; +pub const MAX_CREDENTIAL_FIELD_BYTES: usize = 4 * 1024; +pub const MAX_CREDENTIAL_HEADERS: usize = 256; +pub const MAX_CREDENTIAL_ERROR_BYTES: usize = 4 * 1024; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyCredentialProvider { + Anthropic, + Google, + OpenAi, + Github, + Mcp, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyModelProvider { + Unknown, + Anthropic, + OpenAi, + Google, + Ollama, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProxyCredentialObservation { + pub provider: ProxyCredentialProvider, + pub raw_value: String, + pub source: String, + pub event_type: Option, + pub trace_id: Option, + pub context_json: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProxyBrokeredCredential { + pub provider: ProxyCredentialProvider, + pub credential_ref: String, + pub store_account: String, + pub newly_captured: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProxyHeader { + pub name: String, + #[serde(with = "serde_bytes")] + pub value: Vec, +} + +impl ProxyHeader { + pub fn new(name: impl Into, value: Vec) -> Self { + Self { + name: name.into(), + value, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyCredentialRequest { + Capture { + request_id: u64, + observation: ProxyCredentialObservation, + }, + Substitute { + request_id: u64, + domain: String, + ai_provider: Option, + headers: Vec, + query: Option, + }, +} + +impl ProxyCredentialRequest { + pub const fn request_id(&self) -> u64 { + match self { + Self::Capture { request_id, .. } | Self::Substitute { request_id, .. } => *request_id, + } + } + + pub fn validate(&self) -> Result<(), ProxyCredentialProtocolError> { + if self.request_id() == 0 { + return Err(ProxyCredentialProtocolError::ZeroRequestId); + } + match self { + Self::Capture { observation, .. } => { + bounded(&observation.raw_value, MAX_CREDENTIAL_VALUE_BYTES)?; + bounded(&observation.source, MAX_CREDENTIAL_FIELD_BYTES)?; + bounded_optional(&observation.event_type, MAX_CREDENTIAL_FIELD_BYTES)?; + bounded_optional(&observation.trace_id, MAX_CREDENTIAL_FIELD_BYTES)?; + bounded_optional(&observation.context_json, MAX_CREDENTIAL_VALUE_BYTES) + } + Self::Substitute { + domain, headers, query, .. + } => { + bounded(domain, MAX_CREDENTIAL_FIELD_BYTES)?; + if headers.len() > MAX_CREDENTIAL_HEADERS { + return Err(ProxyCredentialProtocolError::TooManyHeaders); + } + for header in headers { + bounded(&header.name, MAX_CREDENTIAL_FIELD_BYTES)?; + if header.value.len() > MAX_CREDENTIAL_VALUE_BYTES { + return Err(ProxyCredentialProtocolError::FieldTooLong); + } + } + bounded_optional(query, MAX_CREDENTIAL_VALUE_BYTES) + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyCredentialResponse { + Captured { + request_id: u64, + credential: ProxyBrokeredCredential, + }, + Substituted { + request_id: u64, + headers: Vec, + query: Option, + credential_ref: Option, + }, + Rejected { + request_id: u64, + error: String, + }, +} + +impl ProxyCredentialResponse { + pub const fn request_id(&self) -> u64 { + match self { + Self::Captured { request_id, .. } + | Self::Substituted { request_id, .. } + | Self::Rejected { request_id, .. } => *request_id, + } + } + + pub fn rejected(request_id: u64, error: impl Into) -> Result { + if request_id == 0 { + return Err(ProxyCredentialProtocolError::ZeroRequestId); + } + let error = error.into(); + bounded(&error, MAX_CREDENTIAL_ERROR_BYTES)?; + Ok(Self::Rejected { request_id, error }) + } +} + +fn bounded(value: &str, limit: usize) -> Result<(), ProxyCredentialProtocolError> { + if value.len() > limit { + Err(ProxyCredentialProtocolError::FieldTooLong) + } else { + Ok(()) + } +} + +fn bounded_optional(value: &Option, limit: usize) -> Result<(), ProxyCredentialProtocolError> { + match value { + Some(value) => bounded(value, limit), + None => Ok(()), + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum ProxyCredentialProtocolError { + #[error("proxy credential request id must not be zero")] + ZeroRequestId, + #[error("proxy credential field exceeds its bound")] + FieldTooLong, + #[error("proxy credential request has too many headers")] + TooManyHeaders, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bounded_capture_and_substitution_requests_validate() { + let capture = ProxyCredentialRequest::Capture { + request_id: 1, + observation: ProxyCredentialObservation { + provider: ProxyCredentialProvider::OpenAi, + raw_value: "secret".to_string(), + source: "http.header.authorization".to_string(), + event_type: Some("http.request".to_string()), + trace_id: None, + context_json: None, + }, + }; + capture.validate().unwrap(); + + let substitute = ProxyCredentialRequest::Substitute { + request_id: 2, + domain: "api.openai.com".to_string(), + ai_provider: Some(ProxyModelProvider::OpenAi), + headers: vec![ProxyHeader::new( + "authorization", + b"Bearer credential:blake3:abc".to_vec(), + )], + query: Some("key=credential%3Ablake3%3Aabc".to_string()), + }; + substitute.validate().unwrap(); + assert_eq!(capture.request_id(), 1); + assert_eq!(substitute.request_id(), 2); + } + + #[test] + fn oversized_or_uncorrelated_values_fail_closed() { + let request = ProxyCredentialRequest::Capture { + request_id: 0, + observation: ProxyCredentialObservation { + provider: ProxyCredentialProvider::Google, + raw_value: "x".repeat(MAX_CREDENTIAL_VALUE_BYTES + 1), + source: "test".to_string(), + event_type: None, + trace_id: None, + context_json: None, + }, + }; + assert!(request.validate().is_err()); + assert!(ProxyCredentialResponse::rejected(0, "no").is_err()); + assert!(ProxyCredentialResponse::rejected(1, "x".repeat(MAX_CREDENTIAL_ERROR_BYTES + 1)).is_err()); + } +} From 990f828ed46bb3b44d73745ba64647f09c54273b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:38:46 +0000 Subject: [PATCH 136/464] feat(proxy): consume credential capability --- crates/capsem-proxy/src/credential_client.rs | 360 ++++++++++++++++++ .../src/credential_client/tests.rs | 50 +++ crates/capsem-proxy/src/main.rs | 25 ++ 3 files changed, 435 insertions(+) create mode 100644 crates/capsem-proxy/src/credential_client.rs create mode 100644 crates/capsem-proxy/src/credential_client/tests.rs diff --git a/crates/capsem-proxy/src/credential_client.rs b/crates/capsem-proxy/src/credential_client.rs new file mode 100644 index 000000000..969340aec --- /dev/null +++ b/crates/capsem-proxy/src/credential_client.rs @@ -0,0 +1,360 @@ +use std::io; +use std::os::unix::net::UnixStream; +use std::sync::mpsc; +use std::time::Duration; + +use capsem_core::credential_broker::{ + BrokeredCredential, BrokeredUpstreamCredentials, CredentialObservation, CredentialProvider, +}; +use capsem_core::net::ai_traffic::provider::ProviderKind; +use capsem_core::net::proxy_engine::ProxyCredentials; +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_control::ProxyChannelCloseReason; +use capsem_proto::proxy_credentials::{ + ProxyBrokeredCredential, ProxyCredentialObservation, ProxyCredentialProvider, ProxyCredentialRequest, + ProxyCredentialResponse, ProxyHeader, ProxyModelProvider, +}; + +const COMMAND_CAPACITY: usize = 32; +const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +type CaptureReply = mpsc::SyncSender>; +type SubstituteReply = mpsc::SyncSender>; + +enum Command { + Capture { + observation: CredentialObservation, + reply: CaptureReply, + }, + Substitute { + domain: String, + ai_provider: Option, + headers: http::HeaderMap, + query: Option, + reply: SubstituteReply, + }, +} + +impl Command { + fn fail(self, error: String) { + match self { + Self::Capture { reply, .. } => { + let _ = reply.send(Err(error)); + } + Self::Substitute { reply, .. } => { + let _ = reply.send(Err(error)); + } + } + } +} + +pub(super) struct CredentialClient { + commands: tokio::sync::mpsc::Sender, +} + +impl CredentialClient { + pub(super) fn start( + stream: UnixStream, + ) -> io::Result<(Self, tokio::sync::watch::Receiver>)> { + let (commands, inbox) = tokio::sync::mpsc::channel(COMMAND_CAPACITY); + let (closed_tx, closed) = tokio::sync::watch::channel(None); + let (startup_tx, startup_rx) = mpsc::sync_channel(1); + std::thread::Builder::new() + .name("capsem-proxy-credentials".to_string()) + .spawn(move || run(stream, inbox, startup_tx, closed_tx))?; + match startup_rx.recv_timeout(OPERATION_TIMEOUT) { + Ok(Ok(())) => Ok((Self { commands }, closed)), + Ok(Err(error)) => Err(io::Error::other(error)), + Err(mpsc::RecvTimeoutError::Timeout) => Err(io::Error::new( + io::ErrorKind::TimedOut, + "proxy credential client startup timed out", + )), + Err(mpsc::RecvTimeoutError::Disconnected) => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "proxy credential client stopped during startup", + )), + } + } + + fn send(&self, command: Command, response: mpsc::Receiver>) -> Result { + self.commands.try_send(command).map_err(|error| match error { + tokio::sync::mpsc::error::TrySendError::Full(_) => "proxy credential capability is at capacity".to_string(), + tokio::sync::mpsc::error::TrySendError::Closed(_) => "proxy credential capability is closed".to_string(), + })?; + let receive = || { + response + .recv_timeout(OPERATION_TIMEOUT) + .map_err(|_| "proxy credential operation timed out".to_string())? + }; + if tokio::runtime::Handle::try_current().is_ok() { + tokio::task::block_in_place(receive) + } else { + receive() + } + } +} + +impl ProxyCredentials for CredentialClient { + fn capture(&self, observation: &CredentialObservation) -> Result { + let (reply, response) = mpsc::sync_channel(1); + self.send( + Command::Capture { + observation: observation.clone(), + reply, + }, + response, + ) + } + + fn substitute_upstream( + &self, + domain: &str, + ai_provider: Option, + headers: &mut http::HeaderMap, + query: Option<&str>, + ) -> Result { + let (reply, response) = mpsc::sync_channel(1); + let (substituted_headers, result) = self.send( + Command::Substitute { + domain: domain.to_string(), + ai_provider, + headers: headers.clone(), + query: query.map(str::to_string), + reply, + }, + response, + )?; + *headers = substituted_headers; + Ok(result) + } +} + +fn run( + stream: UnixStream, + inbox: tokio::sync::mpsc::Receiver, + startup: mpsc::SyncSender>, + closed: tokio::sync::watch::Sender>, +) { + let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() { + Ok(runtime) => runtime, + Err(error) => { + let _ = startup.send(Err(format!("build proxy credential runtime: {error}"))); + return; + } + }; + let channel = { + let _entered = runtime.enter(); + ipc_channel::channel_from_std::(stream) + }; + let (sender, receiver) = match channel { + Ok(channel) => channel, + Err(error) => { + let _ = startup.send(Err(format!("open proxy credential channel: {error}"))); + return; + } + }; + if startup.send(Ok(())).is_err() { + return; + } + let reason = runtime.block_on(run_actor(sender, receiver, inbox)); + let _ = closed.send(Some(reason)); +} + +async fn run_actor( + sender: ipc_channel::Sender, + receiver: ipc_channel::Receiver, + mut commands: tokio::sync::mpsc::Receiver, +) -> ProxyChannelCloseReason { + let mut request_id = 1_u64; + loop { + tokio::select! { + unsolicited = receiver.recv() => { + return match unsolicited { + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => ProxyChannelCloseReason::Disconnected, + _ => ProxyChannelCloseReason::ProtocolError, + }; + } + command = commands.recv() => { + let Some(command) = command else { + return ProxyChannelCloseReason::Revoked; + }; + if dispatch(&sender, &receiver, request_id, command).await.is_err() { + fail_pending(&mut commands, "proxy credential capability closed"); + return ProxyChannelCloseReason::ProtocolError; + } + let Some(next) = request_id.checked_add(1) else { + fail_pending(&mut commands, "proxy credential request id exhausted"); + return ProxyChannelCloseReason::ProtocolError; + }; + request_id = next; + } + } + } +} + +async fn dispatch( + sender: &ipc_channel::Sender, + receiver: &ipc_channel::Receiver, + request_id: u64, + command: Command, +) -> Result<(), String> { + let request = request_for(request_id, &command); + if let Err(error) = request.validate() { + return fail(command, error.to_string()); + } + if let Err(error) = sender.send(request).await { + return fail(command, format!("send proxy credential request: {error}")); + } + let response = match tokio::time::timeout(OPERATION_TIMEOUT, receiver.recv()).await { + Ok(Ok(response)) => response, + Ok(Err(error)) => return fail(command, format!("receive proxy credential response: {error}")), + Err(_) => return fail(command, "proxy credential response timed out".to_string()), + }; + if response.request_id() != request_id { + return fail( + command, + "proxy credential response id did not match request".to_string(), + ); + } + finish(command, response) +} + +fn request_for(request_id: u64, command: &Command) -> ProxyCredentialRequest { + match command { + Command::Capture { observation, .. } => ProxyCredentialRequest::Capture { + request_id, + observation: wire_observation(observation), + }, + Command::Substitute { + domain, + ai_provider, + headers, + query, + .. + } => ProxyCredentialRequest::Substitute { + request_id, + domain: domain.clone(), + ai_provider: ai_provider.map(wire_model_provider), + headers: wire_headers(headers), + query: query.clone(), + }, + } +} + +fn finish(command: Command, response: ProxyCredentialResponse) -> Result<(), String> { + match (command, response) { + (Command::Capture { reply, .. }, ProxyCredentialResponse::Captured { credential, .. }) => { + let _ = reply.send(Ok(core_credential(credential))); + Ok(()) + } + ( + Command::Substitute { reply, .. }, + ProxyCredentialResponse::Substituted { + headers, + query, + credential_ref, + .. + }, + ) => { + let result = + core_headers(headers).map(|headers| (headers, BrokeredUpstreamCredentials { credential_ref, query })); + let _ = reply.send(result); + Ok(()) + } + (Command::Capture { reply, .. }, ProxyCredentialResponse::Rejected { error, .. }) => { + let _ = reply.send(Err(error)); + Ok(()) + } + (Command::Substitute { reply, .. }, ProxyCredentialResponse::Rejected { error, .. }) => { + let _ = reply.send(Err(error)); + Ok(()) + } + (command, _) => fail( + command, + "proxy credential response kind did not match request".to_string(), + ), + } +} + +fn fail(command: Command, error: String) -> Result { + command.fail(error.clone()); + Err(error) +} + +fn fail_pending(commands: &mut tokio::sync::mpsc::Receiver, error: &str) { + while let Ok(command) = commands.try_recv() { + command.fail(error.to_string()); + } +} + +fn wire_observation(observation: &CredentialObservation) -> ProxyCredentialObservation { + ProxyCredentialObservation { + provider: wire_provider(observation.provider), + raw_value: observation.raw_value.clone(), + source: observation.source.clone(), + event_type: observation.event_type.clone(), + trace_id: observation.trace_id.clone(), + context_json: observation.context_json.clone(), + } +} + +const fn wire_provider(provider: CredentialProvider) -> ProxyCredentialProvider { + match provider { + CredentialProvider::Anthropic => ProxyCredentialProvider::Anthropic, + CredentialProvider::Google => ProxyCredentialProvider::Google, + CredentialProvider::OpenAi => ProxyCredentialProvider::OpenAi, + CredentialProvider::Github => ProxyCredentialProvider::Github, + CredentialProvider::Mcp => ProxyCredentialProvider::Mcp, + } +} + +const fn core_provider(provider: ProxyCredentialProvider) -> CredentialProvider { + match provider { + ProxyCredentialProvider::Anthropic => CredentialProvider::Anthropic, + ProxyCredentialProvider::Google => CredentialProvider::Google, + ProxyCredentialProvider::OpenAi => CredentialProvider::OpenAi, + ProxyCredentialProvider::Github => CredentialProvider::Github, + ProxyCredentialProvider::Mcp => CredentialProvider::Mcp, + } +} + +const fn wire_model_provider(provider: ProviderKind) -> ProxyModelProvider { + match provider { + ProviderKind::Unknown => ProxyModelProvider::Unknown, + ProviderKind::Anthropic => ProxyModelProvider::Anthropic, + ProviderKind::OpenAi => ProxyModelProvider::OpenAi, + ProviderKind::Google => ProxyModelProvider::Google, + ProviderKind::Ollama => ProxyModelProvider::Ollama, + } +} + +fn core_credential(credential: ProxyBrokeredCredential) -> BrokeredCredential { + BrokeredCredential { + provider: core_provider(credential.provider), + credential_ref: credential.credential_ref, + store_account: credential.store_account, + newly_captured: credential.newly_captured, + } +} + +fn wire_headers(headers: &http::HeaderMap) -> Vec { + headers + .iter() + .map(|(name, value)| ProxyHeader::new(name.as_str(), value.as_bytes().to_vec())) + .collect() +} + +fn core_headers(headers: Vec) -> Result { + let mut result = http::HeaderMap::new(); + for header in headers { + let name = http::header::HeaderName::from_bytes(header.name.as_bytes()) + .map_err(|error| format!("invalid proxy credential header name: {error}"))?; + let value = http::header::HeaderValue::from_bytes(&header.value) + .map_err(|error| format!("invalid proxy credential header value: {error}"))?; + result.append(name, value); + } + Ok(result) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proxy/src/credential_client/tests.rs b/crates/capsem-proxy/src/credential_client/tests.rs new file mode 100644 index 000000000..ca7f2cb32 --- /dev/null +++ b/crates/capsem-proxy/src/credential_client/tests.rs @@ -0,0 +1,50 @@ +use std::os::unix::net::UnixStream; + +use capsem_core::net::proxy_engine::ProxyCredentials; +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_credentials::{ProxyCredentialRequest, ProxyCredentialResponse}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn remote_substitution_preserves_correlated_headers_and_reports_disconnect() { + let (service, proxy) = UnixStream::pair().unwrap(); + let (client, mut closed) = super::CredentialClient::start(proxy).unwrap(); + let serving = tokio::spawn(async move { + let (sender, receiver) = + ipc_channel::channel_from_std::(service).unwrap(); + let ProxyCredentialRequest::Substitute { + request_id, + headers, + query, + .. + } = receiver.recv().await.unwrap() + else { + panic!("expected substitution request") + }; + sender + .send(ProxyCredentialResponse::Substituted { + request_id, + headers, + query, + credential_ref: Some("credential:blake3:test".to_string()), + }) + .await + .unwrap(); + }); + let mut headers = http::HeaderMap::new(); + headers.insert("x-test", http::HeaderValue::from_static("ordinary")); + let result = client + .substitute_upstream("example.com", None, &mut headers, Some("page=1")) + .unwrap(); + assert_eq!(headers["x-test"], "ordinary"); + assert_eq!(result.query.as_deref(), Some("page=1")); + assert_eq!(result.credential_ref.as_deref(), Some("credential:blake3:test")); + serving.await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(1), closed.changed()) + .await + .unwrap() + .unwrap(); + assert_eq!( + *closed.borrow(), + Some(capsem_proto::proxy_control::ProxyChannelCloseReason::Disconnected) + ); +} diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 938b1bf8a..f295d6803 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -26,6 +26,8 @@ use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; use clap::Parser; use tokio::task::{JoinError, JoinSet}; +mod credential_client; + const CONTROL_QUEUE_CAPACITY: usize = 16; const GRANT_LIMIT: usize = 71; const TRAFFIC_GRANT_LIMIT: usize = 64; @@ -211,6 +213,18 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result .lock() .unwrap_or_else(|error| error.into_inner()) .attach_ledger(Arc::new(writer)); + } else if capability == ProxyCapability::Credential { + let (credentials, mut closed) = credential_client::CredentialClient::start(stream) + .context("open proxy credential capability")?; + state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .attach_credentials(Arc::new(credentials)); + tasks.spawn(async move { + while closed.borrow().is_none() && closed.changed().await.is_ok() {} + let reason = closed.borrow().unwrap_or(ProxyChannelCloseReason::Disconnected); + (capability, grant_id, reason) + }); } else { descriptors.insert(grant_id, stream); } @@ -315,6 +329,17 @@ impl ProxyRuntimeState { )); } } + + fn attach_credentials(&mut self, credentials: Arc) { + self.credentials = credentials; + if let Some(engine) = self.engine.take() { + self.engine = Some(ProxyEngine::new( + engine.policy().clone(), + Arc::clone(&self.ledger), + Arc::clone(&self.credentials), + )); + } + } } struct UnavailableLedger; From c04bf00477f958746c299cd68d2728267cc92b29 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:41:10 +0000 Subject: [PATCH 137/464] feat(service): broker proxy credentials --- crates/capsem-service/Cargo.toml | 2 +- crates/capsem-service/src/main.rs | 1 + .../capsem-service/src/proxy_credentials.rs | 169 ++++++++++++++++++ .../src/proxy_credentials/tests.rs | 36 ++++ crates/capsem-service/src/proxy_worker.rs | 18 ++ .../src/vm_lifecycle/provision.rs | 4 + .../src/vm_lifecycle/resume_process.rs | 4 + 7 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 crates/capsem-service/src/proxy_credentials.rs create mode 100644 crates/capsem-service/src/proxy_credentials/tests.rs diff --git a/crates/capsem-service/Cargo.toml b/crates/capsem-service/Cargo.toml index b44a5222f..a47346bf3 100644 --- a/crates/capsem-service/Cargo.toml +++ b/crates/capsem-service/Cargo.toml @@ -20,6 +20,7 @@ capsem-guard = { path = "../capsem-guard" } capsem-logger = { path = "../capsem-logger" } capsem-telemetry = { path = "../capsem-telemetry", features = ["export"] } capsem-proto = { path = "../capsem-proto" } +http.workspace = true capsem-router = { path = "../capsem-router" } anyhow.workspace = true tokio.workspace = true @@ -44,7 +45,6 @@ workspace = true [dev-dependencies] async-trait = "0.1" bytes.workspace = true -http.workspace = true metrics = "0.24" opentelemetry = { version = "0.33", default-features = false, features = ["metrics"] } opentelemetry-http = { version = "0.33", default-features = false } diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 954fb01a6..f6c8a2e7e 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -48,6 +48,7 @@ mod mcp_routes; mod plugin_routes; mod private_routes; mod process_control; +mod proxy_credentials; mod proxy_worker; mod sandbox_info; mod session_cleanup; diff --git a/crates/capsem-service/src/proxy_credentials.rs b/crates/capsem-service/src/proxy_credentials.rs new file mode 100644 index 000000000..27cc37f0c --- /dev/null +++ b/crates/capsem-service/src/proxy_credentials.rs @@ -0,0 +1,169 @@ +//! Trusted credential-store operations for one confined proxy capability. + +use std::io; +use std::os::unix::net::UnixStream; + +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_credentials::{ + ProxyBrokeredCredential, ProxyCredentialObservation, ProxyCredentialProvider, ProxyCredentialRequest, + ProxyCredentialResponse, ProxyHeader, ProxyModelProvider, MAX_CREDENTIAL_ERROR_BYTES, +}; + +use capsem_core::credential_broker::{BrokeredCredential, CredentialObservation, CredentialProvider}; +use capsem_core::net::ai_traffic::provider::ProviderKind; + +pub(crate) async fn serve(stream: UnixStream) -> Result<(), String> { + let (sender, receiver) = + ipc_channel::channel_from_std::(stream).map_err(io_string)?; + loop { + let request = match receiver.recv().await { + Ok(request) => request, + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(()), + Err(error) => return Err(format!("receive proxy credential request: {error}")), + }; + let request_id = request.request_id(); + let response = if let Err(error) = request.validate() { + rejected(request_id, error.to_string()) + } else { + match tokio::task::spawn_blocking(move || dispatch(request)).await { + Ok(Ok(response)) => response, + Ok(Err(error)) => rejected(request_id, error), + Err(error) => rejected(request_id, format!("credential broker task failed: {error}")), + } + }; + sender + .send(response) + .await + .map_err(|error| format!("send proxy credential response: {error}"))?; + } +} + +fn dispatch(request: ProxyCredentialRequest) -> Result { + match request { + ProxyCredentialRequest::Capture { + request_id, + observation, + } => { + let credential = + capsem_core::credential_broker::broker_observed_credential(&core_observation(observation))?; + Ok(ProxyCredentialResponse::Captured { + request_id, + credential: wire_credential(credential), + }) + } + ProxyCredentialRequest::Substitute { + request_id, + domain, + ai_provider, + headers, + query, + } => { + let mut headers = core_headers(headers)?; + let substituted = capsem_core::credential_broker::substitute_brokered_upstream_credentials( + &domain, + ai_provider.map(core_model_provider), + &mut headers, + query.as_deref(), + )?; + Ok(ProxyCredentialResponse::Substituted { + request_id, + headers: wire_headers(&headers), + query: substituted.query, + credential_ref: substituted.credential_ref, + }) + } + } +} + +fn core_observation(observation: ProxyCredentialObservation) -> CredentialObservation { + CredentialObservation { + provider: core_provider(observation.provider), + raw_value: observation.raw_value, + source: observation.source, + event_type: observation.event_type, + trace_id: observation.trace_id, + context_json: observation.context_json, + } +} + +const fn core_provider(provider: ProxyCredentialProvider) -> CredentialProvider { + match provider { + ProxyCredentialProvider::Anthropic => CredentialProvider::Anthropic, + ProxyCredentialProvider::Google => CredentialProvider::Google, + ProxyCredentialProvider::OpenAi => CredentialProvider::OpenAi, + ProxyCredentialProvider::Github => CredentialProvider::Github, + ProxyCredentialProvider::Mcp => CredentialProvider::Mcp, + } +} + +const fn wire_provider(provider: CredentialProvider) -> ProxyCredentialProvider { + match provider { + CredentialProvider::Anthropic => ProxyCredentialProvider::Anthropic, + CredentialProvider::Google => ProxyCredentialProvider::Google, + CredentialProvider::OpenAi => ProxyCredentialProvider::OpenAi, + CredentialProvider::Github => ProxyCredentialProvider::Github, + CredentialProvider::Mcp => ProxyCredentialProvider::Mcp, + } +} + +const fn core_model_provider(provider: ProxyModelProvider) -> ProviderKind { + match provider { + ProxyModelProvider::Unknown => ProviderKind::Unknown, + ProxyModelProvider::Anthropic => ProviderKind::Anthropic, + ProxyModelProvider::OpenAi => ProviderKind::OpenAi, + ProxyModelProvider::Google => ProviderKind::Google, + ProxyModelProvider::Ollama => ProviderKind::Ollama, + } +} + +fn wire_credential(credential: BrokeredCredential) -> ProxyBrokeredCredential { + ProxyBrokeredCredential { + provider: wire_provider(credential.provider), + credential_ref: credential.credential_ref, + store_account: credential.store_account, + newly_captured: credential.newly_captured, + } +} + +fn core_headers(headers: Vec) -> Result { + let mut result = http::HeaderMap::new(); + for header in headers { + let name = http::header::HeaderName::from_bytes(header.name.as_bytes()) + .map_err(|error| format!("invalid proxy credential header name: {error}"))?; + let value = http::header::HeaderValue::from_bytes(&header.value) + .map_err(|error| format!("invalid proxy credential header value: {error}"))?; + result.append(name, value); + } + Ok(result) +} + +fn wire_headers(headers: &http::HeaderMap) -> Vec { + headers + .iter() + .map(|(name, value)| ProxyHeader::new(name.as_str(), value.as_bytes().to_vec())) + .collect() +} + +fn rejected(request_id: u64, error: String) -> ProxyCredentialResponse { + let error = truncate_utf8(error, MAX_CREDENTIAL_ERROR_BYTES); + ProxyCredentialResponse::rejected(request_id, error).expect("request id and bounded diagnostic were validated") +} + +fn truncate_utf8(mut value: String, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value; + } + let mut boundary = max_bytes; + while !value.is_char_boundary(boundary) { + boundary -= 1; + } + value.truncate(boundary); + value +} + +fn io_string(error: io::Error) -> String { + error.to_string() +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/proxy_credentials/tests.rs b/crates/capsem-service/src/proxy_credentials/tests.rs new file mode 100644 index 000000000..8a5ec84b2 --- /dev/null +++ b/crates/capsem-service/src/proxy_credentials/tests.rs @@ -0,0 +1,36 @@ +use std::os::unix::net::UnixStream; + +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_credentials::{ + ProxyCredentialRequest, ProxyCredentialResponse, ProxyHeader, ProxyModelProvider, +}; + +#[tokio::test] +async fn substitution_round_trip_preserves_unbrokered_headers_and_query() { + let (service, client) = UnixStream::pair().unwrap(); + let serving = tokio::spawn(super::serve(service)); + let (sender, receiver) = + ipc_channel::channel_from_std::(client).unwrap(); + sender + .send(ProxyCredentialRequest::Substitute { + request_id: 7, + domain: "example.com".to_string(), + ai_provider: Some(ProxyModelProvider::Unknown), + headers: vec![ProxyHeader::new("x-test", b"ordinary".to_vec())], + query: Some("page=1".to_string()), + }) + .await + .unwrap(); + assert_eq!( + receiver.recv().await.unwrap(), + ProxyCredentialResponse::Substituted { + request_id: 7, + headers: vec![ProxyHeader::new("x-test", b"ordinary".to_vec())], + query: Some("page=1".to_string()), + credential_ref: None, + } + ); + drop(sender); + drop(receiver); + serving.await.unwrap().unwrap(); +} diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index f5864f236..5fa1c29ca 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -644,6 +644,24 @@ fn generation_hex(generation: ProxyGeneration) -> String { } impl ServiceState { + pub(crate) async fn grant_proxy_credentials(&self, worker: &ProxyWorker) -> Result<()> { + let (service, proxy) = UnixStream::pair().context("create proxy credential capability")?; + let serving = tokio::spawn(crate::proxy_credentials::serve(service)); + if let Err(error) = worker.grant(ProxyCapability::Credential, proxy).await { + serving.abort(); + let _ = serving.await; + return Err(error.context("grant proxy credential capability")); + } + tokio::spawn(async move { + match serving.await { + Ok(Ok(())) => tracing::debug!("proxy credential capability disconnected"), + Ok(Err(error)) => tracing::warn!(%error, "proxy credential capability failed"), + Err(error) => tracing::warn!(%error, "proxy credential capability task failed"), + } + }); + Ok(()) + } + pub(crate) async fn grant_proxy_ledger( &self, session_id: &str, diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index e9dcbe3ae..9f32da652 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -304,6 +304,10 @@ impl ServiceState { instance_reaper::kill_and_reap(child); return Err(error); } + if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_credentials(&proxy)) { + instance_reaper::kill_and_reap(child); + return Err(error); + } let upstream_broker = upstream_broker.with_proxy(proxy.clone()); // Provisioning runs on a blocking thread that keeps the runtime diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 6ed8bd2ce..ac9c4acea 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -234,6 +234,10 @@ impl ServiceState { instance_reaper::kill_and_reap(child); return Err(error); } + if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_credentials(&proxy)) { + instance_reaper::kill_and_reap(child); + return Err(error); + } let upstream_broker = upstream_broker.with_proxy(proxy.clone()); info!( From fec9fb5e832c84530c57116eae4f9264cd55b153 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:45:54 +0000 Subject: [PATCH 138/464] feat(service): grant proxy upstream broker --- crates/capsem-service/src/upstream_broker.rs | 44 +++++++++++++------ .../src/upstream_broker/tests.rs | 25 ++++++++++- .../src/vm_lifecycle/provision.rs | 14 +++++- .../src/vm_lifecycle/resume_process.rs | 14 +++++- 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index fa8600bff..33ace1f83 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -48,7 +48,7 @@ pub(crate) struct BrokerPolicy { #[derive(Clone)] pub(crate) struct PolicyPublisher { - updates: Option>, + updates: Vec>, } pub(crate) struct PendingBroker { coordinator: UnixStream, @@ -132,17 +132,29 @@ impl BrokerPolicy { impl PolicyPublisher { pub(crate) async fn publish(&self, policy: Arc) -> Result<(), String> { - let Some(updates) = &self.updates else { - return Ok(()); - }; - let (applied, confirmation) = oneshot::channel(); - updates - .send(PolicyUpdate { policy, applied }) - .await - .map_err(|_| "upstream policy broker stopped".to_string())?; - confirmation - .await - .map_err(|_| "upstream policy broker stopped before applying the policy".to_string())? + let mut confirmations = Vec::with_capacity(self.updates.len()); + for updates in &self.updates { + let (applied, confirmation) = oneshot::channel(); + updates + .send(PolicyUpdate { + policy: Arc::clone(&policy), + applied, + }) + .await + .map_err(|_| "upstream policy broker stopped".to_string())?; + confirmations.push(confirmation); + } + for confirmation in confirmations { + confirmation + .await + .map_err(|_| "upstream policy broker stopped before applying the policy".to_string())??; + } + Ok(()) + } + + pub(crate) fn merge(mut self, mut other: Self) -> Self { + self.updates.append(&mut other.updates); + self } } @@ -161,7 +173,7 @@ impl PendingBroker { proxy: None, }, PolicyPublisher { - updates: Some(updates_tx), + updates: vec![updates_tx], }, )) } @@ -201,6 +213,10 @@ impl PendingBroker { Ok(std::process::Stdio::from(descriptor)) } + pub(crate) fn worker_stream(&self) -> io::Result { + self.worker.try_clone() + } + pub(crate) fn start(self, authority: WorkerGrant) -> tokio::task::JoinHandle<()> { tokio::spawn(async move { if let Err(error) = run( @@ -726,7 +742,7 @@ fn revoke_grant(grant_id: u64, grant: ActiveGrant) -> Result<(), String> { #[cfg(test)] pub(crate) fn test_policy_publisher() -> PolicyPublisher { - PolicyPublisher { updates: None } + PolicyPublisher { updates: Vec::new() } } #[cfg(test)] diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index 71d100a6a..f1c9fe2ad 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -109,6 +109,29 @@ fn policy_digest(label: &str) -> String { format!("blake3:{}", blake3::hash(label.as_bytes()).to_hex()) } +#[tokio::test] +async fn merged_policy_publisher_waits_for_every_broker() { + let (first_tx, mut first_rx) = tokio::sync::mpsc::channel(1); + let (second_tx, mut second_rx) = tokio::sync::mpsc::channel(1); + let publisher = PolicyPublisher { + updates: vec![first_tx], + } + .merge(PolicyPublisher { + updates: vec![second_tx], + }); + let first = tokio::spawn(async move { + let update = first_rx.recv().await.unwrap(); + update.applied.send(Ok(())).unwrap(); + }); + let second = tokio::spawn(async move { + let update = second_rx.recv().await.unwrap(); + update.applied.send(Ok(())).unwrap(); + }); + publisher.publish(test_policy("merged", None, vec![])).await.unwrap(); + first.await.unwrap(); + second.await.unwrap(); +} + fn start_broker( policy: Arc, ) -> ( @@ -668,7 +691,7 @@ async fn worker_supplied_descriptor_terminates_before_resolution() { .unwrap() .unwrap() .unwrap_err(); - assert_eq!(error, "worker sent an upstream descriptor"); + assert_eq!(error, "worker upstream request carried 1 descriptors, expected 0"); assert!(client.responses.recv().await.is_err()); } diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index 9f32da652..66e1c3586 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -156,12 +156,16 @@ impl ServiceState { let active_policy = self.materialize_active_policy(&session_dir)?; let active_policy_path = active_policy.path.clone(); - let (upstream_broker, upstream_policy) = crate::upstream_broker::PendingBroker::pair_for_session( + let (upstream_broker, owner_upstream_policy) = crate::upstream_broker::PendingBroker::pair_for_session( active_policy.broker_policy(), session_dir.clone(), )?; let upstream_broker = upstream_broker.with_vm_ledger(Arc::clone(&self.ledger_workers), id, &session_dir); let upstream_stdio = upstream_broker.worker_stdio()?; + let (proxy_upstream_broker, proxy_upstream_policy) = + crate::upstream_broker::PendingBroker::pair(active_policy.broker_policy())?; + let proxy_upstream = proxy_upstream_broker.worker_stream()?; + let upstream_policy = owner_upstream_policy.merge(proxy_upstream_policy); info!(process_binary = %self.process_binary.display(), exists = self.process_binary.exists(), "checking process_binary"); @@ -308,6 +312,12 @@ impl ServiceState { instance_reaper::kill_and_reap(child); return Err(error); } + if let Err(error) = tokio::runtime::Handle::current() + .block_on(proxy.grant(capsem_proto::proxy_control::ProxyCapability::Upstream, proxy_upstream)) + { + instance_reaper::kill_and_reap(child); + return Err(error.context("grant proxy upstream broker")); + } let upstream_broker = upstream_broker.with_proxy(proxy.clone()); // Provisioning runs on a blocking thread that keeps the runtime @@ -360,6 +370,7 @@ impl ServiceState { let authority = crate::instance::WorkerAuthority::default(); let upstream_grant = authority.grant(); + let proxy_upstream_grant = authority.grant(); let mut instances = self.instances.lock().unwrap(); instances.insert( id.to_string(), @@ -389,6 +400,7 @@ impl ServiceState { return Err(error.context("register proxy worker")); } let _upstream_broker = upstream_broker.start(upstream_grant); + let _proxy_upstream_broker = proxy_upstream_broker.start(proxy_upstream_grant); let _reaper = instance_reaper::spawn_exit_reaper( child, id.to_string(), diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index ac9c4acea..b57bb1dbb 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -67,13 +67,17 @@ impl ServiceState { self.validate_persistent_entry(&entry)?; let active_policy = self.materialize_active_policy(&entry.session_dir)?; let active_policy_path = active_policy.path.clone(); - let (upstream_broker, upstream_policy) = crate::upstream_broker::PendingBroker::pair_for_session( + let (upstream_broker, owner_upstream_policy) = crate::upstream_broker::PendingBroker::pair_for_session( active_policy.broker_policy(), entry.session_dir.clone(), )?; let upstream_broker = upstream_broker.with_vm_ledger(Arc::clone(&self.ledger_workers), &vm_id, &entry.session_dir); let upstream_stdio = upstream_broker.worker_stdio()?; + let (proxy_upstream_broker, proxy_upstream_policy) = + crate::upstream_broker::PendingBroker::pair(active_policy.broker_policy())?; + let proxy_upstream = proxy_upstream_broker.worker_stream()?; + let upstream_policy = owner_upstream_policy.merge(proxy_upstream_policy); let scratch_disk_size_gb = session_rootfs_size_gb(&entry)?; let resolved = self.resolve_pinned_asset_paths(&entry.asset_pins)?; self.validate_pinned_asset_files(&resolved, &entry.asset_pins)?; @@ -238,6 +242,12 @@ impl ServiceState { instance_reaper::kill_and_reap(child); return Err(error); } + if let Err(error) = tokio::runtime::Handle::current() + .block_on(proxy.grant(capsem_proto::proxy_control::ProxyCapability::Upstream, proxy_upstream)) + { + instance_reaper::kill_and_reap(child); + return Err(error.context("grant proxy upstream broker")); + } let upstream_broker = upstream_broker.with_proxy(proxy.clone()); info!( @@ -249,6 +259,7 @@ impl ServiceState { let authority = crate::instance::WorkerAuthority::default(); let upstream_grant = authority.grant(); + let proxy_upstream_grant = authority.grant(); let session_dir = entry.session_dir.clone(); let mut instances = self.instances.lock().unwrap(); instances.insert( @@ -279,6 +290,7 @@ impl ServiceState { return Err(error.context("register proxy worker")); } let _upstream_broker = upstream_broker.start(upstream_grant); + let _proxy_upstream_broker = proxy_upstream_broker.start(proxy_upstream_grant); let _reaper = instance_reaper::spawn_exit_reaper( child, vm_id.clone(), From bf667e0da6d453e91d2c771d1c75505d8afd3278 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:49:42 +0000 Subject: [PATCH 139/464] refactor(core): share upstream grant client --- crates/capsem-core/src/net/mod.rs | 1 + crates/capsem-core/src/net/upstream_grant.rs | 657 ++++++++++++++++++ .../src/net/upstream_grant/tests.rs | 485 +++++++++++++ 3 files changed, 1143 insertions(+) create mode 100644 crates/capsem-core/src/net/upstream_grant.rs create mode 100644 crates/capsem-core/src/net/upstream_grant/tests.rs diff --git a/crates/capsem-core/src/net/mod.rs b/crates/capsem-core/src/net/mod.rs index 9aba0d1da..be736fece 100644 --- a/crates/capsem-core/src/net/mod.rs +++ b/crates/capsem-core/src/net/mod.rs @@ -14,3 +14,4 @@ pub mod proxy_engine; pub mod router_process; pub mod switch_host; pub mod upstream_address; +pub mod upstream_grant; diff --git a/crates/capsem-core/src/net/upstream_grant.rs b/crates/capsem-core/src/net/upstream_grant.rs new file mode 100644 index 000000000..743e34370 --- /dev/null +++ b/crates/capsem-core/src/net/upstream_grant.rs @@ -0,0 +1,657 @@ +//! Process-side client for coordinator-minted upstream descriptors. + +use std::io; +use std::os::fd::{AsFd as _, AsRawFd as _, BorrowedFd, OwnedFd}; +use std::os::unix::net::UnixStream; +use std::time::Duration; + +use crate::net::dns::{DnsDatagram, DnsGrantFuture, DnsUpstreamGrants}; +use crate::net::mitm_proxy::{ + protocol::Protocol, GrantedTcpStream, TcpConnectGrantFuture, TcpGrantSelection, TcpResolveGrantFuture, + TcpUpstreamGrants, +}; +use anyhow::{anyhow, Result}; +use capsem_foundation::unix::fd::{self, SocketShutdown}; +use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; +use capsem_proto::ledger::LedgerChannelGrant; +use capsem_proto::upstream_grant::{ + decode_upstream_grant_response, encode_upstream_grant_request, ProxyTrafficService, UpstreamDescriptorKind, + UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, +}; +use tokio::sync::{mpsc, oneshot}; + +const COMMAND_CAPACITY: usize = 64; +// The coordinator permits a TCP dial to take ten seconds. Leave room for its +// response framing while still bounding a wedged generation channel. +const WIRE_TIMEOUT: Duration = Duration::from_secs(12); + +type WireSender = DescriptorSender; +type WireReceiver = DescriptorReceiver; + +enum Command { + AttachProxyTraffic { + service: ProxyTrafficService, + descriptor: OwnedFd, + reply: oneshot::Sender>, + }, + OpenLedger { + reply: oneshot::Sender>, + }, + SetGuestMode { + relative_path: Vec, + mode: u16, + reply: oneshot::Sender>, + }, + OpenDns { + upstream_index: u16, + policy_digest: String, + reply: oneshot::Sender>, + }, + ResolveTcp { + protocol: Protocol, + host: String, + port: u16, + policy_digest: String, + reply: oneshot::Sender>, + }, + ConnectTcp { + selection_id: u64, + policy_digest: String, + reply: oneshot::Sender>, + }, +} + +/// Serializes grant protocol traffic over the inherited generation channel. +#[derive(Clone)] +pub struct UpstreamGrantClient { + commands: mpsc::Sender, + stopped: tokio::sync::watch::Receiver>, +} + +impl UpstreamGrantClient { + pub fn start(socket: UnixStream) -> io::Result { + let sender = WireSender::new(socket.try_clone()?)?; + let receiver = WireReceiver::new(socket.try_clone()?)?; + let (commands_tx, commands) = mpsc::channel(COMMAND_CAPACITY); + let (actor_releases, releases) = mpsc::unbounded_channel(); + let (stopped_tx, stopped) = tokio::sync::watch::channel(None); + tokio::spawn(async move { + let result = run_actor(sender, receiver, commands, releases, actor_releases).await; + let reason = result + .err() + .unwrap_or_else(|| "upstream grant channel closed".to_string()); + tracing::warn!(%reason, "upstream grant channel stopped"); + if let Err(error) = fd::shutdown(socket.as_fd(), SocketShutdown::Both) { + tracing::debug!(%error, "failed to shut down upstream grant channel"); + } + let _ = stopped_tx.send(Some(reason)); + }); + Ok(Self { + commands: commands_tx, + stopped, + }) + } + + pub fn stop_receiver(&self) -> tokio::sync::watch::Receiver> { + self.stopped.clone() + } + + pub async fn open_ledger(&self) -> Result<(UnixStream, LedgerChannelGrant)> { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::OpenLedger { reply }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + } + + pub async fn attach_proxy_traffic(&self, service: ProxyTrafficService, descriptor: OwnedFd) -> Result<()> { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::AttachProxyTraffic { + service, + descriptor, + reply, + }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + } +} + +impl crate::GuestMetadataAuthority for UpstreamGrantClient { + fn set_mode(&self, relative_path: &[u8], mode: u16) -> io::Result<()> { + let (reply, result) = oneshot::channel(); + self.commands + .blocking_send(Command::SetGuestMode { + relative_path: relative_path.to_vec(), + mode, + reply, + }) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; + result + .blocking_recv() + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? + .map_err(io::Error::other) + } +} + +impl DnsUpstreamGrants for UpstreamGrantClient { + fn open(&self, upstream_index: u16, policy_digest: &str) -> DnsGrantFuture<'_> { + let policy_digest = policy_digest.to_owned(); + Box::pin(async move { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::OpenDns { + upstream_index, + policy_digest, + reply, + }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + }) + } +} + +impl TcpUpstreamGrants for UpstreamGrantClient { + fn resolve(&self, protocol: Protocol, host: &str, port: u16, policy_digest: &str) -> TcpResolveGrantFuture<'_> { + let host = host.to_owned(); + let policy_digest = policy_digest.to_owned(); + Box::pin(async move { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::ResolveTcp { + protocol, + host, + port, + policy_digest, + reply, + }) + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; + result + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? + .map_err(io::Error::other) + }) + } + + fn connect(&self, selection_id: u64, policy_digest: &str) -> TcpConnectGrantFuture<'_> { + let policy_digest = policy_digest.to_owned(); + Box::pin(async move { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::ConnectTcp { + selection_id, + policy_digest, + reply, + }) + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; + result + .await + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? + .map_err(io::Error::other) + }) + } +} + +async fn run_actor( + sender: WireSender, + receiver: WireReceiver, + mut commands: mpsc::Receiver, + mut releases: mpsc::UnboundedReceiver, + release_sender: mpsc::UnboundedSender, +) -> Result<(), String> { + let mut next_request_id = 1_u64; + loop { + tokio::select! { + biased; + release = releases.recv() => { + let Some(resource_id) = release else { + if commands.is_closed() { + return Ok(()); + } + continue; + }; + send_request(&sender, &UpstreamGrantRequest::Release { resource_id }).await?; + } + command = commands.recv() => { + let Some(command) = command else { + return Ok(()); + }; + let request_id = next_request_id; + next_request_id = next_request_id.checked_add(1).ok_or("upstream request id exhausted")?; + dispatch(&sender, &receiver, &release_sender, request_id, command).await?; + } + } + } +} + +async fn dispatch( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + command: Command, +) -> Result<(), String> { + match command { + Command::AttachProxyTraffic { + service, + descriptor, + reply, + } => finish( + reply, + attach_proxy_traffic(sender, receiver, request_id, service, descriptor).await, + ), + Command::OpenLedger { reply } => finish(reply, open_ledger(sender, receiver, request_id).await), + Command::SetGuestMode { + relative_path, + mode, + reply, + } => finish( + reply, + set_guest_mode(sender, receiver, request_id, relative_path, mode).await, + ), + Command::OpenDns { + upstream_index, + policy_digest, + reply, + } => finish( + reply, + open_dns(sender, receiver, releases, request_id, upstream_index, &policy_digest).await, + ), + Command::ResolveTcp { + protocol, + host, + port, + policy_digest, + reply, + } => finish( + reply, + resolve_tcp( + sender, + receiver, + releases, + request_id, + protocol, + &host, + port, + &policy_digest, + ) + .await, + ), + Command::ConnectTcp { + selection_id, + policy_digest, + reply, + } => finish( + reply, + connect_tcp(sender, receiver, releases, request_id, selection_id, &policy_digest).await, + ), + } +} + +async fn attach_proxy_traffic( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, + service: ProxyTrafficService, + descriptor: OwnedFd, +) -> Result, String> { + let request = UpstreamGrantRequest::AttachProxyTraffic { request_id, service }; + let bytes = + encode_upstream_grant_request(&request).map_err(|error| format!("encode proxy traffic handoff: {error:#}"))?; + tokio::time::timeout(WIRE_TIMEOUT, sender.send(&bytes, &[descriptor.as_raw_fd()])) + .await + .map_err(|_| "send proxy traffic handoff timed out".to_string())? + .map_err(|error| format!("send proxy traffic handoff: {error}"))?; + drop(descriptor); + let (response, fds) = receive_response(receiver).await?; + debug_assert!(fds.is_empty()); + match response { + UpstreamGrantResponse::ProxyTrafficAdopted { + request_id: response_id, + } if response_id == request_id => Ok(Ok(())), + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("proxy traffic handoff denied: {reason:?}"))), + response => Err(format!( + "unexpected proxy traffic response for request {request_id}: {response:?}" + )), + } +} + +async fn open_ledger( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, +) -> Result, String> { + send_request(sender, &UpstreamGrantRequest::OpenLedger { request_id }).await?; + let (response, mut fds) = receive_response(receiver).await?; + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("session ledger grant denied: {reason:?}"))), + UpstreamGrantResponse::LedgerGranted { + request_id: response_id, + grant, + } if response_id == request_id => { + let descriptor = fds.pop().ok_or("ledger grant omitted its descriptor")?; + send_request( + sender, + &UpstreamGrantRequest::Adopted { + grant_id: grant.client_id(), + }, + ) + .await?; + let stream = UnixStream::from(descriptor); + stream + .peer_addr() + .map_err(|error| format!("adopt granted ledger channel: {error}"))?; + Ok(Ok((stream, grant))) + } + response => Err(format!( + "unexpected session ledger response for request {request_id}: {response:?}" + )), + } +} + +async fn set_guest_mode( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, + relative_path: Vec, + mode: u16, +) -> Result, String> { + send_request( + sender, + &UpstreamGrantRequest::SetGuestMode { + request_id, + relative_path, + mode, + }, + ) + .await?; + let (response, fds) = receive_response(receiver).await?; + if !fds.is_empty() { + return Err("guest mode response carried a descriptor".into()); + } + match response { + UpstreamGrantResponse::GuestModeSet { + request_id: response_id, + } if response_id == request_id => Ok(Ok(())), + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("guest mode change denied: {reason:?}"))), + response => Err(format!( + "unexpected guest mode response for request {request_id}: {response:?}" + )), + } +} + +fn finish( + reply: oneshot::Sender>, + result: Result, String>, +) -> Result<(), String> { + match result { + Ok(result) => { + let _ = reply.send(result); + Ok(()) + } + Err(error) => { + let _ = reply.send(Err(error.clone())); + Err(error) + } + } +} + +async fn open_dns( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + upstream_index: u16, + expected_policy_digest: &str, +) -> Result, String> { + send_request( + sender, + &UpstreamGrantRequest::OpenDns { + request_id, + upstream_index, + }, + ) + .await?; + let (response, fds) = receive_response(receiver).await?; + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("upstream DNS grant denied: {reason:?}"))), + UpstreamGrantResponse::DescriptorGranted { + request_id: response_id, + grant_id, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: response_digest, + } if response_id == request_id => { + let descriptor = fds.into_iter().next().ok_or("DNS grant omitted its descriptor")?; + send_request(sender, &UpstreamGrantRequest::Adopted { grant_id }).await?; + if response_digest != expected_policy_digest { + send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; + return Ok(Err(format!( + "upstream DNS grant policy mismatch: expected {expected_policy_digest}, received {response_digest}" + ))); + } + let socket = datagram_from_descriptor(descriptor)?; + let releases = releases.clone(); + Ok(Ok(DnsDatagram::new(socket, move || { + let _ = releases.send(grant_id); + }))) + } + response => Err(format!( + "unexpected upstream DNS grant response for request {request_id}: {response:?}" + )), + } +} + +#[allow(clippy::too_many_arguments)] +async fn resolve_tcp( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + protocol: Protocol, + host: &str, + port: u16, + expected_policy_digest: &str, +) -> Result, String> { + let protocol = match wire_protocol(protocol) { + Ok(protocol) => protocol, + Err(error) => return Ok(Err(error)), + }; + send_request( + sender, + &UpstreamGrantRequest::ResolveTcp { + request_id, + protocol, + host: host.to_owned(), + port, + }, + ) + .await?; + let (response, _fds) = receive_response(receiver).await?; + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("upstream TCP selection denied: {reason:?}"))), + UpstreamGrantResponse::TcpResolved { + request_id: response_id, + selection_id, + protocol, + judged_ip, + policy_digest, + } if response_id == request_id => { + if policy_digest != expected_policy_digest { + send_request( + sender, + &UpstreamGrantRequest::Release { + resource_id: selection_id, + }, + ) + .await?; + return Ok(Err(format!( + "upstream TCP selection policy mismatch: expected {expected_policy_digest}, received {policy_digest}" + ))); + } + let releases = releases.clone(); + Ok(Ok(TcpGrantSelection::new( + selection_id, + core_protocol(protocol), + judged_ip, + move || { + let _ = releases.send(selection_id); + }, + ))) + } + response => Err(format!( + "unexpected upstream TCP selection response for request {request_id}: {response:?}" + )), + } +} + +async fn connect_tcp( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + selection_id: u64, + expected_policy_digest: &str, +) -> Result, String> { + send_request( + sender, + &UpstreamGrantRequest::ConnectTcp { + request_id, + selection_id, + }, + ) + .await?; + let (response, fds) = receive_response(receiver).await?; + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("upstream TCP connection denied: {reason:?}"))), + UpstreamGrantResponse::DescriptorGranted { + request_id: response_id, + grant_id, + kind: UpstreamDescriptorKind::Tcp, + policy_digest, + } if response_id == request_id => { + let descriptor = fds.into_iter().next().ok_or("TCP grant omitted its descriptor")?; + send_request(sender, &UpstreamGrantRequest::Adopted { grant_id }).await?; + if policy_digest != expected_policy_digest { + send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; + return Ok(Err(format!( + "upstream TCP grant policy mismatch: expected {expected_policy_digest}, received {policy_digest}" + ))); + } + let stream = stream_from_descriptor(descriptor)?; + let releases = releases.clone(); + Ok(Ok(GrantedTcpStream::new(stream, move || { + let _ = releases.send(grant_id); + }))) + } + response => Err(format!( + "unexpected upstream TCP grant response for request {request_id}: {response:?}" + )), + } +} + +async fn receive_response(receiver: &WireReceiver) -> Result<(UpstreamGrantResponse, Vec), String> { + let frame = tokio::time::timeout(WIRE_TIMEOUT, receiver.recv()) + .await + .map_err(|_| "receive upstream grant response timed out".to_string())? + .map_err(|error| format!("receive upstream grant response: {error}"))?; + let response = decode_upstream_grant_response(&frame.bytes) + .map_err(|error| format!("decode upstream grant response: {error:#}"))?; + if frame.fds.len() != response.expected_descriptor_count() { + return Err(format!( + "upstream grant response carried {} descriptors, expected {}", + frame.fds.len(), + response.expected_descriptor_count() + )); + } + Ok((response, frame.fds)) +} + +fn wire_protocol(protocol: Protocol) -> Result { + match protocol { + Protocol::Http => Ok(UpstreamProtocol::Http), + Protocol::Tls => Ok(UpstreamProtocol::Tls), + Protocol::McpFrame | Protocol::Unknown => { + Err(format!("protocol {} cannot select a TCP upstream", protocol.label())) + } + } +} + +fn core_protocol(protocol: UpstreamProtocol) -> Protocol { + match protocol { + UpstreamProtocol::Http => Protocol::Http, + UpstreamProtocol::Tls => Protocol::Tls, + } +} + +async fn send_request(sender: &WireSender, request: &UpstreamGrantRequest) -> Result<(), String> { + let bytes = + encode_upstream_grant_request(request).map_err(|error| format!("encode upstream grant request: {error:#}"))?; + tokio::time::timeout(WIRE_TIMEOUT, sender.send(&bytes, &[])) + .await + .map_err(|_| "send upstream grant request timed out".to_string())? + .map_err(|error| format!("send upstream grant request: {error}"))?; + Ok(()) +} + +fn datagram_from_descriptor(descriptor: OwnedFd) -> Result { + let socket = std::net::UdpSocket::from(descriptor); + socket + .set_nonblocking(true) + .map_err(|error| format!("make granted DNS socket nonblocking: {error}"))?; + tokio::net::UdpSocket::from_std(socket).map_err(|error| format!("adopt granted DNS socket: {error}")) +} + +fn stream_from_descriptor(descriptor: OwnedFd) -> Result { + let stream = std::net::TcpStream::from(descriptor); + stream + .set_nonblocking(true) + .map_err(|error| format!("make granted TCP stream nonblocking: {error}"))?; + stream + .set_nodelay(true) + .map_err(|error| format!("set granted TCP stream nodelay: {error}"))?; + tokio::net::TcpStream::from_std(stream).map_err(|error| format!("adopt granted TCP stream: {error}")) +} + +/// Duplicate the inherited generation channel before runtime threads exist. +pub fn adopt_inherited(descriptor: BorrowedFd<'_>) -> io::Result { + fd::set_close_on_exec(descriptor)?; + let owned = fd::duplicate(descriptor)?; + let socket = UnixStream::from(owned); + socket.peer_addr()?; + Ok(socket) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-core/src/net/upstream_grant/tests.rs b/crates/capsem-core/src/net/upstream_grant/tests.rs new file mode 100644 index 000000000..3cecb83be --- /dev/null +++ b/crates/capsem-core/src/net/upstream_grant/tests.rs @@ -0,0 +1,485 @@ +use std::io::{Read as _, Write as _}; +use std::os::fd::{AsFd as _, AsRawFd as _, RawFd}; +use std::sync::Arc; + +use crate::net::dns::{DnsResolver, DnsUpstreamGrants}; +use crate::net::mitm_proxy::{TcpUpstreamGrants, UpstreamTarget}; +use crate::GuestMetadataAuthority as _; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; +use capsem_proto::upstream_grant::{ + decode_upstream_grant_request, encode_upstream_grant_response, ProxyTrafficService, UpstreamDescriptorKind, + UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, +}; + +use super::*; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + +const POLICY_A: &str = "blake3:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const POLICY_B: &str = "blake3:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + +#[tokio::test] +async fn ledger_channel_preserves_grant_and_is_adopted_before_use() { + let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x6c; 16]), 44, LedgerClientRole::VmOwner).unwrap(); + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let (ledger, mut ledger_peer) = UnixStream::pair().unwrap(); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenLedger { request_id: 1 } + ); + send_response( + &responses, + &UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }, + Some(ledger.as_raw_fd()), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { + grant_id: grant.client_id(), + } + ); + }); + + let (mut stream, received) = client.open_ledger().await.unwrap(); + assert_eq!(received, grant); + stream.write_all(b"hello").unwrap(); + let mut message = [0_u8; 5]; + ledger_peer.read_exact(&mut message).unwrap(); + assert_eq!(&message, b"hello"); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn proxy_traffic_descriptor_is_surrendered_before_adoption_returns() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let (traffic, mut guest) = UnixStream::pair().unwrap(); + let broker_task = tokio::spawn(async move { + let frame = requests.recv().await.unwrap(); + assert_eq!(frame.fds.len(), 1); + assert_eq!( + decode_upstream_grant_request(&frame.bytes).unwrap(), + UpstreamGrantRequest::AttachProxyTraffic { + request_id: 1, + service: ProxyTrafficService::Dns, + } + ); + let mut adopted = UnixStream::from(frame.fds.into_iter().next().unwrap()); + adopted.write_all(b"owned").unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 1 }, + None, + ) + .await; + }); + + client + .attach_proxy_traffic(ProxyTrafficService::Dns, traffic.into()) + .await + .unwrap(); + let mut proof = [0; 5]; + guest.read_exact(&mut proof).unwrap(); + assert_eq!(&proof, b"owned"); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn ledger_denial_and_missing_descriptor_fail_closed() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let denied = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenLedger { request_id: 1 } + ); + send_response( + &responses, + &UpstreamGrantResponse::Denied { + request_id: 1, + reason: UpstreamGrantDenial::NotConfigured, + }, + None, + ) + .await; + }); + let error = client.open_ledger().await.unwrap_err().to_string(); + assert!(error.contains("NotConfigured"), "{error}"); + denied.await.unwrap(); + + let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x7d; 16]), 45, LedgerClientRole::VmOwner).unwrap(); + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let malformed = tokio::spawn(async move { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenLedger { request_id: 1 } + )); + send_response( + &responses, + &UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }, + None, + ) + .await; + }); + let error = client.open_ledger().await.unwrap_err().to_string(); + assert!(error.contains("carried 0 descriptors"), "{error}"); + malformed.await.unwrap(); +} + +async fn open_error(client: &UpstreamGrantClient) -> String { + match client.open(0, POLICY_A).await { + Ok(_) => panic!("unexpected DNS grant"), + Err(error) => error.to_string(), + } +} + +fn channels(socket: UnixStream) -> (WireReceiver, WireSender) { + let receiver = WireReceiver::new(socket.try_clone().unwrap()).unwrap(); + let sender = WireSender::new(socket).unwrap(); + (receiver, sender) +} + +async fn receive_request(receiver: &WireReceiver) -> UpstreamGrantRequest { + let frame = receiver.recv().await.unwrap(); + assert!(frame.fds.is_empty()); + decode_upstream_grant_request(&frame.bytes).unwrap() +} + +async fn send_response(sender: &WireSender, response: &UpstreamGrantResponse, descriptor: Option) { + let bytes = encode_upstream_grant_response(response).unwrap(); + let fds = descriptor.map_or_else(Vec::new, |descriptor| vec![descriptor]); + sender.send(&bytes, &fds).await.unwrap(); +} + +fn query() -> Vec { + let mut query = vec![0x12, 0x34, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; + query.extend_from_slice(&[7, b'e', b'x', b'a', b'm', b'p', b'l', b'e', 3, b'c', b'o', b'm', 0]); + query.extend_from_slice(&[0, 1, 0, 1]); + query +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guest_mode_changes_use_the_serialized_coordinator_channel() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = Arc::new(UpstreamGrantClient::start(worker).unwrap()); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::SetGuestMode { + request_id: 1, + relative_path: b"workspace/pyvenv/bin/activate".to_vec(), + mode: 0o755, + } + ); + send_response(&responses, &UpstreamGrantResponse::GuestModeSet { request_id: 1 }, None).await; + }); + let client_for_mode = Arc::clone(&client); + tokio::task::spawn_blocking(move || { + client_for_mode + .set_mode(b"workspace/pyvenv/bin/activate", 0o755) + .unwrap(); + }) + .await + .unwrap(); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { + let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let upstream_address = upstream.local_addr().unwrap(); + let (broker, worker) = UnixStream::pair().unwrap(); + let client = Arc::new(UpstreamGrantClient::start(worker).unwrap()); + let (requests, responses) = channels(broker); + let expected_query = query(); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenDns { + request_id: 1, + upstream_index: 0, + } + ); + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.connect(upstream_address).unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 41, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_A.into(), + }, + Some(socket.as_raw_fd()), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { grant_id: 41 } + ); + let mut received = [0_u8; 512]; + let (count, peer) = upstream.recv_from(&mut received).await.unwrap(); + assert_eq!(&received[..count], expected_query); + let mut answer = received[..count].to_vec(); + answer[2] = 0x81; + answer[3] = 0x80; + upstream.send_to(&answer, peer).await.unwrap(); + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { resource_id: 41 } + ); + }); + + let resolver = DnsResolver::with_grants(vec![upstream_address], client); + let (answer, _) = resolver.resolve_for_policy(&query(), POLICY_A).await.unwrap(); + assert_eq!(&answer[..2], &[0x12, 0x34]); + assert_eq!(&answer[2..4], &[0x81, 0x80]); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn stale_policy_grant_is_adopted_released_and_channel_remains_usable() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenDns { request_id: 1, .. } + )); + let peer = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + socket.connect(peer.local_addr().unwrap()).unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 51, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_B.into(), + }, + Some(socket.as_raw_fd()), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { grant_id: 51 } + ); + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { resource_id: 51 } + ); + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenDns { request_id: 2, .. } + )); + send_response( + &responses, + &UpstreamGrantResponse::Denied { + request_id: 2, + reason: UpstreamGrantDenial::NotConfigured, + }, + None, + ) + .await; + }); + + let error = open_error(&client).await; + assert!(error.contains("policy mismatch"), "{error}"); + let error = open_error(&client).await; + assert!(error.contains("NotConfigured"), "{error}"); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn malformed_or_revoked_grant_channel_fails_closed() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenDns { request_id: 1, .. } + )); + send_response( + &responses, + &UpstreamGrantResponse::DescriptorGranted { + request_id: 1, + grant_id: 61, + kind: UpstreamDescriptorKind::DnsUdp, + policy_digest: POLICY_A.into(), + }, + None, + ) + .await; + }); + let error = open_error(&client).await; + assert!(error.contains("carried 0 descriptors"), "{error}"); + broker_task.await.unwrap(); + let error = open_error(&client).await; + assert!(error.contains("closed") || error.contains("stopped"), "{error}"); + + let (broker, worker) = UnixStream::pair().unwrap(); + let revoked = UpstreamGrantClient::start(worker).unwrap(); + drop(broker); + assert!(!open_error(&revoked).await.is_empty()); +} + +#[tokio::test] +async fn tcp_selection_and_connected_stream_are_brokered_and_released() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream_address = listener.local_addr().unwrap(); + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::ResolveTcp { + request_id: 1, + protocol: UpstreamProtocol::Tls, + host: "api.example".into(), + port: 443, + } + ); + send_response( + &responses, + &UpstreamGrantResponse::TcpResolved { + request_id: 1, + selection_id: 71, + protocol: UpstreamProtocol::Http, + judged_ip: Some("127.0.0.1".parse().unwrap()), + policy_digest: POLICY_A.into(), + }, + None, + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::ConnectTcp { + request_id: 2, + selection_id: 71, + } + ); + let socket = tokio::net::TcpStream::connect(upstream_address) + .await + .unwrap() + .into_std() + .unwrap(); + let (mut upstream, _) = listener.accept().await.unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::DescriptorGranted { + request_id: 2, + grant_id: 72, + kind: UpstreamDescriptorKind::Tcp, + policy_digest: POLICY_A.into(), + }, + Some(socket.as_raw_fd()), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { grant_id: 72 } + ); + let mut request = [0_u8; 4]; + upstream.read_exact(&mut request).await.unwrap(); + assert_eq!(&request, b"ping"); + upstream.write_all(b"pong").await.unwrap(); + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { resource_id: 72 } + ); + }); + + let selection = client + .resolve(Protocol::Tls, "api.example", 443, POLICY_A) + .await + .unwrap(); + assert_eq!(selection.protocol, Protocol::Http); + assert_eq!(selection.judged_ip, Some("127.0.0.1".parse().unwrap())); + let target = UpstreamTarget::Granted { + host: "api.example".into(), + port: 443, + guest_protocol: Protocol::Tls, + protocol: selection.protocol, + judged_ip: selection.judged_ip, + selection: Some(selection), + }; + let (mut stream, pinned) = target.connect_with_grants(Some(&client), POLICY_A).await.unwrap(); + assert!(matches!(pinned, UpstreamTarget::Granted { selection: None, .. })); + stream.write_all(b"ping").await.unwrap(); + let mut response = [0_u8; 4]; + stream.read_exact(&mut response).await.unwrap(); + assert_eq!(&response, b"pong"); + drop(stream); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn unused_or_stale_tcp_selections_are_released_without_connecting() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + for (request_id, selection_id, digest) in [(1, 81, POLICY_A), (2, 82, POLICY_B)] { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::ResolveTcp { + request_id: actual, + .. + } if actual == request_id + )); + send_response( + &responses, + &UpstreamGrantResponse::TcpResolved { + request_id, + selection_id, + protocol: UpstreamProtocol::Tls, + judged_ip: Some("93.184.216.34".parse().unwrap()), + policy_digest: digest.into(), + }, + None, + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { + resource_id: selection_id, + } + ); + } + }); + + let unused = client + .resolve(Protocol::Tls, "unused.example", 443, POLICY_A) + .await + .unwrap(); + drop(unused); + let error = client + .resolve(Protocol::Tls, "stale.example", 443, POLICY_A) + .await + .unwrap_err(); + assert!(error.to_string().contains("policy mismatch"), "{error}"); + broker_task.await.unwrap(); +} + +#[test] +fn inherited_channel_is_duplicated_cloexec_and_connected() { + let (inherited, peer) = UnixStream::pair().unwrap(); + let mut adopted = adopt_inherited(inherited.as_fd()).unwrap(); + let mut peer = peer; + assert!(fd::set_close_on_exec(inherited.as_fd()).unwrap()); + adopted.write_all(b"x").unwrap(); + let mut byte = [0_u8; 1]; + peer.read_exact(&mut byte).unwrap(); + assert_eq!(byte, [b'x']); +} From cba37157449425016b78d0fa053c2e3d2ca0ba3b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:51:43 +0000 Subject: [PATCH 140/464] refactor(process): use shared upstream client --- crates/capsem-process/src/main.rs | 11 +- crates/capsem-process/src/upstream_grant.rs | 654 ------------------ .../src/upstream_grant/tests.rs | 485 ------------- 3 files changed, 5 insertions(+), 1145 deletions(-) delete mode 100644 crates/capsem-process/src/upstream_grant.rs delete mode 100644 crates/capsem-process/src/upstream_grant/tests.rs diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 61934ac7b..a3e592b41 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -10,12 +10,12 @@ mod private_names; mod private_seats; mod runtime_config; mod terminal; -mod upstream_grant; mod vsock; use anyhow::{Context, Result}; use capsem_core::fs_monitor::FsMonitor; use capsem_core::net::dns::{DnsAnswerCache, DnsResolver}; +use capsem_core::net::upstream_grant::{adopt_inherited, UpstreamGrantClient}; use capsem_core::{boot_vm, BootOptions, VirtioFsShare, VsockConnection}; use capsem_logger::DbWriter; use capsem_proto::ipc::{ProcessToService, ServiceToProcess}; @@ -358,7 +358,7 @@ fn prepare_owner_sandbox_attestation(session_dir: &Path) -> Result, + upstream_grants: Arc, ) -> Result<()> { use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; @@ -408,8 +408,7 @@ fn main() -> Result<()> { // SAFETY: process entry precedes argument parsing, telemetry, descriptor // owners and runtime threads. Broker grants will be named here explicitly. unsafe { capsem_foundation::unix::fd::close_inherited_descriptors()? }; - let upstream_socket = - upstream_grant::adopt_inherited(std::io::stdin().as_fd()).context("adopt inherited upstream grant channel")?; + let upstream_socket = adopt_inherited(std::io::stdin().as_fd()).context("adopt inherited upstream grant channel")?; let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { service: "capsem-process", sink: capsem_foundation::telemetry::LogSink::Stderr, @@ -444,7 +443,7 @@ fn main() -> Result<()> { let runtime_config = runtime_source.load()?; let upstream_grants = { let _runtime = rt.enter(); - Arc::new(upstream_grant::UpstreamGrantClient::start(upstream_socket)?) + Arc::new(UpstreamGrantClient::start(upstream_socket)?) }; info!(id = %args.id, "capsem-sandbox-process starting"); @@ -618,7 +617,7 @@ async fn run_async_main_loop( shutdown: Arc>, runtime_source: runtime_config::RuntimePolicySource, runtime_config: runtime_config::RuntimePolicyConfig, - upstream_grants: Arc, + upstream_grants: Arc, ) -> Result<()> { let terminal_output = Arc::new(capsem_core::TerminalOutputQueue::new()); diff --git a/crates/capsem-process/src/upstream_grant.rs b/crates/capsem-process/src/upstream_grant.rs deleted file mode 100644 index 91dcd6153..000000000 --- a/crates/capsem-process/src/upstream_grant.rs +++ /dev/null @@ -1,654 +0,0 @@ -//! Process-side client for coordinator-minted upstream descriptors. - -use std::io; -use std::os::fd::{AsFd as _, AsRawFd as _, BorrowedFd, OwnedFd}; -use std::os::unix::net::UnixStream; -use std::time::Duration; - -use anyhow::{anyhow, Result}; -use capsem_core::net::dns::{DnsDatagram, DnsGrantFuture, DnsUpstreamGrants}; -use capsem_core::net::mitm_proxy::{ - protocol::Protocol, GrantedTcpStream, TcpConnectGrantFuture, TcpGrantSelection, TcpResolveGrantFuture, - TcpUpstreamGrants, -}; -use capsem_foundation::unix::fd::{self, SocketShutdown}; -use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; -use capsem_proto::ledger::LedgerChannelGrant; -use capsem_proto::upstream_grant::{ - decode_upstream_grant_response, encode_upstream_grant_request, ProxyTrafficService, UpstreamDescriptorKind, - UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, -}; -use tokio::sync::{mpsc, oneshot}; - -const COMMAND_CAPACITY: usize = 64; -// The coordinator permits a TCP dial to take ten seconds. Leave room for its -// response framing while still bounding a wedged generation channel. -const WIRE_TIMEOUT: Duration = Duration::from_secs(12); - -type WireSender = DescriptorSender; -type WireReceiver = DescriptorReceiver; - -enum Command { - #[cfg_attr( - not(test), - expect(dead_code, reason = "guest dispatch uses this after proxy traffic handlers land") - )] - AttachProxyTraffic { - service: ProxyTrafficService, - descriptor: OwnedFd, - reply: oneshot::Sender>, - }, - OpenLedger { - reply: oneshot::Sender>, - }, - SetGuestMode { - relative_path: Vec, - mode: u16, - reply: oneshot::Sender>, - }, - OpenDns { - upstream_index: u16, - policy_digest: String, - reply: oneshot::Sender>, - }, - ResolveTcp { - protocol: Protocol, - host: String, - port: u16, - policy_digest: String, - reply: oneshot::Sender>, - }, - ConnectTcp { - selection_id: u64, - policy_digest: String, - reply: oneshot::Sender>, - }, -} - -/// Serializes grant protocol traffic over the inherited generation channel. -#[derive(Clone)] -pub(crate) struct UpstreamGrantClient { - commands: mpsc::Sender, -} - -impl UpstreamGrantClient { - pub(crate) fn start(socket: UnixStream) -> io::Result { - let sender = WireSender::new(socket.try_clone()?)?; - let receiver = WireReceiver::new(socket.try_clone()?)?; - let (commands_tx, commands) = mpsc::channel(COMMAND_CAPACITY); - let (actor_releases, releases) = mpsc::unbounded_channel(); - tokio::spawn(async move { - let result = run_actor(sender, receiver, commands, releases, actor_releases).await; - if let Err(error) = result { - tracing::warn!(%error, "upstream grant channel stopped"); - } - if let Err(error) = fd::shutdown(socket.as_fd(), SocketShutdown::Both) { - tracing::debug!(%error, "failed to shut down upstream grant channel"); - } - }); - Ok(Self { commands: commands_tx }) - } - - pub(crate) async fn open_ledger(&self) -> Result<(UnixStream, LedgerChannelGrant)> { - let (reply, result) = oneshot::channel(); - self.commands - .send(Command::OpenLedger { reply }) - .await - .map_err(|_| anyhow!("upstream grant channel is closed"))?; - result - .await - .map_err(|_| anyhow!("upstream grant channel stopped"))? - .map_err(anyhow::Error::msg) - } - - #[cfg_attr( - not(test), - expect(dead_code, reason = "guest dispatch uses this after proxy traffic handlers land") - )] - pub(crate) async fn attach_proxy_traffic(&self, service: ProxyTrafficService, descriptor: OwnedFd) -> Result<()> { - let (reply, result) = oneshot::channel(); - self.commands - .send(Command::AttachProxyTraffic { - service, - descriptor, - reply, - }) - .await - .map_err(|_| anyhow!("upstream grant channel is closed"))?; - result - .await - .map_err(|_| anyhow!("upstream grant channel stopped"))? - .map_err(anyhow::Error::msg) - } -} - -impl capsem_core::GuestMetadataAuthority for UpstreamGrantClient { - fn set_mode(&self, relative_path: &[u8], mode: u16) -> io::Result<()> { - let (reply, result) = oneshot::channel(); - self.commands - .blocking_send(Command::SetGuestMode { - relative_path: relative_path.to_vec(), - mode, - reply, - }) - .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; - result - .blocking_recv() - .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? - .map_err(io::Error::other) - } -} - -impl DnsUpstreamGrants for UpstreamGrantClient { - fn open(&self, upstream_index: u16, policy_digest: &str) -> DnsGrantFuture<'_> { - let policy_digest = policy_digest.to_owned(); - Box::pin(async move { - let (reply, result) = oneshot::channel(); - self.commands - .send(Command::OpenDns { - upstream_index, - policy_digest, - reply, - }) - .await - .map_err(|_| anyhow!("upstream grant channel is closed"))?; - result - .await - .map_err(|_| anyhow!("upstream grant channel stopped"))? - .map_err(anyhow::Error::msg) - }) - } -} - -impl TcpUpstreamGrants for UpstreamGrantClient { - fn resolve(&self, protocol: Protocol, host: &str, port: u16, policy_digest: &str) -> TcpResolveGrantFuture<'_> { - let host = host.to_owned(); - let policy_digest = policy_digest.to_owned(); - Box::pin(async move { - let (reply, result) = oneshot::channel(); - self.commands - .send(Command::ResolveTcp { - protocol, - host, - port, - policy_digest, - reply, - }) - .await - .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; - result - .await - .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? - .map_err(io::Error::other) - }) - } - - fn connect(&self, selection_id: u64, policy_digest: &str) -> TcpConnectGrantFuture<'_> { - let policy_digest = policy_digest.to_owned(); - Box::pin(async move { - let (reply, result) = oneshot::channel(); - self.commands - .send(Command::ConnectTcp { - selection_id, - policy_digest, - reply, - }) - .await - .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; - result - .await - .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? - .map_err(io::Error::other) - }) - } -} - -async fn run_actor( - sender: WireSender, - receiver: WireReceiver, - mut commands: mpsc::Receiver, - mut releases: mpsc::UnboundedReceiver, - release_sender: mpsc::UnboundedSender, -) -> Result<(), String> { - let mut next_request_id = 1_u64; - loop { - tokio::select! { - biased; - release = releases.recv() => { - let Some(resource_id) = release else { - if commands.is_closed() { - return Ok(()); - } - continue; - }; - send_request(&sender, &UpstreamGrantRequest::Release { resource_id }).await?; - } - command = commands.recv() => { - let Some(command) = command else { - return Ok(()); - }; - let request_id = next_request_id; - next_request_id = next_request_id.checked_add(1).ok_or("upstream request id exhausted")?; - dispatch(&sender, &receiver, &release_sender, request_id, command).await?; - } - } - } -} - -async fn dispatch( - sender: &WireSender, - receiver: &WireReceiver, - releases: &mpsc::UnboundedSender, - request_id: u64, - command: Command, -) -> Result<(), String> { - match command { - Command::AttachProxyTraffic { - service, - descriptor, - reply, - } => finish( - reply, - attach_proxy_traffic(sender, receiver, request_id, service, descriptor).await, - ), - Command::OpenLedger { reply } => finish(reply, open_ledger(sender, receiver, request_id).await), - Command::SetGuestMode { - relative_path, - mode, - reply, - } => finish( - reply, - set_guest_mode(sender, receiver, request_id, relative_path, mode).await, - ), - Command::OpenDns { - upstream_index, - policy_digest, - reply, - } => finish( - reply, - open_dns(sender, receiver, releases, request_id, upstream_index, &policy_digest).await, - ), - Command::ResolveTcp { - protocol, - host, - port, - policy_digest, - reply, - } => finish( - reply, - resolve_tcp( - sender, - receiver, - releases, - request_id, - protocol, - &host, - port, - &policy_digest, - ) - .await, - ), - Command::ConnectTcp { - selection_id, - policy_digest, - reply, - } => finish( - reply, - connect_tcp(sender, receiver, releases, request_id, selection_id, &policy_digest).await, - ), - } -} - -async fn attach_proxy_traffic( - sender: &WireSender, - receiver: &WireReceiver, - request_id: u64, - service: ProxyTrafficService, - descriptor: OwnedFd, -) -> Result, String> { - let request = UpstreamGrantRequest::AttachProxyTraffic { request_id, service }; - let bytes = - encode_upstream_grant_request(&request).map_err(|error| format!("encode proxy traffic handoff: {error:#}"))?; - tokio::time::timeout(WIRE_TIMEOUT, sender.send(&bytes, &[descriptor.as_raw_fd()])) - .await - .map_err(|_| "send proxy traffic handoff timed out".to_string())? - .map_err(|error| format!("send proxy traffic handoff: {error}"))?; - drop(descriptor); - let (response, fds) = receive_response(receiver).await?; - debug_assert!(fds.is_empty()); - match response { - UpstreamGrantResponse::ProxyTrafficAdopted { - request_id: response_id, - } if response_id == request_id => Ok(Ok(())), - UpstreamGrantResponse::Denied { - request_id: response_id, - reason, - } if response_id == request_id => Ok(Err(format!("proxy traffic handoff denied: {reason:?}"))), - response => Err(format!( - "unexpected proxy traffic response for request {request_id}: {response:?}" - )), - } -} - -async fn open_ledger( - sender: &WireSender, - receiver: &WireReceiver, - request_id: u64, -) -> Result, String> { - send_request(sender, &UpstreamGrantRequest::OpenLedger { request_id }).await?; - let (response, mut fds) = receive_response(receiver).await?; - match response { - UpstreamGrantResponse::Denied { - request_id: response_id, - reason, - } if response_id == request_id => Ok(Err(format!("session ledger grant denied: {reason:?}"))), - UpstreamGrantResponse::LedgerGranted { - request_id: response_id, - grant, - } if response_id == request_id => { - let descriptor = fds.pop().ok_or("ledger grant omitted its descriptor")?; - send_request( - sender, - &UpstreamGrantRequest::Adopted { - grant_id: grant.client_id(), - }, - ) - .await?; - let stream = UnixStream::from(descriptor); - stream - .peer_addr() - .map_err(|error| format!("adopt granted ledger channel: {error}"))?; - Ok(Ok((stream, grant))) - } - response => Err(format!( - "unexpected session ledger response for request {request_id}: {response:?}" - )), - } -} - -async fn set_guest_mode( - sender: &WireSender, - receiver: &WireReceiver, - request_id: u64, - relative_path: Vec, - mode: u16, -) -> Result, String> { - send_request( - sender, - &UpstreamGrantRequest::SetGuestMode { - request_id, - relative_path, - mode, - }, - ) - .await?; - let (response, fds) = receive_response(receiver).await?; - if !fds.is_empty() { - return Err("guest mode response carried a descriptor".into()); - } - match response { - UpstreamGrantResponse::GuestModeSet { - request_id: response_id, - } if response_id == request_id => Ok(Ok(())), - UpstreamGrantResponse::Denied { - request_id: response_id, - reason, - } if response_id == request_id => Ok(Err(format!("guest mode change denied: {reason:?}"))), - response => Err(format!( - "unexpected guest mode response for request {request_id}: {response:?}" - )), - } -} - -fn finish( - reply: oneshot::Sender>, - result: Result, String>, -) -> Result<(), String> { - match result { - Ok(result) => { - let _ = reply.send(result); - Ok(()) - } - Err(error) => { - let _ = reply.send(Err(error.clone())); - Err(error) - } - } -} - -async fn open_dns( - sender: &WireSender, - receiver: &WireReceiver, - releases: &mpsc::UnboundedSender, - request_id: u64, - upstream_index: u16, - expected_policy_digest: &str, -) -> Result, String> { - send_request( - sender, - &UpstreamGrantRequest::OpenDns { - request_id, - upstream_index, - }, - ) - .await?; - let (response, fds) = receive_response(receiver).await?; - match response { - UpstreamGrantResponse::Denied { - request_id: response_id, - reason, - } if response_id == request_id => Ok(Err(format!("upstream DNS grant denied: {reason:?}"))), - UpstreamGrantResponse::DescriptorGranted { - request_id: response_id, - grant_id, - kind: UpstreamDescriptorKind::DnsUdp, - policy_digest: response_digest, - } if response_id == request_id => { - let descriptor = fds.into_iter().next().ok_or("DNS grant omitted its descriptor")?; - send_request(sender, &UpstreamGrantRequest::Adopted { grant_id }).await?; - if response_digest != expected_policy_digest { - send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; - return Ok(Err(format!( - "upstream DNS grant policy mismatch: expected {expected_policy_digest}, received {response_digest}" - ))); - } - let socket = datagram_from_descriptor(descriptor)?; - let releases = releases.clone(); - Ok(Ok(DnsDatagram::new(socket, move || { - let _ = releases.send(grant_id); - }))) - } - response => Err(format!( - "unexpected upstream DNS grant response for request {request_id}: {response:?}" - )), - } -} - -#[allow(clippy::too_many_arguments)] -async fn resolve_tcp( - sender: &WireSender, - receiver: &WireReceiver, - releases: &mpsc::UnboundedSender, - request_id: u64, - protocol: Protocol, - host: &str, - port: u16, - expected_policy_digest: &str, -) -> Result, String> { - let protocol = match wire_protocol(protocol) { - Ok(protocol) => protocol, - Err(error) => return Ok(Err(error)), - }; - send_request( - sender, - &UpstreamGrantRequest::ResolveTcp { - request_id, - protocol, - host: host.to_owned(), - port, - }, - ) - .await?; - let (response, _fds) = receive_response(receiver).await?; - match response { - UpstreamGrantResponse::Denied { - request_id: response_id, - reason, - } if response_id == request_id => Ok(Err(format!("upstream TCP selection denied: {reason:?}"))), - UpstreamGrantResponse::TcpResolved { - request_id: response_id, - selection_id, - protocol, - judged_ip, - policy_digest, - } if response_id == request_id => { - if policy_digest != expected_policy_digest { - send_request( - sender, - &UpstreamGrantRequest::Release { - resource_id: selection_id, - }, - ) - .await?; - return Ok(Err(format!( - "upstream TCP selection policy mismatch: expected {expected_policy_digest}, received {policy_digest}" - ))); - } - let releases = releases.clone(); - Ok(Ok(TcpGrantSelection::new( - selection_id, - core_protocol(protocol), - judged_ip, - move || { - let _ = releases.send(selection_id); - }, - ))) - } - response => Err(format!( - "unexpected upstream TCP selection response for request {request_id}: {response:?}" - )), - } -} - -async fn connect_tcp( - sender: &WireSender, - receiver: &WireReceiver, - releases: &mpsc::UnboundedSender, - request_id: u64, - selection_id: u64, - expected_policy_digest: &str, -) -> Result, String> { - send_request( - sender, - &UpstreamGrantRequest::ConnectTcp { - request_id, - selection_id, - }, - ) - .await?; - let (response, fds) = receive_response(receiver).await?; - match response { - UpstreamGrantResponse::Denied { - request_id: response_id, - reason, - } if response_id == request_id => Ok(Err(format!("upstream TCP connection denied: {reason:?}"))), - UpstreamGrantResponse::DescriptorGranted { - request_id: response_id, - grant_id, - kind: UpstreamDescriptorKind::Tcp, - policy_digest, - } if response_id == request_id => { - let descriptor = fds.into_iter().next().ok_or("TCP grant omitted its descriptor")?; - send_request(sender, &UpstreamGrantRequest::Adopted { grant_id }).await?; - if policy_digest != expected_policy_digest { - send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; - return Ok(Err(format!( - "upstream TCP grant policy mismatch: expected {expected_policy_digest}, received {policy_digest}" - ))); - } - let stream = stream_from_descriptor(descriptor)?; - let releases = releases.clone(); - Ok(Ok(GrantedTcpStream::new(stream, move || { - let _ = releases.send(grant_id); - }))) - } - response => Err(format!( - "unexpected upstream TCP grant response for request {request_id}: {response:?}" - )), - } -} - -async fn receive_response(receiver: &WireReceiver) -> Result<(UpstreamGrantResponse, Vec), String> { - let frame = tokio::time::timeout(WIRE_TIMEOUT, receiver.recv()) - .await - .map_err(|_| "receive upstream grant response timed out".to_string())? - .map_err(|error| format!("receive upstream grant response: {error}"))?; - let response = decode_upstream_grant_response(&frame.bytes) - .map_err(|error| format!("decode upstream grant response: {error:#}"))?; - if frame.fds.len() != response.expected_descriptor_count() { - return Err(format!( - "upstream grant response carried {} descriptors, expected {}", - frame.fds.len(), - response.expected_descriptor_count() - )); - } - Ok((response, frame.fds)) -} - -fn wire_protocol(protocol: Protocol) -> Result { - match protocol { - Protocol::Http => Ok(UpstreamProtocol::Http), - Protocol::Tls => Ok(UpstreamProtocol::Tls), - Protocol::McpFrame | Protocol::Unknown => { - Err(format!("protocol {} cannot select a TCP upstream", protocol.label())) - } - } -} - -fn core_protocol(protocol: UpstreamProtocol) -> Protocol { - match protocol { - UpstreamProtocol::Http => Protocol::Http, - UpstreamProtocol::Tls => Protocol::Tls, - } -} - -async fn send_request(sender: &WireSender, request: &UpstreamGrantRequest) -> Result<(), String> { - let bytes = - encode_upstream_grant_request(request).map_err(|error| format!("encode upstream grant request: {error:#}"))?; - tokio::time::timeout(WIRE_TIMEOUT, sender.send(&bytes, &[])) - .await - .map_err(|_| "send upstream grant request timed out".to_string())? - .map_err(|error| format!("send upstream grant request: {error}"))?; - Ok(()) -} - -fn datagram_from_descriptor(descriptor: OwnedFd) -> Result { - let socket = std::net::UdpSocket::from(descriptor); - socket - .set_nonblocking(true) - .map_err(|error| format!("make granted DNS socket nonblocking: {error}"))?; - tokio::net::UdpSocket::from_std(socket).map_err(|error| format!("adopt granted DNS socket: {error}")) -} - -fn stream_from_descriptor(descriptor: OwnedFd) -> Result { - let stream = std::net::TcpStream::from(descriptor); - stream - .set_nonblocking(true) - .map_err(|error| format!("make granted TCP stream nonblocking: {error}"))?; - stream - .set_nodelay(true) - .map_err(|error| format!("set granted TCP stream nodelay: {error}"))?; - tokio::net::TcpStream::from_std(stream).map_err(|error| format!("adopt granted TCP stream: {error}")) -} - -/// Duplicate the inherited generation channel before runtime threads exist. -pub(crate) fn adopt_inherited(descriptor: BorrowedFd<'_>) -> io::Result { - fd::set_close_on_exec(descriptor)?; - let owned = fd::duplicate(descriptor)?; - let socket = UnixStream::from(owned); - socket.peer_addr()?; - Ok(socket) -} - -#[cfg(test)] -mod tests; diff --git a/crates/capsem-process/src/upstream_grant/tests.rs b/crates/capsem-process/src/upstream_grant/tests.rs deleted file mode 100644 index ae17e19bf..000000000 --- a/crates/capsem-process/src/upstream_grant/tests.rs +++ /dev/null @@ -1,485 +0,0 @@ -use std::io::{Read as _, Write as _}; -use std::os::fd::{AsFd as _, AsRawFd as _, RawFd}; -use std::sync::Arc; - -use capsem_core::net::dns::{DnsResolver, DnsUpstreamGrants}; -use capsem_core::net::mitm_proxy::{TcpUpstreamGrants, UpstreamTarget}; -use capsem_core::GuestMetadataAuthority as _; -use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; -use capsem_proto::upstream_grant::{ - decode_upstream_grant_request, encode_upstream_grant_response, ProxyTrafficService, UpstreamDescriptorKind, - UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, -}; - -use super::*; -use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; - -const POLICY_A: &str = "blake3:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; -const POLICY_B: &str = "blake3:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; - -#[tokio::test] -async fn ledger_channel_preserves_grant_and_is_adopted_before_use() { - let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x6c; 16]), 44, LedgerClientRole::VmOwner).unwrap(); - let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker).unwrap(); - let (requests, responses) = channels(broker); - let (ledger, mut ledger_peer) = UnixStream::pair().unwrap(); - let broker_task = tokio::spawn(async move { - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::OpenLedger { request_id: 1 } - ); - send_response( - &responses, - &UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }, - Some(ledger.as_raw_fd()), - ) - .await; - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::Adopted { - grant_id: grant.client_id(), - } - ); - }); - - let (mut stream, received) = client.open_ledger().await.unwrap(); - assert_eq!(received, grant); - stream.write_all(b"hello").unwrap(); - let mut message = [0_u8; 5]; - ledger_peer.read_exact(&mut message).unwrap(); - assert_eq!(&message, b"hello"); - broker_task.await.unwrap(); -} - -#[tokio::test] -async fn proxy_traffic_descriptor_is_surrendered_before_adoption_returns() { - let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker).unwrap(); - let (requests, responses) = channels(broker); - let (traffic, mut guest) = UnixStream::pair().unwrap(); - let broker_task = tokio::spawn(async move { - let frame = requests.recv().await.unwrap(); - assert_eq!(frame.fds.len(), 1); - assert_eq!( - decode_upstream_grant_request(&frame.bytes).unwrap(), - UpstreamGrantRequest::AttachProxyTraffic { - request_id: 1, - service: ProxyTrafficService::Dns, - } - ); - let mut adopted = UnixStream::from(frame.fds.into_iter().next().unwrap()); - adopted.write_all(b"owned").unwrap(); - send_response( - &responses, - &UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 1 }, - None, - ) - .await; - }); - - client - .attach_proxy_traffic(ProxyTrafficService::Dns, traffic.into()) - .await - .unwrap(); - let mut proof = [0; 5]; - guest.read_exact(&mut proof).unwrap(); - assert_eq!(&proof, b"owned"); - broker_task.await.unwrap(); -} - -#[tokio::test] -async fn ledger_denial_and_missing_descriptor_fail_closed() { - let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker).unwrap(); - let (requests, responses) = channels(broker); - let denied = tokio::spawn(async move { - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::OpenLedger { request_id: 1 } - ); - send_response( - &responses, - &UpstreamGrantResponse::Denied { - request_id: 1, - reason: UpstreamGrantDenial::NotConfigured, - }, - None, - ) - .await; - }); - let error = client.open_ledger().await.unwrap_err().to_string(); - assert!(error.contains("NotConfigured"), "{error}"); - denied.await.unwrap(); - - let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x7d; 16]), 45, LedgerClientRole::VmOwner).unwrap(); - let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker).unwrap(); - let (requests, responses) = channels(broker); - let malformed = tokio::spawn(async move { - assert!(matches!( - receive_request(&requests).await, - UpstreamGrantRequest::OpenLedger { request_id: 1 } - )); - send_response( - &responses, - &UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }, - None, - ) - .await; - }); - let error = client.open_ledger().await.unwrap_err().to_string(); - assert!(error.contains("carried 0 descriptors"), "{error}"); - malformed.await.unwrap(); -} - -async fn open_error(client: &UpstreamGrantClient) -> String { - match client.open(0, POLICY_A).await { - Ok(_) => panic!("unexpected DNS grant"), - Err(error) => error.to_string(), - } -} - -fn channels(socket: UnixStream) -> (WireReceiver, WireSender) { - let receiver = WireReceiver::new(socket.try_clone().unwrap()).unwrap(); - let sender = WireSender::new(socket).unwrap(); - (receiver, sender) -} - -async fn receive_request(receiver: &WireReceiver) -> UpstreamGrantRequest { - let frame = receiver.recv().await.unwrap(); - assert!(frame.fds.is_empty()); - decode_upstream_grant_request(&frame.bytes).unwrap() -} - -async fn send_response(sender: &WireSender, response: &UpstreamGrantResponse, descriptor: Option) { - let bytes = encode_upstream_grant_response(response).unwrap(); - let fds = descriptor.map_or_else(Vec::new, |descriptor| vec![descriptor]); - sender.send(&bytes, &fds).await.unwrap(); -} - -fn query() -> Vec { - let mut query = vec![0x12, 0x34, 0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00]; - query.extend_from_slice(&[7, b'e', b'x', b'a', b'm', b'p', b'l', b'e', 3, b'c', b'o', b'm', 0]); - query.extend_from_slice(&[0, 1, 0, 1]); - query -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn guest_mode_changes_use_the_serialized_coordinator_channel() { - let (broker, worker) = UnixStream::pair().unwrap(); - let client = Arc::new(UpstreamGrantClient::start(worker).unwrap()); - let (requests, responses) = channels(broker); - let broker_task = tokio::spawn(async move { - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::SetGuestMode { - request_id: 1, - relative_path: b"workspace/pyvenv/bin/activate".to_vec(), - mode: 0o755, - } - ); - send_response(&responses, &UpstreamGrantResponse::GuestModeSet { request_id: 1 }, None).await; - }); - let client_for_mode = Arc::clone(&client); - tokio::task::spawn_blocking(move || { - client_for_mode - .set_mode(b"workspace/pyvenv/bin/activate", 0o755) - .unwrap(); - }) - .await - .unwrap(); - broker_task.await.unwrap(); -} - -#[tokio::test] -async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { - let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); - let upstream_address = upstream.local_addr().unwrap(); - let (broker, worker) = UnixStream::pair().unwrap(); - let client = Arc::new(UpstreamGrantClient::start(worker).unwrap()); - let (requests, responses) = channels(broker); - let expected_query = query(); - let broker_task = tokio::spawn(async move { - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::OpenDns { - request_id: 1, - upstream_index: 0, - } - ); - let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); - socket.connect(upstream_address).unwrap(); - send_response( - &responses, - &UpstreamGrantResponse::DescriptorGranted { - request_id: 1, - grant_id: 41, - kind: UpstreamDescriptorKind::DnsUdp, - policy_digest: POLICY_A.into(), - }, - Some(socket.as_raw_fd()), - ) - .await; - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::Adopted { grant_id: 41 } - ); - let mut received = [0_u8; 512]; - let (count, peer) = upstream.recv_from(&mut received).await.unwrap(); - assert_eq!(&received[..count], expected_query); - let mut answer = received[..count].to_vec(); - answer[2] = 0x81; - answer[3] = 0x80; - upstream.send_to(&answer, peer).await.unwrap(); - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::Release { resource_id: 41 } - ); - }); - - let resolver = DnsResolver::with_grants(vec![upstream_address], client); - let (answer, _) = resolver.resolve_for_policy(&query(), POLICY_A).await.unwrap(); - assert_eq!(&answer[..2], &[0x12, 0x34]); - assert_eq!(&answer[2..4], &[0x81, 0x80]); - broker_task.await.unwrap(); -} - -#[tokio::test] -async fn stale_policy_grant_is_adopted_released_and_channel_remains_usable() { - let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker).unwrap(); - let (requests, responses) = channels(broker); - let broker_task = tokio::spawn(async move { - assert!(matches!( - receive_request(&requests).await, - UpstreamGrantRequest::OpenDns { request_id: 1, .. } - )); - let peer = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); - let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); - socket.connect(peer.local_addr().unwrap()).unwrap(); - send_response( - &responses, - &UpstreamGrantResponse::DescriptorGranted { - request_id: 1, - grant_id: 51, - kind: UpstreamDescriptorKind::DnsUdp, - policy_digest: POLICY_B.into(), - }, - Some(socket.as_raw_fd()), - ) - .await; - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::Adopted { grant_id: 51 } - ); - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::Release { resource_id: 51 } - ); - assert!(matches!( - receive_request(&requests).await, - UpstreamGrantRequest::OpenDns { request_id: 2, .. } - )); - send_response( - &responses, - &UpstreamGrantResponse::Denied { - request_id: 2, - reason: UpstreamGrantDenial::NotConfigured, - }, - None, - ) - .await; - }); - - let error = open_error(&client).await; - assert!(error.contains("policy mismatch"), "{error}"); - let error = open_error(&client).await; - assert!(error.contains("NotConfigured"), "{error}"); - broker_task.await.unwrap(); -} - -#[tokio::test] -async fn malformed_or_revoked_grant_channel_fails_closed() { - let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker).unwrap(); - let (requests, responses) = channels(broker); - let broker_task = tokio::spawn(async move { - assert!(matches!( - receive_request(&requests).await, - UpstreamGrantRequest::OpenDns { request_id: 1, .. } - )); - send_response( - &responses, - &UpstreamGrantResponse::DescriptorGranted { - request_id: 1, - grant_id: 61, - kind: UpstreamDescriptorKind::DnsUdp, - policy_digest: POLICY_A.into(), - }, - None, - ) - .await; - }); - let error = open_error(&client).await; - assert!(error.contains("carried 0 descriptors"), "{error}"); - broker_task.await.unwrap(); - let error = open_error(&client).await; - assert!(error.contains("closed") || error.contains("stopped"), "{error}"); - - let (broker, worker) = UnixStream::pair().unwrap(); - let revoked = UpstreamGrantClient::start(worker).unwrap(); - drop(broker); - assert!(!open_error(&revoked).await.is_empty()); -} - -#[tokio::test] -async fn tcp_selection_and_connected_stream_are_brokered_and_released() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let upstream_address = listener.local_addr().unwrap(); - let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker).unwrap(); - let (requests, responses) = channels(broker); - let broker_task = tokio::spawn(async move { - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::ResolveTcp { - request_id: 1, - protocol: UpstreamProtocol::Tls, - host: "api.example".into(), - port: 443, - } - ); - send_response( - &responses, - &UpstreamGrantResponse::TcpResolved { - request_id: 1, - selection_id: 71, - protocol: UpstreamProtocol::Http, - judged_ip: Some("127.0.0.1".parse().unwrap()), - policy_digest: POLICY_A.into(), - }, - None, - ) - .await; - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::ConnectTcp { - request_id: 2, - selection_id: 71, - } - ); - let socket = tokio::net::TcpStream::connect(upstream_address) - .await - .unwrap() - .into_std() - .unwrap(); - let (mut upstream, _) = listener.accept().await.unwrap(); - send_response( - &responses, - &UpstreamGrantResponse::DescriptorGranted { - request_id: 2, - grant_id: 72, - kind: UpstreamDescriptorKind::Tcp, - policy_digest: POLICY_A.into(), - }, - Some(socket.as_raw_fd()), - ) - .await; - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::Adopted { grant_id: 72 } - ); - let mut request = [0_u8; 4]; - upstream.read_exact(&mut request).await.unwrap(); - assert_eq!(&request, b"ping"); - upstream.write_all(b"pong").await.unwrap(); - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::Release { resource_id: 72 } - ); - }); - - let selection = client - .resolve(Protocol::Tls, "api.example", 443, POLICY_A) - .await - .unwrap(); - assert_eq!(selection.protocol, Protocol::Http); - assert_eq!(selection.judged_ip, Some("127.0.0.1".parse().unwrap())); - let target = UpstreamTarget::Granted { - host: "api.example".into(), - port: 443, - guest_protocol: Protocol::Tls, - protocol: selection.protocol, - judged_ip: selection.judged_ip, - selection: Some(selection), - }; - let (mut stream, pinned) = target.connect_with_grants(Some(&client), POLICY_A).await.unwrap(); - assert!(matches!(pinned, UpstreamTarget::Granted { selection: None, .. })); - stream.write_all(b"ping").await.unwrap(); - let mut response = [0_u8; 4]; - stream.read_exact(&mut response).await.unwrap(); - assert_eq!(&response, b"pong"); - drop(stream); - broker_task.await.unwrap(); -} - -#[tokio::test] -async fn unused_or_stale_tcp_selections_are_released_without_connecting() { - let (broker, worker) = UnixStream::pair().unwrap(); - let client = UpstreamGrantClient::start(worker).unwrap(); - let (requests, responses) = channels(broker); - let broker_task = tokio::spawn(async move { - for (request_id, selection_id, digest) in [(1, 81, POLICY_A), (2, 82, POLICY_B)] { - assert!(matches!( - receive_request(&requests).await, - UpstreamGrantRequest::ResolveTcp { - request_id: actual, - .. - } if actual == request_id - )); - send_response( - &responses, - &UpstreamGrantResponse::TcpResolved { - request_id, - selection_id, - protocol: UpstreamProtocol::Tls, - judged_ip: Some("93.184.216.34".parse().unwrap()), - policy_digest: digest.into(), - }, - None, - ) - .await; - assert_eq!( - receive_request(&requests).await, - UpstreamGrantRequest::Release { - resource_id: selection_id, - } - ); - } - }); - - let unused = client - .resolve(Protocol::Tls, "unused.example", 443, POLICY_A) - .await - .unwrap(); - drop(unused); - let error = client - .resolve(Protocol::Tls, "stale.example", 443, POLICY_A) - .await - .unwrap_err(); - assert!(error.to_string().contains("policy mismatch"), "{error}"); - broker_task.await.unwrap(); -} - -#[test] -fn inherited_channel_is_duplicated_cloexec_and_connected() { - let (inherited, peer) = UnixStream::pair().unwrap(); - let mut adopted = adopt_inherited(inherited.as_fd()).unwrap(); - let mut peer = peer; - assert!(fd::set_close_on_exec(inherited.as_fd()).unwrap()); - adopted.write_all(b"x").unwrap(); - let mut byte = [0_u8; 1]; - peer.read_exact(&mut byte).unwrap(); - assert_eq!(byte, [b'x']); -} From 8725f56a2a448669c00b458bfd4124ad5e7e370f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:53:06 +0000 Subject: [PATCH 141/464] feat(proxy): adopt upstream authority --- crates/capsem-proxy/src/main.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index f295d6803..b5f3f9220 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -225,6 +225,23 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result let reason = closed.borrow().unwrap_or(ProxyChannelCloseReason::Disconnected); (capability, grant_id, reason) }); + } else if capability == ProxyCapability::Upstream { + let grants = Arc::new( + capsem_core::net::upstream_grant::UpstreamGrantClient::start(stream) + .context("open proxy upstream capability")?, + ); + let mut stopped = grants.stop_receiver(); + let mut runtime = state.lock().unwrap_or_else(|error| error.into_inner()); + runtime.upstream_grants = Some(grants); + tracing::debug!( + upstream_capability = runtime.upstream_grants.is_some(), + "proxy upstream authority attached" + ); + drop(runtime); + tasks.spawn(async move { + while stopped.borrow().is_none() && stopped.changed().await.is_ok() {} + (capability, grant_id, ProxyChannelCloseReason::Disconnected) + }); } else { descriptors.insert(grant_id, stream); } @@ -284,6 +301,7 @@ struct ProxyRuntimeState { engine: Option, ledger: Arc, credentials: Arc, + upstream_grants: Option>, dns_upstreams: Vec, mcp: capsem_core::mcp::policy::McpConfig, } @@ -294,6 +312,7 @@ impl Default for ProxyRuntimeState { engine: None, ledger: Arc::new(UnavailableLedger), credentials: Arc::new(UnavailableCredentials), + upstream_grants: None, dns_upstreams: Vec::new(), mcp: capsem_core::mcp::policy::McpConfig::default(), } From 11d12de40f769dacd4c461aa44e66345929efbbf Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:53:13 +0000 Subject: [PATCH 142/464] style(process): format upstream adoption --- crates/capsem-process/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index a3e592b41..37986cd4c 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -408,7 +408,8 @@ fn main() -> Result<()> { // SAFETY: process entry precedes argument parsing, telemetry, descriptor // owners and runtime threads. Broker grants will be named here explicitly. unsafe { capsem_foundation::unix::fd::close_inherited_descriptors()? }; - let upstream_socket = adopt_inherited(std::io::stdin().as_fd()).context("adopt inherited upstream grant channel")?; + let upstream_socket = + adopt_inherited(std::io::stdin().as_fd()).context("adopt inherited upstream grant channel")?; let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { service: "capsem-process", sink: capsem_foundation::telemetry::LogSink::Stderr, From 762abff24fa36dc0331d924b9b95e4a1d2781f52 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:54:35 +0000 Subject: [PATCH 143/464] feat(proto): define private-name capability --- crates/capsem-proto/src/lib.rs | 1 + .../capsem-proto/src/proxy_private_names.rs | 121 ++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 crates/capsem-proto/src/proxy_private_names.rs diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 8b2b87bfa..1d2d9338f 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -29,6 +29,7 @@ pub mod privatelink; pub mod proxy_control; pub mod proxy_credentials; pub mod proxy_policy; +pub mod proxy_private_names; pub mod repeated; pub mod router; mod sparse; diff --git a/crates/capsem-proto/src/proxy_private_names.rs b/crates/capsem-proto/src/proxy_private_names.rs new file mode 100644 index 000000000..6413d06b4 --- /dev/null +++ b/crates/capsem-proto/src/proxy_private_names.rs @@ -0,0 +1,121 @@ +use std::net::Ipv4Addr; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub const MAX_PRIVATE_NAME_BYTES: usize = 253; +pub const MAX_PRIVATE_NAME_ERROR_BYTES: usize = 4 * 1024; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyPrivateNameRequest { + AddressOf { request_id: u64, name: String }, + NameOf { request_id: u64, address: Ipv4Addr }, +} + +impl ProxyPrivateNameRequest { + pub const fn request_id(&self) -> u64 { + match self { + Self::AddressOf { request_id, .. } | Self::NameOf { request_id, .. } => *request_id, + } + } + + pub fn validate(&self) -> Result<(), ProxyPrivateNameProtocolError> { + if self.request_id() == 0 { + return Err(ProxyPrivateNameProtocolError::ZeroRequestId); + } + if let Self::AddressOf { name, .. } = self { + if name.is_empty() || name.len() > MAX_PRIVATE_NAME_BYTES { + return Err(ProxyPrivateNameProtocolError::InvalidName); + } + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyPrivateNameResponse { + Address { request_id: u64, address: Ipv4Addr }, + Name { request_id: u64, name: String }, + NotFound { request_id: u64 }, + Rejected { request_id: u64, error: String }, +} + +impl ProxyPrivateNameResponse { + pub const fn request_id(&self) -> u64 { + match self { + Self::Address { request_id, .. } + | Self::Name { request_id, .. } + | Self::NotFound { request_id } + | Self::Rejected { request_id, .. } => *request_id, + } + } + + pub fn rejected(request_id: u64, error: impl Into) -> Result { + if request_id == 0 { + return Err(ProxyPrivateNameProtocolError::ZeroRequestId); + } + let error = error.into(); + if error.len() > MAX_PRIVATE_NAME_ERROR_BYTES { + return Err(ProxyPrivateNameProtocolError::ErrorTooLong); + } + Ok(Self::Rejected { request_id, error }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] +pub enum ProxyPrivateNameProtocolError { + #[error("proxy private-name request id must not be zero")] + ZeroRequestId, + #[error("proxy private name is empty or exceeds its bound")] + InvalidName, + #[error("proxy private-name diagnostic exceeds its bound")] + ErrorTooLong, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn requests_and_correlated_answers_are_bounded() { + let address = ProxyPrivateNameRequest::AddressOf { + request_id: 1, + name: "box.dev.capsem.internal".to_string(), + }; + let name = ProxyPrivateNameRequest::NameOf { + request_id: 2, + address: Ipv4Addr::new(10, 0, 0, 2), + }; + address.validate().unwrap(); + name.validate().unwrap(); + assert_eq!(address.request_id(), 1); + assert_eq!(name.request_id(), 2); + assert_eq!( + ProxyPrivateNameResponse::Address { + request_id: 1, + address: Ipv4Addr::new(10, 0, 0, 2), + } + .request_id(), + 1 + ); + } + + #[test] + fn malformed_authority_and_oversized_values_fail_closed() { + assert!(ProxyPrivateNameRequest::AddressOf { + request_id: 0, + name: String::new(), + } + .validate() + .is_err()); + assert!(ProxyPrivateNameRequest::AddressOf { + request_id: 1, + name: "x".repeat(MAX_PRIVATE_NAME_BYTES + 1), + } + .validate() + .is_err()); + assert!(ProxyPrivateNameResponse::rejected(1, "x".repeat(MAX_PRIVATE_NAME_ERROR_BYTES + 1)).is_err()); + } +} From 4f05d24ba557771532b8b6b81df3eb3b481841a3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 06:58:23 +0000 Subject: [PATCH 144/464] feat(service): grant private-name capability --- crates/capsem-service/src/main.rs | 1 + crates/capsem-service/src/private_routes.rs | 13 ++- .../capsem-service/src/proxy_private_names.rs | 86 +++++++++++++++++++ .../src/proxy_private_names/tests.rs | 27 ++++++ crates/capsem-service/src/proxy_worker.rs | 26 ++++++ .../src/vm_lifecycle/provision.rs | 4 + .../src/vm_lifecycle/resume_process.rs | 4 + 7 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 crates/capsem-service/src/proxy_private_names.rs create mode 100644 crates/capsem-service/src/proxy_private_names/tests.rs diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index f6c8a2e7e..491937d70 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -49,6 +49,7 @@ mod plugin_routes; mod private_routes; mod process_control; mod proxy_credentials; +mod proxy_private_names; mod proxy_worker; mod sandbox_info; mod session_cleanup; diff --git a/crates/capsem-service/src/private_routes.rs b/crates/capsem-service/src/private_routes.rs index 67effbfcd..9d1c14890 100644 --- a/crates/capsem-service/src/private_routes.rs +++ b/crates/capsem-service/src/private_routes.rs @@ -74,7 +74,14 @@ pub(super) async fn handle_private_resolve( Json(request): Json, ) -> Result, AppError> { owner_of(&state, &request.source_vm, peer)?; - let visible = visible_to(&state, &request.source_vm).await; + resolve_private(&state, request).await.map(Json) +} + +pub(crate) async fn resolve_private( + state: &ServiceState, + request: PrivateResolveRequest, +) -> Result { + let visible = visible_to(state, &request.source_vm).await; let found = match (&request.name, request.address) { (Some(name), None) => { let name = name.trim_matches('.').to_ascii_lowercase(); @@ -102,10 +109,10 @@ pub(super) async fn handle_private_resolve( } }; let member = found.ok_or_else(|| AppError(StatusCode::NOT_FOUND, "no such member".into()))?; - Ok(Json(PrivateResolveResponse { + Ok(PrivateResolveResponse { name: member.full_name(), address: member.address, vm: member.vm_id.clone(), network: member.network_name.clone(), - })) + }) } diff --git a/crates/capsem-service/src/proxy_private_names.rs b/crates/capsem-service/src/proxy_private_names.rs new file mode 100644 index 000000000..38a52ff7b --- /dev/null +++ b/crates/capsem-service/src/proxy_private_names.rs @@ -0,0 +1,86 @@ +//! Session-bound private DNS lookups for one confined proxy worker. + +use std::io; +use std::os::unix::net::UnixStream; +use std::sync::Arc; + +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_private_names::{ + ProxyPrivateNameRequest, ProxyPrivateNameResponse, MAX_PRIVATE_NAME_ERROR_BYTES, +}; + +use crate::api::PrivateResolveRequest; +use crate::ServiceState; + +pub(crate) async fn serve(state: Arc, vm_id: String, stream: UnixStream) -> Result<(), String> { + let (sender, receiver) = ipc_channel::channel_from_std::(stream) + .map_err(|error| error.to_string())?; + loop { + let request = match receiver.recv().await { + Ok(request) => request, + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(()), + Err(error) => return Err(format!("receive proxy private-name request: {error}")), + }; + let request_id = request.request_id(); + let response = if let Err(error) = request.validate() { + rejected(request_id, error.to_string()) + } else { + let address_lookup = matches!(&request, ProxyPrivateNameRequest::AddressOf { .. }); + let query = match request { + ProxyPrivateNameRequest::AddressOf { name, .. } => PrivateResolveRequest { + source_vm: vm_id.clone(), + name: Some(name), + address: None, + }, + ProxyPrivateNameRequest::NameOf { address, .. } => PrivateResolveRequest { + source_vm: vm_id.clone(), + name: None, + address: Some(address), + }, + }; + match crate::private_routes::resolve_private(&state, query).await { + Ok(answer) => { + if address_lookup { + ProxyPrivateNameResponse::Address { + request_id, + address: answer.address, + } + } else { + ProxyPrivateNameResponse::Name { + request_id, + name: answer.name, + } + } + } + Err(error) if error.0 == axum::http::StatusCode::NOT_FOUND => { + ProxyPrivateNameResponse::NotFound { request_id } + } + Err(error) => rejected(request_id, error.1), + } + }; + sender + .send(response) + .await + .map_err(|error| format!("send proxy private-name response: {error}"))?; + } +} + +fn rejected(request_id: u64, error: String) -> ProxyPrivateNameResponse { + let error = truncate_utf8(error, MAX_PRIVATE_NAME_ERROR_BYTES); + ProxyPrivateNameResponse::rejected(request_id, error).expect("request id and diagnostic were bounded") +} + +fn truncate_utf8(mut value: String, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value; + } + let mut boundary = max_bytes; + while !value.is_char_boundary(boundary) { + boundary -= 1; + } + value.truncate(boundary); + value +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/proxy_private_names/tests.rs b/crates/capsem-service/src/proxy_private_names/tests.rs new file mode 100644 index 000000000..f4fd324d4 --- /dev/null +++ b/crates/capsem-service/src/proxy_private_names/tests.rs @@ -0,0 +1,27 @@ +use std::os::unix::net::UnixStream; + +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_private_names::{ProxyPrivateNameRequest, ProxyPrivateNameResponse}; + +#[tokio::test] +async fn session_bound_lookup_returns_not_found_without_visible_members() { + let state = crate::tests::make_test_state(); + let (service, client) = UnixStream::pair().unwrap(); + let serving = tokio::spawn(super::serve(state, "vm-a".to_string(), service)); + let (sender, receiver) = + ipc_channel::channel_from_std::(client).unwrap(); + sender + .send(ProxyPrivateNameRequest::AddressOf { + request_id: 8, + name: "missing.dev.capsem.internal".to_string(), + }) + .await + .unwrap(); + assert_eq!( + receiver.recv().await.unwrap(), + ProxyPrivateNameResponse::NotFound { request_id: 8 } + ); + drop(sender); + drop(receiver); + serving.await.unwrap().unwrap(); +} diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index 5fa1c29ca..c3edbbf7d 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -644,6 +644,32 @@ fn generation_hex(generation: ProxyGeneration) -> String { } impl ServiceState { + pub(crate) async fn grant_proxy_private_names( + self: &std::sync::Arc, + vm_id: &str, + worker: &ProxyWorker, + ) -> Result<()> { + let (service, proxy) = UnixStream::pair().context("create proxy private-name capability")?; + let serving = tokio::spawn(crate::proxy_private_names::serve( + std::sync::Arc::clone(self), + vm_id.to_string(), + service, + )); + if let Err(error) = worker.grant(ProxyCapability::PrivateNames, proxy).await { + serving.abort(); + let _ = serving.await; + return Err(error.context("grant proxy private-name capability")); + } + tokio::spawn(async move { + match serving.await { + Ok(Ok(())) => tracing::debug!("proxy private-name capability disconnected"), + Ok(Err(error)) => tracing::warn!(%error, "proxy private-name capability failed"), + Err(error) => tracing::warn!(%error, "proxy private-name capability task failed"), + } + }); + Ok(()) + } + pub(crate) async fn grant_proxy_credentials(&self, worker: &ProxyWorker) -> Result<()> { let (service, proxy) = UnixStream::pair().context("create proxy credential capability")?; let serving = tokio::spawn(crate::proxy_credentials::serve(service)); diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index 66e1c3586..71c4e9217 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -312,6 +312,10 @@ impl ServiceState { instance_reaper::kill_and_reap(child); return Err(error); } + if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_private_names(id, &proxy)) { + instance_reaper::kill_and_reap(child); + return Err(error); + } if let Err(error) = tokio::runtime::Handle::current() .block_on(proxy.grant(capsem_proto::proxy_control::ProxyCapability::Upstream, proxy_upstream)) { diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index b57bb1dbb..50f081b42 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -242,6 +242,10 @@ impl ServiceState { instance_reaper::kill_and_reap(child); return Err(error); } + if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_private_names(&vm_id, &proxy)) { + instance_reaper::kill_and_reap(child); + return Err(error); + } if let Err(error) = tokio::runtime::Handle::current() .block_on(proxy.grant(capsem_proto::proxy_control::ProxyCapability::Upstream, proxy_upstream)) { From b3273b55388bb3462541e2441862e1b3d57e6608 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:00:36 +0000 Subject: [PATCH 145/464] feat(proxy): consume private-name capability --- crates/capsem-proxy/src/main.rs | 19 ++ .../capsem-proxy/src/private_names_client.rs | 171 ++++++++++++++++++ .../src/private_names_client/tests.rs | 44 +++++ 3 files changed, 234 insertions(+) create mode 100644 crates/capsem-proxy/src/private_names_client.rs create mode 100644 crates/capsem-proxy/src/private_names_client/tests.rs diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index b5f3f9220..4bb2a2639 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -27,6 +27,7 @@ use clap::Parser; use tokio::task::{JoinError, JoinSet}; mod credential_client; +mod private_names_client; const CONTROL_QUEUE_CAPACITY: usize = 16; const GRANT_LIMIT: usize = 71; @@ -242,6 +243,22 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result while stopped.borrow().is_none() && stopped.changed().await.is_ok() {} (capability, grant_id, ProxyChannelCloseReason::Disconnected) }); + } else if capability == ProxyCapability::PrivateNames { + let (private_names, mut closed) = + private_names_client::PrivateNameClient::start(stream) + .context("open proxy private-name capability")?; + let mut runtime = state.lock().unwrap_or_else(|error| error.into_inner()); + runtime.private_names = Some(Arc::new(private_names)); + tracing::debug!( + private_names_capability = runtime.private_names.is_some(), + "proxy private-name authority attached" + ); + drop(runtime); + tasks.spawn(async move { + while closed.borrow().is_none() && closed.changed().await.is_ok() {} + let reason = closed.borrow().unwrap_or(ProxyChannelCloseReason::Disconnected); + (capability, grant_id, reason) + }); } else { descriptors.insert(grant_id, stream); } @@ -302,6 +319,7 @@ struct ProxyRuntimeState { ledger: Arc, credentials: Arc, upstream_grants: Option>, + private_names: Option>, dns_upstreams: Vec, mcp: capsem_core::mcp::policy::McpConfig, } @@ -313,6 +331,7 @@ impl Default for ProxyRuntimeState { ledger: Arc::new(UnavailableLedger), credentials: Arc::new(UnavailableCredentials), upstream_grants: None, + private_names: None, dns_upstreams: Vec::new(), mcp: capsem_core::mcp::policy::McpConfig::default(), } diff --git a/crates/capsem-proxy/src/private_names_client.rs b/crates/capsem-proxy/src/private_names_client.rs new file mode 100644 index 000000000..fe07b9026 --- /dev/null +++ b/crates/capsem-proxy/src/private_names_client.rs @@ -0,0 +1,171 @@ +use std::io; +use std::net::Ipv4Addr; +use std::os::unix::net::UnixStream; +use std::time::Duration; + +use capsem_core::net::dns::private::{Lookup, PrivateNames}; +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_control::ProxyChannelCloseReason; +use capsem_proto::proxy_private_names::{ProxyPrivateNameRequest, ProxyPrivateNameResponse}; +use tokio::sync::{mpsc, oneshot, watch}; + +const COMMAND_CAPACITY: usize = 32; +const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +enum Command { + AddressOf { + name: String, + reply: oneshot::Sender>, + }, + NameOf { + address: Ipv4Addr, + reply: oneshot::Sender>, + }, +} + +impl Command { + fn fail(self) { + match self { + Self::AddressOf { reply, .. } => { + let _ = reply.send(None); + } + Self::NameOf { reply, .. } => { + let _ = reply.send(None); + } + } + } +} + +pub(super) struct PrivateNameClient { + commands: mpsc::Sender, +} + +impl PrivateNameClient { + pub(super) fn start(stream: UnixStream) -> io::Result<(Self, watch::Receiver>)> { + let (sender, receiver) = + ipc_channel::channel_from_std::(stream)?; + let (commands, inbox) = mpsc::channel(COMMAND_CAPACITY); + let (closed_tx, closed) = watch::channel(None); + tokio::spawn(async move { + let reason = run(sender, receiver, inbox).await; + let _ = closed_tx.send(Some(reason)); + }); + Ok((Self { commands }, closed)) + } +} + +impl PrivateNames for PrivateNameClient { + fn address_of<'a>(&'a self, name: &'a str) -> Lookup<'a, Ipv4Addr> { + Box::pin(async move { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::AddressOf { + name: name.to_string(), + reply, + }) + .await + .ok()?; + result.await.ok().flatten() + }) + } + + fn name_of(&self, address: Ipv4Addr) -> Lookup<'_, String> { + Box::pin(async move { + let (reply, result) = oneshot::channel(); + self.commands.send(Command::NameOf { address, reply }).await.ok()?; + result.await.ok().flatten() + }) + } +} + +async fn run( + sender: ipc_channel::Sender, + receiver: ipc_channel::Receiver, + mut commands: mpsc::Receiver, +) -> ProxyChannelCloseReason { + let mut request_id = 1_u64; + loop { + tokio::select! { + unsolicited = receiver.recv() => { + return match unsolicited { + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => ProxyChannelCloseReason::Disconnected, + _ => ProxyChannelCloseReason::ProtocolError, + }; + } + command = commands.recv() => { + let Some(command) = command else { + return ProxyChannelCloseReason::Revoked; + }; + if dispatch(&sender, &receiver, request_id, command).await.is_err() { + while let Ok(command) = commands.try_recv() { + command.fail(); + } + return ProxyChannelCloseReason::ProtocolError; + } + let Some(next) = request_id.checked_add(1) else { + return ProxyChannelCloseReason::ProtocolError; + }; + request_id = next; + } + } + } +} + +async fn dispatch( + sender: &ipc_channel::Sender, + receiver: &ipc_channel::Receiver, + request_id: u64, + command: Command, +) -> Result<(), ()> { + let request = match &command { + Command::AddressOf { name, .. } => ProxyPrivateNameRequest::AddressOf { + request_id, + name: name.clone(), + }, + Command::NameOf { address, .. } => ProxyPrivateNameRequest::NameOf { + request_id, + address: *address, + }, + }; + if request.validate().is_err() || sender.send(request).await.is_err() { + command.fail(); + return Err(()); + } + let response = match tokio::time::timeout(OPERATION_TIMEOUT, receiver.recv()).await { + Ok(Ok(response)) if response.request_id() == request_id => response, + _ => { + command.fail(); + return Err(()); + } + }; + match (command, response) { + (Command::AddressOf { reply, .. }, ProxyPrivateNameResponse::Address { address, .. }) => { + let _ = reply.send(Some(address)); + } + (Command::NameOf { reply, .. }, ProxyPrivateNameResponse::Name { name, .. }) => { + let _ = reply.send(Some(name)); + } + (Command::AddressOf { reply, .. }, ProxyPrivateNameResponse::NotFound { .. }) => { + let _ = reply.send(None); + } + (Command::NameOf { reply, .. }, ProxyPrivateNameResponse::NotFound { .. }) => { + let _ = reply.send(None); + } + (Command::AddressOf { reply, .. }, ProxyPrivateNameResponse::Rejected { error, .. }) => { + tracing::warn!(%error, "private address lookup refused"); + let _ = reply.send(None); + } + (Command::NameOf { reply, .. }, ProxyPrivateNameResponse::Rejected { error, .. }) => { + tracing::warn!(%error, "private reverse lookup refused"); + let _ = reply.send(None); + } + (command, _) => { + command.fail(); + return Err(()); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proxy/src/private_names_client/tests.rs b/crates/capsem-proxy/src/private_names_client/tests.rs new file mode 100644 index 000000000..bd6f32dcd --- /dev/null +++ b/crates/capsem-proxy/src/private_names_client/tests.rs @@ -0,0 +1,44 @@ +use std::net::Ipv4Addr; +use std::os::unix::net::UnixStream; + +use capsem_core::net::dns::private::PrivateNames; +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_private_names::{ProxyPrivateNameRequest, ProxyPrivateNameResponse}; + +#[tokio::test] +async fn correlated_lookup_and_idle_disconnect_are_observed() { + let (service, proxy) = UnixStream::pair().unwrap(); + let (client, mut closed) = super::PrivateNameClient::start(proxy).unwrap(); + let serving = tokio::spawn(async move { + let (sender, receiver) = + ipc_channel::channel_from_std::(service).unwrap(); + let request = receiver.recv().await.unwrap(); + assert_eq!( + request, + ProxyPrivateNameRequest::AddressOf { + request_id: 1, + name: "box.dev.capsem.internal".to_string(), + } + ); + sender + .send(ProxyPrivateNameResponse::Address { + request_id: 1, + address: Ipv4Addr::new(10, 0, 0, 2), + }) + .await + .unwrap(); + }); + assert_eq!( + client.address_of("box.dev.capsem.internal").await, + Some(Ipv4Addr::new(10, 0, 0, 2)) + ); + serving.await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(1), closed.changed()) + .await + .unwrap() + .unwrap(); + assert_eq!( + *closed.borrow(), + Some(capsem_proto::proxy_control::ProxyChannelCloseReason::Disconnected) + ); +} From c82a18a253ba66c296f8363d669a0194dae62823 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:03:25 +0000 Subject: [PATCH 146/464] feat(proto): reject premature proxy traffic --- crates/capsem-proto/src/proxy_control.rs | 4 +++- crates/capsem-proto/src/proxy_control/tests.rs | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/capsem-proto/src/proxy_control.rs b/crates/capsem-proto/src/proxy_control.rs index 17a70e5bb..0fcd0322c 100644 --- a/crates/capsem-proto/src/proxy_control.rs +++ b/crates/capsem-proto/src/proxy_control.rs @@ -6,7 +6,7 @@ use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration, Ledg pub const PROXY_CONTROL_FRAME_SIZE: usize = 56; pub const PROXY_CONTROL_MAX_FDS: usize = 1; -pub const PROXY_CONTROL_VERSION: u16 = 3; +pub const PROXY_CONTROL_VERSION: u16 = 4; const MAGIC: [u8; 2] = *b"PX"; const VERSION_RANGE: std::ops::Range = 2..4; @@ -158,6 +158,7 @@ pub enum ProxyControlRejection { DuplicateCapability = 2, DuplicateGrant = 3, ShuttingDown = 4, + NotReady = 5, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -402,6 +403,7 @@ fn decode_rejection(value: u16) -> Result Ok(ProxyControlRejection::DuplicateCapability), 3 => Ok(ProxyControlRejection::DuplicateGrant), 4 => Ok(ProxyControlRejection::ShuttingDown), + 5 => Ok(ProxyControlRejection::NotReady), value => Err(ProxyControlError::InvalidRejection(value)), } } diff --git a/crates/capsem-proto/src/proxy_control/tests.rs b/crates/capsem-proto/src/proxy_control/tests.rs index b76509104..3723dfad1 100644 --- a/crates/capsem-proto/src/proxy_control/tests.rs +++ b/crates/capsem-proto/src/proxy_control/tests.rs @@ -65,6 +65,11 @@ fn events_round_trip_generation_and_grant_identity() { grant_id: 43, reason: ProxyControlRejection::DuplicateCapability, }, + ProxyControlEvent::Rejected { + generation: GENERATION, + grant_id: 45, + reason: ProxyControlRejection::NotReady, + }, ProxyControlEvent::Closed { generation: GENERATION, grant_id: 44, From 7e53e2aed5c9b80bc60265aa1023db675a484c81 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:07:27 +0000 Subject: [PATCH 147/464] feat(proxy): serve authenticated HTTP traffic --- crates/capsem-proxy/src/main.rs | 99 ++++++++++++++++++++++--- crates/capsem-proxy/src/tests.rs | 32 ++++++++ crates/capsem-proxy/tests/subprocess.rs | 51 ++++++++++++- 3 files changed, 170 insertions(+), 12 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 4bb2a2639..8a0b411b4 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, HashSet}; use std::io; -use std::os::fd::AsFd; +use std::os::fd::{AsFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::path::Path; use std::process::Command; @@ -170,6 +170,13 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result Some(ProxyControlRejection::Capacity) } else if !is_traffic(capability) && current != 0 { Some(ProxyControlRejection::DuplicateCapability) + } else if is_traffic(capability) + && !state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .traffic_ready(capability)? + { + Some(ProxyControlRejection::NotReady) } else { None }; @@ -233,7 +240,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result ); let mut stopped = grants.stop_receiver(); let mut runtime = state.lock().unwrap_or_else(|error| error.into_inner()); - runtime.upstream_grants = Some(grants); + runtime.attach_upstream(grants); tracing::debug!( upstream_capability = runtime.upstream_grants.is_some(), "proxy upstream authority attached" @@ -259,6 +266,17 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result let reason = closed.borrow().unwrap_or(ProxyChannelCloseReason::Disconnected); (capability, grant_id, reason) }); + } else if capability == ProxyCapability::HttpTraffic { + let config = state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .http_config()? + .context("proxy HTTP runtime became unavailable after readiness check")?; + tasks.spawn(async move { + capsem_core::net::mitm_proxy::handle_connection(OwnedFd::from(stream), config) + .await; + (capability, grant_id, ProxyChannelCloseReason::Disconnected) + }); } else { descriptors.insert(grant_id, stream); } @@ -315,13 +333,16 @@ fn remove_grant( } struct ProxyRuntimeState { - engine: Option, + engine: Option>, ledger: Arc, + db: Option>, credentials: Arc, + credentials_attached: bool, upstream_grants: Option>, private_names: Option>, dns_upstreams: Vec, mcp: capsem_core::mcp::policy::McpConfig, + http_config: Option>, } impl Default for ProxyRuntimeState { @@ -329,11 +350,14 @@ impl Default for ProxyRuntimeState { Self { engine: None, ledger: Arc::new(UnavailableLedger), + db: None, credentials: Arc::new(UnavailableCredentials), + credentials_attached: false, upstream_grants: None, private_names: None, dns_upstreams: Vec::new(), mcp: capsem_core::mcp::policy::McpConfig::default(), + http_config: None, } } } @@ -346,11 +370,11 @@ impl ProxyRuntimeState { if let Some(engine) = &self.engine { engine.policy().replace(snapshot); } else { - self.engine = Some(ProxyEngine::new( + self.engine = Some(Arc::new(ProxyEngine::new( ProxyPolicyHandle::new(snapshot), Arc::clone(&self.ledger), Arc::clone(&self.credentials), - )); + ))); } self.dns_upstreams = dns_upstreams; self.mcp = mcp; @@ -358,25 +382,80 @@ impl ProxyRuntimeState { } fn attach_ledger(&mut self, ledger: Arc) { - self.ledger = ledger; + self.ledger = ledger.clone(); + self.db = Some(ledger); if let Some(engine) = self.engine.take() { - self.engine = Some(ProxyEngine::new( + self.engine = Some(Arc::new(ProxyEngine::new( engine.policy().clone(), Arc::clone(&self.ledger), Arc::clone(&self.credentials), - )); + ))); } + self.http_config = None; } fn attach_credentials(&mut self, credentials: Arc) { self.credentials = credentials; + self.credentials_attached = true; if let Some(engine) = self.engine.take() { - self.engine = Some(ProxyEngine::new( + self.engine = Some(Arc::new(ProxyEngine::new( engine.policy().clone(), Arc::clone(&self.ledger), Arc::clone(&self.credentials), - )); + ))); + } + self.http_config = None; + } + + fn attach_upstream(&mut self, upstream: Arc) { + self.upstream_grants = Some(upstream); + self.http_config = None; + } + + fn traffic_ready(&mut self, capability: ProxyCapability) -> Result { + match capability { + ProxyCapability::HttpTraffic => Ok(self.http_config()?.is_some()), + ProxyCapability::DnsTraffic => Ok(false), + _ => Ok(true), + } + } + + fn http_config(&mut self) -> Result>> { + if let Some(config) = &self.http_config { + return Ok(Some(Arc::clone(config))); + } + let (Some(engine), Some(db), Some(upstream_grants)) = (&self.engine, &self.db, &self.upstream_grants) else { + return Ok(None); + }; + if !self.credentials_attached { + return Ok(None); } + let ca = Arc::new(capsem_core::net::cert_authority::CertAuthority::load( + capsem_core::vm::boot::CA_KEY_PEM, + capsem_core::vm::boot::CA_CERT_PEM, + )?); + let telemetry = Arc::new(capsem_core::net::mitm_proxy::telemetry_hook::TelemetryDeps { + db: Arc::clone(db), + credentials: engine.credentials(), + pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), + trace_state: Arc::new(Mutex::new(capsem_core::net::ai_traffic::TraceState::new())), + }); + let pipeline = capsem_core::net::mitm_proxy::make_production_pipeline(Arc::clone(&telemetry)); + let upstream_grants: Arc = upstream_grants.clone(); + let config = Arc::new(capsem_core::net::mitm_proxy::MitmProxyConfig { + ca: Arc::clone(&ca), + server_tls: capsem_core::net::mitm_proxy::make_server_tls_config(&ca), + engine: Arc::clone(engine), + db: Arc::clone(db), + upstream_tls: capsem_core::net::mitm_proxy::make_upstream_tls_config(), + telemetry, + pipeline, + mcp_endpoint: None, + upstream_resolver: capsem_core::net::upstream_address::UpstreamResolver::disabled(), + upstream_grants: Some(upstream_grants), + }); + self.http_config = Some(Arc::clone(&config)); + Ok(Some(config)) } } diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index f13939423..196a3b99b 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -34,3 +34,35 @@ match = 'http.host == "worker.example"' assert!(state.apply(b"[network]\nunknown = true").is_err()); assert_eq!(state.engine.as_ref().unwrap().policy().snapshot().digest(), digest); } + +#[tokio::test] +async fn http_runtime_requires_every_host_capability() { + let active_policy = br#" +[network] +[user_rules.profiles.rules.worker_http] +name = "worker_http" +action = "allow" +match = 'http.host == "worker.example"' +[corp_rules] +"#; + let mut state = ProxyRuntimeState::default(); + state.apply(active_policy).unwrap(); + assert!(state.http_config().unwrap().is_none()); + + state.attach_ledger(Arc::new(capsem_logger::DbWriter::open_in_memory(8).unwrap())); + assert!(state.http_config().unwrap().is_none()); + + state.attach_credentials(Arc::new(UnavailableCredentials)); + assert!(state.http_config().unwrap().is_none()); + + let (client, _broker) = UnixStream::pair().unwrap(); + state.attach_upstream(Arc::new( + capsem_core::net::upstream_grant::UpstreamGrantClient::start(client).unwrap(), + )); + let config = state.http_config().unwrap().expect("complete capability set"); + assert!(config.upstream_grants.is_some()); + assert_eq!( + config.engine.policy().snapshot().digest(), + state.engine.as_ref().unwrap().policy().snapshot().digest() + ); +} diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs index 17b0bfa40..0771aa3f1 100644 --- a/crates/capsem-proxy/tests/subprocess.rs +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -1,3 +1,4 @@ +use std::io::Read; use std::os::fd::{AsRawFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::process::{Child, Command, Stdio}; @@ -60,6 +61,54 @@ fn spawn() -> (ChildGuard, ControlSender, ControlReceiver) { (ChildGuard(Some(child)), requests, events) } +#[tokio::test] +async fn worker_rejects_http_traffic_until_runtime_is_ready() { + let (mut child, requests, events) = spawn(); + assert_eq!( + event(&events).await, + ProxyControlEvent::Ready { generation: GENERATION } + ); + let (mut peer, granted) = UnixStream::pair().unwrap(); + let grant = ProxyChannelGrant::new(GENERATION, 1, ProxyCapability::HttpTraffic).unwrap(); + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), + &[granted.as_raw_fd()], + ) + .await + .unwrap(); + drop(granted); + assert_eq!( + event(&events).await, + ProxyControlEvent::Rejected { + generation: GENERATION, + grant_id: 1, + reason: ProxyControlRejection::NotReady, + } + ); + peer.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); + assert_eq!(peer.read(&mut [0; 1]).unwrap(), 0); + + requests + .send( + &encode_proxy_control_request(ProxyControlRequest::Shutdown { generation: GENERATION }), + &[], + ) + .await + .unwrap(); + assert_eq!( + event(&events).await, + ProxyControlEvent::Stopped { generation: GENERATION } + ); + let child = child.take(); + assert!(tokio::task::spawn_blocking(move || child.wait_with_output()) + .await + .unwrap() + .unwrap() + .status + .success()); +} + #[tokio::test] async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { let (mut child, requests, events) = spawn(); @@ -78,8 +127,6 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { let mut peers = Vec::new(); for (index, capability) in [ - ProxyCapability::HttpTraffic, - ProxyCapability::DnsTraffic, ProxyCapability::Upstream, ProxyCapability::Credential, ProxyCapability::PrivateNames, From 497cd708139871fd46b67d8103ac861b4a71b1f5 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:10:53 +0000 Subject: [PATCH 148/464] refactor(core): share guest DNS sessions --- crates/capsem-core/src/net/dns/mod.rs | 1 + crates/capsem-core/src/net/dns/session/mod.rs | 255 ++++++++++++++++++ .../capsem-core/src/net/dns/session/tests.rs | 231 ++++++++++++++++ 3 files changed, 487 insertions(+) create mode 100644 crates/capsem-core/src/net/dns/session/mod.rs create mode 100644 crates/capsem-core/src/net/dns/session/tests.rs diff --git a/crates/capsem-core/src/net/dns/mod.rs b/crates/capsem-core/src/net/dns/mod.rs index a2651387e..b00cfa9c7 100644 --- a/crates/capsem-core/src/net/dns/mod.rs +++ b/crates/capsem-core/src/net/dns/mod.rs @@ -42,6 +42,7 @@ mod coalesce; pub mod private; pub mod resolver; pub mod server; +pub mod session; pub mod telemetry; pub use cache::{DnsAnswerCache, DEFAULT_CAPACITY, DEFAULT_MAX_TTL_SECS}; diff --git a/crates/capsem-core/src/net/dns/session/mod.rs b/crates/capsem-core/src/net/dns/session/mod.rs new file mode 100644 index 000000000..6280c30cf --- /dev/null +++ b/crates/capsem-core/src/net/dns/session/mod.rs @@ -0,0 +1,255 @@ +//! Multiplexed DNS query sessions over the vsock DNS port. +//! +//! Wire shape: +//! guest -> host: `[u32 BE length][rmp DnsRequest]` +//! host -> guest: `[u32 BE length][rmp DnsResponse]` +//! +//! One connection carries many queries at once. Each request names a +//! correlation id; its answer echoes it, so the guest can have hundreds of +//! queries in flight on one connection and take the answers in whatever +//! order the upstreams produce them. The previous shape was lock-step (one +//! query per connection at a time, read on one blocking-pool thread and +//! written on another), which capped the whole guest at eight queries in +//! flight and let one slow lookup hold everything queued behind it. +//! +//! What stays on the reply path: the security decision (inside +//! `DnsHandler::handle`) and the `dns_events` ledger row, which is accepted +//! by the writer before the answer is sent; a query whose row is refused is +//! answered SERVFAIL instead. The rule-ledger rows derived from +//! that event are accepted before completion as well, so shutdown can drain +//! the DB queue without racing detached producers. + +use std::io::{Read as _, Write as _}; +use std::os::fd::{AsFd as _, OwnedFd}; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use tokio::io::unix::AsyncFd; +use tokio::sync::{mpsc, Semaphore}; +use tracing::{debug, warn}; + +/// Queries one connection answers concurrently. Matches the guest's +/// per-session in-flight cap; past it the reader simply waits. +pub const DNS_SESSION_MAX_IN_FLIGHT: usize = 128; + +pub async fn serve_dns_session( + conn: OwnedFd, + handler: Arc, + db: Arc, + policy: crate::net::proxy_engine::ProxyPolicyHandle, +) { + let (read_file, write_file) = match ( + capsem_foundation::unix::fd::duplicate(conn.as_fd()).map(std::fs::File::from), + capsem_foundation::unix::fd::duplicate(conn.as_fd()).map(std::fs::File::from), + ) { + (Ok(read_file), Ok(write_file)) => (read_file, write_file), + (Err(error), _) | (_, Err(error)) => { + warn!(error = %error, "DNS port: cannot duplicate the session descriptor"); + return; + } + }; + let (reader, writer) = match (async_side(read_file), async_side(write_file)) { + (Ok(reader), Ok(writer)) => (reader, writer), + (Err(error), _) | (_, Err(error)) => { + warn!(error = %error, "DNS port: cannot register the session descriptor"); + return; + } + }; + + let (frames, mut frames_rx) = mpsc::channel::>(DNS_SESSION_MAX_IN_FLIGHT); + let write_task = tokio::spawn(async move { + while let Some(frame) = frames_rx.recv().await { + if let Err(error) = write_all(&writer, &frame).await { + debug!(error = %error, "DNS port: write failed"); + return; + } + } + }); + let in_flight = Arc::new(Semaphore::new(DNS_SESSION_MAX_IN_FLIGHT)); + + loop { + let payload = match read_frame(&reader).await { + Ok(Some(payload)) => payload, + Ok(None) => break, + Err(error) => { + debug!(error = %error, "DNS port: read failed"); + break; + } + }; + let request = match capsem_proto::decode_dns_request(&payload) { + Ok(request) => request, + Err(error) => { + debug!(error = %error, "DNS port: decode_dns_request failed"); + break; + } + }; + let Ok(permit) = Arc::clone(&in_flight).acquire_owned().await else { + break; + }; + let frames = frames.clone(); + let handler = Arc::clone(&handler); + let db = Arc::clone(&db); + let policy = policy.clone(); + tokio::spawn(async move { + let frame = answer_one(request, &handler, &db, &policy).await; + if let Some(frame) = frame { + // channel-closed-ok: the writer is gone with the connection. + let _ = frames.send(frame).await; + } + drop(permit); + }); + } + + drop(frames); + // Queries already accepted still get their answers written; the writer + // ends when the last of them has sent its frame. + let _ = write_task.await; + drop(conn); +} + +/// Evaluate one query, record it, and frame its answer under the request's id. +async fn answer_one( + request: capsem_proto::DnsRequest, + handler: &super::DnsHandler, + db: &Arc, + policy: &crate::net::proxy_engine::ProxyPolicyHandle, +) -> Option> { + let result = handler.handle(&request.raw).await; + + // T3.3 -- one `dns_events` row per query. trace_id ties it back to the + // agent action; source_proto distinguishes UDP from TCP DNS at the + // source. The security emitter is awaited so the audit row is accepted + // by the single security/logging rail before the answer is returned. + let event = super::build_dns_event( + &result, + Some(request.proto.as_str()), + request.process_name.clone(), + capsem_foundation::telemetry::ambient_capsem_trace_id(), + ); + // Both the event and its derived rule rows belong to this request. The + // answer is released only with its primary row: a query the ledger + // refused is answered SERVFAIL, so no lookup succeeds unrecorded. + let audited = crate::security_engine::admit_within( + crate::security_engine::SECURITY_ADMISSION_DEADLINE, + Box::pin(emit_dns_security_write_and_rules(db, policy, event)), + ) + .await + .is_some(); + let (raw, decision, rcode) = if audited { + (result.answer_bytes, result.decision, result.rcode) + } else { + warn!(id = request.id, "DNS port: audit row refused; answering SERVFAIL"); + let servfail = crate::net::parsers::dns_parser::build_servfail(&request.raw).unwrap_or_default(); + (servfail, capsem_logger::events::Decision::Error, 2) + }; + + let response = capsem_proto::DnsResponse { + id: request.id, + raw, + decision: decision.as_str().to_string(), + rcode, + }; + match capsem_proto::encode_dns_response(&response) { + Ok(frame) => Some(frame), + Err(error) => { + warn!(error = %error, id = request.id, "DNS port: encode_dns_response failed"); + None + } + } +} + +/// Accept the DNS event and its derived rule rows before completing. A DB +/// flush or shutdown after this call therefore includes the whole emission. +pub async fn emit_dns_security_write_and_rules( + db: &Arc, + policy: &crate::net::proxy_engine::ProxyPolicyHandle, + event: capsem_logger::DnsEvent, +) -> Option { + let security_event = super::security_event_from_dns_event(&event); + let event_id = crate::security_engine::emit_security_write(db, capsem_logger::WriteOp::DnsEvent(event)).await?; + let snapshot = policy.snapshot(); + if let Err(error) = crate::security_engine::emit_matching_security_rules( + db, + event_id.clone(), + crate::security_engine::RuntimeSecurityEventType::DnsQuery, + snapshot.security_rules(), + &security_event, + current_unix_ms(), + ) + .await + { + warn!(error = %error, "failed to emit DNS security rule ledger rows"); + } + Some(event_id) +} + +fn current_unix_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64 +} + +/// One duplicated descriptor of the connection, registered with the +/// reactor. Reads and writes go through `try_io`, so a `WouldBlock` clears +/// readiness and the task parks instead of spinning or blocking a thread. +fn async_side(file: std::fs::File) -> std::io::Result> { + capsem_foundation::unix::fd::set_nonblocking(file.as_fd(), true)?; + AsyncFd::new(file) +} + +async fn read_exact(fd: &AsyncFd, buf: &mut [u8]) -> std::io::Result { + let mut filled = 0; + while filled < buf.len() { + let mut guard = fd.readable().await?; + match guard.try_io(|inner| inner.get_ref().read(&mut buf[filled..])) { + Ok(Ok(0)) => return Ok(filled), + Ok(Ok(n)) => filled += n, + Ok(Err(error)) => return Err(error), + Err(_would_block) => continue, + } + } + Ok(filled) +} + +/// One length-prefixed frame; `None` at a clean end of stream. +async fn read_frame(fd: &AsyncFd) -> Result>> { + let mut len_buf = [0u8; 4]; + match read_exact(fd, &mut len_buf).await.context("DNS port: read length")? { + 0 => return Ok(None), + 4 => {} + short => anyhow::bail!("DNS port: length prefix cut short after {short} bytes"), + } + let len = u32::from_be_bytes(len_buf); + if len > capsem_proto::MAX_FRAME_SIZE { + anyhow::bail!("DNS port: frame too large ({len} > MAX_FRAME_SIZE)"); + } + let mut payload = vec![0u8; len as usize]; + let filled = read_exact(fd, &mut payload).await.context("DNS port: read payload")?; + if filled != payload.len() { + anyhow::bail!("DNS port: frame cut short ({filled} of {} bytes)", payload.len()); + } + debug!(len, "DNS port: frame received"); + Ok(Some(payload)) +} + +async fn write_all(fd: &AsyncFd, mut data: &[u8]) -> std::io::Result<()> { + while !data.is_empty() { + let mut guard = fd.writable().await?; + match guard.try_io(|inner| inner.get_ref().write(data)) { + Ok(Ok(0)) => { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "DNS port: wrote nothing", + )) + } + Ok(Ok(n)) => data = &data[n..], + Ok(Err(error)) => return Err(error), + Err(_would_block) => continue, + } + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-core/src/net/dns/session/tests.rs b/crates/capsem-core/src/net/dns/session/tests.rs new file mode 100644 index 000000000..16f050506 --- /dev/null +++ b/crates/capsem-core/src/net/dns/session/tests.rs @@ -0,0 +1,231 @@ +use super::*; +use crate::net::dns::{DnsHandler, DnsResolver}; +use crate::net::proxy_engine::{ProxyPolicyHandle, ProxyPolicySnapshot}; +use std::os::fd::OwnedFd; +use std::os::unix::net::UnixStream; + +fn handler() -> Arc { + let policy = ProxyPolicyHandle::new(ProxyPolicySnapshot::new( + "blake3:test".into(), + crate::net::policy::NetworkMechanics::default(), + crate::net::policy_config::SecurityRuleSet::new(Vec::new()), + std::collections::BTreeMap::new(), + crate::net::policy_config::ModelEndpointRegistry::default(), + )); + Arc::new(DnsHandler::new( + policy, + Arc::new(DnsResolver::with_upstreams(Vec::new())), + )) +} + +fn policy() -> ProxyPolicyHandle { + ProxyPolicyHandle::new(ProxyPolicySnapshot::new( + "blake3:test".into(), + crate::net::policy::NetworkMechanics::default(), + crate::net::policy_config::SecurityRuleSet::new(Vec::new()), + std::collections::BTreeMap::new(), + crate::net::policy_config::ModelEndpointRegistry::default(), + )) +} + +/// A `.capsem-bogus` A query the handler answers locally with NXDOMAIN. +fn query(id: u16, label: &str) -> Vec { + let mut msg = vec![0u8; 12]; + msg[0..2].copy_from_slice(&id.to_be_bytes()); + msg[2] = 0x01; + msg[5] = 1; + for part in [label, "capsem-bogus"] { + msg.push(part.len() as u8); + msg.extend_from_slice(part.as_bytes()); + } + msg.push(0); + msg.extend_from_slice(&[0, 1, 0, 1]); + msg +} + +fn request_frame(id: u32, raw: Vec) -> Vec { + capsem_proto::encode_dns_request(&capsem_proto::DnsRequest { + id, + raw, + proto: "udp".into(), + process_name: None, + }) + .unwrap() +} + +fn read_response(guest: &mut UnixStream) -> Option { + let mut len_buf = [0u8; 4]; + guest.read_exact(&mut len_buf).ok()?; + let mut payload = vec![0u8; u32::from_be_bytes(len_buf) as usize]; + guest.read_exact(&mut payload).ok()?; + Some(capsem_proto::decode_dns_response(&payload).unwrap()) +} + +/// Serve one session over a socket pair; returns the guest end and the +/// server task. +fn session(db: Arc) -> (UnixStream, tokio::task::JoinHandle<()>) { + let (guest, host) = UnixStream::pair().unwrap(); + let task = tokio::spawn(serve_dns_session(OwnedFd::from(host), handler(), db, policy())); + (guest, task) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn thirty_two_queries_in_flight_are_all_answered_under_their_own_ids() { + let db = Arc::new(capsem_logger::DbWriter::open_in_memory(64).unwrap()); + let (mut guest, task) = session(db); + let queries: Vec<(u32, Vec)> = (1..=32u32) + .map(|id| (id, query(id as u16, &format!("q{id}")))) + .collect(); + let mut writer = guest.try_clone().unwrap(); + let frames: Vec> = queries + .iter() + .map(|(id, raw)| request_frame(*id, raw.clone())) + .collect(); + let send = tokio::task::spawn_blocking(move || { + for frame in frames { + writer.write_all(&frame).unwrap(); + } + }); + let answers = tokio::task::spawn_blocking(move || { + let mut seen = std::collections::HashMap::new(); + for _ in 0..32 { + let response = read_response(&mut guest).expect("an answer for every query"); + seen.insert(response.id, response); + } + (guest, seen) + }); + send.await.unwrap(); + let (guest, seen) = answers.await.unwrap(); + assert_eq!(seen.len(), 32, "every correlation id answered exactly once"); + for (id, raw) in &queries { + let response = &seen[id]; + assert_eq!(response.rcode, 3, "local NXDOMAIN fixture"); + assert_eq!( + &response.raw[0..2], + &raw[0..2], + "answer keeps the query's transaction id" + ); + assert_eq!(&response.raw[12..], &raw[12..], "answer echoes the query's question"); + } + drop(guest); + tokio::time::timeout(std::time::Duration::from_secs(5), task) + .await + .expect("session ends at EOF") + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_oversized_frame_ends_the_session() { + let db = Arc::new(capsem_logger::DbWriter::open_in_memory(16).unwrap()); + let (mut guest, task) = session(db); + let mut writer = guest.try_clone().unwrap(); + tokio::task::spawn_blocking(move || { + writer + .write_all(&(capsem_proto::MAX_FRAME_SIZE + 1).to_be_bytes()) + .unwrap(); + let _ = writer.write_all(&[0u8; 32]); + }) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), task) + .await + .expect("session refuses the frame and ends") + .unwrap(); + let eof = tokio::task::spawn_blocking(move || read_response(&mut guest)) + .await + .unwrap(); + assert!(eof.is_none(), "nothing is answered after the connection is dropped"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_truncated_frame_ends_the_session_without_an_answer() { + let db = Arc::new(capsem_logger::DbWriter::open_in_memory(16).unwrap()); + let (guest, task) = session(db); + let mut writer = guest.try_clone().unwrap(); + tokio::task::spawn_blocking(move || { + writer.write_all(&64u32.to_be_bytes()).unwrap(); + writer.write_all(&[1, 2, 3]).unwrap(); + drop(writer); + }) + .await + .unwrap(); + // Close our end fully so the reader sees EOF mid-frame. + let guest_fd_holder = guest.try_clone().unwrap(); + drop(guest); + drop(guest_fd_holder); + tokio::time::timeout(std::time::Duration::from_secs(5), task) + .await + .expect("session ends on the cut frame") + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn every_answered_query_leaves_a_dns_event_row_before_its_answer() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("session.db"); + let db = Arc::new(capsem_logger::DbWriter::open(&db_path, 64).unwrap()); + let (mut guest, task) = session(Arc::clone(&db)); + let mut writer = guest.try_clone().unwrap(); + tokio::task::spawn_blocking(move || { + for id in 1..=8u32 { + writer.write_all(&request_frame(id, query(id as u16, "row"))).unwrap(); + } + }) + .await + .unwrap(); + let guest = tokio::task::spawn_blocking(move || { + for _ in 0..8 { + read_response(&mut guest).expect("answer"); + } + guest + }) + .await + .unwrap(); + drop(guest); + task.await.unwrap(); + let db_for_flush = Arc::clone(&db); + tokio::task::spawn_blocking(move || db_for_flush.shutdown_blocking()) + .await + .unwrap(); + let reader = capsem_logger::DbReader::open(&db_path).unwrap(); + let rows: serde_json::Value = + serde_json::from_str(&reader.query_raw("SELECT COUNT(*) FROM dns_events").unwrap()).unwrap(); + assert_eq!(rows["rows"][0][0].as_i64(), Some(8)); +} + +/// An answer is released only with its audit row. When the ledger refuses the +/// query's primary `dns_events` row, the guest gets SERVFAIL rather than the +/// resolved answer: a lookup the session cannot account for does not succeed. +/// The answer used to go out anyway, with nothing recorded. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_query_whose_audit_row_is_refused_is_answered_servfail() { + let db = Arc::new(capsem_logger::DbWriter::open_in_memory(8).unwrap()); + let closed = Arc::clone(&db); + tokio::task::spawn_blocking(move || closed.shutdown_blocking()) + .await + .unwrap(); + let (mut guest, task) = session(db); + let raw = query(7, "unaudited"); + let mut writer = guest.try_clone().unwrap(); + let frame = request_frame(7, raw.clone()); + tokio::task::spawn_blocking(move || writer.write_all(&frame).unwrap()) + .await + .unwrap(); + let (response, guest) = tokio::task::spawn_blocking(move || (read_response(&mut guest), guest)) + .await + .unwrap(); + drop(guest); + task.await.unwrap(); + + let response = response.expect("the query is still answered"); + assert_eq!(response.id, 7); + assert_eq!( + response.rcode, 2, + "refused audit must answer SERVFAIL, not the resolved answer" + ); + assert_eq!( + response.raw, + crate::net::parsers::dns_parser::build_servfail(&raw).unwrap(), + "no byte of the resolved answer may reach the guest" + ); +} From f4695809d66d9c68236f368fd22d2d0522716a54 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:13:38 +0000 Subject: [PATCH 149/464] refactor(process): use shared DNS sessions --- crates/capsem-process/src/vsock.rs | 13 +- crates/capsem-process/src/vsock/dns.rs | 273 ------------------- crates/capsem-process/src/vsock/dns/tests.rs | 229 ---------------- crates/capsem-process/src/vsock/tests.rs | 14 +- 4 files changed, 22 insertions(+), 507 deletions(-) delete mode 100644 crates/capsem-process/src/vsock/dns.rs delete mode 100644 crates/capsem-process/src/vsock/dns/tests.rs diff --git a/crates/capsem-process/src/vsock.rs b/crates/capsem-process/src/vsock.rs index d52e78c0e..d4c25d83a 100644 --- a/crates/capsem-process/src/vsock.rs +++ b/crates/capsem-process/src/vsock.rs @@ -17,7 +17,6 @@ mod clone_state; #[cfg(test)] use audit::handle_audit_frame; use audit::serve_audit_records; -mod dns; mod handshake; mod streams; use handshake::{collect_terminal_control_pair, is_retryable_handshake_error, perform_handshake}; @@ -856,7 +855,17 @@ fn dispatch_aux_connection( Some(HostVsockService::Publication) => job_store.publisher.accept(conn), Some(HostVsockService::Network) => streams::serve_network(conn, job_store, vm_id), Some(HostVsockService::SniProxy) => streams::serve_mitm(conn, Arc::clone(mitm_config)), - Some(HostVsockService::DnsProxy) => dns::serve(conn, dns_handler, db, security_rules), + Some(HostVsockService::DnsProxy) => match conn.try_clone_fd() { + Ok(descriptor) => { + let handler = Arc::clone(dns_handler); + let db = Arc::clone(db); + let policy = mitm_config.engine.policy().clone(); + tokio::spawn(async move { + capsem_core::net::dns::session::serve_dns_session(descriptor, handler, db, policy).await; + }); + } + Err(error) => warn!(%error, "DNS port: cannot duplicate the session descriptor"), + }, Some(HostVsockService::Exec) => { let js = Arc::clone(job_store); std::thread::spawn(move || { diff --git a/crates/capsem-process/src/vsock/dns.rs b/crates/capsem-process/src/vsock/dns.rs deleted file mode 100644 index 4e22cfe50..000000000 --- a/crates/capsem-process/src/vsock/dns.rs +++ /dev/null @@ -1,273 +0,0 @@ -//! Multiplexed DNS query sessions over the vsock DNS port. -//! -//! Wire shape: -//! guest -> host: `[u32 BE length][rmp DnsRequest]` -//! host -> guest: `[u32 BE length][rmp DnsResponse]` -//! -//! One connection carries many queries at once. Each request names a -//! correlation id; its answer echoes it, so the guest can have hundreds of -//! queries in flight on one connection and take the answers in whatever -//! order the upstreams produce them. The previous shape was lock-step (one -//! query per connection at a time, read on one blocking-pool thread and -//! written on another), which capped the whole guest at eight queries in -//! flight and let one slow lookup hold everything queued behind it. -//! -//! What stays on the reply path: the security decision (inside -//! `DnsHandler::handle`) and the `dns_events` ledger row, which is accepted -//! by the writer before the answer is sent; a query whose row is refused is -//! answered SERVFAIL instead. The rule-ledger rows derived from -//! that event are accepted before completion as well, so shutdown can drain -//! the DB queue without racing detached producers. - -use std::io::{Read as _, Write as _}; -use std::os::fd::AsFd as _; -use std::sync::Arc; - -use anyhow::{Context, Result}; -use capsem_core::VsockConnection; -use tokio::io::unix::AsyncFd; -use tokio::sync::{mpsc, Semaphore}; -use tracing::{debug, warn}; - -use crate::helpers::clone_fd; - -/// Queries one connection answers concurrently. Matches the guest's -/// per-session in-flight cap; past it the reader simply waits. -pub(super) const DNS_SESSION_MAX_IN_FLIGHT: usize = 128; - -type SecurityRulesHandle = Arc>>; - -/// Serve one guest DNS session on its own task: long-lived, many queries at -/// once, each a `dns_events` row under the ambient trace id. -pub(super) fn serve( - conn: VsockConnection, - handler: &Arc, - db: &Arc, - security_rules: &SecurityRulesHandle, -) { - tokio::spawn(serve_dns_session( - conn, - Arc::clone(handler), - Arc::clone(db), - Arc::clone(security_rules), - )); -} - -pub(super) async fn serve_dns_session( - conn: VsockConnection, - handler: Arc, - db: Arc, - security_rules: SecurityRulesHandle, -) { - let Some(read_file) = clone_fd(&conn, "duplicate-dns-vsock-reader") else { - return; - }; - let Some(write_file) = clone_fd(&conn, "duplicate-dns-vsock-writer") else { - return; - }; - let (reader, writer) = match (async_side(read_file), async_side(write_file)) { - (Ok(reader), Ok(writer)) => (reader, writer), - (Err(error), _) | (_, Err(error)) => { - warn!(error = %error, "DNS port: cannot register the session descriptor"); - return; - } - }; - - let (frames, mut frames_rx) = mpsc::channel::>(DNS_SESSION_MAX_IN_FLIGHT); - let write_task = tokio::spawn(async move { - while let Some(frame) = frames_rx.recv().await { - if let Err(error) = write_all(&writer, &frame).await { - debug!(error = %error, "DNS port: write failed"); - return; - } - } - }); - let in_flight = Arc::new(Semaphore::new(DNS_SESSION_MAX_IN_FLIGHT)); - - loop { - let payload = match read_frame(&reader).await { - Ok(Some(payload)) => payload, - Ok(None) => break, - Err(error) => { - debug!(error = %error, "DNS port: read failed"); - break; - } - }; - let request = match capsem_proto::decode_dns_request(&payload) { - Ok(request) => request, - Err(error) => { - debug!(error = %error, "DNS port: decode_dns_request failed"); - break; - } - }; - let Ok(permit) = Arc::clone(&in_flight).acquire_owned().await else { - break; - }; - let frames = frames.clone(); - let handler = Arc::clone(&handler); - let db = Arc::clone(&db); - let security_rules = Arc::clone(&security_rules); - tokio::spawn(async move { - let frame = answer_one(request, &handler, &db, &security_rules).await; - if let Some(frame) = frame { - // channel-closed-ok: the writer is gone with the connection. - let _ = frames.send(frame).await; - } - drop(permit); - }); - } - - drop(frames); - // Queries already accepted still get their answers written; the writer - // ends when the last of them has sent its frame. - let _ = write_task.await; - drop(conn); -} - -/// Evaluate one query, record it, and frame its answer under the request's id. -async fn answer_one( - request: capsem_proto::DnsRequest, - handler: &capsem_core::net::dns::DnsHandler, - db: &Arc, - security_rules: &SecurityRulesHandle, -) -> Option> { - let result = handler.handle(&request.raw).await; - - // T3.3 -- one `dns_events` row per query. trace_id ties it back to the - // agent action; source_proto distinguishes UDP from TCP DNS at the - // source. The security emitter is awaited so the audit row is accepted - // by the single security/logging rail before the answer is returned. - let event = capsem_core::net::dns::build_dns_event( - &result, - Some(request.proto.as_str()), - request.process_name.clone(), - capsem_foundation::telemetry::ambient_capsem_trace_id(), - ); - // Both the event and its derived rule rows belong to this request. The - // answer is released only with its primary row: a query the ledger - // refused is answered SERVFAIL, so no lookup succeeds unrecorded. - let audited = capsem_core::security_engine::admit_within( - capsem_core::security_engine::SECURITY_ADMISSION_DEADLINE, - Box::pin(emit_dns_security_write_and_rules(db, security_rules, event)), - ) - .await - .is_some(); - let (raw, decision, rcode) = if audited { - (result.answer_bytes, result.decision, result.rcode) - } else { - warn!(id = request.id, "DNS port: audit row refused; answering SERVFAIL"); - let servfail = capsem_core::net::parsers::dns_parser::build_servfail(&request.raw).unwrap_or_default(); - (servfail, capsem_logger::events::Decision::Error, 2) - }; - - let response = capsem_proto::DnsResponse { - id: request.id, - raw, - decision: decision.as_str().to_string(), - rcode, - }; - match capsem_proto::encode_dns_response(&response) { - Ok(frame) => Some(frame), - Err(error) => { - warn!(error = %error, id = request.id, "DNS port: encode_dns_response failed"); - None - } - } -} - -/// Accept the DNS event and its derived rule rows before completing. A DB -/// flush or shutdown after this call therefore includes the whole emission. -pub(super) async fn emit_dns_security_write_and_rules( - db: &Arc, - security_rules: &SecurityRulesHandle, - event: capsem_logger::DnsEvent, -) -> Option { - let security_event = capsem_core::net::dns::security_event_from_dns_event(&event); - let event_id = - capsem_core::security_engine::emit_security_write(db, capsem_logger::WriteOp::DnsEvent(event)).await?; - let rules = security_rules.read().unwrap().clone(); - if let Err(error) = capsem_core::security_engine::emit_matching_security_rules( - db, - event_id.clone(), - capsem_core::security_engine::RuntimeSecurityEventType::DnsQuery, - &rules, - &security_event, - current_unix_ms(), - ) - .await - { - warn!(error = %error, "failed to emit DNS security rule ledger rows"); - } - Some(event_id) -} - -fn current_unix_ms() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::SystemTime::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as i64 -} - -/// One duplicated descriptor of the connection, registered with the -/// reactor. Reads and writes go through `try_io`, so a `WouldBlock` clears -/// readiness and the task parks instead of spinning or blocking a thread. -fn async_side(file: std::fs::File) -> std::io::Result> { - capsem_foundation::unix::fd::set_nonblocking(file.as_fd(), true)?; - AsyncFd::new(file) -} - -async fn read_exact(fd: &AsyncFd, buf: &mut [u8]) -> std::io::Result { - let mut filled = 0; - while filled < buf.len() { - let mut guard = fd.readable().await?; - match guard.try_io(|inner| inner.get_ref().read(&mut buf[filled..])) { - Ok(Ok(0)) => return Ok(filled), - Ok(Ok(n)) => filled += n, - Ok(Err(error)) => return Err(error), - Err(_would_block) => continue, - } - } - Ok(filled) -} - -/// One length-prefixed frame; `None` at a clean end of stream. -async fn read_frame(fd: &AsyncFd) -> Result>> { - let mut len_buf = [0u8; 4]; - match read_exact(fd, &mut len_buf).await.context("DNS port: read length")? { - 0 => return Ok(None), - 4 => {} - short => anyhow::bail!("DNS port: length prefix cut short after {short} bytes"), - } - let len = u32::from_be_bytes(len_buf); - if len > capsem_proto::MAX_FRAME_SIZE { - anyhow::bail!("DNS port: frame too large ({len} > MAX_FRAME_SIZE)"); - } - let mut payload = vec![0u8; len as usize]; - let filled = read_exact(fd, &mut payload).await.context("DNS port: read payload")?; - if filled != payload.len() { - anyhow::bail!("DNS port: frame cut short ({filled} of {} bytes)", payload.len()); - } - debug!(len, "DNS port: frame received"); - Ok(Some(payload)) -} - -async fn write_all(fd: &AsyncFd, mut data: &[u8]) -> std::io::Result<()> { - while !data.is_empty() { - let mut guard = fd.writable().await?; - match guard.try_io(|inner| inner.get_ref().write(data)) { - Ok(Ok(0)) => { - return Err(std::io::Error::new( - std::io::ErrorKind::WriteZero, - "DNS port: wrote nothing", - )) - } - Ok(Ok(n)) => data = &data[n..], - Ok(Err(error)) => return Err(error), - Err(_would_block) => continue, - } - } - Ok(()) -} - -#[cfg(test)] -mod tests; diff --git a/crates/capsem-process/src/vsock/dns/tests.rs b/crates/capsem-process/src/vsock/dns/tests.rs deleted file mode 100644 index bef047c7d..000000000 --- a/crates/capsem-process/src/vsock/dns/tests.rs +++ /dev/null @@ -1,229 +0,0 @@ -use super::*; -use capsem_core::net::dns::{DnsHandler, DnsResolver}; -use std::os::fd::AsRawFd; -use std::os::unix::net::UnixStream; - -fn handler() -> Arc { - let policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new( - capsem_core::net::proxy_engine::ProxyPolicySnapshot::new( - "blake3:test".into(), - capsem_core::net::policy::NetworkMechanics::default(), - capsem_core::net::policy_config::SecurityRuleSet::new(Vec::new()), - std::collections::BTreeMap::new(), - capsem_core::net::policy_config::ModelEndpointRegistry::default(), - ), - ); - Arc::new(DnsHandler::new( - policy, - Arc::new(DnsResolver::with_upstreams(Vec::new())), - )) -} - -fn rules() -> SecurityRulesHandle { - Arc::new(std::sync::RwLock::new(Arc::new( - capsem_core::net::policy_config::SecurityRuleSet::new(Vec::new()), - ))) -} - -/// A `.capsem-bogus` A query the handler answers locally with NXDOMAIN. -fn query(id: u16, label: &str) -> Vec { - let mut msg = vec![0u8; 12]; - msg[0..2].copy_from_slice(&id.to_be_bytes()); - msg[2] = 0x01; - msg[5] = 1; - for part in [label, "capsem-bogus"] { - msg.push(part.len() as u8); - msg.extend_from_slice(part.as_bytes()); - } - msg.push(0); - msg.extend_from_slice(&[0, 1, 0, 1]); - msg -} - -fn request_frame(id: u32, raw: Vec) -> Vec { - capsem_proto::encode_dns_request(&capsem_proto::DnsRequest { - id, - raw, - proto: "udp".into(), - process_name: None, - }) - .unwrap() -} - -fn read_response(guest: &mut UnixStream) -> Option { - let mut len_buf = [0u8; 4]; - guest.read_exact(&mut len_buf).ok()?; - let mut payload = vec![0u8; u32::from_be_bytes(len_buf) as usize]; - guest.read_exact(&mut payload).ok()?; - Some(capsem_proto::decode_dns_response(&payload).unwrap()) -} - -/// Serve one session over a socket pair; returns the guest end and the -/// server task. -fn session(db: Arc) -> (UnixStream, tokio::task::JoinHandle<()>) { - let (guest, host) = UnixStream::pair().unwrap(); - let conn = VsockConnection::new(host.as_raw_fd(), capsem_proto::VSOCK_PORT_DNS_PROXY, Box::new(host)); - let task = tokio::spawn(serve_dns_session(conn, handler(), db, rules())); - (guest, task) -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn thirty_two_queries_in_flight_are_all_answered_under_their_own_ids() { - let db = Arc::new(capsem_logger::DbWriter::open_in_memory(64).unwrap()); - let (mut guest, task) = session(db); - let queries: Vec<(u32, Vec)> = (1..=32u32) - .map(|id| (id, query(id as u16, &format!("q{id}")))) - .collect(); - let mut writer = guest.try_clone().unwrap(); - let frames: Vec> = queries - .iter() - .map(|(id, raw)| request_frame(*id, raw.clone())) - .collect(); - let send = tokio::task::spawn_blocking(move || { - for frame in frames { - writer.write_all(&frame).unwrap(); - } - }); - let answers = tokio::task::spawn_blocking(move || { - let mut seen = std::collections::HashMap::new(); - for _ in 0..32 { - let response = read_response(&mut guest).expect("an answer for every query"); - seen.insert(response.id, response); - } - (guest, seen) - }); - send.await.unwrap(); - let (guest, seen) = answers.await.unwrap(); - assert_eq!(seen.len(), 32, "every correlation id answered exactly once"); - for (id, raw) in &queries { - let response = &seen[id]; - assert_eq!(response.rcode, 3, "local NXDOMAIN fixture"); - assert_eq!( - &response.raw[0..2], - &raw[0..2], - "answer keeps the query's transaction id" - ); - assert_eq!(&response.raw[12..], &raw[12..], "answer echoes the query's question"); - } - drop(guest); - tokio::time::timeout(std::time::Duration::from_secs(5), task) - .await - .expect("session ends at EOF") - .unwrap(); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn an_oversized_frame_ends_the_session() { - let db = Arc::new(capsem_logger::DbWriter::open_in_memory(16).unwrap()); - let (mut guest, task) = session(db); - let mut writer = guest.try_clone().unwrap(); - tokio::task::spawn_blocking(move || { - writer - .write_all(&(capsem_proto::MAX_FRAME_SIZE + 1).to_be_bytes()) - .unwrap(); - let _ = writer.write_all(&[0u8; 32]); - }) - .await - .unwrap(); - tokio::time::timeout(std::time::Duration::from_secs(5), task) - .await - .expect("session refuses the frame and ends") - .unwrap(); - let eof = tokio::task::spawn_blocking(move || read_response(&mut guest)) - .await - .unwrap(); - assert!(eof.is_none(), "nothing is answered after the connection is dropped"); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_truncated_frame_ends_the_session_without_an_answer() { - let db = Arc::new(capsem_logger::DbWriter::open_in_memory(16).unwrap()); - let (guest, task) = session(db); - let mut writer = guest.try_clone().unwrap(); - tokio::task::spawn_blocking(move || { - writer.write_all(&64u32.to_be_bytes()).unwrap(); - writer.write_all(&[1, 2, 3]).unwrap(); - drop(writer); - }) - .await - .unwrap(); - // Close our end fully so the reader sees EOF mid-frame. - let guest_fd_holder = guest.try_clone().unwrap(); - drop(guest); - drop(guest_fd_holder); - tokio::time::timeout(std::time::Duration::from_secs(5), task) - .await - .expect("session ends on the cut frame") - .unwrap(); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn every_answered_query_leaves_a_dns_event_row_before_its_answer() { - let dir = tempfile::tempdir().unwrap(); - let db_path = dir.path().join("session.db"); - let db = Arc::new(capsem_logger::DbWriter::open(&db_path, 64).unwrap()); - let (mut guest, task) = session(Arc::clone(&db)); - let mut writer = guest.try_clone().unwrap(); - tokio::task::spawn_blocking(move || { - for id in 1..=8u32 { - writer.write_all(&request_frame(id, query(id as u16, "row"))).unwrap(); - } - }) - .await - .unwrap(); - let guest = tokio::task::spawn_blocking(move || { - for _ in 0..8 { - read_response(&mut guest).expect("answer"); - } - guest - }) - .await - .unwrap(); - drop(guest); - task.await.unwrap(); - let db_for_flush = Arc::clone(&db); - tokio::task::spawn_blocking(move || db_for_flush.shutdown_blocking()) - .await - .unwrap(); - let reader = capsem_logger::DbReader::open(&db_path).unwrap(); - let rows: serde_json::Value = - serde_json::from_str(&reader.query_raw("SELECT COUNT(*) FROM dns_events").unwrap()).unwrap(); - assert_eq!(rows["rows"][0][0].as_i64(), Some(8)); -} - -/// An answer is released only with its audit row. When the ledger refuses the -/// query's primary `dns_events` row, the guest gets SERVFAIL rather than the -/// resolved answer: a lookup the session cannot account for does not succeed. -/// The answer used to go out anyway, with nothing recorded. -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_query_whose_audit_row_is_refused_is_answered_servfail() { - let db = Arc::new(capsem_logger::DbWriter::open_in_memory(8).unwrap()); - let closed = Arc::clone(&db); - tokio::task::spawn_blocking(move || closed.shutdown_blocking()) - .await - .unwrap(); - let (mut guest, task) = session(db); - let raw = query(7, "unaudited"); - let mut writer = guest.try_clone().unwrap(); - let frame = request_frame(7, raw.clone()); - tokio::task::spawn_blocking(move || writer.write_all(&frame).unwrap()) - .await - .unwrap(); - let (response, guest) = tokio::task::spawn_blocking(move || (read_response(&mut guest), guest)) - .await - .unwrap(); - drop(guest); - task.await.unwrap(); - - let response = response.expect("the query is still answered"); - assert_eq!(response.id, 7); - assert_eq!( - response.rcode, 2, - "refused audit must answer SERVFAIL, not the resolved answer" - ); - assert_eq!( - response.raw, - capsem_core::net::parsers::dns_parser::build_servfail(&raw).unwrap(), - "no byte of the resolved answer may reach the guest" - ); -} diff --git a/crates/capsem-process/src/vsock/tests.rs b/crates/capsem-process/src/vsock/tests.rs index 6fc4278be..2e3a35444 100644 --- a/crates/capsem-process/src/vsock/tests.rs +++ b/crates/capsem-process/src/vsock/tests.rs @@ -1,5 +1,5 @@ -use super::dns::emit_dns_security_write_and_rules; use super::*; +use capsem_core::net::dns::session::emit_dns_security_write_and_rules; mod ports; @@ -686,7 +686,15 @@ match = 'dns.qname == "api.openai.com" && dns.qtype == "1"' capsem_core::net::policy_config::SecurityRuleSource::User, ) .expect("rules compile"); - let security_rules = Arc::new(std::sync::RwLock::new(Arc::new(rules))); + let policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new( + capsem_core::net::proxy_engine::ProxyPolicySnapshot::new( + "blake3:dns-test".into(), + capsem_core::net::policy::NetworkMechanics::default(), + rules, + std::collections::BTreeMap::new(), + capsem_core::net::policy_config::ModelEndpointRegistry::default(), + ), + ); let event = capsem_logger::DnsEvent { event_id: None, timestamp: std::time::SystemTime::now(), @@ -708,7 +716,7 @@ match = 'dns.qname == "api.openai.com" && dns.qtype == "1"' credential_ref: None, }; - let event_id = emit_dns_security_write_and_rules(&db, &security_rules, event) + let event_id = emit_dns_security_write_and_rules(&db, &policy, event) .await .expect("event id allocated"); // The caller has completed its write and can shut down immediately. From d58463cce7f3ec379c97dfde0ad0fb47d36604e6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:15:42 +0000 Subject: [PATCH 150/464] feat(proxy): serve authenticated DNS traffic --- crates/capsem-proxy/src/main.rs | 68 +++++++++++++++++++++++++++++++- crates/capsem-proxy/src/tests.rs | 38 ++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 8a0b411b4..532fab7c6 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -255,7 +255,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result private_names_client::PrivateNameClient::start(stream) .context("open proxy private-name capability")?; let mut runtime = state.lock().unwrap_or_else(|error| error.into_inner()); - runtime.private_names = Some(Arc::new(private_names)); + runtime.attach_private_names(Arc::new(private_names)); tracing::debug!( private_names_capability = runtime.private_names.is_some(), "proxy private-name authority attached" @@ -277,6 +277,22 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result .await; (capability, grant_id, ProxyChannelCloseReason::Disconnected) }); + } else if capability == ProxyCapability::DnsTraffic { + let runtime = state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .dns_runtime() + .context("proxy DNS runtime became unavailable after readiness check")?; + tasks.spawn(async move { + capsem_core::net::dns::session::serve_dns_session( + OwnedFd::from(stream), + Arc::clone(&runtime.handler), + Arc::clone(&runtime.db), + runtime.policy.clone(), + ) + .await; + (capability, grant_id, ProxyChannelCloseReason::Disconnected) + }); } else { descriptors.insert(grant_id, stream); } @@ -343,6 +359,7 @@ struct ProxyRuntimeState { dns_upstreams: Vec, mcp: capsem_core::mcp::policy::McpConfig, http_config: Option>, + dns_runtime: Option>, } impl Default for ProxyRuntimeState { @@ -358,6 +375,7 @@ impl Default for ProxyRuntimeState { dns_upstreams: Vec::new(), mcp: capsem_core::mcp::policy::McpConfig::default(), http_config: None, + dns_runtime: None, } } } @@ -378,6 +396,7 @@ impl ProxyRuntimeState { } self.dns_upstreams = dns_upstreams; self.mcp = mcp; + self.dns_runtime = None; Ok(digest) } @@ -392,6 +411,7 @@ impl ProxyRuntimeState { ))); } self.http_config = None; + self.dns_runtime = None; } fn attach_credentials(&mut self, credentials: Arc) { @@ -410,12 +430,18 @@ impl ProxyRuntimeState { fn attach_upstream(&mut self, upstream: Arc) { self.upstream_grants = Some(upstream); self.http_config = None; + self.dns_runtime = None; + } + + fn attach_private_names(&mut self, private_names: Arc) { + self.private_names = Some(private_names); + self.dns_runtime = None; } fn traffic_ready(&mut self, capability: ProxyCapability) -> Result { match capability { ProxyCapability::HttpTraffic => Ok(self.http_config()?.is_some()), - ProxyCapability::DnsTraffic => Ok(false), + ProxyCapability::DnsTraffic => Ok(self.dns_runtime().is_some()), _ => Ok(true), } } @@ -457,6 +483,44 @@ impl ProxyRuntimeState { self.http_config = Some(Arc::clone(&config)); Ok(Some(config)) } + + fn dns_runtime(&mut self) -> Option> { + if let Some(runtime) = &self.dns_runtime { + return Some(Arc::clone(runtime)); + } + let (Some(engine), Some(db), Some(upstream_grants), Some(private_names)) = + (&self.engine, &self.db, &self.upstream_grants, &self.private_names) + else { + return None; + }; + let grants: Arc = upstream_grants.clone(); + let resolver = Arc::new(capsem_core::net::dns::DnsResolver::with_grants( + self.dns_upstreams.clone(), + grants, + )); + let policy = engine.policy().clone(); + let handler = Arc::new( + capsem_core::net::dns::DnsHandler::with_cache( + policy.clone(), + resolver, + Arc::new(capsem_core::net::dns::DnsAnswerCache::default()), + ) + .with_private_names(Arc::clone(private_names)), + ); + let runtime = Arc::new(DnsRuntime { + handler, + db: Arc::clone(db), + policy, + }); + self.dns_runtime = Some(Arc::clone(&runtime)); + Some(runtime) + } +} + +struct DnsRuntime { + handler: Arc, + db: Arc, + policy: ProxyPolicyHandle, } struct UnavailableLedger; diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index 196a3b99b..c621f1c65 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -66,3 +66,41 @@ match = 'http.host == "worker.example"' state.engine.as_ref().unwrap().policy().snapshot().digest() ); } + +struct EmptyPrivateNames; + +impl capsem_core::net::dns::private::PrivateNames for EmptyPrivateNames { + fn address_of<'a>(&'a self, _name: &'a str) -> capsem_core::net::dns::private::Lookup<'a, std::net::Ipv4Addr> { + Box::pin(async { None }) + } + + fn name_of(&self, _address: std::net::Ipv4Addr) -> capsem_core::net::dns::private::Lookup<'_, String> { + Box::pin(async { None }) + } +} + +#[tokio::test] +async fn dns_runtime_requires_ledger_upstream_and_private_names() { + let mut state = ProxyRuntimeState::default(); + state + .apply( + br#" +[network] +[user_rules.profiles.rules.worker_dns] +name = "worker_dns" +action = "allow" +match = 'dns.qname == "worker.example"' +[corp_rules] +"#, + ) + .unwrap(); + state.attach_ledger(Arc::new(capsem_logger::DbWriter::open_in_memory(8).unwrap())); + let (client, _broker) = UnixStream::pair().unwrap(); + state.attach_upstream(Arc::new( + capsem_core::net::upstream_grant::UpstreamGrantClient::start(client).unwrap(), + )); + assert!(state.dns_runtime().is_none()); + + state.attach_private_names(Arc::new(EmptyPrivateNames)); + assert!(state.dns_runtime().is_some()); +} From 95f0f40aff6084ff86dcc18c4456469a41f36cc7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:18:03 +0000 Subject: [PATCH 151/464] security(process): surrender guest DNS traffic --- CHANGELOG.md | 4 + crates/capsem-process/src/ipc.rs | 2 - crates/capsem-process/src/ipc/tests.rs | 8 -- crates/capsem-process/src/main.rs | 33 +------ crates/capsem-process/src/private_names.rs | 54 ------------ .../capsem-process/src/private_names/tests.rs | 88 ------------------- crates/capsem-process/src/tests.rs | 7 +- crates/capsem-process/src/vsock.rs | 28 +++--- 8 files changed, 23 insertions(+), 201 deletions(-) delete mode 100644 crates/capsem-process/src/private_names.rs delete mode 100644 crates/capsem-process/src/private_names/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8671ebedf..39862da22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Move guest DNS framing, private-name resolution, policy enforcement and + audit admission into the confined per-session proxy, with the VM owner + surrendering each connection through its generation-bound broker. + - Serve live and stopped-session inspection through coordinator-granted, role-bound ledger channels and bounded named operations, eliminating the service's per-session SQLite readers and closing ledger workers before a diff --git a/crates/capsem-process/src/ipc.rs b/crates/capsem-process/src/ipc.rs index 458d09b60..cee6442e8 100644 --- a/crates/capsem-process/src/ipc.rs +++ b/crates/capsem-process/src/ipc.rs @@ -142,7 +142,6 @@ pub(crate) async fn handle_ipc_connection( mcp_builtin_binary: Option, mcp_builtin_env: HashMap, vm_ready: Arc, - dns_resolver: Arc, ) -> Result<()> { // First frame on every IPC connection is a Hello -- detect cross-version // mixes (capsem-service built before X, capsem-process built after) in @@ -685,7 +684,6 @@ pub(crate) async fn handle_ipc_connection( let reply = match runtime_source.load() { Ok(runtime_config) => { let digest = runtime_config.active_policy_digest.clone(); - dns_resolver.replace_upstreams(runtime_config.dns_upstreams.clone()); runtime_config.apply(&net_state, &mcp_runtime); ProcessToService::ConfigReloadResult { id, diff --git a/crates/capsem-process/src/ipc/tests.rs b/crates/capsem-process/src/ipc/tests.rs index 8b28b2cc4..69c8f8e2b 100644 --- a/crates/capsem-process/src/ipc/tests.rs +++ b/crates/capsem-process/src/ipc/tests.rs @@ -100,7 +100,6 @@ pub(in crate::ipc) struct Dispatcher { pub(in crate::ipc) mcp_runtime: Arc, runtime_source: RuntimePolicySource, pub(in crate::ipc) ready: Arc, - dns_resolver: Arc, } impl Dispatcher { @@ -201,7 +200,6 @@ impl Dispatcher { let job_store = Arc::new(JobStore::new()); let (ctrl_tx, ctrl_rx) = mpsc::channel(16); let (events_tx, _) = broadcast::channel(16); - let dns_resolver = Arc::new(capsem_core::net::dns::DnsResolver::with_upstreams(Vec::new())); ( Self { job_store, @@ -212,7 +210,6 @@ impl Dispatcher { mcp_runtime, runtime_source: RuntimePolicySource::new(active_policy), ready: Arc::new(AtomicBool::new(true)), - dns_resolver, }, ctrl_rx, ) @@ -233,7 +230,6 @@ impl Dispatcher { None, HashMap::new(), Arc::clone(&self.ready), - Arc::clone(&self.dns_resolver), )) } } @@ -590,10 +586,6 @@ async fn negotiated_dispatcher_covers_stream_jobs_queries_and_lifecycle() { if digest == applied )); assert_eq!(dispatcher.mcp_runtime.proxy_policy.snapshot().digest(), applied); - assert_eq!( - dispatcher.dns_resolver.upstreams(), - vec!["127.0.0.1:5353".parse().unwrap()] - ); service_tx .send(ServiceToProcess::McpListServers { id: 15 }) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 37986cd4c..f633c8520 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -5,8 +5,6 @@ mod ipc; mod job_store; mod mcp_runtime; mod metric_export; -mod owner; -mod private_names; mod private_seats; mod runtime_config; mod terminal; @@ -14,7 +12,6 @@ mod vsock; use anyhow::{Context, Result}; use capsem_core::fs_monitor::FsMonitor; -use capsem_core::net::dns::{DnsAnswerCache, DnsResolver}; use capsem_core::net::upstream_grant::{adopt_inherited, UpstreamGrantClient}; use capsem_core::{boot_vm, BootOptions, VirtioFsShare, VsockConnection}; use capsem_logger::DbWriter; @@ -697,11 +694,6 @@ async fn run_async_main_loop( &job_store, ctrl_tx.clone(), )?; - let private_service = Arc::new( - capsem_core::service_uds::Client::connect(&seats.service_socket) - .await - .context("prepare private-name service channel")?, - ); let metric_service = if args.metric_broker { Some( tokio::net::UnixStream::connect(&seats.service_socket) @@ -843,25 +835,6 @@ async fn run_async_main_loop( upstream_grants: Some(Arc::clone(&upstream_grants) as Arc), }); - // DNS and HTTP clone the same immutable revision once per request. - let dns_resolver = Arc::new(DnsResolver::with_grants( - runtime_config.dns_upstreams.clone(), - Arc::clone(&upstream_grants) as Arc, - )); - // The private zone is the service's to answer, for this VM's networks. - let private_names = Arc::new(private_names::ServicePrivateNames::new( - private_service, - args.id.clone(), - )); - let dns_handler = Arc::new( - capsem_core::net::dns::DnsHandler::with_cache( - proxy_policy, - Arc::clone(&dns_resolver), - Arc::new(DnsAnswerCache::default()), - ) - .with_private_names(private_names), - ); - let ipc_tx_clone = ipc_tx.clone(); let job_store_clone = Arc::clone(&job_store); let terminal_output_clone = Arc::clone(&terminal_output); @@ -873,7 +846,7 @@ async fn run_async_main_loop( let net_state_clone = Arc::clone(&net_state); let mitm_config_clone = Arc::clone(&mitm_config); - let dns_handler_clone = Arc::clone(&dns_handler); + let upstream_grants_for_vsock = Arc::clone(&upstream_grants); // Parse --env KEY=VALUE pairs for guest injection let cli_env: Vec<(String, String)> = args @@ -942,7 +915,7 @@ async fn run_async_main_loop( cli_env, guest_config, mitm_config: mitm_config_clone, - dns_handler: dns_handler_clone, + upstream_grants: upstream_grants_for_vsock, security_rules: Arc::clone(&security_rules), plugin_policy: Arc::clone(&plugin_policy), _net_state: net_state_clone, @@ -991,7 +964,6 @@ async fn run_async_main_loop( let builtin_bin_c = builtin_bin.clone(); let builtin_env_c = builtin_env.clone(); let ready_c = Arc::clone(&vm_ready); - let dns_resolver_c = Arc::clone(&dns_resolver); tokio::spawn(async move { if let Err(e) = ipc::handle_ipc_connection( @@ -1007,7 +979,6 @@ async fn run_async_main_loop( builtin_bin_c, builtin_env_c, ready_c, - dns_resolver_c, ) .await { diff --git a/crates/capsem-process/src/private_names.rs b/crates/capsem-process/src/private_names.rs deleted file mode 100644 index 7237b5557..000000000 --- a/crates/capsem-process/src/private_names.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! This VM's view of the private zone: every question is the service's to -//! answer, with the members of the networks this VM is in and nothing else. -//! No cache here: the answer carries no TTL and a membership that changed -//! is gone from the next answer. -use capsem_core::net::dns::private::{Lookup, PrivateNames}; -use std::net::Ipv4Addr; -use std::sync::Arc; - -pub(crate) struct ServicePrivateNames { - service: Arc, - vm_id: String, -} - -impl ServicePrivateNames { - pub(crate) fn new(service: Arc, vm_id: String) -> Self { - Self { service, vm_id } - } - - async fn ask(&self, question: serde_json::Value) -> Option { - let mut request = question; - request["source_vm"] = serde_json::Value::String(self.vm_id.clone()); - match self.service.post_json("/networks/private/resolve", &request).await { - Ok((200, answer)) => Some(answer), - Ok((404, _)) => None, - Ok((status, answer)) => { - tracing::warn!(status, %answer, "private name lookup refused"); - None - } - Err(error) => { - tracing::warn!(%error, "private name lookup failed"); - None - } - } - } -} - -impl PrivateNames for ServicePrivateNames { - fn address_of<'a>(&'a self, name: &'a str) -> Lookup<'a, Ipv4Addr> { - Box::pin(async move { - let answer = self.ask(serde_json::json!({ "name": name })).await?; - answer["address"].as_str()?.parse().ok() - }) - } - - fn name_of(&self, address: Ipv4Addr) -> Lookup<'_, String> { - Box::pin(async move { - let answer = self.ask(serde_json::json!({ "address": address.to_string() })).await?; - answer["name"].as_str().map(str::to_string) - }) - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/capsem-process/src/private_names/tests.rs b/crates/capsem-process/src/private_names/tests.rs deleted file mode 100644 index ff32a1ab2..000000000 --- a/crates/capsem-process/src/private_names/tests.rs +++ /dev/null @@ -1,88 +0,0 @@ -use super::*; -use std::path::PathBuf; -use std::sync::{Arc, Mutex}; - -/// The service's resolve route as a fake: answers from a table, records -/// what it was asked. -async fn fake_service(socket: PathBuf, seen: Arc>>) { - use axum::{routing::post, Router}; - let app = Router::new().route( - "/networks/private/resolve", - post(move |axum::Json(body): axum::Json| { - let seen = Arc::clone(&seen); - async move { - seen.lock().unwrap().push(body.clone()); - match (body["name"].as_str(), body["address"].as_str()) { - (Some("beta.team"), None) => ( - axum::http::StatusCode::OK, - axum::Json(serde_json::json!({ - "name": "beta.team.capsem.internal", "address": "10.128.0.3", "vm": "vm-b", "network": "team" - })), - ), - (None, Some("10.128.0.3")) => ( - axum::http::StatusCode::OK, - axum::Json(serde_json::json!({ - "name": "beta.team.capsem.internal", "address": "10.128.0.3", "vm": "vm-b", "network": "team" - })), - ), - _ => (axum::http::StatusCode::NOT_FOUND, axum::Json(serde_json::json!({}))), - } - } - }), - ); - let listener = tokio::net::UnixListener::bind(&socket).unwrap(); - tokio::spawn(async move { - let _ = axum::serve(listener, app).await; - }); -} - -#[tokio::test] -async fn names_and_addresses_are_the_services_answers_and_misses_are_none() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("service.sock"); - let seen = Arc::new(Mutex::new(Vec::new())); - fake_service(socket.clone(), Arc::clone(&seen)).await; - let service = Arc::new(capsem_core::service_uds::Client::connect(&socket).await.unwrap()); - let names = ServicePrivateNames::new(service, "vm-a".into()); - assert_eq!(names.address_of("beta.team").await, Some(Ipv4Addr::new(10, 128, 0, 3))); - assert_eq!(names.address_of("nobody.team").await, None); - assert_eq!( - names.name_of(Ipv4Addr::new(10, 128, 0, 3)).await.as_deref(), - Some("beta.team.capsem.internal") - ); - assert_eq!(names.name_of(Ipv4Addr::new(10, 128, 0, 9)).await, None); - let asked = seen.lock().unwrap().clone(); - assert_eq!(asked.len(), 4); - assert!(asked.iter().all(|ask| ask["source_vm"] == "vm-a")); - assert!(asked.iter().all(|ask| ask.get("owner_secret").is_none())); - assert_eq!(asked[0]["name"], "beta.team"); - assert_eq!(asked[2]["address"], "10.128.0.3"); -} - -#[tokio::test] -async fn a_refusal_or_a_closed_service_is_a_miss_not_a_failure() { - let dir = tempfile::tempdir().unwrap(); - let socket = dir.path().join("service.sock"); - let listener = tokio::net::UnixListener::bind(&socket).unwrap(); - tokio::spawn(async move { - let app = axum::Router::new().route( - "/networks/private/resolve", - axum::routing::post(|| async { axum::http::StatusCode::FORBIDDEN }), - ); - let _ = axum::serve(listener, app).await; - }); - let service = Arc::new(capsem_core::service_uds::Client::connect(&socket).await.unwrap()); - let refused = ServicePrivateNames::new(service, "vm-a".into()); - assert_eq!(refused.address_of("beta.team").await, None); - - let closed_socket = dir.path().join("closed.sock"); - let listener = tokio::net::UnixListener::bind(&closed_socket).unwrap(); - let closer = tokio::spawn(async move { - let (stream, _) = listener.accept().await.unwrap(); - drop(stream); - }); - let closed = Arc::new(capsem_core::service_uds::Client::connect(&closed_socket).await.unwrap()); - closer.await.unwrap(); - let closed = ServicePrivateNames::new(closed, "vm-a".into()); - assert_eq!(closed.address_of("beta.team").await, None); -} diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index a270e246d..b1b378df7 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -42,10 +42,9 @@ fn platform_confinement_and_attestation_precede_owner_readiness() { #[test] fn private_name_broker_carries_no_session_bearer() { - for source in [include_str!("private_names.rs"), include_str!("private_seats.rs")] { - assert!(!source.contains("owner_secret")); - assert!(!source.contains("owner-secret")); - } + let source = include_str!("private_seats.rs"); + assert!(!source.contains("owner_secret")); + assert!(!source.contains("owner-secret")); } #[test] diff --git a/crates/capsem-process/src/vsock.rs b/crates/capsem-process/src/vsock.rs index d4c25d83a..997d9fd22 100644 --- a/crates/capsem-process/src/vsock.rs +++ b/crates/capsem-process/src/vsock.rs @@ -71,10 +71,7 @@ pub(crate) struct VsockOptions { pub(crate) cli_env: Vec<(String, String)>, pub(crate) guest_config: capsem_core::net::policy_config::GuestConfig, pub(crate) mitm_config: Arc, - /// Handler for DNS queries forwarded over vsock port 5007. DNS - /// NXDOMAIN decisions come from the shared security rules; the network - /// policy handle remains for resolver mechanics such as redirects/cache. - pub(crate) dns_handler: Arc, + pub(crate) upstream_grants: Arc, pub(crate) security_rules: SecurityRulesHandle, pub(crate) plugin_policy: PluginPolicyHandle, pub(crate) _net_state: Arc, @@ -101,7 +98,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { cli_env, guest_config, mitm_config, - dns_handler, + upstream_grants, security_rules, plugin_policy, is_restore, @@ -723,7 +720,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { // 4. Central Dispatcher Loop (Vsock -> Hub) // ----------------------------------------------------------------------- let mitm_config_loop = Arc::clone(&mitm_config); - let dns_handler_loop = Arc::clone(&dns_handler); + let upstream_grants_loop = Arc::clone(&upstream_grants); let security_rules_loop = Arc::clone(&security_rules); let db_for_audit = Arc::clone(&db); let ipc_tx_lifecycle = ipc_tx.clone(); @@ -742,7 +739,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { dispatch_aux_connection( conn, &mitm_config_loop, - &dns_handler_loop, + &upstream_grants_loop, &security_rules_loop, &job_store_vsock, &db_for_audit, @@ -779,7 +776,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { dispatch_aux_connection( aux_conn, &mitm_config_loop, - &dns_handler_loop, + &upstream_grants_loop, &security_rules_loop, &job_store_vsock, &db_for_audit, @@ -819,7 +816,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { dispatch_aux_connection( conn, &mitm_config_loop, - &dns_handler_loop, + &upstream_grants_loop, &security_rules_loop, &job_store_vsock, &db_for_audit, @@ -843,7 +840,7 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { fn dispatch_aux_connection( conn: VsockConnection, mitm_config: &Arc, - dns_handler: &Arc, + upstream_grants: &Arc, security_rules: &Arc>>, job_store: &Arc, db: &Arc, @@ -857,11 +854,14 @@ fn dispatch_aux_connection( Some(HostVsockService::SniProxy) => streams::serve_mitm(conn, Arc::clone(mitm_config)), Some(HostVsockService::DnsProxy) => match conn.try_clone_fd() { Ok(descriptor) => { - let handler = Arc::clone(dns_handler); - let db = Arc::clone(db); - let policy = mitm_config.engine.policy().clone(); + let grants = Arc::clone(upstream_grants); tokio::spawn(async move { - capsem_core::net::dns::session::serve_dns_session(descriptor, handler, db, policy).await; + if let Err(error) = grants + .attach_proxy_traffic(capsem_proto::upstream_grant::ProxyTrafficService::Dns, descriptor) + .await + { + warn!(%error, "DNS port: proxy worker refused the session descriptor"); + } }); } Err(error) => warn!(%error, "DNS port: cannot duplicate the session descriptor"), From c6eb5ca63fd81d792c45b0c6ea2d426f1163fa9a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:22:27 +0000 Subject: [PATCH 152/464] refactor(core): unify MCP policy snapshots --- .../src/net/mitm_proxy/mcp_endpoint.rs | 56 +++++++++++++++++-- .../src/net/mitm_proxy/mcp_endpoint/tests.rs | 29 ++++++++++ .../src/net/mitm_proxy/mcp_frame.rs | 20 +++---- .../src/net/mitm_proxy/mcp_frame/tests.rs | 2 +- 4 files changed, 91 insertions(+), 16 deletions(-) diff --git a/crates/capsem-core/src/net/mitm_proxy/mcp_endpoint.rs b/crates/capsem-core/src/net/mitm_proxy/mcp_endpoint.rs index 689325f27..1c3506fe0 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mcp_endpoint.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mcp_endpoint.rs @@ -5,7 +5,8 @@ use std::time::Duration; use tokio::sync::RwLock; use tracing::warn; -use crate::net::policy_config::{SecurityRuleSet, SharedPluginPolicy}; +use crate::net::policy_config::{snapshot_plugin_policy, PluginPolicySnapshot, SecurityRuleSet, SharedPluginPolicy}; +use crate::net::proxy_engine::ProxyPolicyHandle; use capsem_logger::DbWriter; use capsem_proto::mcp_aggregator::AggregatorClient; use capsem_proto::mcp_contracts::{builtin_ledger, parse_namespaced, JsonRpcRequest, JsonRpcResponse, McpToolDef}; @@ -63,14 +64,21 @@ pub struct McpEndpointState { pub aggregator: AggregatorClient, builtin_ledger: Option>, builtin_servers: std::sync::RwLock>, - pub security_rules: Arc>>, - pub plugin_policy: SharedPluginPolicy, + policy: McpEndpointPolicy, pub inflight: Arc, pub timeouts: McpTimeouts, scoped_tools: Option>, tool_timeout_overrides: RwLock>, } +enum McpEndpointPolicy { + Legacy { + security_rules: Arc>>, + plugin_policy: SharedPluginPolicy, + }, + Unified(ProxyPolicyHandle), +} + /// Tools implemented by the current VM owner rather than an MCP subprocess. /// /// The endpoint still applies the normal MCP admission and logging around @@ -98,8 +106,10 @@ impl McpEndpointState { aggregator, builtin_ledger: None, builtin_servers: std::sync::RwLock::new(BTreeSet::new()), - security_rules, - plugin_policy, + policy: McpEndpointPolicy::Legacy { + security_rules, + plugin_policy, + }, inflight, timeouts, scoped_tools: None, @@ -107,6 +117,42 @@ impl McpEndpointState { } } + /// Build an endpoint whose rules and plugins are read from one atomic + /// proxy-policy revision for each security decision. + pub fn with_proxy_policy( + aggregator: AggregatorClient, + policy: ProxyPolicyHandle, + inflight: Arc, + timeouts: McpTimeouts, + ) -> Self { + Self { + aggregator, + builtin_ledger: None, + builtin_servers: std::sync::RwLock::new(BTreeSet::new()), + policy: McpEndpointPolicy::Unified(policy), + inflight, + timeouts, + scoped_tools: None, + tool_timeout_overrides: RwLock::new(HashMap::new()), + } + } + + pub(super) fn policy_snapshot(&self) -> (Arc, PluginPolicySnapshot) { + match &self.policy { + McpEndpointPolicy::Legacy { + security_rules, + plugin_policy, + } => ( + security_rules.read().unwrap().clone(), + snapshot_plugin_policy(plugin_policy), + ), + McpEndpointPolicy::Unified(policy) => { + let snapshot = policy.snapshot(); + (Arc::clone(snapshot.security_rules()), Arc::clone(snapshot.plugins())) + } + } + } + pub fn with_scoped_tools(mut self, scoped_tools: Arc) -> Self { self.scoped_tools = Some(scoped_tools); self diff --git a/crates/capsem-core/src/net/mitm_proxy/mcp_endpoint/tests.rs b/crates/capsem-core/src/net/mitm_proxy/mcp_endpoint/tests.rs index 2d37a899c..264b5c35f 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mcp_endpoint/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mcp_endpoint/tests.rs @@ -10,6 +10,35 @@ use capsem_proto::mcp_contracts::{JsonRpcRequest, McpPromptDef, McpResourceDef, use super::*; +fn empty_proxy_snapshot(digest: &str) -> crate::net::proxy_engine::ProxyPolicySnapshot { + crate::net::proxy_engine::ProxyPolicySnapshot::new( + digest.to_string(), + crate::net::policy::NetworkMechanics::default(), + SecurityRuleSet::new(Vec::new()), + BTreeMap::new(), + crate::net::policy_config::ModelEndpointRegistry::default(), + ) +} + +#[test] +fn unified_endpoint_reads_rules_and_plugins_from_one_live_revision() { + let (aggregator, _rx) = capsem_proto::mcp_aggregator::AggregatorClient::channel(1); + let policy = ProxyPolicyHandle::new(empty_proxy_snapshot("first")); + let endpoint = McpEndpointState::with_proxy_policy( + aggregator, + policy.clone(), + Arc::new(tokio::sync::Semaphore::new(1)), + McpTimeouts::default(), + ); + let (first_rules, first_plugins) = endpoint.policy_snapshot(); + + policy.replace(empty_proxy_snapshot("second")); + let (second_rules, second_plugins) = endpoint.policy_snapshot(); + + assert!(!Arc::ptr_eq(&first_rules, &second_rules)); + assert!(!Arc::ptr_eq(&first_plugins, &second_plugins)); +} + struct ScopedToolsFixture { calls: Arc>>, fail: bool, diff --git a/crates/capsem-core/src/net/mitm_proxy/mcp_frame.rs b/crates/capsem-core/src/net/mitm_proxy/mcp_frame.rs index 4adb9dac9..871b134c7 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mcp_frame.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mcp_frame.rs @@ -16,7 +16,8 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tracing::{debug, warn}; use crate::net::dns::private::MCP_HOST; -use crate::net::policy_config::{snapshot_plugin_policy, SecurityRuleSet}; +#[cfg(test)] +use crate::net::policy_config::SecurityRuleSet; use crate::security_engine::{ emit_evaluated_security_rules, emit_security_write, evaluate_security_boundary, McpSecurityEvent, ProcessSecurityEvent, RuntimeSecurityEventType, SecurityEnforcementAction, SecurityEnforcementDecision, @@ -112,7 +113,7 @@ pub async fn dispatch_logged_mcp_request( let response = policy_blocked_response(request.id.clone(), "request", &request_decision); let emission = log_mcp_call_with_policy( Arc::clone(&db), - &endpoint.security_rules, + &endpoint, &request, &response, caller, @@ -151,7 +152,7 @@ pub async fn dispatch_logged_mcp_request( }; let emission = log_mcp_call_with_policy( Arc::clone(&db), - &endpoint.security_rules, + &endpoint, &request, &response, caller, @@ -366,7 +367,7 @@ where }; log_mcp_call_with_policy( db_h, - &endpoint_h.security_rules, + &endpoint_h, &request_h, &response, McpCaller::frame(&process_name_h), @@ -383,7 +384,7 @@ where let response = policy_blocked_response(request.id.clone(), "request", &request_decision); log_mcp_call_with_policy( Arc::clone(&db), - &endpoint.security_rules, + &endpoint, &dispatch_request, &response, McpCaller::frame(&process_name), @@ -448,7 +449,7 @@ where let policy_fields = McpCallPolicyFields::from(&final_decision); log_mcp_call_with_policy( db_h, - &endpoint_h.security_rules, + &endpoint_h, &dispatch_request, &response, McpCaller::frame(&process_name_h), @@ -639,7 +640,7 @@ struct LoggedMcpEmission { async fn log_mcp_call_with_policy( db: Arc, - security_rules: &Arc>>, + endpoint: &McpEndpointState, req: &JsonRpcRequest, resp: &JsonRpcResponse, caller: McpCaller<'_>, @@ -710,7 +711,7 @@ async fn log_mcp_call_with_policy( if let Some(event_id) = emit_security_write(&db, WriteOp::McpCall(call)).await { // Complete the derived rule emission before replying so an immediate // shutdown cannot lose audit rows for an already-completed call. - let rules = security_rules.read().unwrap().clone(); + let (rules, _) = endpoint.policy_snapshot(); emit_evaluated_security_rules( Arc::clone(&db), event_id.clone(), @@ -824,8 +825,7 @@ fn ensure_mcp_request_identity(mcp: &mut McpSecurityEvent, request_id: Option SecurityEnforcementDecision { - let rules = endpoint.security_rules.read().unwrap().clone(); - let plugin_policy = snapshot_plugin_policy(&endpoint.plugin_policy); + let (rules, plugin_policy) = endpoint.policy_snapshot(); match evaluate_security_boundary(&rules, plugin_policy, event) { Ok(evaluation) => evaluation.enforcement, Err(error) => { diff --git a/crates/capsem-core/src/net/mitm_proxy/mcp_frame/tests.rs b/crates/capsem-core/src/net/mitm_proxy/mcp_frame/tests.rs index 48770c659..1f26cc5c4 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mcp_frame/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mcp_frame/tests.rs @@ -709,7 +709,7 @@ async fn completing_mcp_logging_then_shutting_down_preserves_rule_rows() { let response = ok_response(json!({"content":[{"type":"text","text":"pong"}]})); let logged = log_mcp_call_with_policy( Arc::clone(&db), - &endpoint.security_rules, + &endpoint, &req, &response, McpCaller::frame("codex"), From ee6ea59a3910a01770302c6af4e9aa50f85dc137 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:23:43 +0000 Subject: [PATCH 153/464] feat(proto): bootstrap proxy MCP capability --- crates/capsem-proto/src/lib.rs | 1 + crates/capsem-proto/src/proxy_mcp.rs | 65 ++++++++++++++++++++++ crates/capsem-proto/src/proxy_mcp/tests.rs | 39 +++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 crates/capsem-proto/src/proxy_mcp.rs create mode 100644 crates/capsem-proto/src/proxy_mcp/tests.rs diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 1d2d9338f..e643519f2 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -28,6 +28,7 @@ pub mod poll; pub mod privatelink; pub mod proxy_control; pub mod proxy_credentials; +pub mod proxy_mcp; pub mod proxy_policy; pub mod proxy_private_names; pub mod repeated; diff --git a/crates/capsem-proto/src/proxy_mcp.rs b/crates/capsem-proto/src/proxy_mcp.rs new file mode 100644 index 000000000..ae0017e8b --- /dev/null +++ b/crates/capsem-proto/src/proxy_mcp.rs @@ -0,0 +1,65 @@ +//! Bootstrap metadata for one proxy worker's MCP capability. + +use std::collections::HashSet; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub const PROXY_MCP_VERSION: u16 = 1; +pub const PROXY_MCP_MAX_BUILTIN_SERVERS: usize = 64; +pub const PROXY_MCP_MAX_SERVER_NAME_BYTES: usize = 128; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProxyMcpHello { + pub version: u16, + pub builtin_servers: Vec, +} + +impl ProxyMcpHello { + pub fn new(builtin_servers: Vec) -> Result { + let hello = Self { + version: PROXY_MCP_VERSION, + builtin_servers, + }; + hello.validate()?; + Ok(hello) + } + + pub fn validate(&self) -> Result<(), ProxyMcpError> { + if self.version != PROXY_MCP_VERSION { + return Err(ProxyMcpError::Version { + expected: PROXY_MCP_VERSION, + actual: self.version, + }); + } + if self.builtin_servers.len() > PROXY_MCP_MAX_BUILTIN_SERVERS { + return Err(ProxyMcpError::TooManyBuiltinServers(self.builtin_servers.len())); + } + let mut seen = HashSet::with_capacity(self.builtin_servers.len()); + for name in &self.builtin_servers { + if name.is_empty() || name.len() > PROXY_MCP_MAX_SERVER_NAME_BYTES { + return Err(ProxyMcpError::InvalidServerName); + } + if !seen.insert(name) { + return Err(ProxyMcpError::DuplicateServerName); + } + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +pub enum ProxyMcpError { + #[error("unsupported proxy MCP version {actual}, expected {expected}")] + Version { expected: u16, actual: u16 }, + #[error("proxy MCP capability names {0} builtin servers, exceeding the limit")] + TooManyBuiltinServers(usize), + #[error("proxy MCP capability contains an invalid builtin server name")] + InvalidServerName, + #[error("proxy MCP capability contains a duplicate builtin server name")] + DuplicateServerName, +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/proxy_mcp/tests.rs b/crates/capsem-proto/src/proxy_mcp/tests.rs new file mode 100644 index 000000000..2d14cf493 --- /dev/null +++ b/crates/capsem-proto/src/proxy_mcp/tests.rs @@ -0,0 +1,39 @@ +use super::*; + +#[test] +fn hello_accepts_one_bounded_unique_builtin_set() { + let hello = ProxyMcpHello::new(vec!["capsem".into(), "files".into()]).unwrap(); + assert_eq!(hello.version, PROXY_MCP_VERSION); + assert_eq!(hello.builtin_servers, ["capsem", "files"]); +} + +#[test] +fn hello_rejects_untrusted_shape() { + assert_eq!( + ProxyMcpHello { + version: PROXY_MCP_VERSION + 1, + builtin_servers: Vec::new(), + } + .validate(), + Err(ProxyMcpError::Version { + expected: PROXY_MCP_VERSION, + actual: PROXY_MCP_VERSION + 1, + }) + ); + assert_eq!( + ProxyMcpHello::new(vec!["capsem".into(), "capsem".into()]), + Err(ProxyMcpError::DuplicateServerName) + ); + assert_eq!( + ProxyMcpHello::new(vec![String::new()]), + Err(ProxyMcpError::InvalidServerName) + ); + assert_eq!( + ProxyMcpHello::new(vec!["x".repeat(PROXY_MCP_MAX_SERVER_NAME_BYTES + 1)]), + Err(ProxyMcpError::InvalidServerName) + ); + assert_eq!( + ProxyMcpHello::new(vec!["x".into(); PROXY_MCP_MAX_BUILTIN_SERVERS + 1]), + Err(ProxyMcpError::TooManyBuiltinServers(PROXY_MCP_MAX_BUILTIN_SERVERS + 1)) + ); +} From 59b49ba6e6d20ba66ee6537cb4f89dc699d5772b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:26:10 +0000 Subject: [PATCH 154/464] refactor(core): share MCP aggregator driver --- .../capsem-core/src/mcp/aggregator_driver.rs | 128 ++++++++++++ .../src/mcp/aggregator_driver/tests.rs | 187 ++++++++++++++++++ crates/capsem-core/src/mcp/mod.rs | 1 + 3 files changed, 316 insertions(+) create mode 100644 crates/capsem-core/src/mcp/aggregator_driver.rs create mode 100644 crates/capsem-core/src/mcp/aggregator_driver/tests.rs diff --git a/crates/capsem-core/src/mcp/aggregator_driver.rs b/crates/capsem-core/src/mcp/aggregator_driver.rs new file mode 100644 index 000000000..9c7b1fa1e --- /dev/null +++ b/crates/capsem-core/src/mcp/aggregator_driver.rs @@ -0,0 +1,128 @@ +//! Driver between `AggregatorClient` callers and the aggregator subprocess. +//! +//! Callers hand the driver `(request, oneshot)` pairs over an mpsc channel. +//! The writer task frames each request onto the subprocess stdin and parks +//! the oneshot in the pending map; the reader task frames responses off the +//! subprocess stdout and routes each to its parked oneshot by `id`. +//! +//! The pending map is the only place a caller can get stuck. Two exits keep +//! it honest: when the subprocess stdout closes, every parked oneshot is +//! dropped so its caller fails at once instead of waiting out the endpoint +//! timeout, and the map is closed so later requests fail the same way. And a +//! caller that gave up (its receiver dropped) is pruned on the next insert, so +//! a remote MCP server that never answers cannot grow the map for the life of +//! the VM. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use capsem_proto::mcp_aggregator::{read_frame, write_frame, AggregatorResponse, ClientMessage}; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::sync::{mpsc, oneshot}; +use tracing::{error, info}; + +#[derive(Default)] +struct Pending { + waiters: HashMap>, + closed: bool, +} + +impl Pending { + /// Fail every parked caller and refuse new ones. Dropping a sender wakes + /// its caller with "aggregator response channel dropped". + fn close(&mut self) { + self.closed = true; + self.waiters.clear(); + } + + /// Park a caller unless the subprocess is gone. Returns false, dropping + /// the sender, when the request cannot be answered. + fn park(&mut self, id: u64, tx: oneshot::Sender) -> bool { + if self.closed { + return false; + } + self.waiters.retain(|_, waiter| !waiter.is_closed()); + self.waiters.insert(id, tx); + true + } +} + +/// The callers parked between writing their request and reading its response. +pub struct Inflight(Arc>); + +impl Inflight { + /// Fail every parked caller and refuse new ones. The reader does this on + /// EOF; the child monitor does it on exit, for the case where a grandchild + /// inherited the stdout pipe and EOF never comes. + pub fn close(&self) { + self.0.lock().unwrap_or_else(|e| e.into_inner()).close(); + } + + #[cfg(test)] + pub(crate) fn count(&self) -> usize { + self.0.lock().unwrap_or_else(|e| e.into_inner()).waiters.len() + } +} + +/// Spawn the reader and writer tasks for one aggregator subprocess. +pub fn spawn(mut rx: mpsc::Receiver, mut stdin: W, mut stdout: R) -> Inflight +where + W: AsyncWrite + Unpin + Send + 'static, + R: AsyncRead + Unpin + Send + 'static, +{ + let pending = Arc::new(Mutex::new(Pending::default())); + + let pending_reader = Arc::clone(&pending); + tokio::spawn(async move { + info!("aggregator reader task started"); + loop { + match read_frame::<_, AggregatorResponse>(&mut stdout).await { + Ok(Some(resp)) => { + let waiter = pending_reader + .lock() + .unwrap_or_else(|e| e.into_inner()) + .waiters + .remove(&resp.id); + if let Some(tx) = waiter { + crate::try_send!("aggregator_oneshot", tx.send(resp)); + } + } + Ok(None) => { + info!("aggregator stdout closed (EOF)"); + break; + } + Err(e) => { + error!(error = %e, "failed to read aggregator response frame"); + break; + } + } + } + pending_reader.lock().unwrap_or_else(|e| e.into_inner()).close(); + info!("aggregator reader task ending; in-flight callers failed"); + }); + + let pending_writer = Arc::clone(&pending); + tokio::spawn(async move { + info!("aggregator writer task started"); + while let Some((req, resp_tx)) = rx.recv().await { + if !pending_writer + .lock() + .unwrap_or_else(|e| e.into_inner()) + .park(req.id, resp_tx) + { + break; + } + if let Err(e) = write_frame(&mut stdin, &req).await { + error!(error = %e, "failed to write aggregator request frame"); + pending_writer.lock().unwrap_or_else(|e| e.into_inner()).close(); + break; + } + } + info!("aggregator writer task ending"); + }); + + Inflight(pending) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-core/src/mcp/aggregator_driver/tests.rs b/crates/capsem-core/src/mcp/aggregator_driver/tests.rs new file mode 100644 index 000000000..a50ee152c --- /dev/null +++ b/crates/capsem-core/src/mcp/aggregator_driver/tests.rs @@ -0,0 +1,187 @@ +use std::time::Duration; + +use capsem_proto::mcp_aggregator::{ + read_frame, write_frame, AggregatorClient, AggregatorMethod, AggregatorRequest, AggregatorResponse, + AggregatorResult, +}; +use tokio::io::DuplexStream; + +use super::*; + +/// A fake aggregator: the driver owns one end of each pipe, the test the other. +struct FakeAggregator { + /// The subprocess's stdin, as the subprocess reads it. + stdin: DuplexStream, + /// The subprocess's stdout, as the subprocess writes it. Dropping it is + /// the subprocess exiting. + stdout: Option, + client: AggregatorClient, + inflight: Inflight, +} + +fn spawn_fake() -> FakeAggregator { + let (client, rx) = AggregatorClient::channel(8); + let (driver_stdin, fake_stdin) = tokio::io::duplex(64 * 1024); + let (fake_stdout, driver_stdout) = tokio::io::duplex(64 * 1024); + let inflight = spawn(rx, driver_stdin, driver_stdout); + FakeAggregator { + stdin: fake_stdin, + stdout: Some(fake_stdout), + client, + inflight, + } +} + +impl FakeAggregator { + async fn next_request(&mut self) -> AggregatorRequest { + read_frame(&mut self.stdin) + .await + .expect("frame from driver") + .expect("driver stdin still open") + } + + async fn reply(&mut self, id: u64) { + let resp = AggregatorResponse { + id, + body: AggregatorResult::Ok { ok: true }, + }; + write_frame(self.stdout.as_mut().expect("stdout open"), &resp) + .await + .expect("write response"); + } + + fn exit(&mut self) { + self.stdout.take(); + } +} + +const PROMPT: Duration = Duration::from_secs(2); + +#[tokio::test] +async fn responses_route_back_to_their_caller() { + let mut fake = spawn_fake(); + let client = fake.client.clone(); + let call = tokio::spawn(async move { client.request(AggregatorMethod::ListServers).await }); + + let req = fake.next_request().await; + fake.reply(req.id).await; + + let body = tokio::time::timeout(PROMPT, call) + .await + .expect("prompt") + .unwrap() + .unwrap(); + assert!(matches!(body, AggregatorResult::Ok { ok: true })); +} + +// The aggregator can die with requests in flight: a panic, an OOM kill, or +// a session teardown racing a late tool call. Its stdout closes, the driver's +// reader task ends, and every caller parked in the pending map used to wait +// forever because nothing dropped their oneshots. The guest saw each such +// call hang for the full endpoint timeout instead of failing immediately. + +#[tokio::test] +async fn subprocess_exit_fails_inflight_callers_immediately() { + let mut fake = spawn_fake(); + let client = fake.client.clone(); + let call = tokio::spawn(async move { client.request(AggregatorMethod::ListTools).await }); + + fake.next_request().await; + fake.exit(); + + let result = tokio::time::timeout(PROMPT, call) + .await + .expect("an in-flight call must fail promptly when the aggregator exits") + .unwrap(); + assert!(result.is_err(), "no aggregator can answer this call: {result:?}"); +} + +#[tokio::test] +async fn requests_after_subprocess_exit_fail_immediately() { + let mut fake = spawn_fake(); + fake.exit(); + // Let the reader observe EOF before the next request is issued. + tokio::task::yield_now().await; + + let result = tokio::time::timeout(PROMPT, fake.client.request(AggregatorMethod::ListPrompts)) + .await + .expect("a call issued after the aggregator exited must fail promptly"); + assert!(result.is_err(), "{result:?}"); +} + +// A caller that gives up (the endpoint's per-request timeout) drops its +// receiver, but its sender stayed parked in the pending map until a response +// with that id arrived. A remote MCP server that never answers therefore grew +// the map by one entry per timed-out call for the life of the VM. + +#[tokio::test] +async fn abandoned_callers_are_pruned_from_the_pending_map() { + let mut fake = spawn_fake(); + let client = fake.client.clone(); + let abandoned = tokio::time::timeout( + Duration::from_millis(20), + client.request(AggregatorMethod::CallTool { + name: "slow__tool".into(), + arguments: serde_json::json!({}), + timeout_ms: None, + }), + ) + .await; + assert!(abandoned.is_err(), "the caller gave up before any response arrived"); + fake.next_request().await; + + let client = fake.client.clone(); + let live = tokio::spawn(async move { client.request(AggregatorMethod::ListServers).await }); + let req = fake.next_request().await; + assert_eq!(fake.inflight.count(), 1, "only the live request may remain parked"); + fake.reply(req.id).await; + tokio::time::timeout(PROMPT, live).await.unwrap().unwrap().unwrap(); +} + +// ----------------------------------------------------------------------- +// Misbehaving aggregator output +// ----------------------------------------------------------------------- + +#[tokio::test] +async fn unknown_and_duplicate_response_ids_are_ignored_and_live_calls_still_complete() { + let mut fake = spawn_fake(); + let client = fake.client.clone(); + let call = tokio::spawn(async move { client.request(AggregatorMethod::ListServers).await }); + let req = fake.next_request().await; + + fake.reply(u64::MAX).await; + fake.reply(req.id.wrapping_add(1)).await; + fake.reply(req.id).await; + fake.reply(req.id).await; + + let body = tokio::time::timeout(PROMPT, call) + .await + .expect("prompt") + .unwrap() + .unwrap(); + assert!(matches!(body, AggregatorResult::Ok { ok: true })); + assert_eq!(fake.inflight.count(), 0, "a duplicate reply must not repark anything"); +} + +#[tokio::test] +async fn a_corrupt_frame_from_the_aggregator_fails_callers_instead_of_misrouting() { + use tokio::io::AsyncWriteExt; + let mut fake = spawn_fake(); + let client = fake.client.clone(); + let call = tokio::spawn(async move { client.request(AggregatorMethod::ListTools).await }); + fake.next_request().await; + + let stdout = fake.stdout.as_mut().unwrap(); + stdout.write_all(&(u32::MAX).to_be_bytes()).await.unwrap(); + stdout.write_all(b"garbage").await.unwrap(); + + let result = tokio::time::timeout(PROMPT, call) + .await + .expect("a corrupt stream must not leave the caller waiting") + .unwrap(); + assert!(result.is_err(), "{result:?}"); + let late = tokio::time::timeout(PROMPT, fake.client.request(AggregatorMethod::ListPrompts)) + .await + .expect("prompt"); + assert!(late.is_err(), "the driver is closed after a framing error: {late:?}"); +} diff --git a/crates/capsem-core/src/mcp/mod.rs b/crates/capsem-core/src/mcp/mod.rs index 4754cd42e..e0df51346 100644 --- a/crates/capsem-core/src/mcp/mod.rs +++ b/crates/capsem-core/src/mcp/mod.rs @@ -1,3 +1,4 @@ +pub mod aggregator_driver; pub mod builtin_ledger; pub mod builtin_tools; pub mod policy; From 1ca6bd760fa146b62ba58424446da059c7af1402 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:28:02 +0000 Subject: [PATCH 155/464] refactor(process): use shared MCP driver --- .../capsem-process/src/aggregator_driver.rs | 128 ------------ .../src/aggregator_driver/tests.rs | 187 ------------------ crates/capsem-process/src/main.rs | 3 +- 3 files changed, 1 insertion(+), 317 deletions(-) delete mode 100644 crates/capsem-process/src/aggregator_driver.rs delete mode 100644 crates/capsem-process/src/aggregator_driver/tests.rs diff --git a/crates/capsem-process/src/aggregator_driver.rs b/crates/capsem-process/src/aggregator_driver.rs deleted file mode 100644 index 062cb2903..000000000 --- a/crates/capsem-process/src/aggregator_driver.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! Driver between `AggregatorClient` callers and the aggregator subprocess. -//! -//! Callers hand the driver `(request, oneshot)` pairs over an mpsc channel. -//! The writer task frames each request onto the subprocess stdin and parks -//! the oneshot in the pending map; the reader task frames responses off the -//! subprocess stdout and routes each to its parked oneshot by `id`. -//! -//! The pending map is the only place a caller can get stuck. Two exits keep -//! it honest: when the subprocess stdout closes, every parked oneshot is -//! dropped so its caller fails at once instead of waiting out the endpoint -//! timeout, and the map is closed so later requests fail the same way. And a -//! caller that gave up (its receiver dropped) is pruned on the next insert, so -//! a remote MCP server that never answers cannot grow the map for the life of -//! the VM. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use capsem_proto::mcp_aggregator::{read_frame, write_frame, AggregatorResponse, ClientMessage}; -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::sync::{mpsc, oneshot}; -use tracing::{error, info}; - -#[derive(Default)] -struct Pending { - waiters: HashMap>, - closed: bool, -} - -impl Pending { - /// Fail every parked caller and refuse new ones. Dropping a sender wakes - /// its caller with "aggregator response channel dropped". - fn close(&mut self) { - self.closed = true; - self.waiters.clear(); - } - - /// Park a caller unless the subprocess is gone. Returns false, dropping - /// the sender, when the request cannot be answered. - fn park(&mut self, id: u64, tx: oneshot::Sender) -> bool { - if self.closed { - return false; - } - self.waiters.retain(|_, waiter| !waiter.is_closed()); - self.waiters.insert(id, tx); - true - } -} - -/// The callers parked between writing their request and reading its response. -pub(crate) struct Inflight(Arc>); - -impl Inflight { - /// Fail every parked caller and refuse new ones. The reader does this on - /// EOF; the child monitor does it on exit, for the case where a grandchild - /// inherited the stdout pipe and EOF never comes. - pub(crate) fn close(&self) { - self.0.lock().unwrap_or_else(|e| e.into_inner()).close(); - } - - #[cfg(test)] - pub(crate) fn count(&self) -> usize { - self.0.lock().unwrap_or_else(|e| e.into_inner()).waiters.len() - } -} - -/// Spawn the reader and writer tasks for one aggregator subprocess. -pub(crate) fn spawn(mut rx: mpsc::Receiver, mut stdin: W, mut stdout: R) -> Inflight -where - W: AsyncWrite + Unpin + Send + 'static, - R: AsyncRead + Unpin + Send + 'static, -{ - let pending = Arc::new(Mutex::new(Pending::default())); - - let pending_reader = Arc::clone(&pending); - tokio::spawn(async move { - info!("aggregator reader task started"); - loop { - match read_frame::<_, AggregatorResponse>(&mut stdout).await { - Ok(Some(resp)) => { - let waiter = pending_reader - .lock() - .unwrap_or_else(|e| e.into_inner()) - .waiters - .remove(&resp.id); - if let Some(tx) = waiter { - capsem_core::try_send!("aggregator_oneshot", tx.send(resp)); - } - } - Ok(None) => { - info!("aggregator stdout closed (EOF)"); - break; - } - Err(e) => { - error!(error = %e, "failed to read aggregator response frame"); - break; - } - } - } - pending_reader.lock().unwrap_or_else(|e| e.into_inner()).close(); - info!("aggregator reader task ending; in-flight callers failed"); - }); - - let pending_writer = Arc::clone(&pending); - tokio::spawn(async move { - info!("aggregator writer task started"); - while let Some((req, resp_tx)) = rx.recv().await { - if !pending_writer - .lock() - .unwrap_or_else(|e| e.into_inner()) - .park(req.id, resp_tx) - { - break; - } - if let Err(e) = write_frame(&mut stdin, &req).await { - error!(error = %e, "failed to write aggregator request frame"); - pending_writer.lock().unwrap_or_else(|e| e.into_inner()).close(); - break; - } - } - info!("aggregator writer task ending"); - }); - - Inflight(pending) -} - -#[cfg(test)] -mod tests; diff --git a/crates/capsem-process/src/aggregator_driver/tests.rs b/crates/capsem-process/src/aggregator_driver/tests.rs deleted file mode 100644 index a50ee152c..000000000 --- a/crates/capsem-process/src/aggregator_driver/tests.rs +++ /dev/null @@ -1,187 +0,0 @@ -use std::time::Duration; - -use capsem_proto::mcp_aggregator::{ - read_frame, write_frame, AggregatorClient, AggregatorMethod, AggregatorRequest, AggregatorResponse, - AggregatorResult, -}; -use tokio::io::DuplexStream; - -use super::*; - -/// A fake aggregator: the driver owns one end of each pipe, the test the other. -struct FakeAggregator { - /// The subprocess's stdin, as the subprocess reads it. - stdin: DuplexStream, - /// The subprocess's stdout, as the subprocess writes it. Dropping it is - /// the subprocess exiting. - stdout: Option, - client: AggregatorClient, - inflight: Inflight, -} - -fn spawn_fake() -> FakeAggregator { - let (client, rx) = AggregatorClient::channel(8); - let (driver_stdin, fake_stdin) = tokio::io::duplex(64 * 1024); - let (fake_stdout, driver_stdout) = tokio::io::duplex(64 * 1024); - let inflight = spawn(rx, driver_stdin, driver_stdout); - FakeAggregator { - stdin: fake_stdin, - stdout: Some(fake_stdout), - client, - inflight, - } -} - -impl FakeAggregator { - async fn next_request(&mut self) -> AggregatorRequest { - read_frame(&mut self.stdin) - .await - .expect("frame from driver") - .expect("driver stdin still open") - } - - async fn reply(&mut self, id: u64) { - let resp = AggregatorResponse { - id, - body: AggregatorResult::Ok { ok: true }, - }; - write_frame(self.stdout.as_mut().expect("stdout open"), &resp) - .await - .expect("write response"); - } - - fn exit(&mut self) { - self.stdout.take(); - } -} - -const PROMPT: Duration = Duration::from_secs(2); - -#[tokio::test] -async fn responses_route_back_to_their_caller() { - let mut fake = spawn_fake(); - let client = fake.client.clone(); - let call = tokio::spawn(async move { client.request(AggregatorMethod::ListServers).await }); - - let req = fake.next_request().await; - fake.reply(req.id).await; - - let body = tokio::time::timeout(PROMPT, call) - .await - .expect("prompt") - .unwrap() - .unwrap(); - assert!(matches!(body, AggregatorResult::Ok { ok: true })); -} - -// The aggregator can die with requests in flight: a panic, an OOM kill, or -// a session teardown racing a late tool call. Its stdout closes, the driver's -// reader task ends, and every caller parked in the pending map used to wait -// forever because nothing dropped their oneshots. The guest saw each such -// call hang for the full endpoint timeout instead of failing immediately. - -#[tokio::test] -async fn subprocess_exit_fails_inflight_callers_immediately() { - let mut fake = spawn_fake(); - let client = fake.client.clone(); - let call = tokio::spawn(async move { client.request(AggregatorMethod::ListTools).await }); - - fake.next_request().await; - fake.exit(); - - let result = tokio::time::timeout(PROMPT, call) - .await - .expect("an in-flight call must fail promptly when the aggregator exits") - .unwrap(); - assert!(result.is_err(), "no aggregator can answer this call: {result:?}"); -} - -#[tokio::test] -async fn requests_after_subprocess_exit_fail_immediately() { - let mut fake = spawn_fake(); - fake.exit(); - // Let the reader observe EOF before the next request is issued. - tokio::task::yield_now().await; - - let result = tokio::time::timeout(PROMPT, fake.client.request(AggregatorMethod::ListPrompts)) - .await - .expect("a call issued after the aggregator exited must fail promptly"); - assert!(result.is_err(), "{result:?}"); -} - -// A caller that gives up (the endpoint's per-request timeout) drops its -// receiver, but its sender stayed parked in the pending map until a response -// with that id arrived. A remote MCP server that never answers therefore grew -// the map by one entry per timed-out call for the life of the VM. - -#[tokio::test] -async fn abandoned_callers_are_pruned_from_the_pending_map() { - let mut fake = spawn_fake(); - let client = fake.client.clone(); - let abandoned = tokio::time::timeout( - Duration::from_millis(20), - client.request(AggregatorMethod::CallTool { - name: "slow__tool".into(), - arguments: serde_json::json!({}), - timeout_ms: None, - }), - ) - .await; - assert!(abandoned.is_err(), "the caller gave up before any response arrived"); - fake.next_request().await; - - let client = fake.client.clone(); - let live = tokio::spawn(async move { client.request(AggregatorMethod::ListServers).await }); - let req = fake.next_request().await; - assert_eq!(fake.inflight.count(), 1, "only the live request may remain parked"); - fake.reply(req.id).await; - tokio::time::timeout(PROMPT, live).await.unwrap().unwrap().unwrap(); -} - -// ----------------------------------------------------------------------- -// Misbehaving aggregator output -// ----------------------------------------------------------------------- - -#[tokio::test] -async fn unknown_and_duplicate_response_ids_are_ignored_and_live_calls_still_complete() { - let mut fake = spawn_fake(); - let client = fake.client.clone(); - let call = tokio::spawn(async move { client.request(AggregatorMethod::ListServers).await }); - let req = fake.next_request().await; - - fake.reply(u64::MAX).await; - fake.reply(req.id.wrapping_add(1)).await; - fake.reply(req.id).await; - fake.reply(req.id).await; - - let body = tokio::time::timeout(PROMPT, call) - .await - .expect("prompt") - .unwrap() - .unwrap(); - assert!(matches!(body, AggregatorResult::Ok { ok: true })); - assert_eq!(fake.inflight.count(), 0, "a duplicate reply must not repark anything"); -} - -#[tokio::test] -async fn a_corrupt_frame_from_the_aggregator_fails_callers_instead_of_misrouting() { - use tokio::io::AsyncWriteExt; - let mut fake = spawn_fake(); - let client = fake.client.clone(); - let call = tokio::spawn(async move { client.request(AggregatorMethod::ListTools).await }); - fake.next_request().await; - - let stdout = fake.stdout.as_mut().unwrap(); - stdout.write_all(&(u32::MAX).to_be_bytes()).await.unwrap(); - stdout.write_all(b"garbage").await.unwrap(); - - let result = tokio::time::timeout(PROMPT, call) - .await - .expect("a corrupt stream must not leave the caller waiting") - .unwrap(); - assert!(result.is_err(), "{result:?}"); - let late = tokio::time::timeout(PROMPT, fake.client.request(AggregatorMethod::ListPrompts)) - .await - .expect("prompt"); - assert!(late.is_err(), "the driver is closed after a framing error: {late:?}"); -} diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index f633c8520..cca756fb9 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -1,4 +1,3 @@ -mod aggregator_driver; mod cables; mod helpers; mod ipc; @@ -1070,7 +1069,7 @@ async fn spawn_mcp_aggregator( let defs_vec = servers.to_vec(); write_frame(&mut child_stdin, &defs_vec).await?; - let inflight = aggregator_driver::spawn(rx, child_stdin, child_stdout); + let inflight = capsem_core::mcp::aggregator_driver::spawn(rx, child_stdin, child_stdout); // Monitor child process. tokio::spawn(async move { From b446559599cfba360472f415e04a6c535be7ff52 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:29:43 +0000 Subject: [PATCH 156/464] feat(core): observe MCP driver closure --- .../capsem-core/src/mcp/aggregator_driver.rs | 21 +++++++++++++++---- .../src/mcp/aggregator_driver/tests.rs | 3 +++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/capsem-core/src/mcp/aggregator_driver.rs b/crates/capsem-core/src/mcp/aggregator_driver.rs index 9c7b1fa1e..d24d131ee 100644 --- a/crates/capsem-core/src/mcp/aggregator_driver.rs +++ b/crates/capsem-core/src/mcp/aggregator_driver.rs @@ -48,19 +48,27 @@ impl Pending { } /// The callers parked between writing their request and reading its response. -pub struct Inflight(Arc>); +pub struct Inflight { + pending: Arc>, + stopped: tokio::sync::watch::Sender, +} impl Inflight { /// Fail every parked caller and refuse new ones. The reader does this on /// EOF; the child monitor does it on exit, for the case where a grandchild /// inherited the stdout pipe and EOF never comes. pub fn close(&self) { - self.0.lock().unwrap_or_else(|e| e.into_inner()).close(); + self.pending.lock().unwrap_or_else(|e| e.into_inner()).close(); + self.stopped.send_replace(true); + } + + pub fn stop_receiver(&self) -> tokio::sync::watch::Receiver { + self.stopped.subscribe() } #[cfg(test)] pub(crate) fn count(&self) -> usize { - self.0.lock().unwrap_or_else(|e| e.into_inner()).waiters.len() + self.pending.lock().unwrap_or_else(|e| e.into_inner()).waiters.len() } } @@ -71,8 +79,10 @@ where R: AsyncRead + Unpin + Send + 'static, { let pending = Arc::new(Mutex::new(Pending::default())); + let (stopped, _) = tokio::sync::watch::channel(false); let pending_reader = Arc::clone(&pending); + let stopped_reader = stopped.clone(); tokio::spawn(async move { info!("aggregator reader task started"); loop { @@ -98,10 +108,12 @@ where } } pending_reader.lock().unwrap_or_else(|e| e.into_inner()).close(); + stopped_reader.send_replace(true); info!("aggregator reader task ending; in-flight callers failed"); }); let pending_writer = Arc::clone(&pending); + let stopped_writer = stopped.clone(); tokio::spawn(async move { info!("aggregator writer task started"); while let Some((req, resp_tx)) = rx.recv().await { @@ -118,10 +130,11 @@ where break; } } + stopped_writer.send_replace(true); info!("aggregator writer task ending"); }); - Inflight(pending) + Inflight { pending, stopped } } #[cfg(test)] diff --git a/crates/capsem-core/src/mcp/aggregator_driver/tests.rs b/crates/capsem-core/src/mcp/aggregator_driver/tests.rs index a50ee152c..037175927 100644 --- a/crates/capsem-core/src/mcp/aggregator_driver/tests.rs +++ b/crates/capsem-core/src/mcp/aggregator_driver/tests.rs @@ -87,6 +87,7 @@ async fn subprocess_exit_fails_inflight_callers_immediately() { let call = tokio::spawn(async move { client.request(AggregatorMethod::ListTools).await }); fake.next_request().await; + let mut stopped = fake.inflight.stop_receiver(); fake.exit(); let result = tokio::time::timeout(PROMPT, call) @@ -94,6 +95,8 @@ async fn subprocess_exit_fails_inflight_callers_immediately() { .expect("an in-flight call must fail promptly when the aggregator exits") .unwrap(); assert!(result.is_err(), "no aggregator can answer this call: {result:?}"); + stopped.changed().await.unwrap(); + assert!(*stopped.borrow()); } #[tokio::test] From 38dd398ab698f3d1e69be19f47bcda3f70adf898 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:30:57 +0000 Subject: [PATCH 157/464] feat(proto): preserve proxy MCP runtime limits --- crates/capsem-proto/src/proxy_mcp.rs | 34 +++++++++++++++++++++- crates/capsem-proto/src/proxy_mcp/tests.rs | 22 ++++++++++---- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/crates/capsem-proto/src/proxy_mcp.rs b/crates/capsem-proto/src/proxy_mcp.rs index ae0017e8b..30c2d963d 100644 --- a/crates/capsem-proto/src/proxy_mcp.rs +++ b/crates/capsem-proto/src/proxy_mcp.rs @@ -8,19 +8,35 @@ use thiserror::Error; pub const PROXY_MCP_VERSION: u16 = 1; pub const PROXY_MCP_MAX_BUILTIN_SERVERS: usize = 64; pub const PROXY_MCP_MAX_SERVER_NAME_BYTES: usize = 128; +pub const PROXY_MCP_MAX_INFLIGHT: u16 = 4096; +pub const PROXY_MCP_MAX_TIMEOUT_MS: u64 = 3_600_000; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ProxyMcpHello { pub version: u16, pub builtin_servers: Vec, + pub inflight_cap: u16, + pub default_timeout_ms: u64, + pub tool_call_default_ms: u64, + pub tool_call_ceiling_ms: u64, } impl ProxyMcpHello { - pub fn new(builtin_servers: Vec) -> Result { + pub fn new( + builtin_servers: Vec, + inflight_cap: u16, + default_timeout_ms: u64, + tool_call_default_ms: u64, + tool_call_ceiling_ms: u64, + ) -> Result { let hello = Self { version: PROXY_MCP_VERSION, builtin_servers, + inflight_cap, + default_timeout_ms, + tool_call_default_ms, + tool_call_ceiling_ms, }; hello.validate()?; Ok(hello) @@ -36,6 +52,18 @@ impl ProxyMcpHello { if self.builtin_servers.len() > PROXY_MCP_MAX_BUILTIN_SERVERS { return Err(ProxyMcpError::TooManyBuiltinServers(self.builtin_servers.len())); } + if self.inflight_cap == 0 || self.inflight_cap > PROXY_MCP_MAX_INFLIGHT { + return Err(ProxyMcpError::InvalidInflightCap(self.inflight_cap)); + } + for timeout_ms in [ + self.default_timeout_ms, + self.tool_call_default_ms, + self.tool_call_ceiling_ms, + ] { + if timeout_ms == 0 || timeout_ms > PROXY_MCP_MAX_TIMEOUT_MS { + return Err(ProxyMcpError::InvalidTimeout(timeout_ms)); + } + } let mut seen = HashSet::with_capacity(self.builtin_servers.len()); for name in &self.builtin_servers { if name.is_empty() || name.len() > PROXY_MCP_MAX_SERVER_NAME_BYTES { @@ -59,6 +87,10 @@ pub enum ProxyMcpError { InvalidServerName, #[error("proxy MCP capability contains a duplicate builtin server name")] DuplicateServerName, + #[error("proxy MCP in-flight cap {0} is outside the supported range")] + InvalidInflightCap(u16), + #[error("proxy MCP timeout {0}ms is outside the supported range")] + InvalidTimeout(u64), } #[cfg(test)] diff --git a/crates/capsem-proto/src/proxy_mcp/tests.rs b/crates/capsem-proto/src/proxy_mcp/tests.rs index 2d14cf493..476682de5 100644 --- a/crates/capsem-proto/src/proxy_mcp/tests.rs +++ b/crates/capsem-proto/src/proxy_mcp/tests.rs @@ -2,7 +2,7 @@ use super::*; #[test] fn hello_accepts_one_bounded_unique_builtin_set() { - let hello = ProxyMcpHello::new(vec!["capsem".into(), "files".into()]).unwrap(); + let hello = ProxyMcpHello::new(vec!["capsem".into(), "files".into()], 32, 30_000, 60_000, 120_000).unwrap(); assert_eq!(hello.version, PROXY_MCP_VERSION); assert_eq!(hello.builtin_servers, ["capsem", "files"]); } @@ -13,6 +13,10 @@ fn hello_rejects_untrusted_shape() { ProxyMcpHello { version: PROXY_MCP_VERSION + 1, builtin_servers: Vec::new(), + inflight_cap: 1, + default_timeout_ms: 1, + tool_call_default_ms: 1, + tool_call_ceiling_ms: 1, } .validate(), Err(ProxyMcpError::Version { @@ -21,19 +25,27 @@ fn hello_rejects_untrusted_shape() { }) ); assert_eq!( - ProxyMcpHello::new(vec!["capsem".into(), "capsem".into()]), + ProxyMcpHello::new(vec!["capsem".into(), "capsem".into()], 1, 1, 1, 1), Err(ProxyMcpError::DuplicateServerName) ); assert_eq!( - ProxyMcpHello::new(vec![String::new()]), + ProxyMcpHello::new(vec![String::new()], 1, 1, 1, 1), Err(ProxyMcpError::InvalidServerName) ); assert_eq!( - ProxyMcpHello::new(vec!["x".repeat(PROXY_MCP_MAX_SERVER_NAME_BYTES + 1)]), + ProxyMcpHello::new(vec!["x".repeat(PROXY_MCP_MAX_SERVER_NAME_BYTES + 1)], 1, 1, 1, 1,), Err(ProxyMcpError::InvalidServerName) ); assert_eq!( - ProxyMcpHello::new(vec!["x".into(); PROXY_MCP_MAX_BUILTIN_SERVERS + 1]), + ProxyMcpHello::new(vec!["x".into(); PROXY_MCP_MAX_BUILTIN_SERVERS + 1], 1, 1, 1, 1,), Err(ProxyMcpError::TooManyBuiltinServers(PROXY_MCP_MAX_BUILTIN_SERVERS + 1)) ); + assert_eq!( + ProxyMcpHello::new(Vec::new(), 0, 1, 1, 1), + Err(ProxyMcpError::InvalidInflightCap(0)) + ); + assert_eq!( + ProxyMcpHello::new(Vec::new(), 1, 0, 1, 1), + Err(ProxyMcpError::InvalidTimeout(0)) + ); } From 2269837f26b08949f74734a2fb1cc0c34f76aa88 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:33:29 +0000 Subject: [PATCH 158/464] feat(proxy): consume MCP capability --- crates/capsem-proxy/src/main.rs | 45 ++++++++++++++++++++- crates/capsem-proxy/src/mcp_client.rs | 34 ++++++++++++++++ crates/capsem-proxy/src/mcp_client/tests.rs | 40 ++++++++++++++++++ crates/capsem-proxy/src/tests.rs | 7 ++++ crates/capsem-proxy/tests/subprocess.rs | 1 - 5 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 crates/capsem-proxy/src/mcp_client.rs create mode 100644 crates/capsem-proxy/src/mcp_client/tests.rs diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 532fab7c6..e8dbe63e7 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -27,6 +27,7 @@ use clap::Parser; use tokio::task::{JoinError, JoinSet}; mod credential_client; +mod mcp_client; mod private_names_client; const CONTROL_QUEUE_CAPACITY: usize = 16; @@ -266,6 +267,17 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result let reason = closed.borrow().unwrap_or(ProxyChannelCloseReason::Disconnected); (capability, grant_id, reason) }); + } else if capability == ProxyCapability::Mcp { + let (client, hello, mut closed) = + mcp_client::start(stream).await.context("open proxy MCP capability")?; + state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .attach_mcp(client, hello); + tasks.spawn(async move { + while !*closed.borrow() && closed.changed().await.is_ok() {} + (capability, grant_id, ProxyChannelCloseReason::Disconnected) + }); } else if capability == ProxyCapability::HttpTraffic { let config = state .lock() @@ -358,6 +370,10 @@ struct ProxyRuntimeState { private_names: Option>, dns_upstreams: Vec, mcp: capsem_core::mcp::policy::McpConfig, + mcp_client: Option<( + capsem_proto::mcp_aggregator::AggregatorClient, + capsem_proto::proxy_mcp::ProxyMcpHello, + )>, http_config: Option>, dns_runtime: Option>, } @@ -374,6 +390,7 @@ impl Default for ProxyRuntimeState { private_names: None, dns_upstreams: Vec::new(), mcp: capsem_core::mcp::policy::McpConfig::default(), + mcp_client: None, http_config: None, dns_runtime: None, } @@ -438,6 +455,15 @@ impl ProxyRuntimeState { self.dns_runtime = None; } + fn attach_mcp( + &mut self, + client: capsem_proto::mcp_aggregator::AggregatorClient, + hello: capsem_proto::proxy_mcp::ProxyMcpHello, + ) { + self.mcp_client = Some((client, hello)); + self.http_config = None; + } + fn traffic_ready(&mut self, capability: ProxyCapability) -> Result { match capability { ProxyCapability::HttpTraffic => Ok(self.http_config()?.is_some()), @@ -450,7 +476,9 @@ impl ProxyRuntimeState { if let Some(config) = &self.http_config { return Ok(Some(Arc::clone(config))); } - let (Some(engine), Some(db), Some(upstream_grants)) = (&self.engine, &self.db, &self.upstream_grants) else { + let (Some(engine), Some(db), Some(upstream_grants), Some((mcp_client, mcp_hello))) = + (&self.engine, &self.db, &self.upstream_grants, &self.mcp_client) + else { return Ok(None); }; if !self.credentials_attached { @@ -468,6 +496,19 @@ impl ProxyRuntimeState { }); let pipeline = capsem_core::net::mitm_proxy::make_production_pipeline(Arc::clone(&telemetry)); let upstream_grants: Arc = upstream_grants.clone(); + let mcp_endpoint = Arc::new( + capsem_core::net::mitm_proxy::McpEndpointState::with_proxy_policy( + mcp_client.clone(), + engine.policy().clone(), + Arc::new(tokio::sync::Semaphore::new(usize::from(mcp_hello.inflight_cap))), + capsem_core::net::mitm_proxy::McpTimeouts { + default_timeout: Duration::from_millis(mcp_hello.default_timeout_ms), + tool_call_default: Duration::from_millis(mcp_hello.tool_call_default_ms), + tool_call_ceiling: Duration::from_millis(mcp_hello.tool_call_ceiling_ms), + }, + ) + .with_builtin_ledger(Arc::clone(db), mcp_hello.builtin_servers.iter().cloned().collect()), + ); let config = Arc::new(capsem_core::net::mitm_proxy::MitmProxyConfig { ca: Arc::clone(&ca), server_tls: capsem_core::net::mitm_proxy::make_server_tls_config(&ca), @@ -476,7 +517,7 @@ impl ProxyRuntimeState { upstream_tls: capsem_core::net::mitm_proxy::make_upstream_tls_config(), telemetry, pipeline, - mcp_endpoint: None, + mcp_endpoint: Some(mcp_endpoint), upstream_resolver: capsem_core::net::upstream_address::UpstreamResolver::disabled(), upstream_grants: Some(upstream_grants), }); diff --git a/crates/capsem-proxy/src/mcp_client.rs b/crates/capsem-proxy/src/mcp_client.rs new file mode 100644 index 000000000..f6883cd3f --- /dev/null +++ b/crates/capsem-proxy/src/mcp_client.rs @@ -0,0 +1,34 @@ +use std::os::fd::AsFd as _; +use std::os::unix::net::UnixStream; +use std::time::Duration; + +use anyhow::{Context, Result}; +use capsem_proto::mcp_aggregator::AggregatorClient; +use capsem_proto::proxy_mcp::ProxyMcpHello; + +const HELLO_TIMEOUT: Duration = Duration::from_secs(5); + +pub(crate) async fn start( + socket: UnixStream, +) -> Result<(AggregatorClient, ProxyMcpHello, tokio::sync::watch::Receiver)> { + capsem_foundation::unix::fd::set_nonblocking(socket.as_fd(), true)?; + let reader = tokio::net::UnixStream::from_std(socket.try_clone()?)?; + let writer = tokio::net::UnixStream::from_std(socket)?; + let mut reader = reader; + let hello = tokio::time::timeout( + HELLO_TIMEOUT, + capsem_proto::mcp_aggregator::read_frame::<_, ProxyMcpHello>(&mut reader), + ) + .await + .context("proxy MCP hello timed out")?? + .context("proxy MCP capability closed before hello")?; + hello.validate()?; + + let (client, requests) = AggregatorClient::channel(64); + let driver = capsem_core::mcp::aggregator_driver::spawn(requests, writer, reader); + let stopped = driver.stop_receiver(); + Ok((client, hello, stopped)) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proxy/src/mcp_client/tests.rs b/crates/capsem-proxy/src/mcp_client/tests.rs new file mode 100644 index 000000000..2e8f148bd --- /dev/null +++ b/crates/capsem-proxy/src/mcp_client/tests.rs @@ -0,0 +1,40 @@ +use std::os::fd::AsFd as _; +use std::os::unix::net::UnixStream; + +use capsem_proto::mcp_aggregator::{ + read_frame, write_frame, AggregatorMethod, AggregatorRequest, AggregatorResponse, AggregatorResult, +}; +use capsem_proto::proxy_mcp::ProxyMcpHello; + +#[tokio::test] +async fn authenticated_channel_routes_correlated_calls_and_reports_disconnect() { + let (worker, broker) = UnixStream::pair().unwrap(); + capsem_foundation::unix::fd::set_nonblocking(broker.as_fd(), true).unwrap(); + let mut broker_read = tokio::net::UnixStream::from_std(broker.try_clone().unwrap()).unwrap(); + let mut broker_write = tokio::net::UnixStream::from_std(broker).unwrap(); + let hello = ProxyMcpHello::new(vec!["local".into()], 7, 2_000, 3_000, 4_000).unwrap(); + let broker_task = tokio::spawn(async move { + write_frame(&mut broker_write, &hello).await.unwrap(); + let request = read_frame::<_, AggregatorRequest>(&mut broker_read) + .await + .unwrap() + .unwrap(); + assert!(matches!(request.method, AggregatorMethod::ListTools)); + write_frame( + &mut broker_write, + &AggregatorResponse { + id: request.id, + body: AggregatorResult::Tools { tools: Vec::new() }, + }, + ) + .await + .unwrap(); + }); + + let (client, hello, mut stopped) = super::start(worker).await.unwrap(); + assert_eq!(hello.builtin_servers, ["local"]); + assert!(client.list_tools().await.unwrap().is_empty()); + broker_task.await.unwrap(); + stopped.changed().await.unwrap(); + assert!(*stopped.borrow()); +} diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index c621f1c65..5758dee53 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -59,8 +59,15 @@ match = 'http.host == "worker.example"' state.attach_upstream(Arc::new( capsem_core::net::upstream_grant::UpstreamGrantClient::start(client).unwrap(), )); + assert!(state.http_config().unwrap().is_none()); + let (mcp, _requests) = capsem_proto::mcp_aggregator::AggregatorClient::channel(1); + state.attach_mcp( + mcp, + capsem_proto::proxy_mcp::ProxyMcpHello::new(Vec::new(), 1, 1, 1, 1).unwrap(), + ); let config = state.http_config().unwrap().expect("complete capability set"); assert!(config.upstream_grants.is_some()); + assert!(config.mcp_endpoint.is_some()); assert_eq!( config.engine.policy().snapshot().digest(), state.engine.as_ref().unwrap().policy().snapshot().digest() diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs index 0771aa3f1..72fa0073f 100644 --- a/crates/capsem-proxy/tests/subprocess.rs +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -130,7 +130,6 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { ProxyCapability::Upstream, ProxyCapability::Credential, ProxyCapability::PrivateNames, - ProxyCapability::Mcp, ProxyCapability::Telemetry, ProxyCapability::Policy, ] From 98e3b3b08596186894f65de5fec0bc271eec6b24 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:34:54 +0000 Subject: [PATCH 159/464] feat(proto): hand off proxy MCP channels --- crates/capsem-proto/src/upstream_grant.rs | 40 +++++++++++++++++-- .../capsem-proto/src/upstream_grant/tests.rs | 10 +++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index 8287cc02f..6a1642293 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -22,7 +22,7 @@ pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; pub const MAX_GUEST_SHARE_PATH_BYTES: usize = 4096; const MAGIC: [u8; 2] = *b"UG"; -const VERSION: u8 = 3; +const VERSION: u8 = 4; const MAGIC_RANGE: std::ops::Range = 0..2; const VERSION_OFFSET: usize = 2; const KIND_OFFSET: usize = 3; @@ -46,12 +46,14 @@ const RELEASE: u8 = 5; const SET_GUEST_MODE: u8 = 6; const OPEN_LEDGER: u8 = 7; const ATTACH_PROXY_TRAFFIC: u8 = 8; +const ATTACH_PROXY_MCP: u8 = 9; const TCP_RESOLVED: u8 = 101; const DESCRIPTOR_GRANTED: u8 = 102; const DENIED: u8 = 103; const GUEST_MODE_SET: u8 = 104; const LEDGER_GRANTED: u8 = 105; const PROXY_TRAFFIC_ADOPTED: u8 = 106; +const PROXY_MCP_ADOPTED: u8 = 107; /// Application protocol spoken over a granted TCP stream. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -200,6 +202,9 @@ pub enum UpstreamGrantRequest { request_id: u64, service: ProxyTrafficService, }, + AttachProxyMcp { + request_id: u64, + }, } /// Coordinator-to-worker record. Successful descriptor grants carry one fd. @@ -232,13 +237,16 @@ pub enum UpstreamGrantResponse { ProxyTrafficAdopted { request_id: u64, }, + ProxyMcpAdopted { + request_id: u64, + }, } impl UpstreamGrantRequest { /// Requests never carry descriptors from the untrusted worker. pub const fn expected_descriptor_count(&self) -> usize { match self { - Self::AttachProxyTraffic { .. } => 1, + Self::AttachProxyTraffic { .. } | Self::AttachProxyMcp { .. } => 1, _ => 0, } } @@ -252,7 +260,8 @@ impl UpstreamGrantResponse { Self::TcpResolved { .. } | Self::Denied { .. } | Self::GuestModeSet { .. } - | Self::ProxyTrafficAdopted { .. } => 0, + | Self::ProxyTrafficAdopted { .. } + | Self::ProxyMcpAdopted { .. } => 0, } } } @@ -333,6 +342,11 @@ pub fn encode_upstream_grant_request(request: &UpstreamGrantRequest) -> Result<[ put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); put_u16(&mut frame, DETAIL_RANGE, service.code()); } + UpstreamGrantRequest::AttachProxyMcp { request_id } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = ATTACH_PROXY_MCP; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + } } Ok(frame) } @@ -434,6 +448,15 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> service: ProxyTrafficService::decode(detail)?, }) } + ATTACH_PROXY_MCP => { + require_nonzero("request id", request_id)?; + if resource_id != 0 { + bail!("proxy MCP request carries a resource"); + } + require_empty_fields(port, detail, &name)?; + require_empty_path(&relative_path)?; + Ok(UpstreamGrantRequest::AttachProxyMcp { request_id }) + } kind => bail!("invalid upstream request kind {kind}"), } } @@ -498,6 +521,11 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul frame[KIND_OFFSET] = PROXY_TRAFFIC_ADOPTED; put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); } + UpstreamGrantResponse::ProxyMcpAdopted { request_id } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = PROXY_MCP_ADOPTED; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + } } Ok(frame) } @@ -583,6 +611,12 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - } Ok(UpstreamGrantResponse::ProxyTrafficAdopted { request_id }) } + PROXY_MCP_ADOPTED => { + if resource_id != 0 || detail != 0 || !name.is_empty() || policy_digest.is_some() { + bail!("proxy MCP adoption carries unrelated fields"); + } + Ok(UpstreamGrantResponse::ProxyMcpAdopted { request_id }) + } kind => bail!("invalid upstream response kind {kind}"), } } diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index 9d2adb87d..976cdefa5 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -45,6 +45,7 @@ fn every_request_round_trips_exactly() { request_id: 7, service: ProxyTrafficService::Dns, }, + UpstreamGrantRequest::AttachProxyMcp { request_id: 8 }, UpstreamGrantRequest::Adopted { grant_id: 51 }, UpstreamGrantRequest::Release { resource_id: 61 }, UpstreamGrantRequest::SetGuestMode { @@ -103,6 +104,7 @@ fn every_response_round_trips_exactly() { grant: LedgerChannelGrant::new(LedgerGeneration::new([0x5a; 16]), 16, LedgerClientRole::VmOwner).unwrap(), }, UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 9 }, + UpstreamGrantResponse::ProxyMcpAdopted { request_id: 10 }, ] { response_roundtrip(response); } @@ -160,10 +162,18 @@ fn descriptor_expectations_are_explicit() { .expected_descriptor_count(), 1 ); + assert_eq!( + UpstreamGrantRequest::AttachProxyMcp { request_id: 10 }.expected_descriptor_count(), + 1 + ); assert_eq!( UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 9 }.expected_descriptor_count(), 0 ); + assert_eq!( + UpstreamGrantResponse::ProxyMcpAdopted { request_id: 10 }.expected_descriptor_count(), + 0 + ); assert_eq!( UpstreamGrantResponse::LedgerGranted { request_id: 2, From 03eea143e35f09e10619b8f416ecb99e001dcec5 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:37:15 +0000 Subject: [PATCH 160/464] feat(core): surrender proxy MCP capability --- crates/capsem-core/src/net/upstream_grant.rs | 48 +++++++++++++++++++ .../src/net/upstream_grant/tests.rs | 30 ++++++++++++ 2 files changed, 78 insertions(+) diff --git a/crates/capsem-core/src/net/upstream_grant.rs b/crates/capsem-core/src/net/upstream_grant.rs index 743e34370..c69f71101 100644 --- a/crates/capsem-core/src/net/upstream_grant.rs +++ b/crates/capsem-core/src/net/upstream_grant.rs @@ -29,6 +29,10 @@ type WireSender = DescriptorSender; enum Command { + AttachProxyMcp { + descriptor: OwnedFd, + reply: oneshot::Sender>, + }, AttachProxyTraffic { service: ProxyTrafficService, descriptor: OwnedFd, @@ -123,6 +127,18 @@ impl UpstreamGrantClient { .map_err(|_| anyhow!("upstream grant channel stopped"))? .map_err(anyhow::Error::msg) } + + pub async fn attach_proxy_mcp(&self, descriptor: OwnedFd) -> Result<()> { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::AttachProxyMcp { descriptor, reply }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + } } impl crate::GuestMetadataAuthority for UpstreamGrantClient { @@ -246,6 +262,9 @@ async fn dispatch( command: Command, ) -> Result<(), String> { match command { + Command::AttachProxyMcp { descriptor, reply } => { + finish(reply, attach_proxy_mcp(sender, receiver, request_id, descriptor).await) + } Command::AttachProxyTraffic { service, descriptor, @@ -302,6 +321,35 @@ async fn dispatch( } } +async fn attach_proxy_mcp( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, + descriptor: OwnedFd, +) -> Result, String> { + let bytes = encode_upstream_grant_request(&UpstreamGrantRequest::AttachProxyMcp { request_id }) + .map_err(|error| format!("encode proxy MCP handoff: {error:#}"))?; + tokio::time::timeout(WIRE_TIMEOUT, sender.send(&bytes, &[descriptor.as_raw_fd()])) + .await + .map_err(|_| "send proxy MCP handoff timed out".to_string())? + .map_err(|error| format!("send proxy MCP handoff: {error}"))?; + drop(descriptor); + let (response, fds) = receive_response(receiver).await?; + debug_assert!(fds.is_empty()); + match response { + UpstreamGrantResponse::ProxyMcpAdopted { + request_id: response_id, + } if response_id == request_id => Ok(Ok(())), + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("proxy MCP handoff denied: {reason:?}"))), + response => Err(format!( + "unexpected proxy MCP response for request {request_id}: {response:?}" + )), + } +} + async fn attach_proxy_traffic( sender: &WireSender, receiver: &WireReceiver, diff --git a/crates/capsem-core/src/net/upstream_grant/tests.rs b/crates/capsem-core/src/net/upstream_grant/tests.rs index 3cecb83be..d5973c01a 100644 --- a/crates/capsem-core/src/net/upstream_grant/tests.rs +++ b/crates/capsem-core/src/net/upstream_grant/tests.rs @@ -88,6 +88,36 @@ async fn proxy_traffic_descriptor_is_surrendered_before_adoption_returns() { broker_task.await.unwrap(); } +#[tokio::test] +async fn proxy_mcp_descriptor_is_surrendered_before_adoption_returns() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let (capability, mut proxy) = UnixStream::pair().unwrap(); + let broker_task = tokio::spawn(async move { + let frame = requests.recv().await.unwrap(); + assert_eq!(frame.fds.len(), 1); + assert_eq!( + decode_upstream_grant_request(&frame.bytes).unwrap(), + UpstreamGrantRequest::AttachProxyMcp { request_id: 1 } + ); + let mut adopted = UnixStream::from(frame.fds.into_iter().next().unwrap()); + adopted.write_all(b"mcp").unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::ProxyMcpAdopted { request_id: 1 }, + None, + ) + .await; + }); + + client.attach_proxy_mcp(capability.into()).await.unwrap(); + let mut proof = [0; 3]; + proxy.read_exact(&mut proof).unwrap(); + assert_eq!(&proof, b"mcp"); + broker_task.await.unwrap(); +} + #[tokio::test] async fn ledger_denial_and_missing_descriptor_fail_closed() { let (broker, worker) = UnixStream::pair().unwrap(); From 71a43ae82bfbddb94f0cec71410a2ea567603302 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:40:16 +0000 Subject: [PATCH 161/464] feat(service): broker proxy MCP capability --- crates/capsem-service/src/upstream_broker.rs | 23 +++++++++++++++++++ .../src/upstream_broker/tests.rs | 6 +++++ 2 files changed, 29 insertions(+) diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 33ace1f83..748e4ee2f 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -615,6 +615,29 @@ async fn run( } } } + UpstreamGrantRequest::AttachProxyMcp { request_id } => { + let Some(proxy) = proxy.as_ref() else { + send_denied(&responses, request_id, UpstreamGrantDenial::NotConfigured).await?; + continue; + }; + let descriptor = frame.fds.into_iter().next().expect("one validated MCP descriptor"); + match proxy + .grant( + capsem_proto::proxy_control::ProxyCapability::Mcp, + UnixStream::from(descriptor), + ) + .await + { + Ok(()) => { + send_response(&responses, &UpstreamGrantResponse::ProxyMcpAdopted { request_id }, None) + .await?; + } + Err(error) => { + warn!(%error, "proxy worker refused MCP descriptor"); + send_denied(&responses, request_id, UpstreamGrantDenial::Revoked).await?; + } + } + } } } } diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index f1c9fe2ad..a9e40cb29 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -85,6 +85,12 @@ async fn typed_guest_traffic_is_adopted_by_the_registered_proxy_generation() { .await; assert_eq!(response, UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 1 }); + let (mcp, _mcp_peer) = UnixStream::pair().unwrap(); + let response = client + .request_with_descriptor(&UpstreamGrantRequest::AttachProxyMcp { request_id: 2 }, &mcp) + .await; + assert_eq!(response, UpstreamGrantResponse::ProxyMcpAdopted { request_id: 2 }); + authority.revoke(); assert!(task.await.unwrap().unwrap_err().contains("revoked")); } From 853ac8a70d2b5acfd41e0f428890c4d039e1cf7d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:44:11 +0000 Subject: [PATCH 162/464] feat(process): grant scoped MCP to proxy --- crates/capsem-process/src/main.rs | 44 +++++-- crates/capsem-process/src/proxy_mcp.rs | 117 +++++++++++++++++++ crates/capsem-process/src/proxy_mcp/tests.rs | 105 +++++++++++++++++ 3 files changed, 259 insertions(+), 7 deletions(-) create mode 100644 crates/capsem-process/src/proxy_mcp.rs create mode 100644 crates/capsem-process/src/proxy_mcp/tests.rs diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index cca756fb9..63057314a 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -5,6 +5,7 @@ mod job_store; mod mcp_runtime; mod metric_export; mod private_seats; +mod proxy_mcp; mod runtime_config; mod terminal; mod vsock; @@ -37,6 +38,7 @@ pub(crate) struct Shutdown { publisher: Option>, db: Option>, fs_monitor: Option, + proxy_mcp: Option>, } impl Shutdown { @@ -58,6 +60,10 @@ pub(crate) async fn drain_background_owners(shutdown: &Arc>) { // stop the run loop while the first caller is still draining its owners. let mut guard = shutdown.lock().await; let mut owned = std::mem::take(&mut *guard); + if let Some(proxy_mcp) = owned.proxy_mcp.take() { + proxy_mcp.abort(); + let _ = proxy_mcp.await; + } if let Some(publisher) = owned.publisher.take() { publisher.shutdown().await; } @@ -791,28 +797,52 @@ async fn run_async_main_loop( let inflight_cap = capsem_core::mcp::resolve_inflight_cap(); info!(inflight_cap, "MITM MCP endpoint in-flight handler cap"); let mcp_inflight = Arc::new(tokio::sync::Semaphore::new(inflight_cap)); + let mcp_timeouts = capsem_core::net::mitm_proxy::McpTimeouts::from_env(); + let builtin_servers = capsem_core::mcp::builtin_server_names(&mcp_servers); + let scoped_tools: Arc = Arc::new(GuestExposureTools::new( + Arc::clone(&job_store.publisher), + ctrl_tx.clone(), + )); let mcp_endpoint = Arc::new( capsem_core::net::mitm_proxy::McpEndpointState::new( aggregator_client.clone(), Arc::clone(&security_rules), Arc::clone(&plugin_policy), Arc::clone(&mcp_inflight), - capsem_core::net::mitm_proxy::McpTimeouts::from_env(), + mcp_timeouts.clone(), ) - .with_builtin_ledger(Arc::clone(&db), capsem_core::mcp::builtin_server_names(&mcp_servers)) - .with_scoped_tools(Arc::new(GuestExposureTools::new( - Arc::clone(&job_store.publisher), - ctrl_tx.clone(), - ))), + .with_builtin_ledger(Arc::clone(&db), builtin_servers.clone()) + .with_scoped_tools(Arc::clone(&scoped_tools)), ); let mcp_runtime = Arc::new(McpRuntime { - aggregator: aggregator_client, + aggregator: aggregator_client.clone(), endpoint: Arc::clone(&mcp_endpoint), db: Arc::clone(&db), security_rules: Arc::clone(&security_rules), plugin_policy: Arc::clone(&plugin_policy), proxy_policy: proxy_policy.clone(), }); + let (proxy_mcp, owner_mcp) = std::os::unix::net::UnixStream::pair()?; + let proxy_mcp_task = tokio::spawn(async move { + if let Err(error) = proxy_mcp::serve( + owner_mcp, + aggregator_client, + scoped_tools, + builtin_servers, + inflight_cap, + mcp_timeouts, + ) + .await + { + tracing::warn!(%error, "proxy MCP capability stopped"); + } + }); + if let Err(error) = upstream_grants.attach_proxy_mcp(proxy_mcp.into()).await { + proxy_mcp_task.abort(); + let _ = proxy_mcp_task.await; + return Err(error.context("grant proxy MCP capability")); + } + shutdown.lock().await.proxy_mcp = Some(proxy_mcp_task); let telemetry_deps = Arc::new(capsem_core::net::mitm_proxy::telemetry_hook::TelemetryDeps { db: Arc::clone(&db), diff --git a/crates/capsem-process/src/proxy_mcp.rs b/crates/capsem-process/src/proxy_mcp.rs new file mode 100644 index 000000000..fdb1336d3 --- /dev/null +++ b/crates/capsem-process/src/proxy_mcp.rs @@ -0,0 +1,117 @@ +use std::collections::{BTreeSet, HashSet}; +use std::os::fd::AsFd as _; +use std::os::unix::net::UnixStream; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use capsem_core::net::mitm_proxy::{McpTimeouts, ScopedMcpTools}; +use capsem_proto::mcp_aggregator::{ + AggregatorClient, AggregatorMethod, AggregatorRequest, AggregatorResponse, AggregatorResult, +}; +use capsem_proto::proxy_mcp::ProxyMcpHello; + +const RESPONSE_CAPACITY: usize = 64; + +pub(crate) async fn serve( + socket: UnixStream, + aggregator: AggregatorClient, + scoped_tools: Arc, + builtin_servers: BTreeSet, + inflight_cap: usize, + timeouts: McpTimeouts, +) -> Result<()> { + capsem_foundation::unix::fd::set_nonblocking(socket.as_fd(), true)?; + let mut reader = tokio::net::UnixStream::from_std(socket.try_clone()?)?; + let mut writer = tokio::net::UnixStream::from_std(socket)?; + let hello = ProxyMcpHello::new( + builtin_servers.into_iter().collect(), + u16::try_from(inflight_cap).context("proxy MCP in-flight cap exceeds its wire field")?, + timeout_ms(timeouts.default_timeout)?, + timeout_ms(timeouts.tool_call_default)?, + timeout_ms(timeouts.tool_call_ceiling)?, + )?; + capsem_proto::mcp_aggregator::write_frame(&mut writer, &hello).await?; + + let scoped_names: HashSet = scoped_tools + .definitions() + .into_iter() + .map(|definition| definition.namespaced_name) + .collect(); + let (responses, mut response_rx) = tokio::sync::mpsc::channel::(RESPONSE_CAPACITY); + let writer_task = tokio::spawn(async move { + while let Some(response) = response_rx.recv().await { + if capsem_proto::mcp_aggregator::write_frame(&mut writer, &response) + .await + .is_err() + { + break; + } + } + }); + let permits = Arc::new(tokio::sync::Semaphore::new(inflight_cap)); + let mut requests = tokio::task::JoinSet::new(); + + let result = loop { + let request = match capsem_proto::mcp_aggregator::read_frame::<_, AggregatorRequest>(&mut reader).await { + Ok(Some(request)) => request, + Ok(None) => break Ok(()), + Err(error) => break Err(error.context("read proxy MCP request")), + }; + let permit = Arc::clone(&permits).acquire_owned().await?; + let aggregator = aggregator.clone(); + let scoped_tools = Arc::clone(&scoped_tools); + let scoped_names = scoped_names.clone(); + let responses = responses.clone(); + requests.spawn(async move { + let _permit = permit; + let body = dispatch(request.method, aggregator, scoped_tools, &scoped_names).await; + let _ = responses.send(AggregatorResponse { id: request.id, body }).await; + }); + }; + + requests.abort_all(); + while requests.join_next().await.is_some() {} + drop(responses); + let _ = writer_task.await; + result +} + +async fn dispatch( + method: AggregatorMethod, + aggregator: AggregatorClient, + scoped_tools: Arc, + scoped_names: &HashSet, +) -> AggregatorResult { + match method { + AggregatorMethod::ListTools => match aggregator.list_tools().await { + Ok(mut tools) => { + tools.extend(scoped_tools.definitions()); + AggregatorResult::Tools { tools } + } + Err(error) => AggregatorResult::Error { + error: format!("{error:#}"), + }, + }, + AggregatorMethod::CallTool { + name, + arguments, + timeout_ms: _, + } if scoped_names.contains(&name) => match scoped_tools.call_tool(&name, arguments).await { + Ok(result) => AggregatorResult::CallResult { result }, + Err(error) => AggregatorResult::Error { error }, + }, + method => aggregator + .request(method) + .await + .unwrap_or_else(|error| AggregatorResult::Error { + error: format!("{error:#}"), + }), + } +} + +fn timeout_ms(timeout: std::time::Duration) -> Result { + u64::try_from(timeout.as_millis()).context("proxy MCP timeout exceeds its wire field") +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-process/src/proxy_mcp/tests.rs b/crates/capsem-process/src/proxy_mcp/tests.rs new file mode 100644 index 000000000..7e2679d8c --- /dev/null +++ b/crates/capsem-process/src/proxy_mcp/tests.rs @@ -0,0 +1,105 @@ +use std::collections::BTreeSet; +use std::os::fd::AsFd as _; +use std::os::unix::net::UnixStream; +use std::sync::Arc; + +use capsem_core::net::mitm_proxy::ScopedMcpTools; +use capsem_proto::mcp_aggregator::{ + read_frame, write_frame, AggregatorClient, AggregatorMethod, AggregatorRequest, AggregatorResponse, + AggregatorResult, +}; +use capsem_proto::mcp_contracts::McpToolDef; +use capsem_proto::proxy_mcp::ProxyMcpHello; + +struct Scoped; + +impl ScopedMcpTools for Scoped { + fn definitions(&self) -> Vec { + vec![McpToolDef { + namespaced_name: "capsem__fixture".into(), + original_name: "fixture".into(), + description: None, + input_schema: serde_json::json!({"type": "object"}), + server_name: "capsem".into(), + annotations: None, + timeout_secs: None, + }] + } + + fn call_tool<'a>( + &'a self, + _name: &'a str, + arguments: serde_json::Value, + ) -> futures::future::BoxFuture<'a, Result> { + Box::pin(async move { Ok(serde_json::json!({"arguments": arguments})) }) + } +} + +#[tokio::test] +async fn bridge_bootstraps_limits_and_dispatches_scoped_tools_without_aggregator_handoff() { + let (aggregator, mut aggregator_rx) = AggregatorClient::channel(4); + let aggregator_task = tokio::spawn(async move { + let (request, reply) = aggregator_rx.recv().await.unwrap(); + assert!(matches!(request.method, AggregatorMethod::ListTools)); + reply + .send(AggregatorResponse { + id: request.id, + body: AggregatorResult::Tools { tools: Vec::new() }, + }) + .unwrap(); + assert!(matches!( + tokio::time::timeout(std::time::Duration::from_millis(50), aggregator_rx.recv()).await, + Err(_) | Ok(None) + )); + }); + let (proxy, owner) = UnixStream::pair().unwrap(); + let server = tokio::spawn(super::serve( + owner, + aggregator, + Arc::new(Scoped), + BTreeSet::from(["local".into()]), + 7, + capsem_core::net::mitm_proxy::McpTimeouts { + default_timeout: std::time::Duration::from_secs(2), + tool_call_default: std::time::Duration::from_secs(3), + tool_call_ceiling: std::time::Duration::from_secs(4), + }, + )); + capsem_foundation::unix::fd::set_nonblocking(proxy.as_fd(), true).unwrap(); + let mut read = tokio::net::UnixStream::from_std(proxy.try_clone().unwrap()).unwrap(); + let mut write = tokio::net::UnixStream::from_std(proxy).unwrap(); + let hello = read_frame::<_, ProxyMcpHello>(&mut read).await.unwrap().unwrap(); + assert_eq!(hello.inflight_cap, 7); + assert_eq!(hello.builtin_servers, ["local"]); + + write_frame( + &mut write, + &AggregatorRequest { + id: 1, + method: AggregatorMethod::ListTools, + }, + ) + .await + .unwrap(); + let listed = read_frame::<_, AggregatorResponse>(&mut read).await.unwrap().unwrap(); + assert!(matches!(listed.body, AggregatorResult::Tools { tools } if tools.len() == 1)); + write_frame( + &mut write, + &AggregatorRequest { + id: 2, + method: AggregatorMethod::CallTool { + name: "capsem__fixture".into(), + arguments: serde_json::json!({"value": 7}), + timeout_ms: Some(3_000), + }, + }, + ) + .await + .unwrap(); + let called = read_frame::<_, AggregatorResponse>(&mut read).await.unwrap().unwrap(); + assert!(matches!(called.body, AggregatorResult::CallResult { .. })); + drop(read); + drop(write); + server.await.unwrap().unwrap(); + aggregator_task.await.unwrap(); +} From 55c1dfc954e0b6bd92734954c780a1bf9fc814ca Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:46:38 +0000 Subject: [PATCH 163/464] security(process): surrender guest HTTP traffic --- CHANGELOG.md | 7 +-- crates/capsem-process/src/main.rs | 26 ----------- crates/capsem-process/src/vsock.rs | 53 +++++++++++++--------- crates/capsem-process/src/vsock/streams.rs | 20 +------- 4 files changed, 37 insertions(+), 69 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39862da22..b063cf1ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security -- Move guest DNS framing, private-name resolution, policy enforcement and - audit admission into the confined per-session proxy, with the VM owner - surrendering each connection through its generation-bound broker. +- Move guest HTTP interception, DNS framing, private-name resolution, policy + enforcement and audit admission into the confined per-session proxy, with + the VM owner surrendering each connection through its generation-bound + broker. - Serve live and stopped-session inspection through coordinator-granted, role-bound ledger channels and bounded named operations, eliminating the diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 63057314a..0d65fc18b 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -664,10 +664,6 @@ async fn run_async_main_loop( let security_rules = Arc::new(std::sync::RwLock::new(Arc::new(runtime_config.security_rules.clone()))); let plugin_policy = Arc::new(std::sync::RwLock::new(Arc::new(runtime_config.plugins.clone()))); let proxy_policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new(runtime_config.proxy_policy_snapshot()); - let proxy_engine = Arc::new(capsem_core::net::proxy_engine::ProxyEngine::local( - proxy_policy.clone(), - Arc::clone(&db), - )); let job_store = Arc::new(JobStore { publisher: Arc::new( capsem_core::container::publish::Publisher::for_session( @@ -844,26 +840,6 @@ async fn run_async_main_loop( } shutdown.lock().await.proxy_mcp = Some(proxy_mcp_task); - let telemetry_deps = Arc::new(capsem_core::net::mitm_proxy::telemetry_hook::TelemetryDeps { - db: Arc::clone(&db), - credentials: proxy_engine.credentials(), - pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), - trace_state: Arc::clone(&model_trace_state), - }); - let mitm_pipeline = capsem_core::net::mitm_proxy::make_production_pipeline(Arc::clone(&telemetry_deps)); - let mitm_config = Arc::new(capsem_core::net::mitm_proxy::MitmProxyConfig { - ca: Arc::clone(&net_state.ca), - server_tls: capsem_core::net::mitm_proxy::make_server_tls_config(&net_state.ca), - engine: proxy_engine, - db: Arc::clone(&db), - upstream_tls: Arc::clone(&net_state.upstream_tls), - telemetry: telemetry_deps, - pipeline: mitm_pipeline, - mcp_endpoint: Some(mcp_endpoint), - upstream_resolver: capsem_core::net::upstream_address::UpstreamResolver::disabled(), - upstream_grants: Some(Arc::clone(&upstream_grants) as Arc), - }); - let ipc_tx_clone = ipc_tx.clone(); let job_store_clone = Arc::clone(&job_store); let terminal_output_clone = Arc::clone(&terminal_output); @@ -874,7 +850,6 @@ async fn run_async_main_loop( // the first ~100ms of post-resume output. let net_state_clone = Arc::clone(&net_state); - let mitm_config_clone = Arc::clone(&mitm_config); let upstream_grants_for_vsock = Arc::clone(&upstream_grants); // Parse --env KEY=VALUE pairs for guest injection @@ -943,7 +918,6 @@ async fn run_async_main_loop( session_dir: session_dir.clone(), cli_env, guest_config, - mitm_config: mitm_config_clone, upstream_grants: upstream_grants_for_vsock, security_rules: Arc::clone(&security_rules), plugin_policy: Arc::clone(&plugin_policy), diff --git a/crates/capsem-process/src/vsock.rs b/crates/capsem-process/src/vsock.rs index 997d9fd22..a93c4f075 100644 --- a/crates/capsem-process/src/vsock.rs +++ b/crates/capsem-process/src/vsock.rs @@ -70,7 +70,6 @@ pub(crate) struct VsockOptions { pub(crate) session_dir: PathBuf, pub(crate) cli_env: Vec<(String, String)>, pub(crate) guest_config: capsem_core::net::policy_config::GuestConfig, - pub(crate) mitm_config: Arc, pub(crate) upstream_grants: Arc, pub(crate) security_rules: SecurityRulesHandle, pub(crate) plugin_policy: PluginPolicyHandle, @@ -97,7 +96,6 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { session_dir, cli_env, guest_config, - mitm_config, upstream_grants, security_rules, plugin_policy, @@ -719,7 +717,6 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { // ----------------------------------------------------------------------- // 4. Central Dispatcher Loop (Vsock -> Hub) // ----------------------------------------------------------------------- - let mitm_config_loop = Arc::clone(&mitm_config); let upstream_grants_loop = Arc::clone(&upstream_grants); let security_rules_loop = Arc::clone(&security_rules); let db_for_audit = Arc::clone(&db); @@ -738,7 +735,6 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { for conn in pending_aux.drain(..) { dispatch_aux_connection( conn, - &mitm_config_loop, &upstream_grants_loop, &security_rules_loop, &job_store_vsock, @@ -775,7 +771,6 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { for aux_conn in pending_aux.drain(..) { dispatch_aux_connection( aux_conn, - &mitm_config_loop, &upstream_grants_loop, &security_rules_loop, &job_store_vsock, @@ -815,7 +810,6 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { // only thing that requires a successful handshake lock-step. dispatch_aux_connection( conn, - &mitm_config_loop, &upstream_grants_loop, &security_rules_loop, &job_store_vsock, @@ -839,7 +833,6 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { #[allow(clippy::too_many_arguments)] fn dispatch_aux_connection( conn: VsockConnection, - mitm_config: &Arc, upstream_grants: &Arc, security_rules: &Arc>>, job_store: &Arc, @@ -851,21 +844,18 @@ fn dispatch_aux_connection( match HostVsockService::from_port(conn.port) { Some(HostVsockService::Publication) => job_store.publisher.accept(conn), Some(HostVsockService::Network) => streams::serve_network(conn, job_store, vm_id), - Some(HostVsockService::SniProxy) => streams::serve_mitm(conn, Arc::clone(mitm_config)), - Some(HostVsockService::DnsProxy) => match conn.try_clone_fd() { - Ok(descriptor) => { - let grants = Arc::clone(upstream_grants); - tokio::spawn(async move { - if let Err(error) = grants - .attach_proxy_traffic(capsem_proto::upstream_grant::ProxyTrafficService::Dns, descriptor) - .await - { - warn!(%error, "DNS port: proxy worker refused the session descriptor"); - } - }); - } - Err(error) => warn!(%error, "DNS port: cannot duplicate the session descriptor"), - }, + Some(HostVsockService::SniProxy) => surrender_proxy_traffic( + conn, + capsem_proto::upstream_grant::ProxyTrafficService::Http, + "HTTP", + upstream_grants, + ), + Some(HostVsockService::DnsProxy) => surrender_proxy_traffic( + conn, + capsem_proto::upstream_grant::ProxyTrafficService::Dns, + "DNS", + upstream_grants, + ), Some(HostVsockService::Exec) => { let js = Arc::clone(job_store); std::thread::spawn(move || { @@ -979,6 +969,25 @@ fn dispatch_aux_connection( } } +fn surrender_proxy_traffic( + conn: VsockConnection, + service: capsem_proto::upstream_grant::ProxyTrafficService, + label: &'static str, + upstream_grants: &Arc, +) { + match conn.try_clone_fd() { + Ok(descriptor) => { + let grants = Arc::clone(upstream_grants); + tokio::spawn(async move { + if let Err(error) = grants.attach_proxy_traffic(service, descriptor).await { + warn!(%error, "{label} port: proxy worker refused the session descriptor"); + } + }); + } + Err(error) => warn!(%error, "{label} port: cannot duplicate the session descriptor"), + } +} + fn spawn_serial_log_writer(open: F) -> (mpsc::Sender>, std::thread::JoinHandle<()>) where W: std::io::Write + Send + 'static, diff --git a/crates/capsem-process/src/vsock/streams.rs b/crates/capsem-process/src/vsock/streams.rs index 51f605464..4c53aa3ef 100644 --- a/crates/capsem-process/src/vsock/streams.rs +++ b/crates/capsem-process/src/vsock/streams.rs @@ -1,23 +1,7 @@ -//! Guest byte streams handed whole to an async handler: the MITM rail and -//! each network cable's stream. Both duplicate the accepted descriptor and -//! keep the connection alive for as long as the handler runs. +//! Guest network cable streams handed whole to their owning switch. use capsem_core::VsockConnection; use std::sync::Arc; -use tracing::{error, info, warn}; - -pub(super) fn serve_mitm(conn: VsockConnection, config: Arc) { - tokio::spawn(async move { - match conn.try_clone_fd() { - Ok(fd) => capsem_core::net::mitm_proxy::handle_connection(fd, config).await, - Err(error) => error!( - operation = "duplicate-mitm-vsock", - errno = error.raw_os_error(), - error = %error, - "MITM connection descriptor unavailable" - ), - } - }); -} +use tracing::{info, warn}; /// A guest pump's stream for one cable, held for that network's switch once /// the pump has named the cable. The four header bytes are read exactly and From 5fee98e1680227183c18394462ebcd856e189b4e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:48:48 +0000 Subject: [PATCH 164/464] feat(proto): bound proxy metric relay --- crates/capsem-proto/src/lib.rs | 1 + crates/capsem-proto/src/proxy_metrics.rs | 52 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 crates/capsem-proto/src/proxy_metrics.rs diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index e643519f2..6f7a451d4 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -29,6 +29,7 @@ pub mod privatelink; pub mod proxy_control; pub mod proxy_credentials; pub mod proxy_mcp; +pub mod proxy_metrics; pub mod proxy_policy; pub mod proxy_private_names; pub mod repeated; diff --git a/crates/capsem-proto/src/proxy_metrics.rs b/crates/capsem-proto/src/proxy_metrics.rs new file mode 100644 index 000000000..14cac06ae --- /dev/null +++ b/crates/capsem-proto/src/proxy_metrics.rs @@ -0,0 +1,52 @@ +//! Bounded OTLP relay messages for a confined proxy worker. +//! +//! The worker submits only an already encoded metrics body. Collector +//! selection, credentials and transport remain coordinator authority. + +use serde::{Deserialize, Serialize}; + +/// Same payload ceiling as the service HTTP relay. +pub const MAX_PROXY_METRIC_BODY_BYTES: usize = 4 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProxyMetricRequest { + #[serde(with = "serde_bytes")] + pub body: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyMetricResponse { + Relayed { + status: u16, + content_type: Option, + #[serde(with = "serde_bytes")] + body: Vec, + }, + Rejected { + message: String, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn messages_round_trip_binary_bodies_without_collector_authority() { + let request = ProxyMetricRequest { body: vec![0, 0xff, 7] }; + let encoded = rmp_serde::to_vec_named(&request).unwrap(); + assert_eq!(rmp_serde::from_slice::(&encoded).unwrap(), request); + + let response = ProxyMetricResponse::Relayed { + status: 202, + content_type: Some("application/x-protobuf".into()), + body: vec![9, 0, 8], + }; + let encoded = rmp_serde::to_vec_named(&response).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + response + ); + } +} From 69d8a4a7b4658ab945e4ece9eb5c4f3e525165dc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:54:19 +0000 Subject: [PATCH 165/464] feat(service): broker proxy metric export --- .../src/service_runtime/telemetry_export.rs | 112 +++++++++++++++++- .../src/tests/telemetry_export.rs | 104 +++++++++++++++- .../src/vm_lifecycle/provision.rs | 9 +- .../src/vm_lifecycle/resume_process.rs | 9 +- 4 files changed, 228 insertions(+), 6 deletions(-) diff --git a/crates/capsem-service/src/service_runtime/telemetry_export.rs b/crates/capsem-service/src/service_runtime/telemetry_export.rs index ce9e0cb89..cbf727bb6 100644 --- a/crates/capsem-service/src/service_runtime/telemetry_export.rs +++ b/crates/capsem-service/src/service_runtime/telemetry_export.rs @@ -53,6 +53,53 @@ pub(crate) fn grant_metric_broker(command: &mut tokio::process::Command, corp: & } } +/// Grant the confined proxy a generation-bound metrics relay. The channel +/// carries only OTLP bodies; collector identity and credentials stay here. +pub(crate) async fn grant_proxy_metric_broker( + state: &Arc, + id: &str, + generation: uuid::Uuid, + worker: &crate::proxy_worker::ProxyWorker, +) -> anyhow::Result<()> { + let (_, corp) = capsem_core::net::policy_config::load_settings_and_corp_files(); + if Destination::corp(corp.corp_rule_files.open_telemetry.as_deref()).is_none() { + return Ok(()); + } + let owner = { + let instances = state.instances.lock().unwrap(); + let instance = instances + .get(id) + .filter(|instance| instance.generation == generation) + .ok_or_else(|| anyhow::anyhow!("VM owner changed before proxy metric grant"))?; + let owner = owner_connection::OwnerConnection::capture(instance).map_err(anyhow::Error::msg)?; + drop(instances); + owner + }; + let (service, proxy) = std::os::unix::net::UnixStream::pair().context("create proxy metric capability")?; + let serving = tokio::spawn(serve_proxy_metric_channel( + Arc::clone(state), + id.to_string(), + owner, + service, + )); + if let Err(error) = worker + .grant(capsem_proto::proxy_control::ProxyCapability::Telemetry, proxy) + .await + { + serving.abort(); + let _ = serving.await; + return Err(error.context("grant proxy metric capability")); + } + tokio::spawn(async move { + match serving.await { + Ok(Ok(())) => tracing::debug!("proxy metric capability disconnected"), + Ok(Err(error)) => tracing::warn!(%error, "proxy metric capability failed"), + Err(error) => tracing::warn!(%error, "proxy metric capability task failed"), + } + }); + Ok(()) +} + const COLLECTOR_TIMEOUT: Duration = Duration::from_secs(10); static RELAY_CLIENT: OnceLock> = OnceLock::new(); @@ -116,9 +163,31 @@ pub(crate) async fn handle_metric_relay( "metric request body is too large".into(), )); } + let response = relay_metric_body(&state, &id, &owner, body).await?; + Ok((response.status, response.headers, response.body)) +} + +struct MetricRelayResponse { + status: StatusCode, + headers: axum::http::HeaderMap, + body: axum::body::Bytes, +} + +async fn relay_metric_body( + state: &Arc, + id: &str, + owner: &owner_connection::OwnerConnection, + body: axum::body::Bytes, +) -> Result { + if body.len() > capsem_proto::proxy_metrics::MAX_PROXY_METRIC_BODY_BYTES { + return Err(AppError( + StatusCode::PAYLOAD_TOO_LARGE, + "metric request body is too large".into(), + )); + } let collector = current_collector_url()?; owner - .validate(&state, false) + .validate(state, false) .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; let request = relay_client()? .post(collector) @@ -167,13 +236,50 @@ pub(crate) async fn handle_metric_relay( bytes.extend_from_slice(&chunk); } owner - .validate(&state, false) + .validate(state, false) .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; let mut response_headers = axum::http::HeaderMap::new(); if let Some(content_type) = content_type { response_headers.insert(axum::http::header::CONTENT_TYPE, content_type); } - Ok((status, response_headers, axum::body::Bytes::from(bytes))) + Ok(MetricRelayResponse { + status, + headers: response_headers, + body: axum::body::Bytes::from(bytes), + }) +} + +pub(crate) async fn serve_proxy_metric_channel( + state: Arc, + id: String, + owner: owner_connection::OwnerConnection, + stream: std::os::unix::net::UnixStream, +) -> anyhow::Result<()> { + use capsem_proto::proxy_metrics::{ProxyMetricRequest, ProxyMetricResponse}; + + let (responses, requests) = + capsem_foundation::ipc_channel::channel_from_std::(stream) + .context("open proxy metric channel")?; + loop { + let request = match requests.recv().await { + Ok(request) => request, + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()), + Err(error) => return Err(error).context("receive proxy metric request"), + }; + let response = match relay_metric_body(&state, &id, &owner, request.body.into()).await { + Ok(relayed) => ProxyMetricResponse::Relayed { + status: relayed.status.as_u16(), + content_type: relayed + .headers + .get(axum::http::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_string), + body: relayed.body.to_vec(), + }, + Err(AppError(_, message)) => ProxyMetricResponse::Rejected { message }, + }; + responses.send(response).await.context("send proxy metric response")?; + } } /// One session's totals and the attributes its series carry. diff --git a/crates/capsem-service/src/tests/telemetry_export.rs b/crates/capsem-service/src/tests/telemetry_export.rs index 1e8eb7533..1d9b4fc16 100644 --- a/crates/capsem-service/src/tests/telemetry_export.rs +++ b/crates/capsem-service/src/tests/telemetry_export.rs @@ -4,7 +4,9 @@ use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData}; use opentelemetry_sdk::metrics::{InMemoryMetricExporter, SdkMeterProvider}; use super::*; -use crate::service_runtime::telemetry_export::{collect, grant_metric_broker, register, SessionTable}; +use crate::service_runtime::telemetry_export::{ + collect, grant_metric_broker, register, serve_proxy_metric_channel, SessionTable, +}; fn net(decision: capsem_logger::Decision, bytes_sent: u64) -> capsem_logger::WriteOp { capsem_logger::WriteOp::NetEvent( @@ -246,6 +248,106 @@ async fn metric_collector( ) } +#[tokio::test] +async fn proxy_metric_capability_is_bounded_fixed_and_generation_bound() { + use capsem_proto::proxy_metrics::{ProxyMetricRequest, ProxyMetricResponse, MAX_PROXY_METRIC_BODY_BYTES}; + + let _environment = SETTINGS_ENV_LOCK.lock().await; + let settings_dir = tempfile::tempdir().unwrap(); + let (_settings_guard, _, corp_path) = install_empty_settings_env(&settings_dir); + let collector_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let collector_base = format!("http://{}", collector_listener.local_addr().unwrap()); + let (collected_tx, mut collected_rx) = tokio::sync::mpsc::unbounded_channel(); + let collector = tokio::spawn(async move { + axum::serve( + collector_listener, + axum::Router::new() + .route("/v1/metrics", axum::routing::post(metric_collector)) + .with_state(collected_tx), + ) + .await + .unwrap(); + }); + let mut corp = capsem_core::net::policy_config::SettingsFile::default(); + corp.corp_rule_files.open_telemetry = Some(collector_base); + capsem_core::net::policy_config::write_settings_file(&corp_path, &corp).unwrap(); + + let (state, _directory) = make_test_state_with_tempdir(); + insert_fake_instance(&state, "proxy-vm", std::process::id()); + let (generation, owner) = { + let instances = state.instances.lock().unwrap(); + let instance = &instances["proxy-vm"]; + let captured = ( + instance.generation, + crate::owner_connection::OwnerConnection::capture(instance).unwrap(), + ); + drop(instances); + captured + }; + let (service, proxy) = std::os::unix::net::UnixStream::pair().unwrap(); + let serving = tokio::spawn(serve_proxy_metric_channel( + Arc::clone(&state), + "proxy-vm".into(), + owner, + service, + )); + let (requests, responses) = + capsem_foundation::ipc_channel::channel_from_std::(proxy).unwrap(); + + requests + .send(ProxyMetricRequest { + body: vec![0; MAX_PROXY_METRIC_BODY_BYTES + 1], + }) + .await + .unwrap(); + assert!(matches!( + responses.recv().await.unwrap(), + ProxyMetricResponse::Rejected { ref message } if message.contains("too large") + )); + assert!( + collected_rx.try_recv().is_err(), + "oversized proxy metrics reached the collector" + ); + + requests + .send(ProxyMetricRequest { + body: b"proxy otlp".to_vec(), + }) + .await + .unwrap(); + assert_eq!( + responses.recv().await.unwrap(), + ProxyMetricResponse::Relayed { + status: StatusCode::CREATED.as_u16(), + content_type: Some("application/x-protobuf".into()), + body: b"collector response".to_vec(), + } + ); + let (headers, body) = collected_rx.recv().await.unwrap(); + assert_eq!(headers[axum::http::header::CONTENT_TYPE], "application/x-protobuf"); + assert_eq!(body, b"proxy otlp".as_slice()); + + assert!(state.evict_instance("proxy-vm", generation).is_some()); + requests + .send(ProxyMetricRequest { + body: b"stale".to_vec(), + }) + .await + .unwrap(); + assert!(matches!( + responses.recv().await.unwrap(), + ProxyMetricResponse::Rejected { ref message } if message.contains("changed") + )); + assert!( + collected_rx.try_recv().is_err(), + "revoked proxy metrics reached the collector" + ); + + drop(requests); + serving.await.unwrap().unwrap(); + collector.abort(); +} + fn protobuf_contains(body: &[u8], expected: &str) -> bool { body.windows(expected.len()).any(|window| window == expected.as_bytes()) } diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index 71c4e9217..b7385238a 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -398,11 +398,18 @@ impl ServiceState { }, ); drop(instances); - if let Err(error) = self.register_proxy_worker(id, generation, proxy) { + if let Err(error) = self.register_proxy_worker(id, generation, proxy.clone()) { self.evict_instance(id, generation); instance_reaper::kill_and_reap(child); return Err(error.context("register proxy worker")); } + if let Err(error) = tokio::runtime::Handle::current().block_on( + crate::service_runtime::telemetry_export::grant_proxy_metric_broker(self, id, generation, &proxy), + ) { + self.evict_instance(id, generation); + instance_reaper::kill_and_reap(child); + return Err(error); + } let _upstream_broker = upstream_broker.start(upstream_grant); let _proxy_upstream_broker = proxy_upstream_broker.start(proxy_upstream_grant); let _reaper = instance_reaper::spawn_exit_reaper( diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 50f081b42..100604bab 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -288,11 +288,18 @@ impl ServiceState { }, ); drop(instances); - if let Err(error) = self.register_proxy_worker(&vm_id, generation, proxy) { + if let Err(error) = self.register_proxy_worker(&vm_id, generation, proxy.clone()) { self.evict_instance(&vm_id, generation); instance_reaper::kill_and_reap(child); return Err(error.context("register proxy worker")); } + if let Err(error) = tokio::runtime::Handle::current().block_on( + crate::service_runtime::telemetry_export::grant_proxy_metric_broker(self, &vm_id, generation, &proxy), + ) { + self.evict_instance(&vm_id, generation); + instance_reaper::kill_and_reap(child); + return Err(error); + } let _upstream_broker = upstream_broker.start(upstream_grant); let _proxy_upstream_broker = proxy_upstream_broker.start(proxy_upstream_grant); let _reaper = instance_reaper::spawn_exit_reaper( From 9bc855155189de3ee6ce81a470b8733851448c2b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:55:20 +0000 Subject: [PATCH 166/464] feat(proto): identify proxy metric grants --- crates/capsem-proto/src/proxy_metrics.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/capsem-proto/src/proxy_metrics.rs b/crates/capsem-proto/src/proxy_metrics.rs index 14cac06ae..04ff0eca1 100644 --- a/crates/capsem-proto/src/proxy_metrics.rs +++ b/crates/capsem-proto/src/proxy_metrics.rs @@ -14,6 +14,13 @@ pub struct ProxyMetricRequest { pub body: Vec, } +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ProxyMetricBrokerMessage { + Hello { session_id: String }, + Response(ProxyMetricResponse), +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ProxyMetricResponse { @@ -48,5 +55,14 @@ mod tests { rmp_serde::from_slice::(&encoded).unwrap(), response ); + + let hello = ProxyMetricBrokerMessage::Hello { + session_id: "vm-a".into(), + }; + let encoded = rmp_serde::to_vec_named(&hello).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + hello + ); } } From 4e70b89b142b3150163383b86949ce66613f287c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 07:58:16 +0000 Subject: [PATCH 167/464] feat(service): identify proxy metric grants --- .../src/service_runtime/telemetry_export.rs | 13 ++++++++--- .../src/tests/telemetry_export.rs | 23 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/crates/capsem-service/src/service_runtime/telemetry_export.rs b/crates/capsem-service/src/service_runtime/telemetry_export.rs index cbf727bb6..50a267d2d 100644 --- a/crates/capsem-service/src/service_runtime/telemetry_export.rs +++ b/crates/capsem-service/src/service_runtime/telemetry_export.rs @@ -255,11 +255,15 @@ pub(crate) async fn serve_proxy_metric_channel( owner: owner_connection::OwnerConnection, stream: std::os::unix::net::UnixStream, ) -> anyhow::Result<()> { - use capsem_proto::proxy_metrics::{ProxyMetricRequest, ProxyMetricResponse}; + use capsem_proto::proxy_metrics::{ProxyMetricBrokerMessage, ProxyMetricRequest, ProxyMetricResponse}; let (responses, requests) = - capsem_foundation::ipc_channel::channel_from_std::(stream) + capsem_foundation::ipc_channel::channel_from_std::(stream) .context("open proxy metric channel")?; + responses + .send(ProxyMetricBrokerMessage::Hello { session_id: id.clone() }) + .await + .context("send proxy metric identity")?; loop { let request = match requests.recv().await { Ok(request) => request, @@ -278,7 +282,10 @@ pub(crate) async fn serve_proxy_metric_channel( }, Err(AppError(_, message)) => ProxyMetricResponse::Rejected { message }, }; - responses.send(response).await.context("send proxy metric response")?; + responses + .send(ProxyMetricBrokerMessage::Response(response)) + .await + .context("send proxy metric response")?; } } diff --git a/crates/capsem-service/src/tests/telemetry_export.rs b/crates/capsem-service/src/tests/telemetry_export.rs index 1d9b4fc16..bc84d4d28 100644 --- a/crates/capsem-service/src/tests/telemetry_export.rs +++ b/crates/capsem-service/src/tests/telemetry_export.rs @@ -250,7 +250,9 @@ async fn metric_collector( #[tokio::test] async fn proxy_metric_capability_is_bounded_fixed_and_generation_bound() { - use capsem_proto::proxy_metrics::{ProxyMetricRequest, ProxyMetricResponse, MAX_PROXY_METRIC_BODY_BYTES}; + use capsem_proto::proxy_metrics::{ + ProxyMetricBrokerMessage, ProxyMetricRequest, ProxyMetricResponse, MAX_PROXY_METRIC_BODY_BYTES, + }; let _environment = SETTINGS_ENV_LOCK.lock().await; let settings_dir = tempfile::tempdir().unwrap(); @@ -292,7 +294,14 @@ async fn proxy_metric_capability_is_bounded_fixed_and_generation_bound() { service, )); let (requests, responses) = - capsem_foundation::ipc_channel::channel_from_std::(proxy).unwrap(); + capsem_foundation::ipc_channel::channel_from_std::(proxy) + .unwrap(); + assert_eq!( + responses.recv().await.unwrap(), + ProxyMetricBrokerMessage::Hello { + session_id: "proxy-vm".into() + } + ); requests .send(ProxyMetricRequest { @@ -302,7 +311,8 @@ async fn proxy_metric_capability_is_bounded_fixed_and_generation_bound() { .unwrap(); assert!(matches!( responses.recv().await.unwrap(), - ProxyMetricResponse::Rejected { ref message } if message.contains("too large") + ProxyMetricBrokerMessage::Response(ProxyMetricResponse::Rejected { ref message }) + if message.contains("too large") )); assert!( collected_rx.try_recv().is_err(), @@ -317,11 +327,11 @@ async fn proxy_metric_capability_is_bounded_fixed_and_generation_bound() { .unwrap(); assert_eq!( responses.recv().await.unwrap(), - ProxyMetricResponse::Relayed { + ProxyMetricBrokerMessage::Response(ProxyMetricResponse::Relayed { status: StatusCode::CREATED.as_u16(), content_type: Some("application/x-protobuf".into()), body: b"collector response".to_vec(), - } + }) ); let (headers, body) = collected_rx.recv().await.unwrap(); assert_eq!(headers[axum::http::header::CONTENT_TYPE], "application/x-protobuf"); @@ -336,7 +346,8 @@ async fn proxy_metric_capability_is_bounded_fixed_and_generation_bound() { .unwrap(); assert!(matches!( responses.recv().await.unwrap(), - ProxyMetricResponse::Rejected { ref message } if message.contains("changed") + ProxyMetricBrokerMessage::Response(ProxyMetricResponse::Rejected { ref message }) + if message.contains("changed") )); assert!( collected_rx.try_recv().is_err(), From 176f15c53e3cb18b6bd1f5efb8eebf6d895f933c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:01:43 +0000 Subject: [PATCH 168/464] feat(proxy): export brokered session metrics --- CHANGELOG.md | 4 + Cargo.lock | 4 + crates/capsem-proxy/Cargo.toml | 4 + crates/capsem-proxy/src/main.rs | 25 ++++- crates/capsem-proxy/src/metric_client.rs | 91 +++++++++++++++++++ .../capsem-proxy/src/metric_client/tests.rs | 58 ++++++++++++ crates/capsem-proxy/tests/subprocess.rs | 15 ++- 7 files changed, 197 insertions(+), 4 deletions(-) create mode 100644 crates/capsem-proxy/src/metric_client.rs create mode 100644 crates/capsem-proxy/src/metric_client/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b063cf1ba..3b045ca3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Export confined proxy metrics through a bounded, generation-bound broker + capability that fixes session attribution and keeps collector destinations, + credentials and environment state in the trusted service. + - Move guest HTTP interception, DNS framing, private-name resolution, policy enforcement and audit admission into the confined per-session proxy, with the VM owner surrendering each connection through its generation-bound diff --git a/Cargo.lock b/Cargo.lock index 0876e26c3..dadd8ca89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1207,13 +1207,17 @@ name = "capsem-proxy" version = "0.7.0" dependencies = [ "anyhow", + "async-trait", + "bytes", "capsem-core", "capsem-foundation", "capsem-guard", "capsem-logger", "capsem-proto", + "capsem-telemetry", "clap", "http", + "opentelemetry-http", "tempfile", "tokio", "tracing", diff --git a/crates/capsem-proxy/Cargo.toml b/crates/capsem-proxy/Cargo.toml index c12482da4..16ec92b7a 100644 --- a/crates/capsem-proxy/Cargo.toml +++ b/crates/capsem-proxy/Cargo.toml @@ -16,8 +16,12 @@ capsem-foundation = { path = "../capsem-foundation" } capsem-guard = { path = "../capsem-guard" } capsem-logger = { path = "../capsem-logger" } capsem-proto = { path = "../capsem-proto" } +capsem-telemetry = { path = "../capsem-telemetry", features = ["export"] } clap = { workspace = true, features = ["derive"] } +async-trait = "0.1" +bytes.workspace = true http.workspace = true +opentelemetry-http = { version = "0.33", default-features = false } tokio.workspace = true tracing.workspace = true diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index e8dbe63e7..c7ac8f10d 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -28,6 +28,7 @@ use tokio::task::{JoinError, JoinSet}; mod credential_client; mod mcp_client; +mod metric_client; mod private_names_client; const CONTROL_QUEUE_CAPACITY: usize = 16; @@ -119,7 +120,6 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result send_event(&sender, ProxyControlEvent::Ready { generation }).await?; let state = Arc::new(Mutex::new(ProxyRuntimeState::default())); let mut grants = BTreeMap::>::new(); - let mut descriptors = BTreeMap::::new(); let mut grant_ids = HashSet::new(); let mut tasks = JoinSet::::new(); @@ -278,6 +278,20 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result while !*closed.borrow() && closed.changed().await.is_ok() {} (capability, grant_id, ProxyChannelCloseReason::Disconnected) }); + } else if capability == ProxyCapability::Telemetry { + let (client, session_id) = metric_client::start(stream) + .await + .context("open proxy metric capability")?; + let exporter = capsem_telemetry::export::install_with_http_client( + client, + "capsem-proxy", + vec![capsem_telemetry::export::KeyValue::new("session.id", session_id)], + ) + .context("install proxy metric exporter")?; + state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .attach_metric_exporter(exporter); } else if capability == ProxyCapability::HttpTraffic { let config = state .lock() @@ -306,7 +320,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result (capability, grant_id, ProxyChannelCloseReason::Disconnected) }); } else { - descriptors.insert(grant_id, stream); + bail!("proxy capability {capability:?} has no consumer"); } send_event(&sender, ProxyControlEvent::Adopted { generation, grant_id }).await?; } @@ -323,7 +337,6 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result tasks.abort_all(); while tasks.join_next().await.is_some() {} - drop(descriptors); drop(grants); send_event(&sender, ProxyControlEvent::Stopped { generation }).await } @@ -376,6 +389,7 @@ struct ProxyRuntimeState { )>, http_config: Option>, dns_runtime: Option>, + metric_exporter: Option, } impl Default for ProxyRuntimeState { @@ -393,6 +407,7 @@ impl Default for ProxyRuntimeState { mcp_client: None, http_config: None, dns_runtime: None, + metric_exporter: None, } } } @@ -464,6 +479,10 @@ impl ProxyRuntimeState { self.http_config = None; } + fn attach_metric_exporter(&mut self, exporter: capsem_telemetry::export::Exporter) { + self.metric_exporter = Some(exporter); + } + fn traffic_ready(&mut self, capability: ProxyCapability) -> Result { match capability { ProxyCapability::HttpTraffic => Ok(self.http_config()?.is_some()), diff --git a/crates/capsem-proxy/src/metric_client.rs b/crates/capsem-proxy/src/metric_client.rs new file mode 100644 index 000000000..8beca9825 --- /dev/null +++ b/crates/capsem-proxy/src/metric_client.rs @@ -0,0 +1,91 @@ +//! Fixed, typed OTLP transport granted by the session coordinator. + +use std::io; +use std::os::unix::net::UnixStream; + +use anyhow::{bail, Context, Result}; +use capsem_foundation::ipc_channel; +use capsem_proto::proxy_metrics::{ + ProxyMetricBrokerMessage, ProxyMetricRequest, ProxyMetricResponse, MAX_PROXY_METRIC_BODY_BYTES, +}; + +pub(crate) struct MetricClient { + requests: ipc_channel::Sender, + messages: ipc_channel::Receiver, + exchange: tokio::sync::Mutex<()>, +} + +impl std::fmt::Debug for MetricClient { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.debug_struct("MetricClient").finish_non_exhaustive() + } +} + +pub(crate) async fn start(stream: UnixStream) -> Result<(MetricClient, String)> { + let (requests, messages) = ipc_channel::channel_from_std::(stream) + .context("open proxy metric transport")?; + let session_id = match messages.recv().await.context("receive proxy metric identity")? { + ProxyMetricBrokerMessage::Hello { session_id } if !session_id.is_empty() && session_id.len() <= 255 => { + session_id + } + ProxyMetricBrokerMessage::Hello { .. } => bail!("proxy metric identity is invalid"), + ProxyMetricBrokerMessage::Response(_) => bail!("proxy metric broker responded before identity"), + }; + Ok(( + MetricClient { + requests, + messages, + exchange: tokio::sync::Mutex::new(()), + }, + session_id, + )) +} + +#[async_trait::async_trait] +impl opentelemetry_http::HttpClient for MetricClient { + async fn send_bytes( + &self, + request: http::Request, + ) -> std::result::Result, opentelemetry_http::HttpError> { + let body = request.into_body(); + if body.len() > MAX_PROXY_METRIC_BODY_BYTES { + return Err(metric_error("encoded proxy metrics exceed the broker limit")); + } + let _exchange = self.exchange.lock().await; + self.requests + .send(ProxyMetricRequest { body: body.to_vec() }) + .await + .map_err(|error| metric_error(format!("send proxy metrics: {error}")))?; + match self + .messages + .recv() + .await + .map_err(|error| metric_error(format!("receive proxy metric result: {error}")))? + { + ProxyMetricBrokerMessage::Hello { .. } => Err(metric_error("proxy metric broker repeated its identity")), + ProxyMetricBrokerMessage::Response(ProxyMetricResponse::Rejected { message }) => { + Err(metric_error(format!("proxy metric broker rejected export: {message}"))) + } + ProxyMetricBrokerMessage::Response(ProxyMetricResponse::Relayed { + status, + content_type, + body, + }) => { + let mut response = http::Response::builder().status(status); + if let Some(content_type) = content_type { + response = response.header(http::header::CONTENT_TYPE, content_type); + } + response + .body(bytes::Bytes::from(body)) + .map_err(|error| metric_error(format!("build proxy metric response: {error}"))) + } + } + } +} + +fn metric_error(message: impl Into) -> opentelemetry_http::HttpError { + Box::new(io::Error::other(message.into())) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proxy/src/metric_client/tests.rs b/crates/capsem-proxy/src/metric_client/tests.rs new file mode 100644 index 000000000..0288edd6d --- /dev/null +++ b/crates/capsem-proxy/src/metric_client/tests.rs @@ -0,0 +1,58 @@ +use super::*; +use opentelemetry_http::HttpClient as _; + +#[tokio::test] +async fn client_accepts_identity_then_relays_only_the_encoded_body() { + let (broker, proxy) = UnixStream::pair().unwrap(); + let serving = tokio::spawn(async move { + let (messages, requests) = + ipc_channel::channel_from_std::(broker).unwrap(); + messages + .send(ProxyMetricBrokerMessage::Hello { + session_id: "vm-a".into(), + }) + .await + .unwrap(); + assert_eq!(requests.recv().await.unwrap().body, b"encoded otlp"); + messages + .send(ProxyMetricBrokerMessage::Response(ProxyMetricResponse::Relayed { + status: 202, + content_type: Some("application/x-protobuf".into()), + body: b"accepted".to_vec(), + })) + .await + .unwrap(); + }); + + let (client, session_id) = start(proxy).await.unwrap(); + assert_eq!(session_id, "vm-a"); + let response = client + .send_bytes( + http::Request::post("https://attacker.invalid/collector") + .header(http::header::AUTHORIZATION, "must not cross") + .body(bytes::Bytes::from_static(b"encoded otlp")) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), 202); + assert_eq!(response.body().as_ref(), b"accepted"); + serving.await.unwrap(); +} + +#[tokio::test] +async fn client_rejects_a_response_before_the_broker_identity() { + let (broker, proxy) = UnixStream::pair().unwrap(); + let serving = tokio::spawn(async move { + let (messages, _requests) = + ipc_channel::channel_from_std::(broker).unwrap(); + messages + .send(ProxyMetricBrokerMessage::Response(ProxyMetricResponse::Rejected { + message: "not a hello".into(), + })) + .await + .unwrap(); + }); + assert!(start(proxy).await.unwrap_err().to_string().contains("before identity")); + serving.await.unwrap(); +} diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs index 72fa0073f..07ca4ee29 100644 --- a/crates/capsem-proxy/tests/subprocess.rs +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -10,6 +10,7 @@ use capsem_proto::proxy_control::{ decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyChannelGrant, ProxyControlEvent, ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, }; +use capsem_proto::proxy_metrics::{ProxyMetricBrokerMessage, ProxyMetricRequest}; use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; type ControlSender = DescriptorSender; @@ -138,6 +139,19 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { { let grant_id = index as u64 + 1; let (peer, granted) = UnixStream::pair().unwrap(); + if capability == ProxyCapability::Telemetry { + let (messages, _requests) = + capsem_foundation::ipc_channel::channel_from_std::(peer) + .unwrap(); + messages + .send(ProxyMetricBrokerMessage::Hello { + session_id: "subprocess-vm".into(), + }) + .await + .unwrap(); + } else { + peers.push(peer); + } let grant = ProxyChannelGrant::new(GENERATION, grant_id, capability).unwrap(); requests .send( @@ -154,7 +168,6 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { grant_id, } ); - peers.push(peer); } let (duplicate_peer, duplicate) = UnixStream::pair().unwrap(); From 2eb58b3916cfab059c8e83f84a1ac90ded0ec854 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:04:50 +0000 Subject: [PATCH 169/464] feat(proto): coordinate frozen session clones --- crates/capsem-proto/src/ipc.rs | 24 ++++++++++----- crates/capsem-proto/src/ipc/tests.rs | 30 ++++++++++++++----- .../capsem-proto/src/ipc/tests/correlation.rs | 16 +++++----- crates/capsem-proto/src/ipc/tests/wire.rs | 9 ++++-- crates/capsem-proto/src/lib.rs | 6 ++-- 5 files changed, 57 insertions(+), 28 deletions(-) diff --git a/crates/capsem-proto/src/ipc.rs b/crates/capsem-proto/src/ipc.rs index 22f39c2b8..98488e67a 100644 --- a/crates/capsem-proto/src/ipc.rs +++ b/crates/capsem-proto/src/ipc.rs @@ -112,14 +112,20 @@ pub enum ServiceToProcess { Suspend { checkpoint_path: String, }, - /// Clone this sandbox's state into `destination`, an empty session - /// directory the service created. The owner freezes the guest's system - /// filesystem for the copy and always thaws it, so the fork's overlay - /// image is consistent and a service that disappears mid-fork cannot - /// leave the guest frozen. + /// Freeze the guest for a coordinator-owned state clone. The owner emits + /// `CloneStateReady` after the freeze and always thaws before its final + /// result. No source or destination path enters the VM owner. CloneState { id: u64, - destination: String, + }, + /// Complete the coordinator-owned copy while the owner holds the guest + /// frozen. The size or error becomes the owner's final result after thaw. + CloneStateComplete { + id: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + size_bytes: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + error: Option, }, /// Query MCP aggregator for server list with connection status. McpListServers { @@ -320,7 +326,9 @@ pub enum ProcessToService { ShutdownRequested { id: String }, /// Guest requested suspend (forwarded from capsem-sysutil via vsock:5004). SuspendRequested { id: String }, - /// Result of CloneState: the clone's disk usage, or why it failed. + /// The owner froze the guest and is waiting for the coordinator copy. + CloneStateReady { id: u64 }, + /// Result of CloneState after the guest has been thawed. CloneStateResult { id: u64, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -470,7 +478,7 @@ impl ServiceToProcess { | Self::WriteFile { id, .. } | Self::ReadFile { id, .. } | Self::LogFileBoundary { id, .. } - | Self::CloneState { id, .. } + | Self::CloneState { id } | Self::McpListServers { id } | Self::McpListTools { id } | Self::McpRefreshTools { id } diff --git a/crates/capsem-proto/src/ipc/tests.rs b/crates/capsem-proto/src/ipc/tests.rs index ff7a7134c..9a11dca88 100644 --- a/crates/capsem-proto/src/ipc/tests.rs +++ b/crates/capsem-proto/src/ipc/tests.rs @@ -465,18 +465,32 @@ fn suspend_requested_roundtrip() { #[test] fn clone_state_roundtrip() { - let msg = ServiceToProcess::CloneState { - id: 12, - destination: "/run/persistent/fork".into(), - }; + let msg = ServiceToProcess::CloneState { id: 12 }; let bytes = serde_json::to_vec(&msg).unwrap(); match serde_json::from_slice::(&bytes).unwrap() { - ServiceToProcess::CloneState { id, destination } => { - assert_eq!(id, 12); - assert_eq!(destination, "/run/persistent/fork"); - } + ServiceToProcess::CloneState { id } => assert_eq!(id, 12), _ => panic!("wrong variant"), } + let ready = ProcessToService::CloneStateReady { id: 12 }; + let bytes = serde_json::to_vec(&ready).unwrap(); + assert!(matches!( + serde_json::from_slice::(&bytes).unwrap(), + ProcessToService::CloneStateReady { id: 12 } + )); + let complete = ServiceToProcess::CloneStateComplete { + id: 12, + size_bytes: Some(4096), + error: None, + }; + let bytes = serde_json::to_vec(&complete).unwrap(); + assert!(matches!( + serde_json::from_slice::(&bytes).unwrap(), + ServiceToProcess::CloneStateComplete { + id: 12, + size_bytes: Some(4096), + error: None + } + )); let reply = ProcessToService::CloneStateResult { id: 12, size_bytes: Some(4096), diff --git a/crates/capsem-proto/src/ipc/tests/correlation.rs b/crates/capsem-proto/src/ipc/tests/correlation.rs index fa86e4809..7b2cccb5e 100644 --- a/crates/capsem-proto/src/ipc/tests/correlation.rs +++ b/crates/capsem-proto/src/ipc/tests/correlation.rs @@ -22,13 +22,7 @@ fn replies_correlate_to_requests_by_id_and_broadcasts_answer_nothing() { Some(8), ), (ServiceToProcess::McpRefreshTools { id: 10 }, Some(10)), - ( - ServiceToProcess::CloneState { - id: 12, - destination: "/tmp/fork".into(), - }, - Some(12), - ), + (ServiceToProcess::CloneState { id: 12 }, Some(12)), ( ServiceToProcess::LinkDetach { id: 11, @@ -48,6 +42,14 @@ fn replies_correlate_to_requests_by_id_and_broadcasts_answer_nothing() { ), (ServiceToProcess::Ping, None), (ServiceToProcess::ReloadConfig { id: 13 }, Some(13)), + ( + ServiceToProcess::CloneStateComplete { + id: 14, + size_bytes: Some(1), + error: None, + }, + None, + ), ]; for (request, id) in requests { assert_eq!(request.request_id(), id, "{request:?}"); diff --git a/crates/capsem-proto/src/ipc/tests/wire.rs b/crates/capsem-proto/src/ipc/tests/wire.rs index 963707765..056022cb4 100644 --- a/crates/capsem-proto/src/ipc/tests/wire.rs +++ b/crates/capsem-proto/src/ipc/tests/wire.rs @@ -60,9 +60,11 @@ fn service_to_process_variant_names_and_roundtrips_are_stable() { ServiceToProcess::Suspend { checkpoint_path: "/tmp/checkpoint".into(), }, - ServiceToProcess::CloneState { + ServiceToProcess::CloneState { id: 11 }, + ServiceToProcess::CloneStateComplete { id: 11, - destination: "/tmp/fork".into(), + size_bytes: Some(4), + error: None, }, ServiceToProcess::McpListServers { id: 5 }, ServiceToProcess::McpListTools { id: 6 }, @@ -96,6 +98,7 @@ fn service_to_process_variant_names_and_roundtrips_are_stable() { "StopTerminalStream", "Suspend", "CloneState", + "CloneStateComplete", "McpListServers", "McpListTools", "McpRefreshTools", @@ -151,6 +154,7 @@ fn process_to_service_variant_names_and_roundtrips_are_stable() { size_bytes: None, error: Some("frozen".into()), }, + ProcessToService::CloneStateReady { id: 11 }, ProcessToService::McpServersResult { id: 5, servers: vec![] }, ProcessToService::McpToolsResult { id: 6, tools: vec![] }, ProcessToService::McpRefreshResult { @@ -186,6 +190,7 @@ fn process_to_service_variant_names_and_roundtrips_are_stable() { "ShutdownRequested", "SuspendRequested", "CloneStateResult", + "CloneStateReady", "McpServersResult", "McpToolsResult", "McpRefreshResult", diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 6f7a451d4..43a7aaf2f 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -85,9 +85,9 @@ pub const MAX_BOOT_FILES: usize = 64; /// credentials and descriptor handoff admission. Version 10 omits default /// fields from named MessagePack maps while decoding their absence as defaults. /// Version 11 removes the workspace snapshot status query and the unused -/// service-to-owner freeze messages, and adds the owner-run `CloneState` for -/// forks. -/// Version 12 adds host-only memory credential injection and acknowledgement. +/// service-to-owner freeze messages, and adds `CloneState` for forks. +/// Version 12 makes cloning a coordinator operation bracketed by the owner's +/// guest freeze, so no destination path crosses into the VM owner. pub const PROTOCOL_VERSION: u16 = 12; /// Guest loopback port of the agent's DNS proxy (port 53 is redirected here). From fd912195ebdd47ffedc1ebf3ae3db0e4c3c6d1fe Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:08:32 +0000 Subject: [PATCH 170/464] security(process): remove clone path authority --- crates/capsem-process/src/ipc.rs | 14 +++- crates/capsem-process/src/ipc/tests.rs | 5 +- .../src/ipc/tests/classification.rs | 3 +- crates/capsem-process/src/job_store.rs | 18 +++++ crates/capsem-process/src/vsock.rs | 5 +- .../capsem-process/src/vsock/clone_state.rs | 70 +++++++++---------- .../src/vsock/clone_state/tests.rs | 61 +++++++--------- 7 files changed, 91 insertions(+), 85 deletions(-) diff --git a/crates/capsem-process/src/ipc.rs b/crates/capsem-process/src/ipc.rs index cee6442e8..87b8b04f3 100644 --- a/crates/capsem-process/src/ipc.rs +++ b/crates/capsem-process/src/ipc.rs @@ -642,7 +642,7 @@ pub(crate) async fn handle_ipc_connection( message @ ServiceToProcess::LogFileBoundary { .. } => { file_boundary::spawn(&net_state, &mcp_runtime, &ipc_tx_out, message); } - ServiceToProcess::CloneState { id, destination } => { + ServiceToProcess::CloneState { id } => { let job_store = job_store.clone(); let ctrl_tx = ctrl_tx.clone(); let ipc_tx_out = ipc_tx_out.clone(); @@ -651,7 +651,7 @@ pub(crate) async fn handle_ipc_connection( job_store.jobs.lock().unwrap().insert(id, j_tx); capsem_core::try_send!( "ctrl_clone_state", - ctrl_tx.send(ServiceToProcess::CloneState { id, destination }).await + ctrl_tx.send(ServiceToProcess::CloneState { id }).await ); let result = match tokio::time::timeout(CLONE_STATE_TIMEOUT, j_rx).await { Ok(Ok(JobResult::CloneState { result })) => result, @@ -674,6 +674,16 @@ pub(crate) async fn handle_ipc_connection( ); }); } + ServiceToProcess::CloneStateComplete { id, size_bytes, error } => { + let result = match (size_bytes, error) { + (Some(size), None) => Ok(size), + (None, Some(error)) => Err(error), + _ => Err("coordinator returned an invalid clone completion".to_string()), + }; + if let Err(error) = job_store.complete_clone(id, result) { + warn!(id, %error, "clone completion refused"); + } + } ServiceToProcess::ReloadConfig { id } => { info!( active_policy = %runtime_source.active_policy_path().display(), diff --git a/crates/capsem-process/src/ipc/tests.rs b/crates/capsem-process/src/ipc/tests.rs index 69c8f8e2b..e5b37c4b9 100644 --- a/crates/capsem-process/src/ipc/tests.rs +++ b/crates/capsem-process/src/ipc/tests.rs @@ -871,10 +871,7 @@ fn classify_stop_terminal_stream() { #[test] fn classify_clone_state_is_a_job() { assert_eq!( - classify_ipc_message(&ServiceToProcess::CloneState { - id: 1, - destination: "/tmp/fork".into(), - }), + classify_ipc_message(&ServiceToProcess::CloneState { id: 1 }), IpcAction::Job ); } diff --git a/crates/capsem-process/src/ipc/tests/classification.rs b/crates/capsem-process/src/ipc/tests/classification.rs index 2efe5a630..ed4e66a96 100644 --- a/crates/capsem-process/src/ipc/tests/classification.rs +++ b/crates/capsem-process/src/ipc/tests/classification.rs @@ -30,7 +30,8 @@ pub(super) fn classify_ipc_message(msg: &ServiceToProcess) -> IpcAction { | ServiceToProcess::McpListTools { .. } | ServiceToProcess::McpRefreshTools { .. } | ServiceToProcess::McpCallTool { .. } - | ServiceToProcess::CloneState { .. } => IpcAction::Job, + | ServiceToProcess::CloneState { .. } + | ServiceToProcess::CloneStateComplete { .. } => IpcAction::Job, ServiceToProcess::ConnectPort { .. } | ServiceToProcess::AbortPorts { .. } | ServiceToProcess::PlugCable { .. } diff --git a/crates/capsem-process/src/job_store.rs b/crates/capsem-process/src/job_store.rs index 2bde8f137..755841c9d 100644 --- a/crates/capsem-process/src/job_store.rs +++ b/crates/capsem-process/src/job_store.rs @@ -9,6 +9,8 @@ use tracing::{info, warn}; pub(crate) struct JobStore { pub(crate) publisher: Arc, pub(crate) jobs: Mutex>>, + /// Frozen clones awaiting their coordinator-owned copy result. + pub(crate) clone_completions: Mutex>>>, /// Active exec jobs keyed by id, each with captured stdout and a notifier /// the EXEC-port reader thread fires after depositing captured bytes. pub(crate) active_execs: Mutex>, @@ -105,6 +107,7 @@ impl JobStore { Self { publisher: Arc::new(capsem_core::container::publish::Publisher::default()), jobs: Mutex::new(HashMap::new()), + clone_completions: Mutex::new(HashMap::new()), active_execs: Mutex::new(HashMap::new()), active_file_ops: Mutex::new(HashMap::new()), snapshot_ready: Mutex::new(None), @@ -139,6 +142,9 @@ impl JobStore { }) ); } + for (_id, tx) in self.clone_completions.lock().unwrap().drain() { + capsem_core::try_send!("clone_completion_fail_all", tx.send(Err(message.to_string()))); + } if let Some(tx) = self.snapshot_ready.lock().unwrap().take() { capsem_core::try_send!("snapshot_ready_fail_all", tx.send(())); } @@ -149,6 +155,18 @@ impl JobStore { } self.active_file_ops.lock().unwrap().clear(); } + + pub(crate) fn complete_clone(&self, id: u64, result: Result) -> Result<(), String> { + let sender = self + .clone_completions + .lock() + .unwrap() + .remove(&id) + .ok_or_else(|| format!("clone {id} is not waiting for coordinator completion"))?; + sender + .send(result) + .map_err(|_| format!("clone {id} stopped before coordinator completion")) + } } #[derive(Debug)] diff --git a/crates/capsem-process/src/vsock.rs b/crates/capsem-process/src/vsock.rs index a93c4f075..fe74dd245 100644 --- a/crates/capsem-process/src/vsock.rs +++ b/crates/capsem-process/src/vsock.rs @@ -479,7 +479,6 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { let ipc_tx_for_cmd = ipc_tx.clone(); let vm_id_for_cmd = vm_id_original; let vm_handle_for_cmd = vm_handle_original; - let db_for_cmd = Arc::clone(&db); let pty_log_for_cmd = pty_log.clone(); let shutdown_for_cmd = Arc::clone(&shutdown); let exec_dispatch = exec_dispatch::ExecDispatch { @@ -563,8 +562,8 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { .insert(id, ActiveFileOp::Read { path: path.clone() }); capsem_core::try_send!("hub_file_read", hub_tx.send(HostToGuest::FileRead { id, path }).await); } - ServiceToProcess::CloneState { id, destination } => { - clone_state::spawn(&hub_tx, &js_for_cmd, &db_for_cmd, &session_dir, id, destination); + ServiceToProcess::CloneState { id } => { + clone_state::spawn(&hub_tx, &js_for_cmd, &ipc_tx_for_cmd, id); } ServiceToProcess::Suspend { checkpoint_path } => { let full_path = session_dir.join(checkpoint_path); diff --git a/crates/capsem-process/src/vsock/clone_state.rs b/crates/capsem-process/src/vsock/clone_state.rs index 137fdc646..be6b76f91 100644 --- a/crates/capsem-process/src/vsock/clone_state.rs +++ b/crates/capsem-process/src/vsock/clone_state.rs @@ -1,59 +1,53 @@ -//! A fork's clone, run by the sandbox's own process under a guest freeze. +//! Hold a guest filesystem freeze around a coordinator-owned session copy. //! -//! The guest freezes its system filesystem, the session is cloned on the -//! blocking pool, and the guest is thawed whatever the clone did: copying a -//! live ext4 overlay without the freeze can produce an image the fork cannot -//! boot, and running freeze and thaw in one process means no other process's -//! failure can leave the guest frozen. A guest that will not freeze gets no -//! fork. The service owns the destination and removes it when this fails. +//! The VM owner controls freeze and thaw because it owns the guest control +//! channel. It receives no source or destination path and copies no host +//! state. The coordinator reports the copy result, then the owner thaws the +//! guest before answering the original request. -use std::path::PathBuf; use std::sync::Arc; -use capsem_proto::HostToGuest; -use tokio::sync::mpsc; +use capsem_proto::{ipc::ProcessToService, HostToGuest}; +use tokio::sync::{broadcast, mpsc, oneshot}; use crate::job_store::{with_quiescence, JobResult, JobStore}; -/// How long a fork waits for the guest to freeze, as suspend does. const CLONE_FREEZE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); +const COORDINATOR_COPY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(900); -/// Copy the session, with every ledger row accepted so far on disk first. -/// -/// The writer holds accepted rows in memory until its next disk flush, up to -/// five seconds away, and the clone copies the file: without this barrier a -/// fork left out whatever the source recorded in its last seconds (#243). Run -/// under the guest freeze, so nothing the guest does can land between the -/// flush and the copy. A flush that fails fails the fork rather than -/// producing a quietly incomplete one. -async fn flush_then_clone(db: &capsem_logger::DbWriter, source: PathBuf, destination: PathBuf) -> anyhow::Result { - db.flush_checked() - .await - .map_err(|error| anyhow::anyhow!("flush the session ledger before cloning: {error}"))?; - tokio::task::spawn_blocking(move || capsem_core::session::clone_sandbox_state(&source, &destination)) - .await - .map_err(|error| anyhow::anyhow!("clone task failed: {error}"))? +async fn await_coordinator_copy( + jobs: &JobStore, + events: &broadcast::Sender, + id: u64, +) -> anyhow::Result { + let (completed, result) = oneshot::channel(); + if jobs.clone_completions.lock().unwrap().insert(id, completed).is_some() { + anyhow::bail!("clone {id} already awaits coordinator completion"); + } + if events.send(ProcessToService::CloneStateReady { id }).is_err() { + jobs.clone_completions.lock().unwrap().remove(&id); + anyhow::bail!("clone {id} has no coordinator listener"); + } + match tokio::time::timeout(COORDINATOR_COPY_TIMEOUT, result).await { + Ok(Ok(result)) => result.map_err(anyhow::Error::msg), + Ok(Err(_)) => anyhow::bail!("clone {id} coordinator completion channel closed"), + Err(_) => { + jobs.clone_completions.lock().unwrap().remove(&id); + anyhow::bail!("clone {id} coordinator copy timed out") + } + } } -/// Clone `source` into `destination` and answer job `id` with its size. pub(super) fn spawn( hub_tx: &mpsc::Sender, job_store: &Arc, - db: &Arc, - source: &std::path::Path, + events: &broadcast::Sender, id: u64, - destination: String, ) { - let (hub_tx, job_store, db, source) = ( - hub_tx.clone(), - Arc::clone(job_store), - Arc::clone(db), - source.to_path_buf(), - ); + let (hub_tx, job_store, events) = (hub_tx.clone(), Arc::clone(job_store), events.clone()); tokio::spawn(async move { - let destination = PathBuf::from(destination); let result = with_quiescence(&hub_tx, &job_store, CLONE_FREEZE_TIMEOUT, || { - flush_then_clone(&db, source, destination) + await_coordinator_copy(&job_store, &events, id) }) .await .map_err(|error| format!("{error:#}")); diff --git a/crates/capsem-process/src/vsock/clone_state/tests.rs b/crates/capsem-process/src/vsock/clone_state/tests.rs index 5435f6ff4..b67e5b9f2 100644 --- a/crates/capsem-process/src/vsock/clone_state/tests.rs +++ b/crates/capsem-process/src/vsock/clone_state/tests.rs @@ -1,43 +1,30 @@ use super::*; -use capsem_logger::{DbWriter, FileAction, FileEvent, FileKind, WriteOp}; - -/// A fork of a running session carries the rows its writer accepted a moment -/// ago (#243). The writer holds accepted rows in memory until its next disk -/// flush, up to five seconds away, and the clone copies the file. #[tokio::test] -async fn a_fork_carries_rows_the_writer_has_not_flushed_yet() { - let tmp = tempfile::tempdir().unwrap(); - let source = tmp.path().join("src"); - let destination = tmp.path().join("dst"); - std::fs::create_dir_all(source.join("system")).unwrap(); - std::fs::create_dir_all(source.join("guest/workspace")).unwrap(); - std::fs::create_dir(&destination).unwrap(); - - let db = DbWriter::open(&source.join("session.db"), 64).unwrap(); - db.write(WriteOp::FileEvent(FileEvent { - event_id: None, - timestamp: std::time::SystemTime::now(), - action: FileAction::Created, - path: "/root/written-just-before-the-fork".into(), - size: Some(7), - kind: FileKind::File, - trace_id: None, - credential_ref: None, - })) - .await; +async fn frozen_clone_waits_for_the_correlated_coordinator_result() { + let jobs = Arc::new(JobStore::new()); + let (events, mut listener) = broadcast::channel(4); + let waiting = { + let jobs = Arc::clone(&jobs); + let events = events.clone(); + tokio::spawn(async move { await_coordinator_copy(&jobs, &events, 41).await }) + }; - flush_then_clone(&db, source.clone(), destination.clone()) - .await - .unwrap(); + assert!(matches!( + listener.recv().await.unwrap(), + ProcessToService::CloneStateReady { id: 41 } + )); + jobs.complete_clone(41, Ok(8192)).unwrap(); + assert_eq!(waiting.await.unwrap().unwrap(), 8192); + assert!(jobs.clone_completions.lock().unwrap().is_empty()); +} - let cloned = capsem_logger::DbReader::open(&destination.join("session.db")).unwrap(); - let rows = cloned.query_raw("SELECT path FROM fs_events").unwrap(); - assert_eq!( - rows, - r#"{"columns":["path"],"rows":[["/root/written-just-before-the-fork"]]}"# - ); - tokio::task::spawn_blocking(move || db.shutdown_blocking()) - .await - .unwrap(); +#[tokio::test] +async fn clone_without_a_coordinator_listener_keeps_no_completion_slot() { + let jobs = JobStore::new(); + let (events, listener) = broadcast::channel(1); + drop(listener); + let error = await_coordinator_copy(&jobs, &events, 7).await.unwrap_err(); + assert!(error.to_string().contains("no coordinator listener")); + assert!(jobs.clone_completions.lock().unwrap().is_empty()); } From 08ebc81479185f82fdae08eab9d154f2d01dea51 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:10:49 +0000 Subject: [PATCH 171/464] feat(proto): authorize ledger snapshots --- crates/capsem-proto/src/ledger.rs | 10 +++++++--- crates/capsem-proto/src/ledger/tests.rs | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/capsem-proto/src/ledger.rs b/crates/capsem-proto/src/ledger.rs index 22bc8bba0..23759d230 100644 --- a/crates/capsem-proto/src/ledger.rs +++ b/crates/capsem-proto/src/ledger.rs @@ -11,8 +11,8 @@ use thiserror::Error; /// Version of the ledger handshake, request envelopes and logger-owned /// operation/reply enums. Bump it when any of those wire shapes changes. -/// Version 2 adds the fixed coordinator-to-worker descriptor handoff record. -pub const LEDGER_PROTOCOL_VERSION: u16 = 2; +/// Version 3 adds maintainer-only coherent snapshot authority. +pub const LEDGER_PROTOCOL_VERSION: u16 = 3; /// Hash of ledger wire declarations, separate from the guest protocol hash. pub const LEDGER_SCHEMA_HASH: u64 = include!(concat!(env!("OUT_DIR"), "/ledger_schema_hash.txt")); /// Largest diagnostic admitted into a response. @@ -64,6 +64,7 @@ pub enum LedgerCapability { Read, Retain, Export, + Snapshot, Shutdown, } @@ -76,7 +77,10 @@ impl LedgerClientRole { Self::Reader => matches!(capability, LedgerCapability::Read | LedgerCapability::Export), Self::Maintainer => matches!( capability, - LedgerCapability::Read | LedgerCapability::Retain | LedgerCapability::Export + LedgerCapability::Read + | LedgerCapability::Retain + | LedgerCapability::Export + | LedgerCapability::Snapshot ), Self::Supervisor => matches!(capability, LedgerCapability::Shutdown), } diff --git a/crates/capsem-proto/src/ledger/tests.rs b/crates/capsem-proto/src/ledger/tests.rs index af9fba756..0287d1f25 100644 --- a/crates/capsem-proto/src/ledger/tests.rs +++ b/crates/capsem-proto/src/ledger/tests.rs @@ -69,6 +69,20 @@ fn role_rejects_wrong_operation_before_dispatch() { request.authorize(&grant(LedgerClientRole::Reader)).unwrap(); } +#[test] +fn only_maintainers_can_snapshot_a_ledger() { + for role in [ + LedgerClientRole::VmOwner, + LedgerClientRole::Proxy, + LedgerClientRole::Coordinator, + LedgerClientRole::Reader, + LedgerClientRole::Supervisor, + ] { + assert!(!role.permits(LedgerCapability::Snapshot), "{role:?}"); + } + assert!(LedgerClientRole::Maintainer.permits(LedgerCapability::Snapshot)); +} + #[test] fn compatibility_mismatch_fails_closed() { let expected = grant(LedgerClientRole::Reader); From 5266535934d3e1cb9cccc80dc510cc2542142924 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:13:59 +0000 Subject: [PATCH 172/464] feat(logger): own coherent fork snapshots --- crates/capsem-logger/src/ledger_client.rs | 49 ++++++++++++++++++- .../capsem-logger/src/ledger_client/tests.rs | 19 +++++++ crates/capsem-logger/src/ledger_protocol.rs | 12 ++++- .../src/ledger_protocol/tests.rs | 13 +++++ crates/capsem-logger/src/ledger_server.rs | 27 +++++++++- 5 files changed, 117 insertions(+), 3 deletions(-) diff --git a/crates/capsem-logger/src/ledger_client.rs b/crates/capsem-logger/src/ledger_client.rs index bad59f120..12c4b7e80 100644 --- a/crates/capsem-logger/src/ledger_client.rs +++ b/crates/capsem-logger/src/ledger_client.rs @@ -28,6 +28,10 @@ const OPERATION_TIMEOUT: Duration = Duration::from_secs(1); const STREAM_TIMEOUT: Duration = Duration::from_secs(30); #[cfg(test)] const STREAM_TIMEOUT: Duration = Duration::from_millis(250); +#[cfg(not(test))] +const SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(5 * 60 + 5); +#[cfg(test)] +const SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(5); type ClientSender = ipc_channel::Sender; type ClientReceiver = ipc_channel::Receiver; @@ -51,6 +55,10 @@ enum Command { ExportWarc { reply: oneshot::Sender>, }, + Snapshot { + snapshot_id: [u8; 16], + reply: oneshot::Sender>, + }, } /// One authenticated, serial connection to a dedicated session-ledger owner. @@ -137,6 +145,17 @@ impl LedgerClient { receive_result(result, "WARC export").await } + pub async fn snapshot(&self, snapshot_id: [u8; 16]) -> Result { + let (reply, result) = oneshot::channel(); + self.send(Command::Snapshot { snapshot_id, reply }).await?; + receive_result(result, "snapshot").await?; + Ok(self + .path + .parent() + .unwrap_or_else(|| Path::new(".")) + .join(crate::ledger_server::snapshot_directory_name(snapshot_id))) + } + async fn send(&self, command: Command) -> Result<(), String> { self.commands .send(command) @@ -265,6 +284,21 @@ async fn dispatch( finish(reply, result) } Command::ExportWarc { reply } => export_warc(sender, receiver, request_id, reply).await, + Command::Snapshot { snapshot_id, reply } => { + let result = one_reply_with_timeout( + sender, + receiver, + request_id, + LedgerCommand::Snapshot { snapshot_id }, + SNAPSHOT_TIMEOUT, + ) + .await + .and_then(|response| match response { + LedgerReply::Snapshotted => Ok(()), + response => Err(unexpected("snapshot", response)), + }); + finish(reply, result) + } } } @@ -291,9 +325,19 @@ async fn one_reply( receiver: &ClientReceiver, request_id: u64, command: LedgerCommand, +) -> Result { + one_reply_with_timeout(sender, receiver, request_id, command, OPERATION_TIMEOUT).await +} + +async fn one_reply_with_timeout( + sender: &ClientSender, + receiver: &ClientReceiver, + request_id: u64, + command: LedgerCommand, + timeout: Duration, ) -> Result { send_request(sender, request_id, command).await?; - tokio::time::timeout(OPERATION_TIMEOUT, receive_reply(receiver, request_id)) + tokio::time::timeout(timeout, receive_reply(receiver, request_id)) .await .map_err(|_| "ledger client operation timed out".to_string())? } @@ -505,6 +549,9 @@ fn fail_pending(commands: &mut mpsc::Receiver, reason: &str) { Command::ExportWarc { reply } => { let _ = reply.send(Err(reason.to_string())); } + Command::Snapshot { reply, .. } => { + let _ = reply.send(Err(reason.to_string())); + } } } } diff --git a/crates/capsem-logger/src/ledger_client/tests.rs b/crates/capsem-logger/src/ledger_client/tests.rs index b2c83bb29..ad6b19384 100644 --- a/crates/capsem-logger/src/ledger_client/tests.rs +++ b/crates/capsem-logger/src/ledger_client/tests.rs @@ -177,6 +177,25 @@ async fn counters_advance_the_cache_epoch_only_when_the_snapshot_changes() { fixture.stop().await; } +#[tokio::test] +async fn maintainer_snapshot_is_coherent_and_staged_beside_the_owned_ledger() { + let fixture = Fixture::start(LedgerClientRole::Maintainer).await; + fixture.write_network_body().await; + + let snapshot_id = [0x2a; 16]; + let snapshot = fixture.client.snapshot(snapshot_id).await.unwrap(); + assert_eq!( + snapshot.file_name().unwrap().to_str().unwrap(), + crate::ledger_server::snapshot_directory_name(snapshot_id) + ); + let copy = crate::DbHandle::open_external_reader(&snapshot.join("session.db")).unwrap(); + copy.ready().await.unwrap(); + assert_eq!(copy.ledger_counters().await.unwrap().net.total, 1); + assert!(snapshot.join("session.bodies").is_dir()); + + fixture.stop().await; +} + #[tokio::test] async fn server_refusal_is_reported_and_closes_the_failed_client_actor() { let fixture = Fixture::start(LedgerClientRole::Reader).await; diff --git a/crates/capsem-logger/src/ledger_protocol.rs b/crates/capsem-logger/src/ledger_protocol.rs index 7c2401734..fa60c887c 100644 --- a/crates/capsem-logger/src/ledger_protocol.rs +++ b/crates/capsem-logger/src/ledger_protocol.rs @@ -23,6 +23,7 @@ const MAX_LEDGER_CELL_BYTES: usize = 1024 * 1024; const MAX_LEDGER_RESULT_SETS: usize = 16; const MAX_EVENT_ID_BYTES: usize = 12; const MAX_TIMESTAMP_BYTES: usize = 64; +const SNAPSHOT_ID_BYTES: usize = 16; pub type LedgerResponse = capsem_proto::ledger::LedgerResponse; @@ -62,6 +63,10 @@ pub enum LedgerCommand { }, /// Stream a WARC as bounded `WarcChunk` replies followed by `WarcComplete`. ExportWarc, + /// Create a coherent ledger copy in a worker-owned staging directory. + Snapshot { + snapshot_id: [u8; SNAPSHOT_ID_BYTES], + }, Shutdown, } @@ -73,6 +78,7 @@ impl LedgerOperation for LedgerCommand { Self::Query { .. } | Self::ReadBodies { .. } | Self::Counters => LedgerCapability::Read, Self::Retain { .. } => LedgerCapability::Retain, Self::ExportWarc => LedgerCapability::Export, + Self::Snapshot { .. } => LedgerCapability::Snapshot, Self::Shutdown => LedgerCapability::Shutdown, } } @@ -88,6 +94,9 @@ impl LedgerOperation for LedgerCommand { { return Err(LedgerProtocolError::InvalidOperation) } + Self::Snapshot { snapshot_id } if snapshot_id.iter().all(|byte| *byte == 0) => { + return Err(LedgerProtocolError::InvalidOperation) + } _ => {} } let encoded = rmp_serde::to_vec_named(self).map_err(|_| LedgerProtocolError::InvalidOperation)?; @@ -281,6 +290,7 @@ pub enum LedgerReply { WarcComplete { summary: LedgerExportSummary, }, + Snapshotted, ShuttingDown, } @@ -314,7 +324,7 @@ impl LedgerReply { .keys() .all(|reason| !reason.is_empty() && reason.len() <= MAX_LEDGER_COLUMN_BYTES) } - Self::ShuttingDown => true, + Self::Snapshotted | Self::ShuttingDown => true, }; if valid { Ok(()) diff --git a/crates/capsem-logger/src/ledger_protocol/tests.rs b/crates/capsem-logger/src/ledger_protocol/tests.rs index a34f41d7c..d268c297c 100644 --- a/crates/capsem-logger/src/ledger_protocol/tests.rs +++ b/crates/capsem-logger/src/ledger_protocol/tests.rs @@ -56,6 +56,10 @@ fn roles_admit_only_their_operation_classes() { }, ), (LedgerClientRole::Reader, LedgerCommand::ExportWarc), + ( + LedgerClientRole::Maintainer, + LedgerCommand::Snapshot { snapshot_id: [1; 16] }, + ), (LedgerClientRole::Supervisor, LedgerCommand::Shutdown), ] { LedgerRequest::new(1, operation) @@ -65,6 +69,15 @@ fn roles_admit_only_their_operation_classes() { } } +#[test] +fn snapshot_ids_must_be_nonzero_fixed_width_values() { + assert_eq!( + LedgerCommand::Snapshot { snapshot_id: [0; 16] }.validate(), + Err(LedgerProtocolError::InvalidOperation) + ); + LedgerCommand::Snapshot { snapshot_id: [1; 16] }.validate().unwrap(); +} + #[test] fn oversized_event_and_malformed_query_are_rejected_before_dispatch() { let huge = LedgerCommand::Admit { diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index c6cc83241..41e3ae3de 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -2,7 +2,7 @@ use std::io; use std::os::unix::net::UnixStream; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use capsem_foundation::ipc_channel; @@ -34,6 +34,7 @@ struct AdmissionState { /// One process-owned session database shared by its authenticated clients. pub struct LedgerServer { db: Arc, + session_dir: PathBuf, admission: tokio::sync::Mutex, } @@ -41,6 +42,7 @@ impl LedgerServer { pub fn open(path: &Path) -> rusqlite::Result { Ok(Self { db: Arc::new(DbHandle::open(path)?), + session_dir: path.parent().unwrap_or_else(|| Path::new(".")).to_path_buf(), admission: tokio::sync::Mutex::new(AdmissionState::default()), }) } @@ -157,6 +159,24 @@ impl LedgerServer { Err(error) => Err(storage(error)), }, LedgerCommand::ExportWarc => self.send_warc(sender, request_id).await, + LedgerCommand::Snapshot { snapshot_id } => { + let destination = self.session_dir.join(snapshot_directory_name(snapshot_id)); + let source = self.session_dir.clone(); + let admission = self.admission.lock().await; + let snapshot = match self.db.flush().await { + Ok(()) => tokio::task::spawn_blocking(move || { + crate::snapshot_session_ledger(&source, &destination).map_err(storage) + }) + .await + .map_err(|error| format!("ledger snapshot task failed: {error}"))?, + Err(error) => Err(storage(error)), + }; + drop(admission); + match snapshot { + Ok(()) => send_success(sender, request_id, LedgerReply::Snapshotted).await, + Err(error) => Err(error), + } + } LedgerCommand::Shutdown => { send_success(sender, request_id, LedgerReply::ShuttingDown).await?; return Ok(true); @@ -244,6 +264,11 @@ impl LedgerServer { } } +#[must_use] +pub fn snapshot_directory_name(snapshot_id: [u8; 16]) -> String { + format!(".ledger-snapshot-{}", uuid::Uuid::from_bytes(snapshot_id).simple()) +} + /// Execute the same bounded named intent as a descriptor client against an /// already-owned handle. Embedded tests and the worker dispatch share this /// rail so neither can grow a second SQL definition. From 7f86b401e2f963ba6059b2ddf07e511afcd5e1fd Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:16:21 +0000 Subject: [PATCH 173/464] refactor(core): split ledger-free session clone --- crates/capsem-core/src/session/clone.rs | 30 ++++++++++++------- crates/capsem-core/src/session/clone/tests.rs | 15 ++++++++++ crates/capsem-core/src/session/mod.rs | 2 +- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/crates/capsem-core/src/session/clone.rs b/crates/capsem-core/src/session/clone.rs index 953a6c3f8..9bb5f1da1 100644 --- a/crates/capsem-core/src/session/clone.rs +++ b/crates/capsem-core/src/session/clone.rs @@ -23,6 +23,26 @@ use crate::GUEST_SHARE_DIR; /// from `src_session_dir` into the existing, empty `dst_session_dir`, and /// create the `workspace` compat link. Returns the destination's disk usage. pub fn clone_sandbox_state(src_session_dir: &Path, dst_session_dir: &Path) -> anyhow::Result { + clone_sandbox_files(src_session_dir, dst_session_dir)?; + + // The ledger lives at the session root, outside the share. session.db may + // be in WAL mode while the VM runs, so copying the main file alone can + // produce a stale or malformed fork; the logger writes a coherent image + // and brings the body archive beside it, since the database only indexes + // into that file. + if src_session_dir.join("session.db").exists() { + capsem_logger::snapshot_session_ledger(src_session_dir, dst_session_dir) + .context("failed to snapshot the session ledger")?; + } + + Ok(super::disk_usage_bytes(dst_session_dir)) +} + +/// Clone only the sandbox files that are outside the session ledger. +/// +/// The dedicated ledger owner snapshots `session.db` and its body archive; +/// coordinators use this function so they never open or copy database files. +pub fn clone_sandbox_files(src_session_dir: &Path, dst_session_dir: &Path) -> anyhow::Result { adopt_system_overlay(src_session_dir).context("system overlay of the clone source")?; let src_root = ContainedDir::open_root(src_session_dir) .with_context(|| format!("open clone source {}", src_session_dir.display()))?; @@ -65,16 +85,6 @@ pub fn clone_sandbox_state(src_session_dir: &Path, dst_session_dir: &Path) -> an // re-read from anything the guest could have written. super::image_share::carry_image_share(src_session_dir, dst_session_dir).context("carry the image share")?; - // The ledger lives at the session root, outside the share. session.db may - // be in WAL mode while the VM runs, so copying the main file alone can - // produce a stale or malformed fork; the logger writes a coherent image - // and brings the body archive beside it, since the database only indexes - // into that file. - if src_session_dir.join("session.db").exists() { - capsem_logger::snapshot_session_ledger(src_session_dir, dst_session_dir) - .context("failed to snapshot the session ledger")?; - } - let size_bytes = super::disk_usage_bytes(dst_session_dir); info!( cloned_files = stats.cloned_files, diff --git a/crates/capsem-core/src/session/clone/tests.rs b/crates/capsem-core/src/session/clone/tests.rs index 6813ad924..b4fb28e5d 100644 --- a/crates/capsem-core/src/session/clone/tests.rs +++ b/crates/capsem-core/src/session/clone/tests.rs @@ -212,3 +212,18 @@ fn session_db_is_copied_coherently_outside_the_share() { assert_eq!(payload, "committed-in-wal"); drop(conn); } + +#[test] +fn sandbox_file_clone_never_copies_ledger_files() { + let s = sessions(); + std::fs::write(s.src.join("session.db"), b"database").unwrap(); + std::fs::write(s.src.join("session.db-wal"), b"wal").unwrap(); + std::fs::create_dir(s.src.join("session.bodies")).unwrap(); + std::fs::write(s.src.join("session.bodies/generation"), b"bodies").unwrap(); + + clone_sandbox_files(&s.src, &s.dst).unwrap(); + + assert!(!s.dst.join("session.db").exists()); + assert!(!s.dst.join("session.db-wal").exists()); + assert!(!s.dst.join("session.bodies").exists()); +} diff --git a/crates/capsem-core/src/session/mod.rs b/crates/capsem-core/src/session/mod.rs index fe74706e9..3c4d68cd6 100644 --- a/crates/capsem-core/src/session/mod.rs +++ b/crates/capsem-core/src/session/mod.rs @@ -12,7 +12,7 @@ mod workspace; mod tests; pub use capsem_logger::{epoch_to_iso, generate_session_id, is_valid_session_id, now_iso}; -pub use clone::{clone_file, clone_sandbox_state}; +pub use clone::{clone_file, clone_sandbox_files, clone_sandbox_state}; pub use image_share::{ carry_image_share, clear_image_share, image_share_blobs, image_share_path, prepare_image_share, publish_image_share, IMAGE_SHARE_DIR, IMAGE_SHARE_TAG, From 6634c4ac1b9420677f9c507881be85f6f15449ab Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:22:30 +0000 Subject: [PATCH 174/464] security(service): coordinate path-free session clones --- CHANGELOG.md | 5 + .../src/container_setup/relaunch.rs | 2 +- crates/capsem-service/src/tests.rs | 75 +++++--- .../src/tests/db_handle_ownership.rs | 13 ++ crates/capsem-service/src/vm_files/fork.rs | 171 +++++++++++++++--- .../src/vm_lifecycle/provision.rs | 1 + 6 files changed, 224 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b045ca3b..3f857d9fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Coordinate live forks through a path-free freeze/copy/thaw handshake and + copy coherent database state only from the session ledger worker's + maintainer-scoped snapshot, keeping both VM owners and the service from + opening session SQLite storage. + - Export confined proxy metrics through a bounded, generation-bound broker capability that fixes session attribution and keeps collector destinations, credentials and environment state in the trusted service. diff --git a/crates/capsem-service/src/container_setup/relaunch.rs b/crates/capsem-service/src/container_setup/relaunch.rs index 04cc701ec..9722747ed 100644 --- a/crates/capsem-service/src/container_setup/relaunch.rs +++ b/crates/capsem-service/src/container_setup/relaunch.rs @@ -10,7 +10,7 @@ //! The image itself needs nothing from here. A relaunch unpacks from the //! session's own image share, which is host-only and survives a stop, or not //! at all when the unpacked root is already on its overlay; a clone's share is -//! linked from its source's (`capsem_core::session::clone_sandbox_state`). +//! linked from its source's (`capsem_core::session::clone_sandbox_files`). //! Neither reads the workspace, and neither depends on the host's blob cache, //! which may have pruned the image since. diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index fa0115414..d477e1a50 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -233,32 +233,65 @@ pub(crate) fn only_reloads(received: &[ServiceToProcess], count: usize) -> bool .all(|message| matches!(message, ServiceToProcess::ReloadConfig { .. })) } -/// A VM owner that answers `CloneState` the way the real one does once the -/// guest is frozen -- by cloning `source` -- or refuses with `refusal`. +/// A VM owner that freezes, lets the service clone, and then reports thaw. pub(crate) fn spawn_fake_fork_owner( uds_path: &StdPath, - source: PathBuf, + _source: PathBuf, refusal: Option<&'static str>, ) -> tokio::task::JoinHandle> { - spawn_fake_process(uds_path, 1, move |message| { - let reply = match message { - ServiceToProcess::CloneState { id, destination } => { - let (size_bytes, error) = match refusal { - None => ( - Some(capsem_core::session::clone_sandbox_state(&source, StdPath::new(destination)).unwrap()), - None, - ), - Some(reason) => (None, Some(reason.to_string())), - }; - Some(ProcessToService::CloneStateResult { - id: *id, - size_bytes, - error, - }) - } - other => panic!("fork sent an unexpected owner message: {other:?}"), + let _ = std::fs::remove_file(uds_path); + let listener = tokio::net::UnixListener::bind(uds_path).unwrap(); + std::fs::write(uds_path.with_extension("ready"), b"ready").unwrap(); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let std_stream = stream.into_std().unwrap(); + let std_stream = tokio::task::spawn_blocking(move || { + let mut std_stream = std_stream; + capsem_foundation::ipc_handshake::negotiate_responder(&mut std_stream, "capsem-process-test", "")?; + Ok::<_, capsem_proto::handshake::HandshakeError>(std_stream) + }) + .await + .unwrap() + .unwrap(); + let (tx, rx): ( + capsem_foundation::ipc_channel::Sender, + capsem_foundation::ipc_channel::Receiver, + ) = capsem_foundation::ipc_channel::channel_from_std(std_stream).unwrap(); + let request = rx.recv().await.unwrap(); + let ServiceToProcess::CloneState { id } = request else { + panic!("fork sent an unexpected owner message: {request:?}") }; - Box::pin(async move { reply }) + let mut messages = vec![request]; + if let Some(reason) = refusal { + tx.send(ProcessToService::CloneStateResult { + id, + size_bytes: None, + error: Some(reason.to_string()), + }) + .await + .unwrap(); + return messages; + } + tx.send(ProcessToService::CloneStateReady { id }).await.unwrap(); + let completion = rx.recv().await.unwrap(); + let ServiceToProcess::CloneStateComplete { + id: completed_id, + size_bytes, + error, + } = &completion + else { + panic!("fork sent an unexpected completion: {completion:?}") + }; + assert_eq!(*completed_id, id); + tx.send(ProcessToService::CloneStateResult { + id, + size_bytes: *size_bytes, + error: error.clone(), + }) + .await + .unwrap(); + messages.push(completion); + messages }) } diff --git a/crates/capsem-service/src/tests/db_handle_ownership.rs b/crates/capsem-service/src/tests/db_handle_ownership.rs index a6245e0bb..c7d7c28c6 100644 --- a/crates/capsem-service/src/tests/db_handle_ownership.rs +++ b/crates/capsem-service/src/tests/db_handle_ownership.rs @@ -61,3 +61,16 @@ fn provision_never_opens_the_ledger_its_child_is_creating() { 2026-09-24). Routes register the reader lazily once the ledger is ready." ); } + +#[test] +fn fork_coordinates_ledger_snapshot_without_opening_storage() { + let fork = std::fs::read_to_string(concat!(env!("CARGO_MANIFEST_DIR"), "/src/vm_files/fork.rs")) + .expect("fork source must be readable"); + assert!(!fork.contains("clone_sandbox_state")); + assert!(!fork.contains("snapshot_session_ledger")); + assert!(!fork.contains("rusqlite")); + assert!(fork.contains("LedgerClientRole::Maintainer")); + assert!(fork.contains("client.snapshot(")); + assert!(fork.contains("CloneStateReady")); + assert!(fork.contains("CloneStateComplete")); +} diff --git a/crates/capsem-service/src/vm_files/fork.rs b/crates/capsem-service/src/vm_files/fork.rs index 389e199fc..116a6b534 100644 --- a/crates/capsem-service/src/vm_files/fork.rs +++ b/crates/capsem-service/src/vm_files/fork.rs @@ -58,7 +58,7 @@ pub(crate) async fn handle_fork( // separate from the human display name. let vm_id = new_persistent_vm_id(); let new_session_dir = state.run_dir.join("persistent").join(&vm_id); - let size_bytes = clone_session_state(&state, uds_path.as_deref(), session_dir, new_session_dir.clone()) + let size_bytes = clone_session_state(&state, &id, uds_path.as_deref(), session_dir, new_session_dir.clone()) .await .map_err(|e| { capsem_service::app_error_logged!(error, StatusCode::INTERNAL_SERVER_ERROR, "fork: clone failed: {e}") @@ -111,6 +111,7 @@ pub(crate) async fn handle_fork( /// stopped sandbox has no writer and is copied here, off the async workers. pub(crate) async fn clone_session_state( state: &ServiceState, + source_id: &str, running: Option<&std::path::Path>, source: PathBuf, destination: PathBuf, @@ -118,7 +119,13 @@ pub(crate) async fn clone_session_state( tokio::fs::create_dir_all(&destination) .await .map_err(|error| format!("create {}: {error}", destination.display()))?; - let size = clone_guest_state(state, running, source.clone(), destination.clone()).await?; + let size = match clone_guest_state(state, source_id, running, source.clone(), destination.clone()).await { + Ok(size) => size, + Err(error) => { + let _ = tokio::fs::remove_dir_all(&destination).await; + return Err(error); + } + }; // The clone boots the source's staged image; the service must know it // runs one, or exec and the files API would treat it as a bare VM. if let Err(error) = crate::container_setup::carry_launch_record(&source, &destination) { @@ -130,45 +137,167 @@ pub(crate) async fn clone_session_state( async fn clone_guest_state( state: &ServiceState, + source_id: &str, running: Option<&std::path::Path>, source: PathBuf, destination: PathBuf, ) -> Result { let Some(uds_path) = running else { - return tokio::task::spawn_blocking(move || { - capsem_core::session::clone_sandbox_state(&source, &destination).map_err(|error| { - let _ = std::fs::remove_dir_all(&destination); - format!("{error:#}") - }) - }) - .await - .map_err(|error| format!("clone task failed: {error}"))?; + return clone_coordinator_state(state, source_id, source, destination).await; }; let id = state.next_job_id(); - let request = ServiceToProcess::CloneState { - id, - destination: destination.to_string_lossy().into_owned(), + let owner = owner_connection::OwnerConnection::acquire(state, uds_path)?; + let (tx, rx) = owner.open(state, "capsem-service", false).await?; + tx.send(ServiceToProcess::CloneState { id }) + .await + .map_err(|error| format!("send clone freeze request: {error}"))?; + match receive_clone_message(&rx, id, ClonePhase::Ready).await? { + ProcessToService::CloneStateReady { .. } => {} + ProcessToService::CloneStateResult { error, .. } => { + return Err(error.unwrap_or_else(|| "the sandbox thawed before clone readiness".into())); + } + _ => unreachable!(), + } + owner.validate(state, false)?; + + let copied = clone_coordinator_state(state, source_id, source, destination.clone()).await; + let completion = match &copied { + Ok(size_bytes) => ServiceToProcess::CloneStateComplete { + id, + size_bytes: Some(*size_bytes), + error: None, + }, + Err(error) => ServiceToProcess::CloneStateComplete { + id, + size_bytes: None, + error: Some(error.clone()), + }, }; - let result = match send_ipc_command(state, uds_path, request, Some(CLONE_STATE_REPLY_SECS)).await { - Ok(ProcessToService::CloneStateResult { + tx.send(completion) + .await + .map_err(|error| format!("send clone completion: {error}"))?; + let owner_result = match receive_clone_message(&rx, id, ClonePhase::Complete).await? { + ProcessToService::CloneStateResult { size_bytes: Some(size), error: None, .. - }) => Ok(size), - Ok(ProcessToService::CloneStateResult { error, .. }) => { + } => Ok(size), + ProcessToService::CloneStateResult { error, .. } => { Err(error.unwrap_or_else(|| "the sandbox reported no clone size".into())) } - Ok(other) => Err(format!("unexpected clone reply: {other:?}")), - Err(error) => Err(error), + _ => unreachable!(), + }; + owner.validate(state, false)?; + let result = match (copied, owner_result) { + (Err(error), _) => Err(error), + (Ok(expected), Ok(actual)) if expected == actual => Ok(actual), + (Ok(expected), Ok(actual)) => Err(format!( + "the sandbox reported clone size {actual}, coordinator measured {expected}" + )), + (Ok(_), Err(error)) => Err(error), }; if result.is_err() { - // The owner removes what it wrote; this covers an owner that never - // answered, so no half-made fork directory outlives the request. let _ = tokio::fs::remove_dir_all(&destination).await; } result } +#[derive(Clone, Copy)] +enum ClonePhase { + Ready, + Complete, +} + +async fn receive_clone_message( + rx: &capsem_foundation::ipc_channel::Receiver, + id: u64, + phase: ClonePhase, +) -> Result { + let receive = async { + loop { + let message = rx + .recv() + .await + .map_err(|error| format!("clone owner connection closed: {error}"))?; + let matches = match (&phase, &message) { + (ClonePhase::Ready, ProcessToService::CloneStateReady { id: reply }) + | (ClonePhase::Ready, ProcessToService::CloneStateResult { id: reply, .. }) + | (ClonePhase::Complete, ProcessToService::CloneStateResult { id: reply, .. }) => *reply == id, + _ => false, + }; + if matches { + return Ok(message); + } + } + }; + tokio::time::timeout(std::time::Duration::from_secs(CLONE_STATE_REPLY_SECS), receive) + .await + .map_err(|_| format!("clone owner timed out after {CLONE_STATE_REPLY_SECS}s"))? +} + +async fn clone_coordinator_state( + state: &ServiceState, + source_id: &str, + source: PathBuf, + destination: PathBuf, +) -> Result { + let files_source = source.clone(); + let files_destination = destination.clone(); + tokio::task::spawn_blocking(move || { + capsem_core::session::clone_sandbox_files(&files_source, &files_destination) + .map_err(|error| format!("{error:#}")) + }) + .await + .map_err(|error| format!("clone task failed: {error}"))??; + + match tokio::fs::symlink_metadata(source.join("session.db")).await { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(format!("inspect source ledger: {error}")), + Ok(_) => clone_ledger(state, source_id, &source, &destination).await?, + } + Ok(capsem_core::session::disk_usage_bytes(&destination)) +} + +async fn clone_ledger( + state: &ServiceState, + source_id: &str, + source: &std::path::Path, + destination: &std::path::Path, +) -> Result<(), String> { + let database = source.join("session.db"); + let channel = state + .ledger_workers + .acquire( + source_id, + &database, + &source.join("ledger.log"), + capsem_proto::ledger::LedgerClientRole::Maintainer, + ) + .await + .map_err(|error| format!("acquire source ledger owner: {error:#}"))?; + let (stream, grant) = channel.into_parts(); + let client = capsem_logger::ledger_client::LedgerClient::connect(stream, grant, database).await?; + let snapshot = client.snapshot(*uuid::Uuid::new_v4().as_bytes()).await?; + let snapshot_copy = snapshot.clone(); + let destination = destination.to_path_buf(); + let copied = tokio::task::spawn_blocking(move || { + let source = capsem_foundation::unix::contained::ContainedDir::open_root(&snapshot_copy) + .map_err(|error| format!("open staged ledger snapshot: {error}"))?; + let destination = capsem_foundation::unix::contained::ContainedDir::open_root(&destination) + .map_err(|error| format!("open clone destination: {error}"))?; + capsem_foundation::unix::tree_clone::clone_tree(&source, &destination) + .map_err(|error| format!("copy staged ledger snapshot: {error}"))?; + Ok::<(), String>(()) + }) + .await + .map_err(|error| format!("ledger copy task failed: {error}"))?; + let removed = tokio::fs::remove_dir_all(&snapshot) + .await + .map_err(|error| format!("remove staged ledger snapshot {}: {error}", snapshot.display())); + copied?; + removed +} + /// The owner answers within its own 900 s clone bound; wait a little longer /// so its answer, not this deadline, is what the caller sees. const CLONE_STATE_REPLY_SECS: u64 = 930; diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index b7385238a..3a5191286 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -141,6 +141,7 @@ impl ServiceState { tokio::runtime::Handle::current() .block_on(crate::vm_files::clone_session_state( self, + &super::persistent_entry_vm_id(entry), running.as_deref(), entry.session_dir.clone(), session_dir.clone(), From 462a085405dcc5e0c36e0102107545a456ae1f53 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:26:08 +0000 Subject: [PATCH 175/464] refactor(core): isolate mutable owner state --- .../capsem-core/src/container/publish/saved.rs | 16 +++++++++++++++- .../src/container/publish/tests/security.rs | 3 ++- crates/capsem-core/src/session/mod.rs | 11 +++++++++++ crates/capsem-core/src/session/tests.rs | 12 +++++++++++- 4 files changed, 39 insertions(+), 3 deletions(-) diff --git a/crates/capsem-core/src/container/publish/saved.rs b/crates/capsem-core/src/container/publish/saved.rs index c6e429023..2c32bd878 100644 --- a/crates/capsem-core/src/container/publish/saved.rs +++ b/crates/capsem-core/src/container/publish/saved.rs @@ -1,8 +1,11 @@ //! Host-only listener intent; guest workspace and fork snapshots carry no ports. use super::*; use capsem_proto::{ipc::PublicationInfo, PublicationTarget}; +use std::ffi::OsStr; use std::path::{Path, PathBuf}; +const SAVED_PUBLICATIONS_FILE: &str = "published-ports.json"; + /// One declared listener. Records written before targets existed are the /// container's, which is all a publication could reach then. #[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] @@ -52,9 +55,20 @@ fn forget(path: &Path, host: u16) -> Result { impl Publisher { pub fn for_session(session_dir: &Path, budgets: capsem_config::router::RouterConfig) -> Result { + let state_dir = crate::session::prepare_owner_state_dir(session_dir)?; + let root = capsem_foundation::unix::contained::ContainedDir::open_root(session_dir)?; + let state = root.descend(OsStr::new(crate::session::OWNER_STATE_DIR))?; + let legacy = OsStr::new(SAVED_PUBLICATIONS_FILE); + if root.entry_kind(legacy)?.is_some() { + ensure!( + state.entry_kind(legacy)?.is_none(), + "both legacy and owner-state saved publication records exist" + ); + root.rename_to(legacy, &state, legacy)?; + } let mut publisher = Self::configured(budgets)?; publisher.saved = Some(Mappings { - path: session_dir.join("published-ports.json"), + path: state_dir.join(SAVED_PUBLICATIONS_FILE), lock: tokio::sync::Mutex::new(()), }); Ok(publisher) diff --git a/crates/capsem-core/src/container/publish/tests/security.rs b/crates/capsem-core/src/container/publish/tests/security.rs index 95862e668..4a5a7dc08 100644 --- a/crates/capsem-core/src/container/publish/tests/security.rs +++ b/crates/capsem-core/src/container/publish/tests/security.rs @@ -632,9 +632,10 @@ async fn a_saved_exposure_the_rules_now_refuse_is_forgotten_on_restore() { let (control, _requests) = mpsc::channel(8); assert_eq!(owner.restore(control).await.unwrap(), 0); assert_eq!( - std::fs::read_to_string(dir.path().join("published-ports.json")).unwrap(), + std::fs::read_to_string(dir.path().join("owner/published-ports.json")).unwrap(), "[]" ); + assert!(!dir.path().join("published-ports.json").exists()); let rows = lifecycle_rows(&engine, &path).await; assert!(rows.contains("restored"), "{rows}"); std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, port)).expect("nothing listens for a refused restore"); diff --git a/crates/capsem-core/src/session/mod.rs b/crates/capsem-core/src/session/mod.rs index 3c4d68cd6..6440a0db1 100644 --- a/crates/capsem-core/src/session/mod.rs +++ b/crates/capsem-core/src/session/mod.rs @@ -25,6 +25,17 @@ pub use overlay::{ pub use spawn_identity::{read_spawn_identity, write_spawn_identity}; pub use workspace::{open_workspace, WORKSPACE_DIR}; +/// Host-only mutable state retained by the VM owner after confinement. +pub const OWNER_STATE_DIR: &str = "owner"; + +pub fn prepare_owner_state_dir(session_dir: &std::path::Path) -> std::io::Result { + use std::ffi::OsStr; + + let root = capsem_foundation::unix::contained::ContainedDir::open_root(session_dir)?; + root.descend_or_create(OsStr::new(OWNER_STATE_DIR), 0o700)?; + Ok(session_dir.join(OWNER_STATE_DIR)) +} + /// Distil a captured `process.log`/`serial.log` tail down to the one line /// worth putting in front of a human. /// diff --git a/crates/capsem-core/src/session/tests.rs b/crates/capsem-core/src/session/tests.rs index eb8455175..5a992d197 100644 --- a/crates/capsem-core/src/session/tests.rs +++ b/crates/capsem-core/src/session/tests.rs @@ -1,4 +1,14 @@ -use super::boot_failure_summary; +use super::{boot_failure_summary, prepare_owner_state_dir}; + +#[test] +fn owner_state_is_a_private_host_only_subdirectory() { + use std::os::unix::fs::PermissionsExt as _; + + let session = tempfile::tempdir().unwrap(); + let owner = prepare_owner_state_dir(session.path()).unwrap(); + assert_eq!(owner, session.path().join("owner")); + assert_eq!(std::fs::metadata(owner).unwrap().permissions().mode() & 0o777, 0o700); +} #[test] fn boot_failure_summary_returns_the_last_line_that_says_something() { From 7770fc7f5617f8c3583f723d6867e4ce3b75575a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:26:58 +0000 Subject: [PATCH 176/464] feat(proto): fix owner checkpoint location --- crates/capsem-proto/src/ipc.rs | 6 ++---- crates/capsem-proto/src/ipc/tests.rs | 11 ++--------- crates/capsem-proto/src/ipc/tests/wire.rs | 4 +--- crates/capsem-proto/src/lib.rs | 3 ++- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/crates/capsem-proto/src/ipc.rs b/crates/capsem-proto/src/ipc.rs index 98488e67a..11cbde924 100644 --- a/crates/capsem-proto/src/ipc.rs +++ b/crates/capsem-proto/src/ipc.rs @@ -108,10 +108,8 @@ pub enum ServiceToProcess { /// longer reading -- prevents late writes from leaking into the /// user's parent shell after raw mode is restored. StopTerminalStream, - /// Suspend VM and save checkpoint to disk. - Suspend { - checkpoint_path: String, - }, + /// Suspend the VM into its fixed, confined checkpoint location. + Suspend, /// Freeze the guest for a coordinator-owned state clone. The owner emits /// `CloneStateReady` after the freeze and always thaws before its final /// result. No source or destination path enters the VM owner. diff --git a/crates/capsem-proto/src/ipc/tests.rs b/crates/capsem-proto/src/ipc/tests.rs index 9a11dca88..00cea1d76 100644 --- a/crates/capsem-proto/src/ipc/tests.rs +++ b/crates/capsem-proto/src/ipc/tests.rs @@ -428,17 +428,10 @@ fn reload_config_roundtrip() { #[test] fn suspend_roundtrip() { - let msg = ServiceToProcess::Suspend { - checkpoint_path: "/tmp/checkpoint.vzsave".into(), - }; + let msg = ServiceToProcess::Suspend; let bytes = serde_json::to_vec(&msg).unwrap(); let msg2: ServiceToProcess = serde_json::from_slice(&bytes).unwrap(); - match msg2 { - ServiceToProcess::Suspend { checkpoint_path } => { - assert_eq!(checkpoint_path, "/tmp/checkpoint.vzsave"); - } - _ => panic!("wrong variant"), - } + assert!(matches!(msg2, ServiceToProcess::Suspend)); } #[test] diff --git a/crates/capsem-proto/src/ipc/tests/wire.rs b/crates/capsem-proto/src/ipc/tests/wire.rs index 056022cb4..01a6218a8 100644 --- a/crates/capsem-proto/src/ipc/tests/wire.rs +++ b/crates/capsem-proto/src/ipc/tests/wire.rs @@ -57,9 +57,7 @@ fn service_to_process_variant_names_and_roundtrips_are_stable() { ServiceToProcess::ReloadConfig { id: 11 }, ServiceToProcess::StartTerminalStream, ServiceToProcess::StopTerminalStream, - ServiceToProcess::Suspend { - checkpoint_path: "/tmp/checkpoint".into(), - }, + ServiceToProcess::Suspend, ServiceToProcess::CloneState { id: 11 }, ServiceToProcess::CloneStateComplete { id: 11, diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 43a7aaf2f..c7213d05d 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -88,7 +88,8 @@ pub const MAX_BOOT_FILES: usize = 64; /// service-to-owner freeze messages, and adds `CloneState` for forks. /// Version 12 makes cloning a coordinator operation bracketed by the owner's /// guest freeze, so no destination path crosses into the VM owner. -pub const PROTOCOL_VERSION: u16 = 12; +/// Version 13 fixes suspend to the owner's confined checkpoint location. +pub const PROTOCOL_VERSION: u16 = 13; /// Guest loopback port of the agent's DNS proxy (port 53 is redirected here). pub const GUEST_DNS_PROXY_PORT: u16 = 1053; From d20b44c1afba9d22a01615b1fc21c5744ec9c6aa Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:30:00 +0000 Subject: [PATCH 177/464] refactor(service): confine checkpoint state path --- crates/capsem-service/src/main.rs | 2 +- .../capsem-service/src/owner_connection/tests.rs | 3 +-- crates/capsem-service/src/vm_lifecycle.rs | 15 ++++++--------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 491937d70..dc9dbf310 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -89,7 +89,7 @@ use vm_lifecycle::*; /// choice, not ours; every reader takes a bounded tail. const SESSION_LOG_TAIL_MAX_BYTES: usize = 5 * 1024 * 1024; -const RESUME_CHECKPOINT_NAME: &str = "checkpoint.vzsave"; +const RESUME_CHECKPOINT_NAME: &str = "owner/checkpoint.vzsave"; const SUSPEND_CONFIRM_TIMEOUT_SECS: u64 = 45; const AUTOMATIC_UPDATE_INITIAL_DELAY_SECS: u64 = 60; const AUTOMATIC_UPDATE_POLL_SECS: u64 = 60 * 60; diff --git a/crates/capsem-service/src/owner_connection/tests.rs b/crates/capsem-service/src/owner_connection/tests.rs index 1a800049b..af5e3cf58 100644 --- a/crates/capsem-service/src/owner_connection/tests.rs +++ b/crates/capsem-service/src/owner_connection/tests.rs @@ -94,10 +94,9 @@ async fn suspending_owner_child() { .await .unwrap(); let (tx, rx) = channel_from_std::(socket).unwrap(); - let ServiceToProcess::Suspend { checkpoint_path } = rx.recv().await.unwrap() else { + let ServiceToProcess::Suspend = rx.recv().await.unwrap() else { panic!("expected suspend") }; - assert_eq!(checkpoint_path, RESUME_CHECKPOINT_NAME); tx.send(ProcessToService::StateChanged { id: "owner".into(), state: "Suspended".into(), diff --git a/crates/capsem-service/src/vm_lifecycle.rs b/crates/capsem-service/src/vm_lifecycle.rs index 6d47cb94f..010ee355c 100644 --- a/crates/capsem-service/src/vm_lifecycle.rs +++ b/crates/capsem-service/src/vm_lifecycle.rs @@ -446,15 +446,12 @@ pub(super) async fn handle_suspend( .await .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e))?; - let checkpoint_path = RESUME_CHECKPOINT_NAME.to_string(); - tx.send(ServiceToProcess::Suspend { checkpoint_path }) - .await - .map_err(|e| { - AppError( - StatusCode::INTERNAL_SERVER_ERROR, - format!("failed to send suspend command: {e}"), - ) - })?; + tx.send(ServiceToProcess::Suspend).await.map_err(|e| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to send suspend command: {e}"), + ) + })?; // Wait for process exit (channel closed). The process sends StateChanged {"Suspended"} // right before exiting. We must wait for full exit to avoid a race condition where From e88382f74c14f048485804d7bf7ded3e118d1c61 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:32:43 +0000 Subject: [PATCH 178/464] security(process): deny session ledger paths --- CHANGELOG.md | 5 ++ crates/capsem-process/src/ipc.rs | 7 +-- crates/capsem-process/src/ipc/tests.rs | 19 +------ .../src/ipc/tests/classification.rs | 4 +- crates/capsem-process/src/main.rs | 55 ++++++++++++++++++- crates/capsem-process/src/tests.rs | 16 ++++++ crates/capsem-process/src/vsock.rs | 11 +--- 7 files changed, 83 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f857d9fd..f5c10086a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Restrict VM owners to the guest share, system overlay, read-only image share, + and a dedicated mutable owner-state directory; attest after confinement that + the owner cannot open `session.db`, and fix suspend checkpoints to that + confined directory instead of accepting a path over IPC. + - Coordinate live forks through a path-free freeze/copy/thaw handshake and copy coherent database state only from the session ledger worker's maintainer-scoped snapshot, keeping both VM owners and the service from diff --git a/crates/capsem-process/src/ipc.rs b/crates/capsem-process/src/ipc.rs index 87b8b04f3..15ef97180 100644 --- a/crates/capsem-process/src/ipc.rs +++ b/crates/capsem-process/src/ipc.rs @@ -720,12 +720,9 @@ pub(crate) async fn handle_ipc_connection( info!("Received Shutdown command, exiting IPC loop gracefully"); break; } - ServiceToProcess::Suspend { checkpoint_path } => { + ServiceToProcess::Suspend => { info!("Received Suspend command, forwarding to ctrl channel"); - capsem_core::try_send!( - "ctrl_suspend", - ctrl_tx.send(ServiceToProcess::Suspend { checkpoint_path }).await - ); + capsem_core::try_send!("ctrl_suspend", ctrl_tx.send(ServiceToProcess::Suspend).await); } ServiceToProcess::McpListServers { id } => { let mcp = Arc::clone(&mcp_runtime); diff --git a/crates/capsem-process/src/ipc/tests.rs b/crates/capsem-process/src/ipc/tests.rs index e5b37c4b9..a8dcf63e9 100644 --- a/crates/capsem-process/src/ipc/tests.rs +++ b/crates/capsem-process/src/ipc/tests.rs @@ -639,16 +639,8 @@ async fn negotiated_dispatcher_covers_stream_jobs_queries_and_lifecycle() { other => panic!("unexpected MCP call result: {other:?}"), } - service_tx - .send(ServiceToProcess::Suspend { - checkpoint_path: "checkpoint.vzsave".to_string(), - }) - .await - .unwrap(); - assert!(matches!( - ctrl_rx.recv().await.unwrap(), - ServiceToProcess::Suspend { .. } - )); + service_tx.send(ServiceToProcess::Suspend).await.unwrap(); + assert!(matches!(ctrl_rx.recv().await.unwrap(), ServiceToProcess::Suspend)); service_tx.send(ServiceToProcess::Shutdown).await.unwrap(); assert!(matches!(ctrl_rx.recv().await.unwrap(), ServiceToProcess::Shutdown)); handler.await.unwrap().unwrap(); @@ -839,12 +831,7 @@ fn classify_shutdown() { #[test] fn classify_suspend() { - assert_eq!( - classify_ipc_message(&ServiceToProcess::Suspend { - checkpoint_path: "cp.vzsave".into() - }), - IpcAction::Lifecycle - ); + assert_eq!(classify_ipc_message(&ServiceToProcess::Suspend), IpcAction::Lifecycle); } #[test] diff --git a/crates/capsem-process/src/ipc/tests/classification.rs b/crates/capsem-process/src/ipc/tests/classification.rs index ed4e66a96..841549779 100644 --- a/crates/capsem-process/src/ipc/tests/classification.rs +++ b/crates/capsem-process/src/ipc/tests/classification.rs @@ -36,8 +36,8 @@ pub(super) fn classify_ipc_message(msg: &ServiceToProcess) -> IpcAction { | ServiceToProcess::AbortPorts { .. } | ServiceToProcess::PlugCable { .. } | ServiceToProcess::UnplugCable { .. } => IpcAction::Unexpected, - ServiceToProcess::ReloadConfig { .. } | ServiceToProcess::InjectCredentials { .. } => IpcAction::Reload, - ServiceToProcess::Shutdown | ServiceToProcess::Suspend { .. } => IpcAction::Lifecycle, + ServiceToProcess::ReloadConfig { .. } => IpcAction::Reload, + ServiceToProcess::Shutdown | ServiceToProcess::Suspend => IpcAction::Lifecycle, } } diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 0d65fc18b..1b808e35f 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -180,6 +180,14 @@ fn aggregator_log_path(session_dir: &Path) -> PathBuf { session_dir.join("mcp-aggregator.stderr.log") } +const OWNER_CHECKPOINT_FILE: &str = "checkpoint.vzsave"; + +fn owner_checkpoint_path(session_dir: &Path) -> PathBuf { + session_dir + .join(capsem_core::session::OWNER_STATE_DIR) + .join(OWNER_CHECKPOINT_FILE) +} + fn prepare_session_layout(session_dir: &Path, scratch_disk_size_gb: u32) -> Result { capsem_core::create_virtiofs_session(session_dir, scratch_disk_size_gb)?; let guest_dir = capsem_core::guest_share_dir(session_dir); @@ -236,12 +244,37 @@ fn grant_owner_path( Ok(policy.allow(canonical, access)) } +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn grant_owner_session_paths( + mut policy: capsem_foundation::unix::worker_sandbox::Policy, + session_dir: &Path, +) -> Result { + use capsem_foundation::unix::worker_sandbox::Access; + + for (path, access) in [ + (capsem_core::guest_share_dir(session_dir), Access::ReadWrite), + ( + capsem_core::session::system_overlay_image_path(session_dir), + Access::ReadWrite, + ), + (capsem_core::session::image_share_path(session_dir), Access::ReadOnly), + ( + session_dir.join(capsem_core::session::OWNER_STATE_DIR), + Access::ReadWrite, + ), + (session_dir.join("serial.log"), Access::ReadWrite), + ] { + policy = grant_owner_path(policy, &path, access)?; + } + Ok(policy) +} + #[cfg(target_os = "linux")] fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; let mut policy = Policy::new(Role::VmOwner); - policy = grant_owner_path(policy, session_dir, Access::ReadWrite)?; + policy = grant_owner_session_paths(policy, session_dir)?; policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; @@ -278,7 +311,7 @@ fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; let mut policy = Policy::new(Role::VmOwner); - policy = grant_owner_path(policy, session_dir, Access::ReadWrite)?; + policy = grant_owner_session_paths(policy, session_dir)?; policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; @@ -318,6 +351,7 @@ struct OwnerSandboxAttestation { direct_file: std::fs::File, guest_path: PathBuf, guest_relative: Vec, + ledger_path: PathBuf, } #[cfg(any(target_os = "linux", target_os = "macos"))] @@ -341,7 +375,9 @@ fn prepare_owner_sandbox_attestation(session_dir: &Path) -> Result Result Result<()> { info!(id = %args.id, "capsem-sandbox-process starting"); let guest_dir = prepare_session_layout(&session_dir, args.scratch_disk_size_gb)?; + capsem_core::session::prepare_owner_state_dir(&session_dir)?; // The image share is attached to every session, read-only at the device: // a device cannot be added after boot, and an image is pulled only once // the VM runs (its owner admits the pull). It stays empty until the @@ -481,6 +523,13 @@ fn main() -> Result<()> { let system_img = capsem_core::session::system_overlay_image_path(&session_dir); let machine_identifier_path = session_dir.join("machine_identifier"); let serial_log_path = session_dir.join("serial.log"); + drop( + std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(&serial_log_path) + .with_context(|| format!("prepare serial log {}", serial_log_path.display()))?, + ); let (vm, vsock_rx, sm) = boot_vm(BootOptions { assets: &args.assets_dir, kernel_override: args.kernel.as_deref(), diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index b1b378df7..04ccd2f14 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -24,6 +24,22 @@ fn production_vm_owner_never_opens_the_session_database() { ); } +#[test] +fn confinement_grants_exact_runtime_paths_and_attests_ledger_denial() { + let source = include_str!("main.rs"); + assert!(!source.contains("grant_owner_path(policy, session_dir")); + assert!(source.contains("grant_owner_session_paths(policy, session_dir)")); + assert!(source.contains("session_dir.join(\"session.db\")")); + assert!(source.contains("std::fs::File::open(&attestation.ledger_path)")); + assert!(source.contains("open session ledger storage")); +} + +#[test] +fn suspend_checkpoint_stays_in_owner_state() { + let session = Path::new("/tmp/session"); + assert_eq!(owner_checkpoint_path(session), session.join("owner/checkpoint.vzsave")); +} + #[test] fn platform_confinement_and_attestation_precede_owner_readiness() { let source = include_str!("main.rs"); diff --git a/crates/capsem-process/src/vsock.rs b/crates/capsem-process/src/vsock.rs index fe74dd245..64902011b 100644 --- a/crates/capsem-process/src/vsock.rs +++ b/crates/capsem-process/src/vsock.rs @@ -565,8 +565,8 @@ pub(crate) async fn setup_vsock(options: VsockOptions) -> Result<()> { ServiceToProcess::CloneState { id } => { clone_state::spawn(&hub_tx, &js_for_cmd, &ipc_tx_for_cmd, id); } - ServiceToProcess::Suspend { checkpoint_path } => { - let full_path = session_dir.join(checkpoint_path); + ServiceToProcess::Suspend => { + let full_path = crate::owner_checkpoint_path(&session_dir); let complete_path = checkpoint_complete_path(&full_path); let _ = std::fs::remove_file(&complete_path); let checkpoint_path_for_save = full_path.clone(); @@ -933,12 +933,7 @@ fn dispatch_aux_connection( "ipc_lifecycle_suspend", itx.send(ProcessToService::SuspendRequested { id }) ); - capsem_core::try_send!( - "ctrl_lifecycle_suspend", - ctx.blocking_send(ServiceToProcess::Suspend { - checkpoint_path: "checkpoint.vzsave".into() - }) - ); + capsem_core::try_send!("ctrl_lifecycle_suspend", ctx.blocking_send(ServiceToProcess::Suspend)); } other => { // W4: a lifecycle-port frame the host doesn't recognize From 11ed279fa836f431548c439b259b3c2afe26d770 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:43:52 +0000 Subject: [PATCH 179/464] feat(core): pin standalone model upstreams --- crates/capsem-core/src/net/mitm_proxy/mod.rs | 242 +++++++++++++++--- .../capsem-core/src/net/mitm_proxy/tests.rs | 84 ++++++ crates/capsem-core/tests/mitm_integration.rs | 65 +++++ 3 files changed, 360 insertions(+), 31 deletions(-) diff --git a/crates/capsem-core/src/net/mitm_proxy/mod.rs b/crates/capsem-core/src/net/mitm_proxy/mod.rs index 92c60ec11..bc370b858 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mod.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mod.rs @@ -151,6 +151,103 @@ struct ModelTrafficIdentity { protocol: Option, } +/// One configured model endpoint exposed through the standalone HTTP adapter. +/// +/// The trusted policy snapshot supplies every upstream routing field. Client +/// headers never participate in selecting a host, port, transport, provider, +/// or wire protocol. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StandaloneModelTarget { + provider_id: String, + domain: String, + port: u16, + upstream_protocol: Protocol, + base_path: String, + provider: ProviderKind, + model_protocol: ModelProtocol, +} + +impl StandaloneModelTarget { + pub fn from_registry( + registry: &crate::net::policy_config::ModelEndpointRegistry, + provider_id: &str, + ) -> Result { + let endpoint = registry + .get(provider_id) + .ok_or_else(|| format!("model provider {provider_id:?} is not configured"))?; + let parsed = reqwest::Url::parse(&endpoint.upstream_url) + .map_err(|error| format!("model provider {provider_id:?} has an invalid upstream URL: {error}"))?; + let upstream_protocol = match parsed.scheme() { + "http" => Protocol::Http, + "https" => Protocol::Tls, + scheme => { + return Err(format!( + "model provider {provider_id:?} uses unsupported upstream URL scheme {scheme:?}" + )); + } + }; + if !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(format!( + "model provider {provider_id:?} upstream URL must not contain credentials, query, or fragment" + )); + } + let domain = parsed + .host_str() + .filter(|host| !host.is_empty()) + .ok_or_else(|| format!("model provider {provider_id:?} upstream URL has no host"))? + .to_string(); + let port = parsed + .port_or_known_default() + .ok_or_else(|| format!("model provider {provider_id:?} upstream URL has no port"))?; + let base_path = parsed.path().trim_end_matches('/'); + Ok(Self { + provider_id: endpoint.provider_id.clone(), + domain, + port, + upstream_protocol, + base_path: if base_path.is_empty() { + String::new() + } else { + base_path.to_string() + }, + provider: endpoint.provider_kind, + model_protocol: endpoint.protocol, + }) + } + + pub fn provider_id(&self) -> &str { + &self.provider_id + } + + pub fn domain(&self) -> &str { + &self.domain + } + + pub const fn port(&self) -> u16 { + self.port + } + + pub const fn upstream_protocol(&self) -> Protocol { + self.upstream_protocol + } + + pub fn base_path(&self) -> &str { + &self.base_path + } + + pub const fn provider(&self) -> ProviderKind { + self.provider + } + + pub const fn model_protocol(&self) -> ModelProtocol { + self.model_protocol + } +} + fn ai_identity_for_target_or_path( registry: &crate::net::policy_config::ModelEndpointRegistry, domain: &str, @@ -395,6 +492,37 @@ pub async fn handle_connection(vsock_fd: OwnedFd, config: Arc) } } +/// Serve one plaintext OpenAI-compatible client connection while pinning all +/// upstream routing to a provider selected from the trusted policy snapshot. +pub async fn handle_standalone_connection( + client_fd: OwnedFd, + config: Arc, + target: StandaloneModelTarget, +) { + ::metrics::gauge!(m::ACTIVE_CONNECTIONS).increment(1.0); + let _gauge_guard = ConnectionGauge; + + let std_fd = std::fs::File::from(client_fd); + if let Err(error) = capsem_foundation::unix::fd::set_nonblocking(std_fd.as_fd(), true) { + warn!(%error, "standalone proxy could not configure client socket"); + return; + } + let async_fd = match tokio::io::unix::AsyncFd::new(std_fd) { + Ok(fd) => fd, + Err(error) => { + warn!(%error, "standalone proxy could not adopt client socket"); + return; + } + }; + serve_pipeline( + TokioIo::new(AsyncFdStream(async_fd)), + PipelineTarget::Standalone(target), + &config, + Arc::new(None), + ) + .await; +} + /// Inner handler. Returns Ok(domain) on success, Err((domain, decision, reason)) /// on connection-level failure. Per-request telemetry is emitted by `TelemetryHook`. /// Deadline for each pre-classification read from a freshly-accepted guest @@ -608,7 +736,16 @@ async fn serve_tls( .ok_or_else(|| (String::new(), Decision::Denied, "no SNI in ClientHello".into()))?; let io = TokioIo::new(tls_stream); - serve_pipeline(io, domain.clone(), Protocol::Tls, config, process_name).await; + serve_pipeline( + io, + PipelineTarget::Dynamic { + connection_domain: domain.clone(), + protocol: Protocol::Tls, + }, + config, + process_name, + ) + .await; Ok(domain) } @@ -625,7 +762,16 @@ async fn serve_plain_http( ) -> Result { let replay = ReplayReader::new(initial_buf, vsock_stream); let io = TokioIo::new(replay); - serve_pipeline(io, String::new(), Protocol::Http, config, process_name).await; + serve_pipeline( + io, + PipelineTarget::Dynamic { + connection_domain: String::new(), + protocol: Protocol::Http, + }, + config, + process_name, + ) + .await; // Per-request telemetry is emitted by `TelemetryHook`. The // connection-level `NetEvent` `handle_connection` would write on // an Err-return is intentionally skipped on this path -- there @@ -640,10 +786,18 @@ async fn serve_plain_http( /// * HTTP: parsed from the inbound `Host` header per request; falls /// back to `("", 80)` when the header is missing or malformed, /// producing a 502 downstream once `handle_request` runs. +#[derive(Clone)] +enum PipelineTarget { + Dynamic { + connection_domain: String, + protocol: Protocol, + }, + Standalone(StandaloneModelTarget), +} + async fn serve_pipeline( io: IO, - connection_domain: String, - protocol: Protocol, + target: PipelineTarget, config: &Arc, process_name: Arc>, ) where @@ -660,38 +814,62 @@ async fn serve_pipeline( let svc = hyper::service::service_fn(move |req: hyper::Request| { let upstream_tls = Arc::clone(&upstream_tls); - let connection_domain = connection_domain.clone(); + let target = target.clone(); let config_arc = Arc::clone(&config_arc); let process_name = Arc::clone(&process_name); let cached_upstream = Arc::clone(&cached_upstream); async move { - // Resolve the per-request `(domain, upstream_port)`. TLS - // already knows from SNI; HTTP must read the Host header. - let (request_domain, upstream_port) = match protocol { - Protocol::Tls => (connection_domain, 443u16), - Protocol::Http => { - parse_http_host_target(req.headers().get("host")).unwrap_or_else(|| (String::new(), 80)) + let (request_domain, upstream_port, protocol, upstream_protocol, authoritative_host, fixed_identity) = + match target { + PipelineTarget::Dynamic { + connection_domain, + protocol, + } => { + let (domain, port) = match protocol { + Protocol::Tls => (connection_domain, 443u16), + Protocol::Http => { + parse_http_host_target(req.headers().get("host")).unwrap_or_else(|| (String::new(), 80)) + } + Protocol::McpFrame => unreachable!("framed MCP bypasses HTTP pipeline"), + Protocol::Unknown => (String::new(), 0), + }; + (domain, port, protocol, protocol, protocol == Protocol::Tls, None) + } + PipelineTarget::Standalone(target) => ( + target.domain().to_string(), + target.port(), + Protocol::Http, + target.upstream_protocol(), + true, + Some(ModelTrafficIdentity { + provider: Some(target.provider()), + protocol: Some(target.model_protocol()), + }), + ), + }; + if fixed_identity.is_none() { + // Capsem's own zone is answered here, before any HTTP policy, body sniff or dial. + if let Some(route) = mcp_http::internal_route(&request_domain, upstream_port, protocol) { + let (domain, port) = (&request_domain, upstream_port); + return Ok(mcp_http::serve(req, route, domain, port, protocol, &config_arc, &process_name).await); } - Protocol::McpFrame => unreachable!("framed MCP bypasses HTTP pipeline"), - Protocol::Unknown => (String::new(), 0), }; - // Capsem's own zone is answered here, before any HTTP policy, body sniff or dial. - if let Some(route) = mcp_http::internal_route(&request_domain, upstream_port, protocol) { - let (domain, port) = (&request_domain, upstream_port); - return Ok(mcp_http::serve(req, route, domain, port, protocol, &config_arc, &process_name).await); - } let policy_snapshot = config_arc.engine.policy().snapshot(); - let ai_identity = ai_identity_for_target_or_path( - policy_snapshot.model_endpoints(), - &request_domain, - upstream_port, - req.uri().path(), - ); + let ai_identity = fixed_identity.unwrap_or_else(|| { + ai_identity_for_target_or_path( + policy_snapshot.model_endpoints(), + &request_domain, + upstream_port, + req.uri().path(), + ) + }); handle_request( req, &request_domain, protocol, + upstream_protocol, + authoritative_host, upstream_port, &upstream_tls, &config_arc, @@ -798,6 +976,8 @@ async fn handle_request( mut req: hyper::Request, domain: &str, protocol: Protocol, + upstream_protocol: Protocol, + authoritative_host: bool, upstream_port: u16, upstream_tls: &Arc, config: &Arc, @@ -827,7 +1007,7 @@ async fn handle_request( let target = UpstreamTarget::select( resolver, UpstreamPolicy::from(policy_snapshot.as_ref()), - protocol, + upstream_protocol, domain, upstream_port, cached_upstream, @@ -910,7 +1090,7 @@ async fn handle_request( domain: domain.to_string(), process_name: process_name.clone(), port: upstream_port, - protocol, + protocol: upstream_protocol, ai_provider: conn_ai_provider, ai_protocol: conn_ai_protocol, }, @@ -929,7 +1109,7 @@ async fn handle_request( client_upgrade: client_upgrade.expect("websocket upgrade captured before split"), domain, target: &target, - protocol, + protocol: upstream_protocol, upstream_port, upstream_tls, config, @@ -1219,7 +1399,7 @@ async fn handle_request( // Host-side plain-HTTP port allowlist. The port came from the guest's Host // header; enforce it before any upstream dial so a guest reaching the proxy // directly cannot make the host connect to an arbitrary port. - if !http_upstream_port_allowed(policy, protocol, upstream_port) { + if !http_upstream_port_allowed(policy, upstream_protocol, upstream_port) { actions_span.record("decision", "deny"); actions_span.record("status", "ok"); let matched = "security.web.http_upstream_ports"; @@ -1675,7 +1855,7 @@ async fn handle_request( let mut tcp_us = 0u64; let mut tls_us = 0u64; let mut handshake_us = 0u64; - let upstream_protocol = target.protocol(protocol); + let upstream_protocol = target.protocol(upstream_protocol); // What the sender reaches, pinned to the peer once connected: the cache key. let mut connected = target.clone(); @@ -1870,13 +2050,13 @@ async fn handle_request( // HTTP: preserve inbound `host` -- the guest sent it, // and parse_http_host_target already drove our // upstream selection from it. - let drop_host = matches!(protocol, Protocol::Tls) && name == "host"; + let drop_host = authoritative_host && name == "host"; if drop_host || name == "accept-encoding" { continue; } builder = builder.header(name.clone(), value.clone()); } - if matches!(protocol, Protocol::Tls) { + if authoritative_host { builder = builder.header("host", domain); } // Only accept gzip -- we can decompress it; brotli/zstd we cannot. diff --git a/crates/capsem-core/src/net/mitm_proxy/tests.rs b/crates/capsem-core/src/net/mitm_proxy/tests.rs index d6de4aa36..d690dd06f 100644 --- a/crates/capsem-core/src/net/mitm_proxy/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/tests.rs @@ -2,6 +2,90 @@ use super::*; use crate::net::policy::NetworkMechanics; use crate::net::policy_config::{SecurityRuleAction, SecurityRuleProfile, SecurityRuleSet}; +#[test] +fn standalone_target_uses_only_the_configured_provider_authority() { + let profile = crate::net::policy_config::ProviderRuleProfile::parse_toml( + r#" +[ai.openai] +name = "OpenAI" +protocol = "openai" +url = "https://api.openai.com/v1" +listen_ports = [443] + +[ai.openai.rules.allow] +name = "allow" +action = "allow" +match = 'http.host == "api.openai.com"' +"#, + ) + .expect("provider profile parses"); + let registry = profile.endpoint_registry().expect("registry builds"); + + let target = StandaloneModelTarget::from_registry(®istry, "openai").expect("target builds"); + + assert_eq!(target.provider_id(), "openai"); + assert_eq!(target.domain(), "api.openai.com"); + assert_eq!(target.port(), 443); + assert_eq!(target.upstream_protocol(), Protocol::Tls); + assert_eq!(target.base_path(), "/v1"); + assert_eq!(target.provider(), ProviderKind::OpenAi); + assert_eq!(target.model_protocol(), ModelProtocol::OpenAi); +} + +#[test] +fn standalone_target_supports_configured_plain_http_and_explicit_port() { + let profile = crate::net::policy_config::ProviderRuleProfile::parse_toml( + r#" +[ai.local] +name = "Local" +protocol = "openai" +url = "http://model.internal:8080/openai/v1/" +listen_ports = [8080] + +[ai.local.rules.allow] +name = "allow" +action = "allow" +match = 'http.host == "model.internal"' +"#, + ) + .expect("provider profile parses"); + let registry = profile.endpoint_registry().expect("registry builds"); + + let target = StandaloneModelTarget::from_registry(®istry, "local").expect("target builds"); + + assert_eq!(target.domain(), "model.internal"); + assert_eq!(target.port(), 8080); + assert_eq!(target.upstream_protocol(), Protocol::Http); + assert_eq!(target.base_path(), "/openai/v1"); +} + +#[test] +fn standalone_target_rejects_missing_or_unsafe_provider_urls() { + let profile = crate::net::policy_config::ProviderRuleProfile::parse_toml( + r#" +[ai.unsafe] +name = "Unsafe" +protocol = "openai" +url = "https://user:secret@example.com/v1?redirect=other" +listen_ports = [443] + +[ai.unsafe.rules.allow] +name = "allow" +action = "allow" +match = 'http.host == "example.com"' +"#, + ) + .expect("provider profile parses"); + let registry = profile.endpoint_registry().expect("registry builds"); + + assert!(StandaloneModelTarget::from_registry(®istry, "missing") + .unwrap_err() + .contains("not configured")); + assert!(StandaloneModelTarget::from_registry(®istry, "unsafe") + .unwrap_err() + .contains("credentials, query, or fragment")); +} + #[test] fn collected_gzip_chunked_response_headers_are_materialized() { let mut headers = http::HeaderMap::new(); diff --git a/crates/capsem-core/tests/mitm_integration.rs b/crates/capsem-core/tests/mitm_integration.rs index 3934a3550..cd8da671e 100644 --- a/crates/capsem-core/tests/mitm_integration.rs +++ b/crates/capsem-core/tests/mitm_integration.rs @@ -242,6 +242,25 @@ async fn spawn_proxy(config: Arc) -> (tokio::task::JoinHandle<( (handle, addr) } + +/// Spawn the standalone model adapter for exactly one accepted connection. +async fn spawn_standalone_proxy( + config: Arc, + provider_id: &str, +) -> (tokio::task::JoinHandle<()>, std::net::SocketAddr) { + let target = mitm_proxy::StandaloneModelTarget::from_registry( + config.engine.policy().snapshot().model_endpoints(), + provider_id, + ) + .expect("configured standalone provider"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + mitm_proxy::handle_standalone_connection(stream.into_std().unwrap().into(), config, target).await; + }); + (handle, addr) +} #[tokio::test] async fn mitm_proxy_allows_hermetic_upstream() { let (upstream_port, upstream_task) = spawn_fake_upstream(|mut sock| { @@ -1180,6 +1199,52 @@ async fn mitm_proxy_plain_http_preserves_host_header_to_upstream() { ); } +#[tokio::test] +async fn standalone_proxy_ignores_client_host_and_pins_configured_provider() { + let received: Arc>> = Arc::new(std::sync::Mutex::new(Vec::new())); + let received_for_serve = Arc::clone(&received); + let (upstream_port, upstream_task) = spawn_fake_upstream(move |mut sock| { + Box::pin(async move { + let bytes = read_http11_request(&mut sock).await; + *received_for_serve.lock().unwrap() = bytes.clone(); + sock.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") + .await + .unwrap(); + bytes + }) + }) + .await; + let (config, db) = make_proxy_config_with_local_http_upstream("api.openai.com", upstream_port); + let (proxy_task, proxy_addr) = spawn_standalone_proxy(config, "openai").await; + + let mut client = tokio::net::TcpStream::connect(proxy_addr).await.unwrap(); + client + .write_all(b"POST /v1/responses HTTP/1.1\r\nHost: attacker.invalid:22\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + .await + .unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).await.unwrap(); + + upstream_task.await.unwrap(); + proxy_task.await.unwrap(); + db.flush().await; + + assert!(String::from_utf8_lossy(&response).contains("200 OK")); + let request = String::from_utf8_lossy(&received.lock().unwrap()).to_ascii_lowercase(); + assert!( + request.contains("host: api.openai.com"), + "configured Host missing: {request}" + ); + assert!( + !request.contains("attacker.invalid"), + "client Host reached upstream: {request}" + ); + let events = db.reader().unwrap().recent_net_events(10).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].domain, "api.openai.com"); + assert_eq!(events[0].port, 443); +} + /// T2.2: a request to a plain-HTTP upstream that fails to dial /// produces a 502 response and a NetEvent with Decision::Error + /// matched_rule containing the dial error reason. No silent drop. From 787de82becdc6e7f122bce34c2d764f1f3d5d499 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:46:45 +0000 Subject: [PATCH 180/464] feat(proxy): serve fixed provider traffic --- crates/capsem-proxy/src/main.rs | 115 +++++++++++++++++++++++-------- crates/capsem-proxy/src/tests.rs | 27 ++++++++ 2 files changed, 113 insertions(+), 29 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index c7ac8f10d..ea0bb7823 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, HashSet}; use std::io; -use std::os::fd::{AsFd, OwnedFd}; +use std::os::fd::AsFd; use std::os::unix::net::UnixStream; use std::path::Path; use std::process::Command; @@ -49,6 +49,9 @@ struct Args { /// Sixteen-byte worker generation as 32 lowercase hexadecimal digits. #[arg(long, value_parser = parse_generation)] generation: ProxyGeneration, + /// Trusted configured provider selected by the service for standalone mode. + #[arg(long)] + standalone_provider: Option, } fn main() -> Result<()> { @@ -75,7 +78,7 @@ fn main() -> Result<()> { capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Proxy)) .context("confine proxy worker before readiness")?; attest_confinement(&denied_file, args.parent_pid)?; - runtime.block_on(run_control(control, args.generation)) + runtime.block_on(run_control(control, args.generation, args.standalone_provider)) } fn attest_confinement(denied_file: &std::path::Path, parent_pid: u32) -> Result<()> { @@ -109,7 +112,11 @@ fn require_denied(result: io::Result, operation: &str) -> Result<()> { Ok(()) } -async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result<()> { +async fn run_control( + control: UnixStream, + generation: ProxyGeneration, + standalone_provider: Option, +) -> Result<()> { let shutdown = control.try_clone()?; let sender = ControlSender::new(control.try_clone()?)?; let receiver = ControlReceiver::new(control)?; @@ -118,7 +125,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result let outcome = async { send_event(&sender, ProxyControlEvent::Ready { generation }).await?; - let state = Arc::new(Mutex::new(ProxyRuntimeState::default())); + let state = Arc::new(Mutex::new(ProxyRuntimeState::new(standalone_provider))); let mut grants = BTreeMap::>::new(); let mut grant_ids = HashSet::new(); let mut tasks = JoinSet::::new(); @@ -194,16 +201,17 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result continue; } let descriptor = frame.fds.into_iter().next().expect("one checked descriptor"); - let stream = UnixStream::from(descriptor); grants.entry(capability).or_default().push(grant_id); grant_ids.insert(grant_id); if capability == ProxyCapability::Policy { + let stream = UnixStream::from(descriptor); let state = Arc::clone(&state); tasks.spawn(async move { let reason = serve_policy(stream, state).await; (capability, grant_id, reason) }); } else if capability == ProxyCapability::Ledger { + let stream = UnixStream::from(descriptor); let ledger_grant = grant .ledger_grant() .expect("ledger capability was decoded with exact authority"); @@ -223,6 +231,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result .unwrap_or_else(|error| error.into_inner()) .attach_ledger(Arc::new(writer)); } else if capability == ProxyCapability::Credential { + let stream = UnixStream::from(descriptor); let (credentials, mut closed) = credential_client::CredentialClient::start(stream) .context("open proxy credential capability")?; state @@ -235,6 +244,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result (capability, grant_id, reason) }); } else if capability == ProxyCapability::Upstream { + let stream = UnixStream::from(descriptor); let grants = Arc::new( capsem_core::net::upstream_grant::UpstreamGrantClient::start(stream) .context("open proxy upstream capability")?, @@ -252,6 +262,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result (capability, grant_id, ProxyChannelCloseReason::Disconnected) }); } else if capability == ProxyCapability::PrivateNames { + let stream = UnixStream::from(descriptor); let (private_names, mut closed) = private_names_client::PrivateNameClient::start(stream) .context("open proxy private-name capability")?; @@ -268,6 +279,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result (capability, grant_id, reason) }); } else if capability == ProxyCapability::Mcp { + let stream = UnixStream::from(descriptor); let (client, hello, mut closed) = mcp_client::start(stream).await.context("open proxy MCP capability")?; state @@ -279,6 +291,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result (capability, grant_id, ProxyChannelCloseReason::Disconnected) }); } else if capability == ProxyCapability::Telemetry { + let stream = UnixStream::from(descriptor); let (client, session_id) = metric_client::start(stream) .await .context("open proxy metric capability")?; @@ -293,14 +306,23 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result .unwrap_or_else(|error| error.into_inner()) .attach_metric_exporter(exporter); } else if capability == ProxyCapability::HttpTraffic { - let config = state + let runtime = state .lock() .unwrap_or_else(|error| error.into_inner()) - .http_config()? + .http_runtime()? .context("proxy HTTP runtime became unavailable after readiness check")?; tasks.spawn(async move { - capsem_core::net::mitm_proxy::handle_connection(OwnedFd::from(stream), config) + if let Some(target) = runtime.target { + capsem_core::net::mitm_proxy::handle_standalone_connection( + descriptor, + runtime.config, + target, + ) .await; + } else { + capsem_core::net::mitm_proxy::handle_connection(descriptor, runtime.config) + .await; + } (capability, grant_id, ProxyChannelCloseReason::Disconnected) }); } else if capability == ProxyCapability::DnsTraffic { @@ -311,7 +333,7 @@ async fn run_control(control: UnixStream, generation: ProxyGeneration) -> Result .context("proxy DNS runtime became unavailable after readiness check")?; tasks.spawn(async move { capsem_core::net::dns::session::serve_dns_session( - OwnedFd::from(stream), + descriptor, Arc::clone(&runtime.handler), Arc::clone(&runtime.db), runtime.policy.clone(), @@ -374,6 +396,7 @@ fn remove_grant( } struct ProxyRuntimeState { + standalone_provider: Option, engine: Option>, ledger: Arc, db: Option>, @@ -394,7 +417,19 @@ struct ProxyRuntimeState { impl Default for ProxyRuntimeState { fn default() -> Self { + Self::new(None) + } +} + +struct HttpRuntime { + config: Arc, + target: Option, +} + +impl ProxyRuntimeState { + fn new(standalone_provider: Option) -> Self { Self { + standalone_provider, engine: None, ledger: Arc::new(UnavailableLedger), db: None, @@ -410,9 +445,11 @@ impl Default for ProxyRuntimeState { metric_exporter: None, } } -} -impl ProxyRuntimeState { + #[cfg(test)] + fn standalone(provider: String) -> Self { + Self::new(Some(provider)) + } fn apply(&mut self, active_policy: &[u8]) -> Result { let runtime = ProxyRuntimePolicy::compile(active_policy).map_err(anyhow::Error::msg)?; let digest = runtime.snapshot().digest().to_string(); @@ -485,7 +522,7 @@ impl ProxyRuntimeState { fn traffic_ready(&mut self, capability: ProxyCapability) -> Result { match capability { - ProxyCapability::HttpTraffic => Ok(self.http_config()?.is_some()), + ProxyCapability::HttpTraffic => Ok(self.http_runtime()?.is_some()), ProxyCapability::DnsTraffic => Ok(self.dns_runtime().is_some()), _ => Ok(true), } @@ -495,9 +532,7 @@ impl ProxyRuntimeState { if let Some(config) = &self.http_config { return Ok(Some(Arc::clone(config))); } - let (Some(engine), Some(db), Some(upstream_grants), Some((mcp_client, mcp_hello))) = - (&self.engine, &self.db, &self.upstream_grants, &self.mcp_client) - else { + let (Some(engine), Some(db), Some(upstream_grants)) = (&self.engine, &self.db, &self.upstream_grants) else { return Ok(None); }; if !self.credentials_attached { @@ -515,19 +550,23 @@ impl ProxyRuntimeState { }); let pipeline = capsem_core::net::mitm_proxy::make_production_pipeline(Arc::clone(&telemetry)); let upstream_grants: Arc = upstream_grants.clone(); - let mcp_endpoint = Arc::new( - capsem_core::net::mitm_proxy::McpEndpointState::with_proxy_policy( - mcp_client.clone(), - engine.policy().clone(), - Arc::new(tokio::sync::Semaphore::new(usize::from(mcp_hello.inflight_cap))), - capsem_core::net::mitm_proxy::McpTimeouts { - default_timeout: Duration::from_millis(mcp_hello.default_timeout_ms), - tool_call_default: Duration::from_millis(mcp_hello.tool_call_default_ms), - tool_call_ceiling: Duration::from_millis(mcp_hello.tool_call_ceiling_ms), - }, - ) - .with_builtin_ledger(Arc::clone(db), mcp_hello.builtin_servers.iter().cloned().collect()), - ); + let mcp_endpoint = match &self.mcp_client { + Some((mcp_client, mcp_hello)) => Some(Arc::new( + capsem_core::net::mitm_proxy::McpEndpointState::with_proxy_policy( + mcp_client.clone(), + engine.policy().clone(), + Arc::new(tokio::sync::Semaphore::new(usize::from(mcp_hello.inflight_cap))), + capsem_core::net::mitm_proxy::McpTimeouts { + default_timeout: Duration::from_millis(mcp_hello.default_timeout_ms), + tool_call_default: Duration::from_millis(mcp_hello.tool_call_default_ms), + tool_call_ceiling: Duration::from_millis(mcp_hello.tool_call_ceiling_ms), + }, + ) + .with_builtin_ledger(Arc::clone(db), mcp_hello.builtin_servers.iter().cloned().collect()), + )), + None if self.standalone_provider.is_some() => None, + None => return Ok(None), + }; let config = Arc::new(capsem_core::net::mitm_proxy::MitmProxyConfig { ca: Arc::clone(&ca), server_tls: capsem_core::net::mitm_proxy::make_server_tls_config(&ca), @@ -536,7 +575,7 @@ impl ProxyRuntimeState { upstream_tls: capsem_core::net::mitm_proxy::make_upstream_tls_config(), telemetry, pipeline, - mcp_endpoint: Some(mcp_endpoint), + mcp_endpoint, upstream_resolver: capsem_core::net::upstream_address::UpstreamResolver::disabled(), upstream_grants: Some(upstream_grants), }); @@ -544,6 +583,24 @@ impl ProxyRuntimeState { Ok(Some(config)) } + fn http_runtime(&mut self) -> Result> { + let Some(config) = self.http_config()? else { + return Ok(None); + }; + let target = self + .standalone_provider + .as_deref() + .map(|provider| { + capsem_core::net::mitm_proxy::StandaloneModelTarget::from_registry( + config.engine.policy().snapshot().model_endpoints(), + provider, + ) + .map_err(anyhow::Error::msg) + }) + .transpose()?; + Ok(Some(HttpRuntime { config, target })) + } + fn dns_runtime(&mut self) -> Option> { if let Some(runtime) = &self.dns_runtime { return Some(Arc::clone(runtime)); diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index 5758dee53..eebb30144 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -74,6 +74,33 @@ match = 'http.host == "worker.example"' ); } +#[tokio::test] +async fn standalone_http_runtime_pins_provider_without_mcp_capability() { + let active_policy = br#" +[network] +[user_rules.profiles.rules.worker_http] +name = "worker_http" +action = "allow" +match = 'http.host == "api.openai.com"' +[corp_rules] +"#; + let mut state = ProxyRuntimeState::standalone("openai".to_string()); + state.apply(active_policy).unwrap(); + state.attach_ledger(Arc::new(capsem_logger::DbWriter::open_in_memory(8).unwrap())); + state.attach_credentials(Arc::new(UnavailableCredentials)); + let (client, _broker) = UnixStream::pair().unwrap(); + state.attach_upstream(Arc::new( + capsem_core::net::upstream_grant::UpstreamGrantClient::start(client).unwrap(), + )); + + let runtime = state.http_runtime().unwrap().expect("standalone runtime is ready"); + + assert!(runtime.config.mcp_endpoint.is_none()); + let target = runtime.target.expect("standalone target"); + assert_eq!(target.provider_id(), "openai"); + assert_eq!(target.domain(), "api.openai.com"); +} + struct EmptyPrivateNames; impl capsem_core::net::dns::private::PrivateNames for EmptyPrivateNames { From 6f2fd99aac102ffd30e165962fa4d2637e01ea44 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:50:26 +0000 Subject: [PATCH 181/464] feat(api): define standalone proxy leases --- crates/capsem-api/src/document.rs | 8 +- crates/capsem-api/src/lib.rs | 2 + crates/capsem-api/src/proxy.rs | 43 +++++ crates/capsem-api/src/tests.rs | 14 ++ sdk/specification/openapi.json | 294 +++++++++++++++++++++++++----- 5 files changed, 310 insertions(+), 51 deletions(-) create mode 100644 crates/capsem-api/src/proxy.rs diff --git a/crates/capsem-api/src/document.rs b/crates/capsem-api/src/document.rs index 7c56628b9..f2889fa74 100644 --- a/crates/capsem-api/src/document.rs +++ b/crates/capsem-api/src/document.rs @@ -27,6 +27,12 @@ pub fn openapi() -> OpenApi { ); doc.post::("/run", "runVm"); doc.post::("/purge", "purgeVms"); + doc.post::("/proxies", "createProxy"); + doc.post::( + "/proxies/{id}/heartbeat", + "heartbeatProxy", + ); + doc.post::("/proxies/{id}/stop", "stopProxy"); doc.post::("/vms/{id}/save", "persistVm"); doc.get::("/vms/{id}/info", "getVmInfo"); doc.get::("/vms/{id}/status", "getVmStatus"); @@ -96,7 +102,7 @@ struct Document { /// force every generated SDK to be regenerated, and an SDK built against the /// contract keeps working across binary releases. Raise it when the contract /// changes in a way clients must notice. -pub const CONTRACT_VERSION: &str = "3.3.0"; +pub const CONTRACT_VERSION: &str = "3.1.0"; impl Document { fn connections(&mut self) { diff --git a/crates/capsem-api/src/lib.rs b/crates/capsem-api/src/lib.rs index 5b30ab79f..87345900c 100644 --- a/crates/capsem-api/src/lib.rs +++ b/crates/capsem-api/src/lib.rs @@ -46,6 +46,8 @@ mod updates; pub use updates::*; mod restart; pub use restart::*; +mod proxy; +pub use proxy::*; mod networks; pub use networks::*; mod diagnostics; diff --git a/crates/capsem-api/src/proxy.rs b/crates/capsem-api/src/proxy.rs new file mode 100644 index 000000000..7bad553e6 --- /dev/null +++ b/crates/capsem-api/src/proxy.rs @@ -0,0 +1,43 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +/// Start one VM-free, OpenAI-compatible model proxy lease. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)] +pub struct CreateProxyRequest { + /// Provider key from the effective built-in/settings/corp policy. + pub provider: String, + /// Local address on which the unauthenticated data listener binds. + pub bind: String, + /// Requested data-listener port. Zero asks the OS to select one. + pub port: u16, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)] +pub struct CreateProxyResponse { + pub session_id: String, + pub provider: String, + pub bind: String, + pub port: u16, + /// SDK-compatible URL, including the configured provider path prefix. + pub base_url: String, + /// Opaque proof required to renew or stop this lease. + pub lease_token: String, + pub lease_expires_unix_ms: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)] +pub struct ProxyLeaseRequest { + pub lease_token: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)] +pub struct ProxyHeartbeatResponse { + pub session_id: String, + pub lease_expires_unix_ms: u64, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, ToSchema)] +pub struct StopProxyResponse { + pub session_id: String, + pub stopped: bool, +} diff --git a/crates/capsem-api/src/tests.rs b/crates/capsem-api/src/tests.rs index 8a4a8fd75..ed5cd7e77 100644 --- a/crates/capsem-api/src/tests.rs +++ b/crates/capsem-api/src/tests.rs @@ -250,6 +250,20 @@ fn openapi_describes_binary_copy_and_required_vm_identity() { .any(|p| p["name"] == "path" && p["required"] == true)); } +#[test] +fn openapi_describes_standalone_proxy_leases() { + let document = serde_json::to_value(crate::openapi()).unwrap(); + let paths = &document["paths"]; + + assert_eq!(paths["/proxies"]["post"]["operationId"], "createProxy"); + assert_eq!( + paths["/proxies/{id}/heartbeat"]["post"]["operationId"], + "heartbeatProxy" + ); + assert_eq!(paths["/proxies/{id}/stop"]["post"]["operationId"], "stopProxy"); + assert_eq!(crate::CONTRACT_VERSION, "3.1.0"); +} + #[test] fn openapi_exec_output_is_byte_safe_and_retires_ambiguous_file_json() { let document = serde_json::to_value(crate::openapi()).unwrap(); diff --git a/sdk/specification/openapi.json b/sdk/specification/openapi.json index f8af17903..f0e05c624 100644 --- a/sdk/specification/openapi.json +++ b/sdk/specification/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "Capsem Gateway", - "version": "3.3.0" + "version": "3.1.0" }, "paths": { "/assets/status": { @@ -1199,6 +1199,152 @@ } } }, + "/proxies": { + "post": { + "operationId": "createProxy", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateProxyRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/CreateProxyResponse" + } + } + } + }, + "default": { + "description": "HTTP error; inspect status and error body", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/proxies/{id}/heartbeat": { + "post": { + "operationId": "heartbeatProxy", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProxyLeaseRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProxyHeartbeatResponse" + } + } + } + }, + "default": { + "description": "HTTP error; inspect status and error body", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, + "/proxies/{id}/stop": { + "post": { + "operationId": "stopProxy", + "parameters": [ + { + "name": "id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProxyLeaseRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "Success", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StopProxyResponse" + } + } + } + }, + "default": { + "description": "HTTP error; inspect status and error body", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + }, + "text/plain": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/purge": { "post": { "operationId": "purgeVms", @@ -4091,66 +4237,71 @@ } } }, - "CreateSessionGrantRequest": { + "CreateProxyRequest": { "type": "object", + "description": "Start one VM-free, OpenAI-compatible model proxy lease.", "required": [ - "request_id", - "expected_owner_generation", - "connection_id", - "capability_ids", - "expires_at" + "provider", + "bind", + "port" ], "properties": { - "capability_ids": { - "type": "array", - "items": { - "type": "string" - } + "bind": { + "type": "string", + "description": "Local address on which the unauthenticated data listener binds." }, - "connection_id": { - "type": "string" + "port": { + "type": "integer", + "format": "int32", + "description": "Requested data-listener port. Zero asks the OS to select one.", + "minimum": 0 }, - "expected_owner_generation": { + "provider": { "type": "string", - "format": "uuid" + "description": "Provider key from the effective built-in/settings/corp policy." + } + } + }, + "CreateProxyResponse": { + "type": "object", + "required": [ + "session_id", + "provider", + "bind", + "port", + "base_url", + "lease_token", + "lease_expires_unix_ms" + ], + "properties": { + "base_url": { + "type": "string", + "description": "SDK-compatible URL, including the configured provider path prefix." }, - "expires_at": { - "type": [ - "string", - "null" - ], - "format": "date-time" + "bind": { + "type": "string" }, - "request_id": { + "lease_expires_unix_ms": { + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "lease_token": { "type": "string", - "format": "uuid" + "description": "Opaque proof required to renew or stop this lease." + }, + "port": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "provider": { + "type": "string" + }, + "session_id": { + "type": "string" } - }, - "additionalProperties": false - }, - "CredentialErrorCode": { - "type": "string", - "enum": [ - "configuration_required", - "unsupported_capability", - "unknown_scope_profile", - "policy_denied", - "scope_denied", - "connection_unavailable", - "reauth_required", - "storage_unavailable", - "storage_commit_failed", - "transaction_expired", - "transaction_cancelled", - "invalid_callback", - "provider_rejected", - "provider_timeout", - "owner_changed", - "session_unavailable", - "grant_unavailable", - "request_conflict", - "revocation_pending" - ] + } }, "CredentialEvent": { "type": "object", @@ -6868,6 +7019,34 @@ } } }, + "ProxyHeartbeatResponse": { + "type": "object", + "required": [ + "session_id", + "lease_expires_unix_ms" + ], + "properties": { + "lease_expires_unix_ms": { + "type": "integer", + "format": "int64", + "minimum": 0 + }, + "session_id": { + "type": "string" + } + } + }, + "ProxyLeaseRequest": { + "type": "object", + "required": [ + "lease_token" + ], + "properties": { + "lease_token": { + "type": "string" + } + } + }, "PurgeRequest": { "type": "object", "properties": { @@ -7534,6 +7713,21 @@ } } }, + "StopProxyResponse": { + "type": "object", + "required": [ + "session_id", + "stopped" + ], + "properties": { + "session_id": { + "type": "string" + }, + "stopped": { + "type": "boolean" + } + } + }, "StopResponse": { "type": "object", "description": "Stop completes after the VM process exits. Persistent workspace state remains.", From 4da976dec68980b2930b28bccc13e109e441293e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:50:32 +0000 Subject: [PATCH 182/464] feat(python-sdk): add proxy lease operations --- sdk/python/capsem/_operations/__init__.py | 8 +++--- sdk/python/capsem/_operations/create_proxy.py | 24 +++++++++++++++++ .../capsem/_operations/heartbeat_proxy.py | 27 +++++++++++++++++++ sdk/python/capsem/_operations/stop_proxy.py | 27 +++++++++++++++++++ sdk/python/capsem/models/__init__.py | 9 ++++--- .../capsem/models/create_proxy_request.py | 15 +++++++++++ .../capsem/models/create_proxy_response.py | 19 +++++++++++++ .../capsem/models/proxy_heartbeat_response.py | 14 ++++++++++ .../capsem/models/proxy_lease_request.py | 11 ++++++++ .../capsem/models/stop_proxy_response.py | 12 +++++++++ 10 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 sdk/python/capsem/_operations/create_proxy.py create mode 100644 sdk/python/capsem/_operations/heartbeat_proxy.py create mode 100644 sdk/python/capsem/_operations/stop_proxy.py create mode 100644 sdk/python/capsem/models/create_proxy_request.py create mode 100644 sdk/python/capsem/models/create_proxy_response.py create mode 100644 sdk/python/capsem/models/proxy_heartbeat_response.py create mode 100644 sdk/python/capsem/models/proxy_lease_request.py create mode 100644 sdk/python/capsem/models/stop_proxy_response.py diff --git a/sdk/python/capsem/_operations/__init__.py b/sdk/python/capsem/_operations/__init__.py index 5212ba87b..70a7df2da 100644 --- a/sdk/python/capsem/_operations/__init__.py +++ b/sdk/python/capsem/_operations/__init__.py @@ -11,9 +11,7 @@ from .claim_managed_session import claim_managed_session as claim_managed_session from .close_managed_session import close_managed_session as close_managed_session from .create_network import create_network as create_network -from .create_session_credential_grant import ( - create_session_credential_grant as create_session_credential_grant, -) +from .create_proxy import create_proxy as create_proxy from .create_vm import create_vm as create_vm from .create_vm_exposure import create_vm_exposure as create_vm_exposure from .create_vm_preview_session import ( @@ -51,8 +49,7 @@ from .get_vm_stats_summary import get_vm_stats_summary as get_vm_stats_summary from .get_vm_status import get_vm_status as get_vm_status from .get_vm_timeline import get_vm_timeline as get_vm_timeline -from .inject_credential import inject_credential as inject_credential -from .list_connections import list_connections as list_connections +from .heartbeat_proxy import heartbeat_proxy as heartbeat_proxy from .list_images import list_images as list_images from .list_mcp_servers import list_mcp_servers as list_mcp_servers from .list_mcp_tools import list_mcp_tools as list_mcp_tools @@ -78,6 +75,7 @@ ) from .run_vm import run_vm as run_vm from .start_vm import start_vm as start_vm +from .stop_proxy import stop_proxy as stop_proxy from .stop_vm import stop_vm as stop_vm from .update_hypervisor import update_hypervisor as update_hypervisor from .upload_vm_file import upload_vm_file as upload_vm_file diff --git a/sdk/python/capsem/_operations/create_proxy.py b/sdk/python/capsem/_operations/create_proxy.py new file mode 100644 index 000000000..da3aad281 --- /dev/null +++ b/sdk/python/capsem/_operations/create_proxy.py @@ -0,0 +1,24 @@ +"""Generated from Capsem OpenAPI. Do not edit.""" + +from __future__ import annotations + +from pydantic import TypeAdapter + +from .._transport import Method, Transport +from ..models.create_proxy_request import CreateProxyRequest +from ..models.create_proxy_response import CreateProxyResponse + + +async def create_proxy( + transport: Transport, + *, + body: CreateProxyRequest, + request_timeout: float | None = None, +) -> CreateProxyResponse: + payload = await transport.request( + Method.POST, '/proxies', + body=body, + json_body=True, + timeout=request_timeout, + ) + return TypeAdapter(CreateProxyResponse).validate_json(payload) diff --git a/sdk/python/capsem/_operations/heartbeat_proxy.py b/sdk/python/capsem/_operations/heartbeat_proxy.py new file mode 100644 index 000000000..a3c94ee64 --- /dev/null +++ b/sdk/python/capsem/_operations/heartbeat_proxy.py @@ -0,0 +1,27 @@ +"""Generated from Capsem OpenAPI. Do not edit.""" + +from __future__ import annotations + +from pydantic import StrictStr, TypeAdapter + +from .._transport import Method, Transport +from ..models.proxy_heartbeat_response import ProxyHeartbeatResponse +from ..models.proxy_lease_request import ProxyLeaseRequest + + +async def heartbeat_proxy( + transport: Transport, + *, + id: StrictStr, + body: ProxyLeaseRequest, + request_timeout: float | None = None, +) -> ProxyHeartbeatResponse: + id = TypeAdapter(StrictStr).validate_python(id) + payload = await transport.request( + Method.POST, '/proxies/{id}/heartbeat', + path_parameters={'id': id}, + body=body, + json_body=True, + timeout=request_timeout, + ) + return TypeAdapter(ProxyHeartbeatResponse).validate_json(payload) diff --git a/sdk/python/capsem/_operations/stop_proxy.py b/sdk/python/capsem/_operations/stop_proxy.py new file mode 100644 index 000000000..88f6c7f76 --- /dev/null +++ b/sdk/python/capsem/_operations/stop_proxy.py @@ -0,0 +1,27 @@ +"""Generated from Capsem OpenAPI. Do not edit.""" + +from __future__ import annotations + +from pydantic import StrictStr, TypeAdapter + +from .._transport import Method, Transport +from ..models.proxy_lease_request import ProxyLeaseRequest +from ..models.stop_proxy_response import StopProxyResponse + + +async def stop_proxy( + transport: Transport, + *, + id: StrictStr, + body: ProxyLeaseRequest, + request_timeout: float | None = None, +) -> StopProxyResponse: + id = TypeAdapter(StrictStr).validate_python(id) + payload = await transport.request( + Method.POST, '/proxies/{id}/stop', + path_parameters={'id': id}, + body=body, + json_body=True, + timeout=request_timeout, + ) + return TypeAdapter(StopProxyResponse).validate_json(payload) diff --git a/sdk/python/capsem/models/__init__.py b/sdk/python/capsem/models/__init__.py index e40b45d00..6e1edbaf2 100644 --- a/sdk/python/capsem/models/__init__.py +++ b/sdk/python/capsem/models/__init__.py @@ -32,10 +32,8 @@ from .container_surface import ContainerSurface as ContainerSurface from .container_surface_kind import ContainerSurfaceKind as ContainerSurfaceKind from .create_network_request import CreateNetworkRequest as CreateNetworkRequest -from .create_session_grant_request import ( - CreateSessionGrantRequest as CreateSessionGrantRequest, -) -from .credential_error_code import CredentialErrorCode as CredentialErrorCode +from .create_proxy_request import CreateProxyRequest as CreateProxyRequest +from .create_proxy_response import CreateProxyResponse as CreateProxyResponse from .credential_event import CredentialEvent as CredentialEvent from .credential_event_type import CredentialEventType as CredentialEventType from .credential_inject_provider import ( @@ -179,6 +177,8 @@ ) from .provision_request import ProvisionRequest as ProvisionRequest from .provision_response import ProvisionResponse as ProvisionResponse +from .proxy_heartbeat_response import ProxyHeartbeatResponse as ProxyHeartbeatResponse +from .proxy_lease_request import ProxyLeaseRequest as ProxyLeaseRequest from .purge_request import PurgeRequest as PurgeRequest from .purge_response import PurgeResponse as PurgeResponse from .raw_content import RawContent as RawContent @@ -206,6 +206,7 @@ ) from .session_grant_summary import SessionGrantSummary as SessionGrantSummary from .slow_op_event import SlowOpEvent as SlowOpEvent +from .stop_proxy_response import StopProxyResponse as StopProxyResponse from .stop_response import StopResponse as StopResponse from .storage_diagnostics import StorageDiagnostics as StorageDiagnostics from .supply_chain_channel_evidence import ( diff --git a/sdk/python/capsem/models/create_proxy_request.py b/sdk/python/capsem/models/create_proxy_request.py new file mode 100644 index 000000000..5aafd9965 --- /dev/null +++ b/sdk/python/capsem/models/create_proxy_request.py @@ -0,0 +1,15 @@ +"""Generated from Capsem OpenAPI. Do not edit.""" + +from __future__ import annotations + +from typing import Annotated + +from pydantic import Field, StrictInt, StrictStr + +from .model_base import Model + + +class CreateProxyRequest(Model): + bind: StrictStr + port: Annotated[StrictInt, Field(ge=0)] + provider: StrictStr diff --git a/sdk/python/capsem/models/create_proxy_response.py b/sdk/python/capsem/models/create_proxy_response.py new file mode 100644 index 000000000..4eb834b59 --- /dev/null +++ b/sdk/python/capsem/models/create_proxy_response.py @@ -0,0 +1,19 @@ +"""Generated from Capsem OpenAPI. Do not edit.""" + +from __future__ import annotations + +from typing import Annotated + +from pydantic import Field, StrictInt, StrictStr + +from .model_base import Model + + +class CreateProxyResponse(Model): + base_url: StrictStr + bind: StrictStr + lease_expires_unix_ms: Annotated[StrictInt, Field(ge=0)] + lease_token: StrictStr + port: Annotated[StrictInt, Field(ge=0)] + provider: StrictStr + session_id: StrictStr diff --git a/sdk/python/capsem/models/proxy_heartbeat_response.py b/sdk/python/capsem/models/proxy_heartbeat_response.py new file mode 100644 index 000000000..5ee03d7f6 --- /dev/null +++ b/sdk/python/capsem/models/proxy_heartbeat_response.py @@ -0,0 +1,14 @@ +"""Generated from Capsem OpenAPI. Do not edit.""" + +from __future__ import annotations + +from typing import Annotated + +from pydantic import Field, StrictInt, StrictStr + +from .model_base import Model + + +class ProxyHeartbeatResponse(Model): + lease_expires_unix_ms: Annotated[StrictInt, Field(ge=0)] + session_id: StrictStr diff --git a/sdk/python/capsem/models/proxy_lease_request.py b/sdk/python/capsem/models/proxy_lease_request.py new file mode 100644 index 000000000..19a86af24 --- /dev/null +++ b/sdk/python/capsem/models/proxy_lease_request.py @@ -0,0 +1,11 @@ +"""Generated from Capsem OpenAPI. Do not edit.""" + +from __future__ import annotations + +from pydantic import StrictStr + +from .model_base import Model + + +class ProxyLeaseRequest(Model): + lease_token: StrictStr diff --git a/sdk/python/capsem/models/stop_proxy_response.py b/sdk/python/capsem/models/stop_proxy_response.py new file mode 100644 index 000000000..b62512553 --- /dev/null +++ b/sdk/python/capsem/models/stop_proxy_response.py @@ -0,0 +1,12 @@ +"""Generated from Capsem OpenAPI. Do not edit.""" + +from __future__ import annotations + +from pydantic import StrictBool, StrictStr + +from .model_base import Model + + +class StopProxyResponse(Model): + session_id: StrictStr + stopped: StrictBool From faab51861c39982e9fe02334984886c6c1e3c008 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:50:36 +0000 Subject: [PATCH 183/464] feat(typescript-sdk): add proxy lease operations --- .../src/models/CreateProxyRequest.ts | 9 ++++++ .../src/models/CreateProxyResponse.ts | 13 +++++++++ .../src/models/ProxyHeartbeatResponse.ts | 8 ++++++ .../src/models/ProxyLeaseRequest.ts | 7 +++++ .../src/models/StopProxyResponse.ts | 8 ++++++ sdk/typescript/src/models/index.ts | 7 +++-- sdk/typescript/src/operations/createProxy.ts | 25 +++++++++++++++++ .../src/operations/heartbeatProxy.ts | 28 +++++++++++++++++++ sdk/typescript/src/operations/index.ts | 6 ++-- sdk/typescript/src/operations/stopProxy.ts | 28 +++++++++++++++++++ .../src/validation/CreateProxyRequest.ts | 10 +++++++ .../src/validation/CreateProxyResponse.ts | 14 ++++++++++ .../src/validation/ProxyHeartbeatResponse.ts | 9 ++++++ .../src/validation/ProxyLeaseRequest.ts | 8 ++++++ .../src/validation/StopProxyResponse.ts | 9 ++++++ sdk/typescript/src/validation/index.ts | 7 +++-- 16 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 sdk/typescript/src/models/CreateProxyRequest.ts create mode 100644 sdk/typescript/src/models/CreateProxyResponse.ts create mode 100644 sdk/typescript/src/models/ProxyHeartbeatResponse.ts create mode 100644 sdk/typescript/src/models/ProxyLeaseRequest.ts create mode 100644 sdk/typescript/src/models/StopProxyResponse.ts create mode 100644 sdk/typescript/src/operations/createProxy.ts create mode 100644 sdk/typescript/src/operations/heartbeatProxy.ts create mode 100644 sdk/typescript/src/operations/stopProxy.ts create mode 100644 sdk/typescript/src/validation/CreateProxyRequest.ts create mode 100644 sdk/typescript/src/validation/CreateProxyResponse.ts create mode 100644 sdk/typescript/src/validation/ProxyHeartbeatResponse.ts create mode 100644 sdk/typescript/src/validation/ProxyLeaseRequest.ts create mode 100644 sdk/typescript/src/validation/StopProxyResponse.ts diff --git a/sdk/typescript/src/models/CreateProxyRequest.ts b/sdk/typescript/src/models/CreateProxyRequest.ts new file mode 100644 index 000000000..04c7fafc2 --- /dev/null +++ b/sdk/typescript/src/models/CreateProxyRequest.ts @@ -0,0 +1,9 @@ +// Generated from Capsem OpenAPI. Do not edit. + + + +export interface CreateProxyRequest { + "bind": string; + "port": number; + "provider": string; +} diff --git a/sdk/typescript/src/models/CreateProxyResponse.ts b/sdk/typescript/src/models/CreateProxyResponse.ts new file mode 100644 index 000000000..879719bbd --- /dev/null +++ b/sdk/typescript/src/models/CreateProxyResponse.ts @@ -0,0 +1,13 @@ +// Generated from Capsem OpenAPI. Do not edit. + + + +export interface CreateProxyResponse { + "base_url": string; + "bind": string; + "lease_expires_unix_ms": number; + "lease_token": string; + "port": number; + "provider": string; + "session_id": string; +} diff --git a/sdk/typescript/src/models/ProxyHeartbeatResponse.ts b/sdk/typescript/src/models/ProxyHeartbeatResponse.ts new file mode 100644 index 000000000..341f64605 --- /dev/null +++ b/sdk/typescript/src/models/ProxyHeartbeatResponse.ts @@ -0,0 +1,8 @@ +// Generated from Capsem OpenAPI. Do not edit. + + + +export interface ProxyHeartbeatResponse { + "lease_expires_unix_ms": number; + "session_id": string; +} diff --git a/sdk/typescript/src/models/ProxyLeaseRequest.ts b/sdk/typescript/src/models/ProxyLeaseRequest.ts new file mode 100644 index 000000000..c13523358 --- /dev/null +++ b/sdk/typescript/src/models/ProxyLeaseRequest.ts @@ -0,0 +1,7 @@ +// Generated from Capsem OpenAPI. Do not edit. + + + +export interface ProxyLeaseRequest { + "lease_token": string; +} diff --git a/sdk/typescript/src/models/StopProxyResponse.ts b/sdk/typescript/src/models/StopProxyResponse.ts new file mode 100644 index 000000000..21e2dca45 --- /dev/null +++ b/sdk/typescript/src/models/StopProxyResponse.ts @@ -0,0 +1,8 @@ +// Generated from Capsem OpenAPI. Do not edit. + + + +export interface StopProxyResponse { + "session_id": string; + "stopped": boolean; +} diff --git a/sdk/typescript/src/models/index.ts b/sdk/typescript/src/models/index.ts index 707ce56a7..20098d946 100644 --- a/sdk/typescript/src/models/index.ts +++ b/sdk/typescript/src/models/index.ts @@ -26,8 +26,8 @@ export type { ContainerStatusResponse } from "./ContainerStatusResponse.js"; export type { ContainerSurface } from "./ContainerSurface.js"; export { ContainerSurfaceKind } from "./ContainerSurfaceKind.js"; export type { CreateNetworkRequest } from "./CreateNetworkRequest.js"; -export type { CreateSessionGrantRequest } from "./CreateSessionGrantRequest.js"; -export { CredentialErrorCode } from "./CredentialErrorCode.js"; +export type { CreateProxyRequest } from "./CreateProxyRequest.js"; +export type { CreateProxyResponse } from "./CreateProxyResponse.js"; export type { CredentialEvent } from "./CredentialEvent.js"; export { CredentialEventType } from "./CredentialEventType.js"; export { CredentialInjectProvider } from "./CredentialInjectProvider.js"; @@ -139,6 +139,8 @@ export type { ProcessEvent } from "./ProcessEvent.js"; export { ProviderRevocationState } from "./ProviderRevocationState.js"; export type { ProvisionRequest } from "./ProvisionRequest.js"; export type { ProvisionResponse } from "./ProvisionResponse.js"; +export type { ProxyHeartbeatResponse } from "./ProxyHeartbeatResponse.js"; +export type { ProxyLeaseRequest } from "./ProxyLeaseRequest.js"; export type { PurgeRequest } from "./PurgeRequest.js"; export type { PurgeResponse } from "./PurgeResponse.js"; export type { RawContent } from "./RawContent.js"; @@ -160,6 +162,7 @@ export type { SessionDbStatus } from "./SessionDbStatus.js"; export type { SessionGrantListResponse } from "./SessionGrantListResponse.js"; export type { SessionGrantSummary } from "./SessionGrantSummary.js"; export type { SlowOpEvent } from "./SlowOpEvent.js"; +export type { StopProxyResponse } from "./StopProxyResponse.js"; export type { StopResponse } from "./StopResponse.js"; export type { StorageDiagnostics } from "./StorageDiagnostics.js"; export type { SupplyChainChannelEvidence } from "./SupplyChainChannelEvidence.js"; diff --git a/sdk/typescript/src/operations/createProxy.ts b/sdk/typescript/src/operations/createProxy.ts new file mode 100644 index 000000000..d5dad3b59 --- /dev/null +++ b/sdk/typescript/src/operations/createProxy.ts @@ -0,0 +1,25 @@ +// Generated from Capsem OpenAPI. Do not edit. + +import {z} from "zod"; +import {Transport, Method, MediaType, type CallOptions} from "../transport.js"; +import type {CreateProxyRequest} from "../models/CreateProxyRequest.js"; +import type {CreateProxyResponse} from "../models/CreateProxyResponse.js"; +import {CreateProxyRequestSchema} from "../validation/CreateProxyRequest.js"; +import {CreateProxyResponseSchema} from "../validation/CreateProxyResponse.js"; + +export async function createProxy( + transport: Transport, + parameters: { + "body": CreateProxyRequest; + }, + options: CallOptions = {}, +): Promise { + const input = z.object({ + "body": z.lazy(() => CreateProxyRequestSchema), +}).parse(parameters); + const payload = await transport.request(Method.POST, "/proxies", { + signal: options.signal, timeoutMs: options.timeoutMs, accept: MediaType.JSON, + body: JSON.stringify(input.body), contentType: MediaType.JSON, + }); + return z.lazy(() => CreateProxyResponseSchema).parse(JSON.parse(new TextDecoder().decode(payload))); +} diff --git a/sdk/typescript/src/operations/heartbeatProxy.ts b/sdk/typescript/src/operations/heartbeatProxy.ts new file mode 100644 index 000000000..521b2bb6a --- /dev/null +++ b/sdk/typescript/src/operations/heartbeatProxy.ts @@ -0,0 +1,28 @@ +// Generated from Capsem OpenAPI. Do not edit. + +import {z} from "zod"; +import {Transport, Method, MediaType, type CallOptions} from "../transport.js"; +import type {ProxyHeartbeatResponse} from "../models/ProxyHeartbeatResponse.js"; +import type {ProxyLeaseRequest} from "../models/ProxyLeaseRequest.js"; +import {ProxyHeartbeatResponseSchema} from "../validation/ProxyHeartbeatResponse.js"; +import {ProxyLeaseRequestSchema} from "../validation/ProxyLeaseRequest.js"; + +export async function heartbeatProxy( + transport: Transport, + parameters: { + "id": string; + "body": ProxyLeaseRequest; + }, + options: CallOptions = {}, +): Promise { + const input = z.object({ + "id": z.string(), + "body": z.lazy(() => ProxyLeaseRequestSchema), +}).parse(parameters); + const payload = await transport.request(Method.POST, "/proxies/{id}/heartbeat", { + signal: options.signal, timeoutMs: options.timeoutMs, accept: MediaType.JSON, + parameters: {"id": input["id"]}, + body: JSON.stringify(input.body), contentType: MediaType.JSON, + }); + return z.lazy(() => ProxyHeartbeatResponseSchema).parse(JSON.parse(new TextDecoder().decode(payload))); +} diff --git a/sdk/typescript/src/operations/index.ts b/sdk/typescript/src/operations/index.ts index 701ada95c..356526635 100644 --- a/sdk/typescript/src/operations/index.ts +++ b/sdk/typescript/src/operations/index.ts @@ -7,7 +7,7 @@ export {cancelConnectionAuthorization} from "./cancelConnectionAuthorization.js" export {claimManagedSession} from "./claimManagedSession.js"; export {closeManagedSession} from "./closeManagedSession.js"; export {createNetwork} from "./createNetwork.js"; -export {createSessionCredentialGrant} from "./createSessionCredentialGrant.js"; +export {createProxy} from "./createProxy.js"; export {createVm} from "./createVm.js"; export {createVmExposure} from "./createVmExposure.js"; export {createVmPreviewSession} from "./createVmPreviewSession.js"; @@ -41,8 +41,7 @@ export {getVmStatsDetail} from "./getVmStatsDetail.js"; export {getVmStatsSummary} from "./getVmStatsSummary.js"; export {getVmStatus} from "./getVmStatus.js"; export {getVmTimeline} from "./getVmTimeline.js"; -export {injectCredential} from "./injectCredential.js"; -export {listConnections} from "./listConnections.js"; +export {heartbeatProxy} from "./heartbeatProxy.js"; export {listImages} from "./listImages.js"; export {listMcpServers} from "./listMcpServers.js"; export {listMcpTools} from "./listMcpTools.js"; @@ -62,6 +61,7 @@ export {revokeSessionCredentialGrant} from "./revokeSessionCredentialGrant.js"; export {revokeVmPreviewSessions} from "./revokeVmPreviewSessions.js"; export {runVm} from "./runVm.js"; export {startVm} from "./startVm.js"; +export {stopProxy} from "./stopProxy.js"; export {stopVm} from "./stopVm.js"; export {updateHypervisor} from "./updateHypervisor.js"; export {uploadVmFile} from "./uploadVmFile.js"; diff --git a/sdk/typescript/src/operations/stopProxy.ts b/sdk/typescript/src/operations/stopProxy.ts new file mode 100644 index 000000000..777c25c28 --- /dev/null +++ b/sdk/typescript/src/operations/stopProxy.ts @@ -0,0 +1,28 @@ +// Generated from Capsem OpenAPI. Do not edit. + +import {z} from "zod"; +import {Transport, Method, MediaType, type CallOptions} from "../transport.js"; +import type {ProxyLeaseRequest} from "../models/ProxyLeaseRequest.js"; +import type {StopProxyResponse} from "../models/StopProxyResponse.js"; +import {ProxyLeaseRequestSchema} from "../validation/ProxyLeaseRequest.js"; +import {StopProxyResponseSchema} from "../validation/StopProxyResponse.js"; + +export async function stopProxy( + transport: Transport, + parameters: { + "id": string; + "body": ProxyLeaseRequest; + }, + options: CallOptions = {}, +): Promise { + const input = z.object({ + "id": z.string(), + "body": z.lazy(() => ProxyLeaseRequestSchema), +}).parse(parameters); + const payload = await transport.request(Method.POST, "/proxies/{id}/stop", { + signal: options.signal, timeoutMs: options.timeoutMs, accept: MediaType.JSON, + parameters: {"id": input["id"]}, + body: JSON.stringify(input.body), contentType: MediaType.JSON, + }); + return z.lazy(() => StopProxyResponseSchema).parse(JSON.parse(new TextDecoder().decode(payload))); +} diff --git a/sdk/typescript/src/validation/CreateProxyRequest.ts b/sdk/typescript/src/validation/CreateProxyRequest.ts new file mode 100644 index 000000000..82b2f4514 --- /dev/null +++ b/sdk/typescript/src/validation/CreateProxyRequest.ts @@ -0,0 +1,10 @@ +// Generated from Capsem OpenAPI. Do not edit. + +import {z} from "zod"; +import type {CreateProxyRequest} from "../models/CreateProxyRequest.js"; + +export const CreateProxyRequestSchema: z.ZodType = z.object({ + "bind": z.string(), + "port": z.int().min(0), + "provider": z.string(), +}); diff --git a/sdk/typescript/src/validation/CreateProxyResponse.ts b/sdk/typescript/src/validation/CreateProxyResponse.ts new file mode 100644 index 000000000..c78518f07 --- /dev/null +++ b/sdk/typescript/src/validation/CreateProxyResponse.ts @@ -0,0 +1,14 @@ +// Generated from Capsem OpenAPI. Do not edit. + +import {z} from "zod"; +import type {CreateProxyResponse} from "../models/CreateProxyResponse.js"; + +export const CreateProxyResponseSchema: z.ZodType = z.object({ + "base_url": z.string(), + "bind": z.string(), + "lease_expires_unix_ms": z.int().min(0), + "lease_token": z.string(), + "port": z.int().min(0), + "provider": z.string(), + "session_id": z.string(), +}); diff --git a/sdk/typescript/src/validation/ProxyHeartbeatResponse.ts b/sdk/typescript/src/validation/ProxyHeartbeatResponse.ts new file mode 100644 index 000000000..c4b9aa1f8 --- /dev/null +++ b/sdk/typescript/src/validation/ProxyHeartbeatResponse.ts @@ -0,0 +1,9 @@ +// Generated from Capsem OpenAPI. Do not edit. + +import {z} from "zod"; +import type {ProxyHeartbeatResponse} from "../models/ProxyHeartbeatResponse.js"; + +export const ProxyHeartbeatResponseSchema: z.ZodType = z.object({ + "lease_expires_unix_ms": z.int().min(0), + "session_id": z.string(), +}); diff --git a/sdk/typescript/src/validation/ProxyLeaseRequest.ts b/sdk/typescript/src/validation/ProxyLeaseRequest.ts new file mode 100644 index 000000000..c85caa2fa --- /dev/null +++ b/sdk/typescript/src/validation/ProxyLeaseRequest.ts @@ -0,0 +1,8 @@ +// Generated from Capsem OpenAPI. Do not edit. + +import {z} from "zod"; +import type {ProxyLeaseRequest} from "../models/ProxyLeaseRequest.js"; + +export const ProxyLeaseRequestSchema: z.ZodType = z.object({ + "lease_token": z.string(), +}); diff --git a/sdk/typescript/src/validation/StopProxyResponse.ts b/sdk/typescript/src/validation/StopProxyResponse.ts new file mode 100644 index 000000000..386b31206 --- /dev/null +++ b/sdk/typescript/src/validation/StopProxyResponse.ts @@ -0,0 +1,9 @@ +// Generated from Capsem OpenAPI. Do not edit. + +import {z} from "zod"; +import type {StopProxyResponse} from "../models/StopProxyResponse.js"; + +export const StopProxyResponseSchema: z.ZodType = z.object({ + "session_id": z.string(), + "stopped": z.boolean(), +}); diff --git a/sdk/typescript/src/validation/index.ts b/sdk/typescript/src/validation/index.ts index 960efd640..4f065e895 100644 --- a/sdk/typescript/src/validation/index.ts +++ b/sdk/typescript/src/validation/index.ts @@ -26,8 +26,8 @@ export {ContainerStatusResponseSchema} from "./ContainerStatusResponse.js"; export {ContainerSurfaceSchema} from "./ContainerSurface.js"; export {ContainerSurfaceKindSchema} from "./ContainerSurfaceKind.js"; export {CreateNetworkRequestSchema} from "./CreateNetworkRequest.js"; -export {CreateSessionGrantRequestSchema} from "./CreateSessionGrantRequest.js"; -export {CredentialErrorCodeSchema} from "./CredentialErrorCode.js"; +export {CreateProxyRequestSchema} from "./CreateProxyRequest.js"; +export {CreateProxyResponseSchema} from "./CreateProxyResponse.js"; export {CredentialEventSchema} from "./CredentialEvent.js"; export {CredentialEventTypeSchema} from "./CredentialEventType.js"; export {CredentialInjectProviderSchema} from "./CredentialInjectProvider.js"; @@ -139,6 +139,8 @@ export {ProcessEventSchema} from "./ProcessEvent.js"; export {ProviderRevocationStateSchema} from "./ProviderRevocationState.js"; export {ProvisionRequestSchema} from "./ProvisionRequest.js"; export {ProvisionResponseSchema} from "./ProvisionResponse.js"; +export {ProxyHeartbeatResponseSchema} from "./ProxyHeartbeatResponse.js"; +export {ProxyLeaseRequestSchema} from "./ProxyLeaseRequest.js"; export {PurgeRequestSchema} from "./PurgeRequest.js"; export {PurgeResponseSchema} from "./PurgeResponse.js"; export {RawContentSchema} from "./RawContent.js"; @@ -160,6 +162,7 @@ export {SessionDbStatusSchema} from "./SessionDbStatus.js"; export {SessionGrantListResponseSchema} from "./SessionGrantListResponse.js"; export {SessionGrantSummarySchema} from "./SessionGrantSummary.js"; export {SlowOpEventSchema} from "./SlowOpEvent.js"; +export {StopProxyResponseSchema} from "./StopProxyResponse.js"; export {StopResponseSchema} from "./StopResponse.js"; export {StorageDiagnosticsSchema} from "./StorageDiagnostics.js"; export {SupplyChainChannelEvidenceSchema} from "./SupplyChainChannelEvidence.js"; From 2692bbaa958bce4da9eb451c5c2f0bcc84ba94c8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 08:50:39 +0000 Subject: [PATCH 184/464] feat(rust-sdk): add proxy lease operations --- .../src/operations/generated/create_proxy.rs | 21 +++++++++++++++ .../operations/generated/heartbeat_proxy.rs | 26 +++++++++++++++++++ sdk/rust/src/operations/generated/mod.rs | 12 ++++----- .../src/operations/generated/stop_proxy.rs | 26 +++++++++++++++++++ 4 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 sdk/rust/src/operations/generated/create_proxy.rs create mode 100644 sdk/rust/src/operations/generated/heartbeat_proxy.rs create mode 100644 sdk/rust/src/operations/generated/stop_proxy.rs diff --git a/sdk/rust/src/operations/generated/create_proxy.rs b/sdk/rust/src/operations/generated/create_proxy.rs new file mode 100644 index 000000000..62e3cf96b --- /dev/null +++ b/sdk/rust/src/operations/generated/create_proxy.rs @@ -0,0 +1,21 @@ +// Generated from sdk/specification/openapi.json. Do not edit. +use crate::transport::{CallOptions, Request, Transport}; + +#[derive(Debug, Clone, serde::Deserialize)] +pub struct CreateProxyParams { + pub body: capsem_api::CreateProxyRequest, +} + +pub async fn create_proxy( + transport: &Transport, + input: &CreateProxyParams, + options: CallOptions, +) -> crate::Result { + let request = Request { + body: Some(serde_json::to_vec(&input.body)?), + options, + ..Default::default() + }; + let bytes = transport.request(reqwest::Method::POST, "/proxies", request).await?; + Ok(serde_json::from_slice(&bytes)?) +} diff --git a/sdk/rust/src/operations/generated/heartbeat_proxy.rs b/sdk/rust/src/operations/generated/heartbeat_proxy.rs new file mode 100644 index 000000000..61dec2105 --- /dev/null +++ b/sdk/rust/src/operations/generated/heartbeat_proxy.rs @@ -0,0 +1,26 @@ +// Generated from sdk/specification/openapi.json. Do not edit. +use crate::transport::{CallOptions, Request, Transport}; + +#[derive(Debug, Clone, serde::Deserialize)] +pub struct HeartbeatProxyParams { + pub id: String, + pub body: capsem_api::ProxyLeaseRequest, +} + +pub async fn heartbeat_proxy( + transport: &Transport, + input: &HeartbeatProxyParams, + options: CallOptions, +) -> crate::Result { + let path_id = input.id.to_string(); + let request = Request { + parameters: &[("id", path_id.as_str())], + body: Some(serde_json::to_vec(&input.body)?), + options, + ..Default::default() + }; + let bytes = transport + .request(reqwest::Method::POST, "/proxies/{id}/heartbeat", request) + .await?; + Ok(serde_json::from_slice(&bytes)?) +} diff --git a/sdk/rust/src/operations/generated/mod.rs b/sdk/rust/src/operations/generated/mod.rs index 6e5a5e320..e504b4daf 100644 --- a/sdk/rust/src/operations/generated/mod.rs +++ b/sdk/rust/src/operations/generated/mod.rs @@ -13,8 +13,8 @@ mod close_managed_session; pub use close_managed_session::{close_managed_session, CloseManagedSessionParams}; mod create_network; pub use create_network::{create_network, CreateNetworkParams}; -mod create_session_credential_grant; -pub use create_session_credential_grant::{create_session_credential_grant, CreateSessionCredentialGrantParams}; +mod create_proxy; +pub use create_proxy::{create_proxy, CreateProxyParams}; mod create_vm; pub use create_vm::{create_vm, CreateVmParams}; mod create_vm_exposure; @@ -81,10 +81,8 @@ mod get_vm_status; pub use get_vm_status::{get_vm_status, GetVmStatusParams}; mod get_vm_timeline; pub use get_vm_timeline::{get_vm_timeline, GetVmTimelineParams}; -mod inject_credential; -pub use inject_credential::{inject_credential, InjectCredentialParams}; -mod list_connections; -pub use list_connections::{list_connections, ListConnectionsParams}; +mod heartbeat_proxy; +pub use heartbeat_proxy::{heartbeat_proxy, HeartbeatProxyParams}; mod list_images; pub use list_images::{list_images, ListImagesParams}; mod list_mcp_servers; @@ -123,6 +121,8 @@ mod run_vm; pub use run_vm::{run_vm, RunVmParams}; mod start_vm; pub use start_vm::{start_vm, StartVmParams}; +mod stop_proxy; +pub use stop_proxy::{stop_proxy, StopProxyParams}; mod stop_vm; pub use stop_vm::{stop_vm, StopVmParams}; mod update_hypervisor; diff --git a/sdk/rust/src/operations/generated/stop_proxy.rs b/sdk/rust/src/operations/generated/stop_proxy.rs new file mode 100644 index 000000000..3dcf37c54 --- /dev/null +++ b/sdk/rust/src/operations/generated/stop_proxy.rs @@ -0,0 +1,26 @@ +// Generated from sdk/specification/openapi.json. Do not edit. +use crate::transport::{CallOptions, Request, Transport}; + +#[derive(Debug, Clone, serde::Deserialize)] +pub struct StopProxyParams { + pub id: String, + pub body: capsem_api::ProxyLeaseRequest, +} + +pub async fn stop_proxy( + transport: &Transport, + input: &StopProxyParams, + options: CallOptions, +) -> crate::Result { + let path_id = input.id.to_string(); + let request = Request { + parameters: &[("id", path_id.as_str())], + body: Some(serde_json::to_vec(&input.body)?), + options, + ..Default::default() + }; + let bytes = transport + .request(reqwest::Method::POST, "/proxies/{id}/stop", request) + .await?; + Ok(serde_json::from_slice(&bytes)?) +} From 6f483524c98766ec2f60835ac78cb52ad7fcde3c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:15:24 +0000 Subject: [PATCH 185/464] style(api): format proxy lease route --- crates/capsem-api/src/document.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/capsem-api/src/document.rs b/crates/capsem-api/src/document.rs index f2889fa74..336b66cca 100644 --- a/crates/capsem-api/src/document.rs +++ b/crates/capsem-api/src/document.rs @@ -28,10 +28,7 @@ pub fn openapi() -> OpenApi { doc.post::("/run", "runVm"); doc.post::("/purge", "purgeVms"); doc.post::("/proxies", "createProxy"); - doc.post::( - "/proxies/{id}/heartbeat", - "heartbeatProxy", - ); + doc.post::("/proxies/{id}/heartbeat", "heartbeatProxy"); doc.post::("/proxies/{id}/stop", "stopProxy"); doc.post::("/vms/{id}/save", "persistVm"); doc.get::("/vms/{id}/info", "getVmInfo"); From 0b3b14d46a76953ff6e437b2ab1b51dd463606e8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:15:57 +0000 Subject: [PATCH 186/464] security(service): supervise standalone proxy leases --- CHANGELOG.md | 8 + crates/capsem-service/src/active_policy.rs | 14 +- crates/capsem-service/src/instance.rs | 5 + crates/capsem-service/src/instance/tests.rs | 2 + crates/capsem-service/src/main.rs | 2 + crates/capsem-service/src/proxy_worker.rs | 164 ++++-- .../capsem-service/src/proxy_worker/tests.rs | 59 ++- crates/capsem-service/src/router_runtime.rs | 5 +- crates/capsem-service/src/service_runtime.rs | 4 +- .../src/service_runtime/telemetry_export.rs | 102 +++- crates/capsem-service/src/standalone_proxy.rs | 490 ++++++++++++++++++ .../src/standalone_proxy/tests.rs | 50 ++ crates/capsem-service/src/tests.rs | 1 + crates/capsem-service/src/tests/lifecycle.rs | 11 +- crates/capsem-service/src/upstream_broker.rs | 7 +- .../src/vm_lifecycle/provision.rs | 7 +- .../src/vm_lifecycle/resume_process.rs | 7 +- 17 files changed, 866 insertions(+), 72 deletions(-) create mode 100644 crates/capsem-service/src/standalone_proxy.rs create mode 100644 crates/capsem-service/src/standalone_proxy/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index f5c10086a..5149d4911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Bind each standalone model-proxy lease to one service-selected provider and + one revocable worker generation; client traffic receives only connected + descriptors and cannot choose an upstream with `Host` or proxy semantics. + - Restrict VM owners to the guest share, system overlay, read-only image share, and a dedicated mutable owner-state directory; attest after confinement that the owner cannot open `session.db`, and fix suspend checkpoints to that @@ -129,6 +133,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Keep a confined proxy generation alive when an ordinary HTTP or DNS client + disconnects, while still terminating it for unknown, stale or required + capability closure events. + - Keep SQLite writer scratch state in memory so a confined VM owner can flush high-volume session-ledger bursts without ambient temporary-directory access. diff --git a/crates/capsem-service/src/active_policy.rs b/crates/capsem-service/src/active_policy.rs index c185ce2f3..20052f3a3 100644 --- a/crates/capsem-service/src/active_policy.rs +++ b/crates/capsem-service/src/active_policy.rs @@ -13,7 +13,7 @@ pub(crate) struct PublishedActivePolicy { pub(crate) path: PathBuf, pub(crate) digest: String, pub(crate) bytes: Vec, - runtime: Arc, + pub(crate) runtime: Arc, } impl PublishedActivePolicy { @@ -112,7 +112,7 @@ pub(crate) async fn push_policy_to_running_instances( } } let published = state.off_worker(|state| state.refresh_active_policies()).await?; - let total = published.len(); + let mut total = published.len(); let mut failures = Vec::new(); let targets = { @@ -187,14 +187,22 @@ pub(crate) async fn push_policy_to_running_instances( })) .await; failures.extend(results.into_iter().flatten()); + let (standalone_total, standalone_failures) = crate::standalone_proxy::refresh_policies(state).await?; + total += standalone_total; + failures.extend(standalone_failures); if failures.is_empty() { Ok(total) } else { + let subjects = if standalone_total == 0 { + "VMs" + } else { + "VMs and proxy sessions" + }; Err(AppError( StatusCode::INTERNAL_SERVER_ERROR, format!( - "policy applied to {} of {total} running VMs; not applied: {}", + "policy applied to {} of {total} running {subjects}; not applied: {}", total - failures.len(), failures.join(", ") ), diff --git a/crates/capsem-service/src/instance.rs b/crates/capsem-service/src/instance.rs index 691eb23a6..37185a15c 100644 --- a/crates/capsem-service/src/instance.rs +++ b/crates/capsem-service/src/instance.rs @@ -7,6 +7,7 @@ use tokio_util::sync::CancellationToken; pub(crate) struct WorkerAuthority(CancellationToken); /// A non-owning grant canceled when its registered generation is revoked. +#[derive(Clone)] pub(crate) struct WorkerGrant(CancellationToken); impl Default for WorkerAuthority { @@ -32,6 +33,10 @@ impl Drop for WorkerAuthority { } impl WorkerGrant { + pub(crate) fn is_revoked(&self) -> bool { + self.0.is_cancelled() + } + pub(crate) async fn revoked(&self) { self.0.cancelled().await; } diff --git a/crates/capsem-service/src/instance/tests.rs b/crates/capsem-service/src/instance/tests.rs index ea764ad13..b7245f6fe 100644 --- a/crates/capsem-service/src/instance/tests.rs +++ b/crates/capsem-service/src/instance/tests.rs @@ -189,6 +189,7 @@ async fn removing_or_replacing_an_instance_revokes_its_grants() { let grant = original.authority.grant(); let disposable = original.authority.grant(); drop(disposable); + assert!(!grant.is_revoked()); assert!( tokio::time::timeout(std::time::Duration::from_millis(1), grant.revoked()) .await @@ -200,6 +201,7 @@ async fn removing_or_replacing_an_instance_revokes_its_grants() { tokio::time::timeout(std::time::Duration::from_millis(100), grant.revoked()) .await .expect("registry removal revokes the generation"); + assert!(grant.is_revoked()); let replacement_target = test_instance(); let replaced = replacement_target.authority.grant(); diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index dc9dbf310..3d42a9625 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -54,6 +54,7 @@ mod proxy_worker; mod sandbox_info; mod session_cleanup; mod session_db_handles; +mod standalone_proxy; mod switches; use session_db_handles::session_db_path_for_session_dir; mod session_housekeeping; @@ -212,6 +213,7 @@ pub struct ServiceState { connections: connection_routes::Runtime, instances: Mutex>, // instance id to process info proxy_workers: Mutex>, + standalone_proxies: tokio::sync::Mutex>, ledger_workers: Arc, retirements: instance_reaper::Retirements, /// Authenticated ledger channels keyed by session/VM id. Logged-data diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index c3edbbf7d..0ad60c393 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -1,5 +1,6 @@ //! Trusted lifecycle and capability coordinator for one confined proxy worker. +use std::collections::HashMap; use std::fmt::Write as _; use std::io; use std::os::fd::{AsRawFd as _, OwnedFd}; @@ -13,8 +14,8 @@ use capsem_foundation::ipc_channel; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::proxy_control::{ - decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyControlEvent, ProxyControlRequest, - ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, + decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyChannelCloseReason, + ProxyControlEvent, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, }; use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; use tokio::process::{Child, Command}; @@ -52,15 +53,38 @@ impl ProxyWorker { /// Spawn the production worker with an empty environment and no inherited /// descriptors other than its coordinator socket on stdin. pub(crate) async fn spawn(binary: &Path, active_policy: Vec, stdout: Stdio, stderr: Stdio) -> Result { + Self::spawn_mode(binary, active_policy, stdout, stderr, ProxyWorkerMode::Vm).await + } + + pub(crate) async fn spawn_standalone( + binary: &Path, + active_policy: Vec, + stdout: Stdio, + stderr: Stdio, + provider_id: String, + ) -> Result { + Self::spawn_mode( + binary, + active_policy, + stdout, + stderr, + ProxyWorkerMode::Standalone(provider_id), + ) + .await + } + + async fn spawn_mode( + binary: &Path, + active_policy: Vec, + stdout: Stdio, + stderr: Stdio, + mode: ProxyWorkerMode, + ) -> Result { let generation = fresh_generation(); let mut command = Command::new(binary); command.env_clear().stdout(stdout).stderr(stderr).kill_on_drop(true); - launch(command, generation, active_policy, |command, generation| { - command - .arg("--parent-pid") - .arg(std::process::id().to_string()) - .arg("--generation") - .arg(generation_hex(generation)); + launch(command, generation, active_policy, move |command, generation| { + configure_worker_command(command, generation, &mode); }) .await } @@ -69,21 +93,22 @@ impl ProxyWorker { self.generation } - pub(crate) async fn grant(&self, capability: ProxyCapability, stream: UnixStream) -> Result<()> { - self.grant_capability(GrantedCapability::Ordinary(capability), stream) + pub(crate) async fn grant(&self, capability: ProxyCapability, descriptor: OwnedFd) -> Result<()> { + self.grant_capability(GrantedCapability::Ordinary(capability), descriptor) .await } pub(crate) async fn grant_ledger(&self, stream: UnixStream, grant: LedgerChannelGrant) -> Result<()> { - self.grant_capability(GrantedCapability::Ledger(grant), stream).await + self.grant_capability(GrantedCapability::Ledger(grant), stream.into()) + .await } - async fn grant_capability(&self, capability: GrantedCapability, stream: UnixStream) -> Result<()> { + async fn grant_capability(&self, capability: GrantedCapability, descriptor: OwnedFd) -> Result<()> { let (completed, result) = oneshot::channel(); self.commands .send(CommandRequest::Grant { capability, - stream, + descriptor, completed, }) .await @@ -107,10 +132,6 @@ impl ProxyWorker { .map_err(|_| anyhow!(self.stop_reason("proxy worker supervisor stopped with policy pending")))? } - #[cfg_attr( - not(test), - expect(dead_code, reason = "session ownership normally shuts down by dropping the handle") - )] pub(crate) async fn shutdown(self) -> Result<()> { let (completed, result) = oneshot::channel(); self.commands @@ -189,7 +210,7 @@ pub(crate) fn spawn_for_session(binary: &Path, active_policy: Vec, log_path: enum CommandRequest { Grant { capability: GrantedCapability, - stream: UnixStream, + descriptor: OwnedFd, completed: oneshot::Sender>, }, ApplyPolicy { @@ -201,6 +222,22 @@ enum CommandRequest { }, } +enum ProxyWorkerMode { + Vm, + Standalone(String), +} + +fn configure_worker_command(command: &mut Command, generation: ProxyGeneration, mode: &ProxyWorkerMode) { + command + .arg("--parent-pid") + .arg(std::process::id().to_string()) + .arg("--generation") + .arg(generation_hex(generation)); + if let ProxyWorkerMode::Standalone(provider_id) = mode { + command.arg("--standalone-provider").arg(provider_id); + } +} + #[derive(Clone, Copy)] enum GrantedCapability { Ordinary(ProxyCapability), @@ -241,6 +278,7 @@ struct WorkerProcess { policy_rx: ipc_channel::Receiver, next_grant_id: u64, next_request_id: u64, + active_grants: HashMap, } enum PolicyApply { @@ -325,6 +363,7 @@ impl WorkerProcess { policy_rx, next_grant_id, next_request_id: 1, + active_grants: HashMap::new(), }; let expected = capsem_core::net::policy_config::active_policy_digest(&active_policy); let applied = match process.apply_policy(active_policy).await { @@ -345,22 +384,25 @@ impl WorkerProcess { Ok(process) } - async fn grant(&mut self, capability: GrantedCapability, stream: UnixStream) -> Result<()> { + async fn grant(&mut self, capability: GrantedCapability, descriptor: OwnedFd) -> Result<()> { let grant_id = self.allocate_grant_id()?; let grant = capability.channel_grant(self.generation, grant_id)?; self.control_tx .send( &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), - &[stream.as_raw_fd()], + &[descriptor.as_raw_fd()], ) .await .context("send proxy descriptor grant")?; - drop(stream); + drop(descriptor); match self.recv_event().await? { ProxyControlEvent::Adopted { generation, grant_id: adopted, - } if generation == self.generation && adopted == grant_id => Ok(()), + } if generation == self.generation && adopted == grant_id => { + self.active_grants.insert(grant_id, capability.kind()); + Ok(()) + } ProxyControlEvent::Rejected { generation, grant_id: rejected, @@ -429,17 +471,22 @@ impl WorkerProcess { async fn recv_event(&mut self) -> Result { tokio::time::timeout(OPERATION_TIMEOUT, async { - tokio::select! { - status = self.child.wait() => { - let status = status.context("reap proxy worker")?; - bail!("proxy worker exited: {status}") - } - frame = self.control_events.recv() => { - let frame = frame.context("proxy control event pump stopped")??; - if !frame.fds.is_empty() { - bail!("proxy worker returned a descriptor on its control channel"); + loop { + let event = tokio::select! { + status = self.child.wait() => { + let status = status.context("reap proxy worker")?; + bail!("proxy worker exited: {status}") } - decode_proxy_control_event(&frame.bytes).map_err(anyhow::Error::from) + frame = self.control_events.recv() => { + let frame = frame.context("proxy control event pump stopped")??; + if !frame.fds.is_empty() { + bail!("proxy worker returned a descriptor on its control channel"); + } + decode_proxy_control_event(&frame.bytes).map_err(anyhow::Error::from)? + } + }; + if !self.absorb_traffic_close(event)? { + break Ok(event); } } }) @@ -447,6 +494,30 @@ impl WorkerProcess { .map_err(|_| anyhow!("proxy worker control operation timed out"))? } + fn absorb_traffic_close(&mut self, event: ProxyControlEvent) -> Result { + let ProxyControlEvent::Closed { + generation, + grant_id, + reason, + } = event + else { + return Ok(false); + }; + if generation != self.generation { + bail!("proxy worker closed a grant for a stale generation"); + } + let capability = self + .active_grants + .remove(&grant_id) + .with_context(|| format!("proxy worker closed unknown grant {grant_id}"))?; + if !matches!(capability, ProxyCapability::HttpTraffic | ProxyCapability::DnsTraffic) + || reason != ProxyChannelCloseReason::Disconnected + { + bail!("proxy worker closed required {capability:?} grant {grant_id}: {reason:?}"); + } + Ok(true) + } + async fn recv_policy(&mut self) -> Result { tokio::time::timeout(OPERATION_TIMEOUT, async { tokio::select! { @@ -547,15 +618,20 @@ async fn supervise( }; } event = process.control_events.recv() => { - break match event { + let event = match event { Some(Ok(frame)) if frame.fds.is_empty() => match decode_proxy_control_event(&frame.bytes) { - Ok(event) => format!("proxy worker sent unsolicited control event {event:?}"), - Err(error) => format!("proxy worker sent invalid control event: {error}"), + Ok(event) => event, + Err(error) => break format!("proxy worker sent invalid control event: {error}"), }, - Some(Ok(_)) => "proxy worker sent an unsolicited descriptor".to_string(), - Some(Err(error)) => format!("proxy worker control channel failed: {error}"), - None => "proxy worker control event pump stopped".to_string(), + Some(Ok(_)) => break "proxy worker sent an unsolicited descriptor".to_string(), + Some(Err(error)) => break format!("proxy worker control channel failed: {error}"), + None => break "proxy worker control event pump stopped".to_string(), }; + match process.absorb_traffic_close(event) { + Ok(true) => continue, + Ok(false) => break format!("proxy worker sent unsolicited control event {event:?}"), + Err(error) => break format!("proxy worker sent invalid control event: {error:#}"), + } } request = requests.recv() => { let Some(request) = request else { @@ -565,8 +641,12 @@ async fn supervise( }; }; match request { - CommandRequest::Grant { capability, stream, completed } => { - let result = process.grant(capability, stream).await; + CommandRequest::Grant { + capability, + descriptor, + completed, + } => { + let result = process.grant(capability, descriptor).await; let failed = result.is_err(); let diagnostic = result.as_ref().err().map(|error| format!("{error:#}")); let _ = completed.send(result); @@ -655,7 +735,7 @@ impl ServiceState { vm_id.to_string(), service, )); - if let Err(error) = worker.grant(ProxyCapability::PrivateNames, proxy).await { + if let Err(error) = worker.grant(ProxyCapability::PrivateNames, proxy.into()).await { serving.abort(); let _ = serving.await; return Err(error.context("grant proxy private-name capability")); @@ -673,7 +753,7 @@ impl ServiceState { pub(crate) async fn grant_proxy_credentials(&self, worker: &ProxyWorker) -> Result<()> { let (service, proxy) = UnixStream::pair().context("create proxy credential capability")?; let serving = tokio::spawn(crate::proxy_credentials::serve(service)); - if let Err(error) = worker.grant(ProxyCapability::Credential, proxy).await { + if let Err(error) = worker.grant(ProxyCapability::Credential, proxy.into()).await { serving.abort(); let _ = serving.await; return Err(error.context("grant proxy credential capability")); diff --git a/crates/capsem-service/src/proxy_worker/tests.rs b/crates/capsem-service/src/proxy_worker/tests.rs index 16014beae..0829e6f7c 100644 --- a/crates/capsem-service/src/proxy_worker/tests.rs +++ b/crates/capsem-service/src/proxy_worker/tests.rs @@ -24,6 +24,33 @@ fn decode_generation(value: &str) -> ProxyGeneration { ProxyGeneration::new(bytes) } +#[test] +fn standalone_worker_command_carries_trusted_provider_selection() { + let mut command = Command::new("capsem-proxy"); + configure_worker_command( + &mut command, + GENERATION, + &ProxyWorkerMode::Standalone("openai".to_string()), + ); + let args = command + .as_std() + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect::>(); + + assert_eq!( + args, + vec![ + "--parent-pid".to_string(), + std::process::id().to_string(), + "--generation".to_string(), + "09090909090909090909090909090909".to_string(), + "--standalone-provider".to_string(), + "openai".to_string(), + ] + ); +} + #[tokio::test] async fn fake_proxy_worker_child() { let Ok(mode) = std::env::var("CAPSEM_FAKE_PROXY_WORKER") else { @@ -101,6 +128,17 @@ async fn fake_proxy_worker_child() { }), &[], ).await.unwrap(); + if mode == "traffic_close" { + assert_eq!(grant.capability(), ProxyCapability::HttpTraffic); + control_tx.send( + &encode_proxy_control_event(ProxyControlEvent::Closed { + generation, + grant_id: grant.grant_id(), + reason: capsem_proto::proxy_control::ProxyChannelCloseReason::Disconnected, + }), + &[], + ).await.unwrap(); + } } ProxyControlRequest::Shutdown { .. } if mode == "stall_shutdown" => { std::future::pending::<()>().await; @@ -153,8 +191,27 @@ async fn supervisor_applies_policy_grants_connected_descriptors_and_reaps() { capsem_core::net::policy_config::active_policy_digest(ACTIVE_POLICY) ); let (peer, granted) = UnixStream::pair().unwrap(); - worker.grant(ProxyCapability::Telemetry, granted).await.unwrap(); + worker.grant(ProxyCapability::Telemetry, granted.into()).await.unwrap(); + drop(peer); + worker.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn ordinary_traffic_disconnect_does_not_terminate_the_proxy_generation() { + let worker = fake("traffic_close").await.unwrap(); + let (peer, granted) = UnixStream::pair().unwrap(); + worker + .grant(ProxyCapability::HttpTraffic, granted.into()) + .await + .unwrap(); drop(peer); + tokio::task::yield_now().await; + + let digest = worker.apply_policy(ACTIVE_POLICY.to_vec()).await.unwrap(); + assert_eq!( + digest, + capsem_core::net::policy_config::active_policy_digest(ACTIVE_POLICY) + ); worker.shutdown().await.unwrap(); } diff --git a/crates/capsem-service/src/router_runtime.rs b/crates/capsem-service/src/router_runtime.rs index 07043e5de..ee2d7349d 100644 --- a/crates/capsem-service/src/router_runtime.rs +++ b/crates/capsem-service/src/router_runtime.rs @@ -60,8 +60,9 @@ pub(super) fn build_service_router(state: Arc) -> Router { delete(connection_routes::revoke_grant), ) .route("/vms/create", post(handle_provision)) - .route("/vms/managed/{request_id}/claim", post(managed_routes::claim)) - .route("/vms/managed/{request_id}/close", post(managed_routes::close)) + .route("/proxies", post(standalone_proxy::handle_create)) + .route("/proxies/{id}/heartbeat", post(standalone_proxy::handle_heartbeat)) + .route("/proxies/{id}/stop", post(standalone_proxy::handle_stop)) .route("/vms/list", get(handle_list)) .route("/vms/{id}/info", get(handle_info)) .route("/vms/{id}/status", get(handle_vm_status)) diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index bb209eba9..c6652d9f0 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -263,6 +263,7 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { connections, instances: Mutex::new(HashMap::new()), proxy_workers: Mutex::new(HashMap::new()), + standalone_proxies: tokio::sync::Mutex::new(HashMap::new()), ledger_workers: Arc::new(ledger_worker::LedgerWorkers::new(ledger_binary)), retirements: Default::default(), session_db_handles: Mutex::new(HashMap::new()), @@ -459,7 +460,8 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { } } info!("service shutting down, stopping VM processes and draining replies"); - kill_all_vm_processes(&shutdown_state).await; + kill_all_vm_processes(&shutdown_state); + shutdown_state.stop_all_standalone_proxies().await; }) .await; diff --git a/crates/capsem-service/src/service_runtime/telemetry_export.rs b/crates/capsem-service/src/service_runtime/telemetry_export.rs index 50a267d2d..f09613563 100644 --- a/crates/capsem-service/src/service_runtime/telemetry_export.rs +++ b/crates/capsem-service/src/service_runtime/telemetry_export.rs @@ -83,7 +83,7 @@ pub(crate) async fn grant_proxy_metric_broker( service, )); if let Err(error) = worker - .grant(capsem_proto::proxy_control::ProxyCapability::Telemetry, proxy) + .grant(capsem_proto::proxy_control::ProxyCapability::Telemetry, proxy.into()) .await { serving.abort(); @@ -100,6 +100,53 @@ pub(crate) async fn grant_proxy_metric_broker( Ok(()) } +/// Grant a standalone proxy the same collector relay without inventing a VM +/// owner identity. Revocation is tied to that standalone worker generation. +pub(crate) async fn grant_standalone_proxy_metric_broker( + state: &Arc, + id: &str, + authority: crate::instance::WorkerGrant, + worker: &crate::proxy_worker::ProxyWorker, +) -> anyhow::Result<()> { + let (_, corp) = capsem_core::net::policy_config::load_settings_and_corp_files(); + if Destination::corp(corp.corp_rule_files.open_telemetry.as_deref()).is_none() { + return Ok(()); + } + let (service, proxy) = std::os::unix::net::UnixStream::pair().context("create proxy metric capability")?; + let metric_session = id.to_string(); + let serving = tokio::spawn(serve_proxy_metric_channel_authorized( + Arc::clone(state), + metric_session.clone(), + MetricRelayAuthority::Standalone(authority), + service, + )); + if let Err(error) = worker + .grant(capsem_proto::proxy_control::ProxyCapability::Telemetry, proxy.into()) + .await + { + serving.abort(); + let _ = serving.await; + return Err(error.context("grant standalone proxy metric capability")); + } + tokio::spawn(async move { + match serving.await { + Ok(Ok(())) => { + tracing::debug!( + session = metric_session, + "standalone proxy metric capability disconnected" + ) + } + Ok(Err(error)) => { + tracing::warn!(session = metric_session, %error, "standalone proxy metric capability failed") + } + Err(error) => { + tracing::warn!(session = metric_session, %error, "standalone proxy metric capability task failed") + } + } + }); + Ok(()) +} + const COLLECTOR_TIMEOUT: Duration = Duration::from_secs(10); static RELAY_CLIENT: OnceLock> = OnceLock::new(); @@ -163,7 +210,7 @@ pub(crate) async fn handle_metric_relay( "metric request body is too large".into(), )); } - let response = relay_metric_body(&state, &id, &owner, body).await?; + let response = relay_metric_body_authorized(&state, &id, &MetricRelayAuthority::Vm(owner), body).await?; Ok((response.status, response.headers, response.body)) } @@ -173,10 +220,34 @@ struct MetricRelayResponse { body: axum::body::Bytes, } -async fn relay_metric_body( +enum MetricRelayAuthority { + Vm(owner_connection::OwnerConnection), + Standalone(crate::instance::WorkerGrant), +} + +impl MetricRelayAuthority { + fn validate(&self, state: &ServiceState) -> Result<(), String> { + match self { + Self::Vm(owner) => owner.validate(state, false), + Self::Standalone(authority) if authority.is_revoked() => { + Err("standalone proxy authority was revoked".to_string()) + } + Self::Standalone(_) => Ok(()), + } + } + + async fn revoked(&self) { + match self { + Self::Vm(owner) => owner.revoked().await, + Self::Standalone(authority) => authority.revoked().await, + } + } +} + +async fn relay_metric_body_authorized( state: &Arc, id: &str, - owner: &owner_connection::OwnerConnection, + authority: &MetricRelayAuthority, body: axum::body::Bytes, ) -> Result { if body.len() > capsem_proto::proxy_metrics::MAX_PROXY_METRIC_BODY_BYTES { @@ -186,8 +257,8 @@ async fn relay_metric_body( )); } let collector = current_collector_url()?; - owner - .validate(state, false) + authority + .validate(state) .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; let request = relay_client()? .post(collector) @@ -197,8 +268,8 @@ async fn relay_metric_body( tokio::pin!(request); let response = match tokio::select! { biased; - () = owner.revoked() => { - return Err(AppError(StatusCode::FORBIDDEN, "VM owner authority was revoked".into())); + () = authority.revoked() => { + return Err(AppError(StatusCode::FORBIDDEN, "proxy metric authority was revoked".into())); } response = &mut request => response, } { @@ -235,8 +306,8 @@ async fn relay_metric_body( } bytes.extend_from_slice(&chunk); } - owner - .validate(state, false) + authority + .validate(state) .map_err(|error| AppError(StatusCode::FORBIDDEN, error))?; let mut response_headers = axum::http::HeaderMap::new(); if let Some(content_type) = content_type { @@ -254,6 +325,15 @@ pub(crate) async fn serve_proxy_metric_channel( id: String, owner: owner_connection::OwnerConnection, stream: std::os::unix::net::UnixStream, +) -> anyhow::Result<()> { + serve_proxy_metric_channel_authorized(state, id, MetricRelayAuthority::Vm(owner), stream).await +} + +async fn serve_proxy_metric_channel_authorized( + state: Arc, + id: String, + authority: MetricRelayAuthority, + stream: std::os::unix::net::UnixStream, ) -> anyhow::Result<()> { use capsem_proto::proxy_metrics::{ProxyMetricBrokerMessage, ProxyMetricRequest, ProxyMetricResponse}; @@ -270,7 +350,7 @@ pub(crate) async fn serve_proxy_metric_channel( Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(()), Err(error) => return Err(error).context("receive proxy metric request"), }; - let response = match relay_metric_body(&state, &id, &owner, request.body.into()).await { + let response = match relay_metric_body_authorized(&state, &id, &authority, request.body.into()).await { Ok(relayed) => ProxyMetricResponse::Relayed { status: relayed.status.as_u16(), content_type: relayed diff --git a/crates/capsem-service/src/standalone_proxy.rs b/crates/capsem-service/src/standalone_proxy.rs new file mode 100644 index 000000000..65be13c4f --- /dev/null +++ b/crates/capsem-service/src/standalone_proxy.rs @@ -0,0 +1,490 @@ +//! Lifecycle owner for VM-free model proxy sessions. + +use std::net::{IpAddr, SocketAddr}; +use std::path::PathBuf; +use std::process::Stdio; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use axum::extract::{Path, State}; +use axum::Json; +use capsem_api::{ + CreateProxyRequest, CreateProxyResponse, ProxyHeartbeatResponse, ProxyLeaseRequest, StopProxyResponse, +}; +use capsem_proto::proxy_control::ProxyCapability; +use tokio::net::TcpListener; +use tokio::task::JoinHandle; + +use crate::instance::WorkerAuthority; +use crate::{proxy_worker, upstream_broker, AppError, ServiceState}; + +const LEASE_TTL: Duration = Duration::from_secs(15); +const CLEANUP_TIMEOUT: Duration = Duration::from_secs(15); + +pub(crate) struct StandaloneProxy { + lease_token: String, + expires_at: Instant, + worker: proxy_worker::ProxyWorker, + authority: WorkerAuthority, + accept_task: JoinHandle<()>, + broker_task: JoinHandle<()>, + session_dir: PathBuf, + provider_id: String, + upstream_policy: upstream_broker::PolicyPublisher, +} + +impl ServiceState { + async fn create_standalone_proxy( + self: &Arc, + request: CreateProxyRequest, + ) -> Result { + let bind_ip = request.bind.parse::().map_err(|_| { + AppError( + axum::http::StatusCode::BAD_REQUEST, + format!("proxy bind address {:?} is not an IP address", request.bind), + ) + })?; + if request.provider.is_empty() || request.provider.len() > 128 { + return Err(AppError( + axum::http::StatusCode::BAD_REQUEST, + "proxy provider must contain 1 to 128 bytes".to_string(), + )); + } + + let session_id = format!("proxy-{}", uuid::Uuid::new_v4().simple()); + let session_dir = self.run_dir.join("sessions").join(&session_id); + capsem_foundation::unix::fs::ensure_private_dir(&session_dir) + .map_err(|error| internal(format!("create standalone proxy session: {error}")))?; + let started = self + .start_standalone_proxy( + &session_id, + &session_dir, + request.provider.clone(), + bind_ip, + request.port, + ) + .await; + let (entry, local_addr, base_path) = match started { + Ok(started) => started, + Err(error) => { + let _ = tokio::fs::remove_dir_all(&session_dir).await; + return Err(internal(format!("start standalone proxy: {error:#}"))); + } + }; + let lease_token = entry.lease_token.clone(); + let expires_at = entry.expires_at; + let generation = entry.worker.generation(); + let stopped = entry.worker.stop_receiver(); + self.standalone_proxies.lock().await.insert(session_id.clone(), entry); + spawn_lease_monitor(Arc::clone(self), session_id.clone()); + spawn_worker_monitor(Arc::clone(self), session_id.clone(), generation, stopped); + + let advertised_ip = match local_addr.ip() { + IpAddr::V4(ip) if ip.is_unspecified() => IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), + IpAddr::V6(ip) if ip.is_unspecified() => IpAddr::V6(std::net::Ipv6Addr::LOCALHOST), + ip => ip, + }; + let advertised = SocketAddr::new(advertised_ip, local_addr.port()); + Ok(CreateProxyResponse { + session_id, + provider: request.provider, + bind: bind_ip.to_string(), + port: local_addr.port(), + base_url: format!("http://{advertised}{base_path}"), + lease_token, + lease_expires_unix_ms: expiry_unix_ms(expires_at), + }) + } + + async fn start_standalone_proxy( + self: &Arc, + session_id: &str, + session_dir: &std::path::Path, + provider_id: String, + bind_ip: IpAddr, + port: u16, + ) -> Result<(StandaloneProxy, SocketAddr, String)> { + let active_policy = self.materialize_active_policy(session_dir)?; + let target = capsem_core::net::mitm_proxy::StandaloneModelTarget::from_registry( + &active_policy.runtime.model_endpoints, + &provider_id, + ) + .map_err(anyhow::Error::msg)?; + let (broker, upstream_policy) = upstream_broker::PendingBroker::pair(active_policy.broker_policy())?; + let listener = TcpListener::bind(SocketAddr::new(bind_ip, port)) + .await + .context("bind standalone proxy data listener")?; + let local_addr = listener + .local_addr() + .context("read standalone proxy listener address")?; + let log = tokio::fs::OpenOptions::new() + .create(true) + .append(true) + .open(session_dir.join("proxy.log")) + .await + .context("open standalone proxy log")? + .into_std() + .await; + let worker = proxy_worker::ProxyWorker::spawn_standalone( + &self.proxy_binary, + active_policy.bytes, + Stdio::from(log.try_clone()?), + Stdio::from(log), + provider_id.clone(), + ) + .await?; + if let Err(error) = self.grant_proxy_ledger(session_id, session_dir, &worker).await { + let _ = worker.shutdown().await; + return Err(error); + } + if let Err(error) = self.grant_proxy_credentials(&worker).await { + let _ = worker.shutdown().await; + let _ = self.ledger_workers.shutdown(session_id).await; + return Err(error); + } + let proxy_upstream = broker.worker_stream()?; + if let Err(error) = worker.grant(ProxyCapability::Upstream, proxy_upstream.into()).await { + let _ = worker.shutdown().await; + let _ = self.ledger_workers.shutdown(session_id).await; + return Err(error.context("grant standalone proxy upstream broker")); + } + + let authority = WorkerAuthority::default(); + if let Err(error) = crate::service_runtime::telemetry_export::grant_standalone_proxy_metric_broker( + self, + session_id, + authority.grant(), + &worker, + ) + .await + { + authority.revoke(); + let _ = worker.shutdown().await; + let _ = self.ledger_workers.shutdown(session_id).await; + return Err(error); + } + let broker_task = broker.start(authority.grant()); + let accept_task = spawn_accept_loop(listener, worker.clone(), authority.grant()); + let lease_token = uuid::Uuid::new_v4().simple().to_string(); + Ok(( + StandaloneProxy { + lease_token, + expires_at: Instant::now() + LEASE_TTL, + worker, + authority, + accept_task, + broker_task, + session_dir: session_dir.to_path_buf(), + provider_id, + upstream_policy, + }, + local_addr, + target.base_path().to_string(), + )) + } + + async fn renew_standalone_proxy( + &self, + session_id: &str, + request: ProxyLeaseRequest, + ) -> Result { + let mut proxies = self.standalone_proxies.lock().await; + let entry = proxies + .get_mut(session_id) + .ok_or_else(|| AppError(axum::http::StatusCode::NOT_FOUND, "proxy session not found".to_string()))?; + verify_lease(entry, &request)?; + entry.expires_at = Instant::now() + LEASE_TTL; + let expires_at = entry.expires_at; + drop(proxies); + Ok(ProxyHeartbeatResponse { + session_id: session_id.to_string(), + lease_expires_unix_ms: expiry_unix_ms(expires_at), + }) + } + + async fn stop_standalone_proxy( + &self, + session_id: &str, + request: ProxyLeaseRequest, + ) -> Result { + let entry = { + let mut proxies = self.standalone_proxies.lock().await; + let entry = proxies + .get(session_id) + .ok_or_else(|| AppError(axum::http::StatusCode::NOT_FOUND, "proxy session not found".to_string()))?; + verify_lease(entry, &request)?; + proxies.remove(session_id).expect("validated proxy still registered") + }; + cleanup(self, session_id, entry) + .await + .map_err(|error| internal(format!("stop proxy: {error:#}")))?; + Ok(StopProxyResponse { + session_id: session_id.to_string(), + stopped: true, + }) + } + + pub(crate) async fn stop_all_standalone_proxies(&self) { + let entries = std::mem::take(&mut *self.standalone_proxies.lock().await); + for (session_id, entry) in entries { + if let Err(error) = cleanup(self, &session_id, entry).await { + tracing::warn!(%session_id, %error, "standalone proxy shutdown failed"); + } + } + } +} + +pub(crate) async fn refresh_policies(state: &Arc) -> Result<(usize, Vec), AppError> { + let targets = { + let proxies = state.standalone_proxies.lock().await; + proxies + .iter() + .map(|(id, entry)| { + ( + id.clone(), + entry.session_dir.clone(), + entry.provider_id.clone(), + entry.worker.clone(), + entry.worker.generation(), + entry.upstream_policy.clone(), + ) + }) + .collect::>() + }; + let total = targets.len(); + let materialized = state + .off_worker({ + let targets = targets + .iter() + .map(|(id, dir, ..)| (id.clone(), dir.clone())) + .collect::>(); + move |state| { + targets + .into_iter() + .map(|(id, dir)| { + let result = state + .materialize_active_policy(&dir) + .map_err(|error| format!("{error:#}")); + (id, result) + }) + .collect::>() + } + }) + .await?; + + let by_id = materialized.into_iter().collect::>(); + let mut failures = Vec::new(); + for (id, _, provider_id, worker, generation, publisher) in targets { + let still_current = state + .standalone_proxies + .lock() + .await + .get(&id) + .is_some_and(|entry| entry.worker.generation() == generation); + if !still_current { + continue; + } + let Some(published) = by_id.get(&id) else { + continue; + }; + let published = match published { + Ok(published) => published, + Err(error) => { + failures.push(format!("{id}: {error}")); + continue; + } + }; + if let Err(error) = capsem_core::net::mitm_proxy::StandaloneModelTarget::from_registry( + &published.runtime.model_endpoints, + &provider_id, + ) { + failures.push(format!("{id}: {error}")); + continue; + } + match worker.apply_policy(published.bytes.clone()).await { + Ok(applied) if applied == published.digest => { + if let Err(error) = publisher.publish(published.broker_policy()).await { + failures.push(format!("{id}: {error}")); + } + } + Ok(applied) => failures.push(format!( + "{id}: proxy applied active policy {applied}, expected {}", + published.digest + )), + Err(error) => failures.push(format!("{id}: proxy reload refused: {error:#}")), + } + } + Ok((total, failures)) +} + +pub(crate) async fn handle_create( + State(state): State>, + Json(request): Json, +) -> Result, AppError> { + state.create_standalone_proxy(request).await.map(Json) +} + +pub(crate) async fn handle_heartbeat( + State(state): State>, + Path(id): Path, + Json(request): Json, +) -> Result, AppError> { + state.renew_standalone_proxy(&id, request).await.map(Json) +} + +pub(crate) async fn handle_stop( + State(state): State>, + Path(id): Path, + Json(request): Json, +) -> Result, AppError> { + state.stop_standalone_proxy(&id, request).await.map(Json) +} + +fn spawn_accept_loop( + listener: TcpListener, + worker: proxy_worker::ProxyWorker, + authority: crate::instance::WorkerGrant, +) -> JoinHandle<()> { + tokio::spawn(async move { + loop { + let accepted = tokio::select! { + biased; + () = authority.revoked() => return, + accepted = listener.accept() => accepted, + }; + let (stream, peer) = match accepted { + Ok(accepted) => accepted, + Err(error) => { + tracing::warn!(%error, "standalone proxy listener failed"); + return; + } + }; + let descriptor = match stream.into_std() { + Ok(stream) => stream.into(), + Err(error) => { + tracing::warn!(%peer, %error, "standalone proxy could not adopt client"); + continue; + } + }; + if let Err(error) = worker.grant(ProxyCapability::HttpTraffic, descriptor).await { + tracing::debug!(%peer, %error, "standalone proxy worker stopped accepting traffic"); + return; + } + } + }) +} + +fn spawn_lease_monitor(state: Arc, session_id: String) { + tokio::spawn(async move { + loop { + let sleep_for = { + let proxies = state.standalone_proxies.lock().await; + let Some(entry) = proxies.get(&session_id) else { + return; + }; + let sleep_for = entry.expires_at.saturating_duration_since(Instant::now()); + drop(proxies); + sleep_for + }; + tokio::time::sleep(sleep_for).await; + let expired = { + let mut proxies = state.standalone_proxies.lock().await; + if proxies + .get(&session_id) + .is_some_and(|entry| entry.expires_at <= Instant::now()) + { + proxies.remove(&session_id) + } else { + None + } + }; + if let Some(entry) = expired { + if let Err(error) = cleanup(&state, &session_id, entry).await { + tracing::warn!(%session_id, %error, "expired standalone proxy cleanup failed"); + } + return; + } + } + }); +} + +fn spawn_worker_monitor( + state: Arc, + session_id: String, + generation: capsem_proto::proxy_control::ProxyGeneration, + mut stopped: tokio::sync::watch::Receiver>, +) { + tokio::spawn(async move { + if stopped.borrow().is_none() && stopped.changed().await.is_err() { + return; + } + let removed = { + let mut proxies = state.standalone_proxies.lock().await; + if proxies + .get(&session_id) + .is_some_and(|entry| entry.worker.generation() == generation) + { + proxies.remove(&session_id) + } else { + None + } + }; + if let Some(entry) = removed { + let reason = stopped + .borrow() + .clone() + .unwrap_or_else(|| "proxy worker stopped".to_string()); + if let Err(error) = cleanup(&state, &session_id, entry).await { + tracing::warn!(%session_id, %reason, %error, "stopped standalone proxy cleanup failed"); + } + } + }); +} + +async fn cleanup(state: &ServiceState, session_id: &str, entry: StandaloneProxy) -> Result<()> { + entry.authority.revoke(); + entry.accept_task.abort(); + let _ = entry.accept_task.await; + let worker_result = entry.worker.shutdown().await; + let ledger_result = state.ledger_workers.shutdown(session_id).await; + let mut broker_task = entry.broker_task; + if tokio::time::timeout(CLEANUP_TIMEOUT, &mut broker_task).await.is_err() { + broker_task.abort(); + let _ = broker_task.await; + } + let remove_result = tokio::fs::remove_dir_all(&entry.session_dir).await; + worker_result.context("shutdown proxy worker")?; + ledger_result.context("shutdown proxy ledger")?; + remove_result.with_context(|| format!("remove {}", entry.session_dir.display()))?; + Ok(()) +} + +fn verify_lease(entry: &StandaloneProxy, request: &ProxyLeaseRequest) -> Result<(), AppError> { + if entry.lease_token != request.lease_token { + return Err(AppError( + axum::http::StatusCode::FORBIDDEN, + "proxy lease token is invalid".to_string(), + )); + } + Ok(()) +} + +fn expiry_unix_ms(expires_at: Instant) -> u64 { + let remaining = expires_at.saturating_duration_since(Instant::now()); + SystemTime::now() + .checked_add(remaining) + .unwrap_or(SystemTime::now()) + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .try_into() + .unwrap_or(u64::MAX) +} + +fn internal(message: String) -> AppError { + AppError(axum::http::StatusCode::INTERNAL_SERVER_ERROR, message) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/standalone_proxy/tests.rs b/crates/capsem-service/src/standalone_proxy/tests.rs new file mode 100644 index 000000000..66d630552 --- /dev/null +++ b/crates/capsem-service/src/standalone_proxy/tests.rs @@ -0,0 +1,50 @@ +use axum::http::{Method, StatusCode}; +use serde_json::json; + +use super::*; + +#[tokio::test] +async fn create_rejects_a_non_ip_bind_before_starting_any_worker() { + let state = crate::tests::make_test_state(); + let app = crate::router_runtime::build_service_router(Arc::clone(&state)); + + let (status, body) = crate::tests::route_request( + app, + Method::POST, + "/proxies", + Some(json!({"provider": "openai", "bind": "localhost", "port": 0})), + ) + .await; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body["error"].as_str().unwrap().contains("not an IP address")); + assert!(state.standalone_proxies.lock().await.is_empty()); +} + +#[tokio::test] +async fn create_rejects_an_empty_provider_before_starting_any_worker() { + let state = crate::tests::make_test_state(); + let app = crate::router_runtime::build_service_router(Arc::clone(&state)); + + let (status, body) = crate::tests::route_request( + app, + Method::POST, + "/proxies", + Some(json!({"provider": "", "bind": "127.0.0.1", "port": 0})), + ) + .await; + + assert_eq!(status, StatusCode::BAD_REQUEST); + assert!(body["error"].as_str().unwrap().contains("1 to 128 bytes")); + assert!(state.standalone_proxies.lock().await.is_empty()); +} + +#[test] +fn wire_expiry_is_bounded_by_the_lease_ttl() { + let before = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64; + let expiry = expiry_unix_ms(Instant::now() + LEASE_TTL); + let after = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64; + + assert!(expiry >= before + LEASE_TTL.as_millis() as u64 - 10); + assert!(expiry <= after + LEASE_TTL.as_millis() as u64 + 10); +} diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index d477e1a50..9f779dca8 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -62,6 +62,7 @@ pub(crate) fn test_state( connections, instances: Mutex::new(HashMap::new()), proxy_workers: Mutex::new(HashMap::new()), + standalone_proxies: tokio::sync::Mutex::new(HashMap::new()), ledger_workers: Arc::new(ledger_worker::LedgerWorkers::new(PathBuf::from( "/nonexistent/capsem-ledger", ))), diff --git a/crates/capsem-service/src/tests/lifecycle.rs b/crates/capsem-service/src/tests/lifecycle.rs index 0657f04a9..0e3002d5b 100644 --- a/crates/capsem-service/src/tests/lifecycle.rs +++ b/crates/capsem-service/src/tests/lifecycle.rs @@ -1102,7 +1102,16 @@ async fn resume_sandbox_passes_the_session_scratch_disk_size_to_process() { .vms .insert("resume-size".to_string(), entry); - assert_eq!(state.resume_sandbox(&vm_id, None, None).unwrap(), vm_id); + let resume_id = vm_id.clone(); + let error = state + .off_worker(move |state| state.resume_sandbox(&resume_id, None, None)) + .await + .unwrap() + .expect_err("the focused argv fixture has no confined helper binaries"); + assert!( + error.to_string().contains("acquire proxy ledger channel"), + "resume reached an unexpected failure after process spawn: {error:#}" + ); for _ in 0..50 { if argv_path.exists() { break; diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 748e4ee2f..8dc42631f 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -600,7 +600,7 @@ async fn run( ProxyTrafficService::Http => capsem_proto::proxy_control::ProxyCapability::HttpTraffic, ProxyTrafficService::Dns => capsem_proto::proxy_control::ProxyCapability::DnsTraffic, }; - match proxy.grant(capability, UnixStream::from(descriptor)).await { + match proxy.grant(capability, descriptor).await { Ok(()) => { send_response( &responses, @@ -622,10 +622,7 @@ async fn run( }; let descriptor = frame.fds.into_iter().next().expect("one validated MCP descriptor"); match proxy - .grant( - capsem_proto::proxy_control::ProxyCapability::Mcp, - UnixStream::from(descriptor), - ) + .grant(capsem_proto::proxy_control::ProxyCapability::Mcp, descriptor) .await { Ok(()) => { diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index 3a5191286..ead9bd43e 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -317,9 +317,10 @@ impl ServiceState { instance_reaper::kill_and_reap(child); return Err(error); } - if let Err(error) = tokio::runtime::Handle::current() - .block_on(proxy.grant(capsem_proto::proxy_control::ProxyCapability::Upstream, proxy_upstream)) - { + if let Err(error) = tokio::runtime::Handle::current().block_on(proxy.grant( + capsem_proto::proxy_control::ProxyCapability::Upstream, + proxy_upstream.into(), + )) { instance_reaper::kill_and_reap(child); return Err(error.context("grant proxy upstream broker")); } diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 100604bab..6d826b4f6 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -246,9 +246,10 @@ impl ServiceState { instance_reaper::kill_and_reap(child); return Err(error); } - if let Err(error) = tokio::runtime::Handle::current() - .block_on(proxy.grant(capsem_proto::proxy_control::ProxyCapability::Upstream, proxy_upstream)) - { + if let Err(error) = tokio::runtime::Handle::current().block_on(proxy.grant( + capsem_proto::proxy_control::ProxyCapability::Upstream, + proxy_upstream.into(), + )) { instance_reaper::kill_and_reap(child); return Err(error.context("grant proxy upstream broker")); } From 0da722bcf2eef003cdb23a58810db6d8e6a1182b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:17:11 +0000 Subject: [PATCH 187/464] feat(cli): add standalone model proxy --- CHANGELOG.md | 52 ++------------------ crates/capsem/src/grouped_help.rs | 3 ++ crates/capsem/src/main.rs | 12 ++++- crates/capsem/src/proxy_command.rs | 76 ++++++++++++++++++++++++++++++ crates/capsem/src/tests.rs | 32 +++++++++++++ 5 files changed, 125 insertions(+), 50 deletions(-) create mode 100644 crates/capsem/src/proxy_command.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5149d4911..1d477aaef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -180,55 +180,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- TypeScript SDK connection and credential grant resources preserve explicit - consent, typed refusal codes, ownership checks, deadlines and cancellation. - -- Python SDK connection and credential grant resources support explicit consent, - typed refusal codes, ownership checks and deadlines during VM name resolution. - -- Service-owned connection metadata and explicit credential grant routes share - managed ownership fences and enforce the current broker policy. Native AGY - enrollment remains unavailable pending its supported handoff integration. - -- Connection authorization and generation-bound credential grant operations in - the authenticated API contract and generated Rust, Python and TypeScript SDKs. - -- Credential-owned coordination for finite enrollment, redacted connection - state and explicit session grants, including durable request records and - same-owner resume fencing. Service activation and the supported native AGY - adapter remain under integration. - -- Rust SDK connection and session credential-grant resources implement the - nine approved calls with explicit consent, ownership checks and no automatic - replay. Live service activation and native AGY handoff remain pending. - -- TypeScript SDK managed scopes renew finite ownership and await cleanup - through caller abort, lease failure and uncertain creation without replay. - -- Python SDK managed scopes renew finite ownership leases and await the - original session's cleanup on exit, failure and cancellation without replaying create. - -- Ephemeral managed-session creation, capability-authenticated claim/renewal, - and awaited close use a finite service-owned lease and durable replay protection. - -- Desktop credential settings accept explicit API keys or tokens with file or - memory storage, clear submitted input, and return a copyable reference. - -- Rust SDK exposes the matching credential injection resource with file or - memory storage and opaque reference responses. - -- TypeScript SDK exposes credential injection with explicit storage, deadlines - and cancellation through the authenticated gateway. - -- Python SDK exposes credential injection with file or memory storage and - redacted request diagnostics. - -- Credential injection accepts explicitly selected host startup variables, - private file inputs, or an authenticated API. File and memory storage return - opaque references; memory material reaches VM owners only through host IPC. - -- The shared gateway contract describes credential injection with file or - memory storage and responses containing only opaque references. +- `capsem proxy` starts a leased, VM-free OpenAI-compatible endpoint for a + configured provider, prints its base URL, keeps it alive while the command + runs and tears down its confined worker and ledger on exit. - Rust SDK exposes the registry image catalog and prefetch through `Hypervisor.images()`, with typed disk states and per-call registry access. diff --git a/crates/capsem/src/grouped_help.rs b/crates/capsem/src/grouped_help.rs index 8eebe780e..ac9cd23d6 100644 --- a/crates/capsem/src/grouped_help.rs +++ b/crates/capsem/src/grouped_help.rs @@ -22,6 +22,9 @@ pub(crate) const GROUPED_HELP: &str = "\ \x1b[32;1mimages\x1b[0m List the image catalog (`create --image NAME` runs one) \x1b[32;1mimages pull\x1b[0m Pull an image ahead of a create +\x1b[36;1;4mModel Proxy:\x1b[0m + \x1b[32;1mproxy\x1b[0m Run a VM-free OpenAI-compatible API endpoint + \x1b[36;1;4mService:\x1b[0m \x1b[32;1minstall\x1b[0m Install as a system service (LaunchAgent / systemd) \x1b[32;1mstatus\x1b[0m Show service status diff --git a/crates/capsem/src/main.rs b/crates/capsem/src/main.rs index 2113899d8..78444afc8 100644 --- a/crates/capsem/src/main.rs +++ b/crates/capsem/src/main.rs @@ -12,6 +12,7 @@ use grouped_help::GROUPED_HELP; mod network_commands; mod paths; mod platform; +mod proxy_command; mod route_ids; mod service_install; mod session_display; @@ -259,6 +260,9 @@ enum Commands { /// List the image catalog, or pull an image ahead of a create Images(image_commands::ImagesArgs), + /// Run a VM-free OpenAI-compatible model API endpoint + Proxy(proxy_command::ProxyArgs), + #[command(flatten)] Misc(MiscCommands), } @@ -1025,7 +1029,9 @@ fn should_start_background_update_refresh(command: Option<&Commands>) -> bool { fn direct_service_lifetime(command: &Commands) -> client::DirectServiceLifetime { match command { - Commands::Session(SessionCommands::Run(..)) => client::DirectServiceLifetime::BoundToCommand, + Commands::Session(SessionCommands::Run(..)) | Commands::Proxy(..) => { + client::DirectServiceLifetime::BoundToCommand + } _ => client::DirectServiceLifetime::Persistent, } } @@ -1653,6 +1659,10 @@ async fn main() -> Result<()> { } Commands::Network(command) => network_commands::run(&client, command).await?, Commands::Images(args) => image_commands::run(&client, args).await?, + Commands::Proxy(args) => { + let result = proxy_command::run(&client, args).await; + client.finish_direct_request(result).await?; + } Commands::Mcp(McpCommands::Servers) => { let resp: ApiResponse> = client.get("/mcp/servers/list").await?; let servers = resp.into_result()?; diff --git a/crates/capsem/src/proxy_command.rs b/crates/capsem/src/proxy_command.rs new file mode 100644 index 000000000..7eb360f62 --- /dev/null +++ b/crates/capsem/src/proxy_command.rs @@ -0,0 +1,76 @@ +use std::net::IpAddr; +use std::time::Duration; + +use anyhow::Result; +use capsem_api::{ + CreateProxyRequest, CreateProxyResponse, ProxyHeartbeatResponse, ProxyLeaseRequest, StopProxyResponse, +}; +use clap::Args; + +use crate::client::{ApiResponse, UdsClient}; + +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Args, Debug)] +pub(crate) struct ProxyArgs { + /// Provider key from the effective built-in, settings, and corp policy + #[arg(long, default_value = "openai")] + pub(crate) provider: String, + /// Address for the unauthenticated model API listener + #[arg(long, default_value = "127.0.0.1")] + pub(crate) bind: IpAddr, + /// Model API port; zero lets the OS select an available port + #[arg(long, default_value_t = 0)] + pub(crate) port: u16, +} + +pub(crate) async fn run(client: &UdsClient, args: &ProxyArgs) -> Result<()> { + let response: ApiResponse = client + .post( + "/proxies", + &CreateProxyRequest { + provider: args.provider.clone(), + bind: args.bind.to_string(), + port: args.port, + }, + ) + .await?; + let started = response.into_result()?; + println!("Proxy session: {}", started.session_id); + println!("Base URL: {}", started.base_url); + println!("Press Ctrl-C to stop."); + + let request = ProxyLeaseRequest { + lease_token: started.lease_token.clone(), + }; + let mut heartbeat = tokio::time::interval(HEARTBEAT_INTERVAL); + heartbeat.tick().await; + let mut interrupt = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?; + let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + let lease_result = loop { + tokio::select! { + _ = interrupt.recv() => break Ok(()), + _ = terminate.recv() => break Ok(()), + _ = heartbeat.tick() => { + let renewed: Result> = client + .post(&format!("/proxies/{}/heartbeat", started.session_id), &request) + .await; + match renewed.and_then(ApiResponse::into_result) { + Ok(_) => {} + Err(error) => break Err(error.context("standalone proxy lease heartbeat failed")), + } + } + } + }; + + let stopped: Result> = client + .post(&format!("/proxies/{}/stop", started.session_id), &request) + .await; + let stop_result = stopped.and_then(ApiResponse::into_result).map(|_| ()); + match (lease_result, stop_result) { + (Ok(()), Ok(())) => Ok(()), + (Ok(()), Err(error)) => Err(error.context("standalone proxy cleanup failed")), + (Err(error), Ok(())) => Err(error), + (Err(error), Err(cleanup)) => Err(error.context(format!("standalone proxy cleanup also failed: {cleanup:#}"))), + } +} diff --git a/crates/capsem/src/tests.rs b/crates/capsem/src/tests.rs index c245ae793..e5e5445c1 100644 --- a/crates/capsem/src/tests.rs +++ b/crates/capsem/src/tests.rs @@ -482,6 +482,38 @@ fn parse_uds_path_default_none() { assert_eq!(cli.uds_path, None); } +#[test] +fn parse_proxy_defaults_and_explicit_listener() { + let cli = Cli::parse_from(["capsem", "proxy"]); + let Some(Commands::Proxy(defaults)) = cli.command else { + panic!("expected Proxy"); + }; + assert_eq!(defaults.provider, "openai"); + assert_eq!(defaults.bind, "127.0.0.1".parse::().unwrap()); + assert_eq!(defaults.port, 0); + + let cli = Cli::parse_from([ + "capsem", + "proxy", + "--provider", + "local", + "--bind", + "0.0.0.0", + "--port", + "8080", + ]); + let Some(Commands::Proxy(explicit)) = cli.command else { + panic!("expected Proxy"); + }; + assert_eq!(explicit.provider, "local"); + assert_eq!(explicit.bind, "0.0.0.0".parse::().unwrap()); + assert_eq!(explicit.port, 8080); + assert_eq!( + direct_service_lifetime(&Commands::Proxy(explicit)), + client::DirectServiceLifetime::BoundToCommand + ); +} + // RAM conversion #[test] From 68175a07b6d8ea71a76595dcc8dd8e64338c9fe4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:23:53 +0000 Subject: [PATCH 188/464] test(mock-server): add SDK failure controls --- crates/capsem-mock-server/src/main.rs | 45 ++++++++++++++++++++++++-- crates/capsem-mock-server/src/tests.rs | 31 ++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/crates/capsem-mock-server/src/main.rs b/crates/capsem-mock-server/src/main.rs index c2f8d14cd..f20879509 100644 --- a/crates/capsem-mock-server/src/main.rs +++ b/crates/capsem-mock-server/src/main.rs @@ -429,6 +429,9 @@ event: model.done\ndata: {\"finish_reason\":\"stop\"}\n\n", } (&Method::POST, "/v1/chat/completions") => { let payload = parse_json(&request_body); + if let Some(response) = openai_sdk_control(&payload).await { + return response; + } if payload.get("stream").and_then(Value::as_bool) == Some(true) { response(StatusCode::OK, openai_chat_stream(), "text/event-stream") } else { @@ -446,7 +449,21 @@ event: model.done\ndata: {\"finish_reason\":\"stop\"}\n\n", "data": [{"b64_json": "Y2Fwc2VtLW1vY2staW1hZ2U="}], "usage": {"input_tokens": 11, "output_tokens": 17, "total_tokens": 28} })), - (&Method::POST, "/v1/responses") => responses::reply(parse_json(&request_body)), + (&Method::POST, "/v1/responses") => { + let payload = parse_json(&request_body); + if let Some(response) = openai_sdk_control(&payload).await { + return response; + } + if payload.get("stream").and_then(Value::as_bool) == Some(true) { + response( + StatusCode::OK, + responses_stream(&payload, payload_has_function_call_output(&payload)), + "text/event-stream", + ) + } else { + json_response(responses_response(payload)) + } + } (&Method::POST, "/model/shape") => json_response(json!({ "id": "chatcmpl_shape_fixture", "object": "chat.completion", @@ -562,6 +579,26 @@ event: model.done\ndata: {\"finish_reason\":\"stop\"}\n\n", } } +async fn openai_sdk_control(payload: &Value) -> Option> { + match payload.get("model").and_then(Value::as_str) { + Some("capsem-sdk-cancel") => { + tokio::time::sleep(Duration::from_secs(2)).await; + None + } + Some("capsem-sdk-upstream-error") => Some(json_response_with_status( + StatusCode::SERVICE_UNAVAILABLE, + json!({ + "error": { + "message": "hermetic upstream unavailable", + "type": "server_error", + "code": "fixture_unavailable" + } + }), + )), + _ => None, + } +} + fn parse_json(bytes: &[u8]) -> Value { serde_json::from_slice(bytes).unwrap_or_else(|_| json!({})) } @@ -590,8 +627,12 @@ fn echo_response(query: Option<&str>, headers: &HeaderMap, body_size: usize) -> } fn json_response(value: Value) -> Response { + json_response_with_status(StatusCode::OK, value) +} + +fn json_response_with_status(status: StatusCode, value: Value) -> Response { response( - StatusCode::OK, + status, Bytes::from(serde_json::to_vec(&value).unwrap_or_else(|_| b"{}".to_vec())), "application/json", ) diff --git a/crates/capsem-mock-server/src/tests.rs b/crates/capsem-mock-server/src/tests.rs index 2dfc6a59d..8d6fb8970 100644 --- a/crates/capsem-mock-server/src/tests.rs +++ b/crates/capsem-mock-server/src/tests.rs @@ -622,6 +622,37 @@ async fn provider_stream_routes_emit_protocol_terminators() { } } +#[tokio::test] +async fn openai_sdk_controls_expose_cancel_and_upstream_error_fixtures() { + let delayed = tokio::time::timeout( + std::time::Duration::from_millis(50), + routed( + Method::POST, + "/v1/responses", + None, + HeaderMap::new(), + json!({"model": "capsem-sdk-cancel"}), + ), + ) + .await; + assert!( + delayed.is_err(), + "cancel fixture answered before the client could abort" + ); + + let (status, _, body) = routed( + Method::POST, + "/v1/chat/completions", + None, + HeaderMap::new(), + json!({"model": "capsem-sdk-upstream-error"}), + ) + .await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + let body: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["error"]["code"], "fixture_unavailable"); +} + #[tokio::test] async fn auxiliary_post_routes_cover_openai_google_ollama_and_oauth_shapes() { for (path, marker) in [ From 175ae5360fba22cc8698ce76172eba290deba3a8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:32:30 +0000 Subject: [PATCH 189/464] fix(mock-server): escape streamed response text --- crates/capsem-mock-server/src/main.rs | 89 ++++++++++++++++++++++++++ crates/capsem-mock-server/src/tests.rs | 31 +++++++++ 2 files changed, 120 insertions(+) diff --git a/crates/capsem-mock-server/src/main.rs b/crates/capsem-mock-server/src/main.rs index f20879509..2d4ff99e0 100644 --- a/crates/capsem-mock-server/src/main.rs +++ b/crates/capsem-mock-server/src/main.rs @@ -790,6 +790,95 @@ data: [DONE]\n\n", ) } +fn responses_response(payload: Value) -> Value { + let model = payload.get("model").and_then(Value::as_str).unwrap_or("mock-local"); + let has_tool_output = serde_json::to_string(&payload) + .map(|raw| raw.contains("function_call_output")) + .unwrap_or(false); + if !has_tool_output { + return json!({ + "id": "resp_ironbank_tool_01", + "object": "response", + "created_at": 1781205836_u64, + "status": "completed", + "model": model, + "output": [{ + "id": "fc_codex_write_poem", + "type": "function_call", + "status": "completed", + "call_id": "call_codex_write_poem", + "name": "exec_command", + "arguments": "{\"cmd\":\"printf '%s\\\\n' 'Capsem ironbank poem\\\\nledgers count the sparks\\\\nno secret crosses raw' > /root/codex-cli-output.txt\",\"yield_time_ms\":1000,\"max_output_tokens\":2000}" + }], + "usage": {"input_tokens": 31, "output_tokens": 17, "total_tokens": 48} + }); + } + json!({ + "id": "resp_ironbank_01", + "object": "response", + "created_at": 1781205836_u64, + "status": "completed", + "model": model, + "output": [{ + "id": "msg_ironbank_01", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": EXPECTED_POEM, "annotations": []}] + }], + "output_text": EXPECTED_POEM, + "usage": { + "input_tokens": 7, + "output_tokens": 5, + "total_tokens": 12, + "output_tokens_details": {"reasoning_tokens": 2} + } + }) +} + +fn payload_has_function_call_output(payload: &Value) -> bool { + payload + .get("input") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .any(|item| item.get("type").and_then(Value::as_str) == Some("function_call_output")) + }) + .unwrap_or(false) +} + +fn responses_stream(payload: &Value, final_turn: bool) -> Bytes { + if final_turn { + let (token, _) = write_target(payload, "openai-responses"); + let token_json = json_compact(json!(token)); + return Bytes::from( + format!( + "event: response.reasoning_summary_text.delta\ndata: {{\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\"ledger reasoning\"}}\n\n\ +event: response.output_text.delta\ndata: {{\"type\":\"response.output_text.delta\",\"delta\":{token_json}}}\n\n\ +event: response.output_text.done\ndata: {{\"type\":\"response.output_text.done\",\"text\":{token_json}}}\n\n\ +event: response.completed\ndata: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_capsem_mock\",\"status\":\"completed\",\"model\":\"gpt-5-nano\",\"usage\":{{\"input_tokens\":7,\"output_tokens\":5,\"total_tokens\":12,\"output_tokens_details\":{{\"reasoning_tokens\":2}}}}}}}}\n\n" + ), + ); + } + let (token, path) = write_target(payload, "openai-responses"); + let call_id = format!("call_{}", &token[..token.len().min(12)]); + let arguments = json_compact(json!({ + "cmd": shell_write_command(&token, &path), + "yield_time_ms": 1000, + "max_output_tokens": 2000 + })); + let arguments_json = json_compact(json!(arguments)); + Bytes::from(format!( + "event: response.output_item.added\ndata: {{\"type\":\"response.output_item.added\",\"item\":{{\"type\":\"reasoning\",\"id\":\"rs_capsem_mock\"}}}}\n\n\ +event: response.output_item.added\ndata: {{\"type\":\"response.output_item.added\",\"item\":{{\"type\":\"function_call\",\"id\":\"fc_capsem_mock\",\"call_id\":\"{call_id}\",\"name\":\"exec_command\",\"arguments\":{arguments_json}}}}}\n\n\ +event: response.function_call_arguments.delta\ndata: {{\"type\":\"response.function_call_arguments.delta\",\"delta\":{arguments_json}}}\n\n\ +event: response.function_call_arguments.done\ndata: {{\"type\":\"response.function_call_arguments.done\",\"arguments\":{arguments_json}}}\n\n\ +event: response.output_item.done\ndata: {{\"type\":\"response.output_item.done\",\"item\":{{\"type\":\"function_call\",\"id\":\"fc_capsem_mock\",\"status\":\"completed\",\"call_id\":\"{call_id}\",\"name\":\"exec_command\",\"arguments\":{arguments_json}}}}}\n\n\ +event: response.completed\ndata: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_capsem_mock\",\"status\":\"completed\",\"model\":\"gpt-5-nano\",\"usage\":{{\"input_tokens\":31,\"output_tokens\":17,\"total_tokens\":48,\"output_tokens_details\":{{\"reasoning_tokens\":2}}}}}}}}\n\n" + )) +} + /// The code-assist fixtures, carried in the binary rather than read at run time. /// /// These used to be read through `CARGO_MANIFEST_DIR`, which is an absolute diff --git a/crates/capsem-mock-server/src/tests.rs b/crates/capsem-mock-server/src/tests.rs index 8d6fb8970..c7c4b75d7 100644 --- a/crates/capsem-mock-server/src/tests.rs +++ b/crates/capsem-mock-server/src/tests.rs @@ -622,6 +622,37 @@ async fn provider_stream_routes_emit_protocol_terminators() { } } +#[tokio::test] +async fn responses_stream_json_escapes_multiline_output() { + let (_, _, bytes) = routed( + Method::POST, + "/v1/responses", + None, + HeaderMap::new(), + json!({ + "model": "gpt-fixture", + "stream": true, + "input": [{ + "type": "function_call_output", + "call_id": "call_fixture", + "output": "line one\nline two" + }] + }), + ) + .await; + let stream = String::from_utf8(bytes.to_vec()).unwrap(); + let payloads = stream + .lines() + .filter_map(|line| line.strip_prefix("data: ")) + .map(|data| serde_json::from_str::(data).expect("every SSE data field is JSON")) + .collect::>(); + + assert!(payloads.iter().any(|payload| { + payload["type"] == "response.output_text.delta" + && payload["delta"] == "Capsem ironbank poem\nledgers count the sparks\nno secret crosses raw" + })); +} + #[tokio::test] async fn openai_sdk_controls_expose_cancel_and_upstream_error_fixtures() { let delayed = tokio::time::timeout( From 8cadc574200b9180ce9a2a3f67071b4ec342ca86 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:34:35 +0000 Subject: [PATCH 190/464] test(python-sdk): probe standalone OpenAI endpoint --- sdk/python/pyproject.toml | 11 +- sdk/python/tests/standalone_openai_client.py | 155 +++++++++++++ sdk/python/uv.lock | 223 +++++++++++++++++++ 3 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 sdk/python/tests/standalone_openai_client.py diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 7b99a8754..999040e31 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -13,7 +13,16 @@ build-backend = "hatchling.build" packages = ["capsem"] [dependency-groups] -dev = ["pytest>=9", "pytest-cov>=6", "ruff>=0.15.16", "ty>=0.0.46", "build>=1.2", "hatchling==1.29.0", "editables==0.5"] +dev = [ + "pytest>=9", + "pytest-cov>=6", + "ruff>=0.15.16", + "ty>=0.0.46", + "build>=1.2", + "hatchling==1.29.0", + "editables==0.5", + "openai==2.54.0", +] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/sdk/python/tests/standalone_openai_client.py b/sdk/python/tests/standalone_openai_client.py new file mode 100644 index 000000000..5451839e6 --- /dev/null +++ b/sdk/python/tests/standalone_openai_client.py @@ -0,0 +1,155 @@ +"""Official Python OpenAI client probe for the standalone Capsem endpoint.""" + +from __future__ import annotations + +import argparse +import asyncio +import json + +from openai import APIStatusError, AsyncOpenAI + + +async def probe(base_url: str) -> dict[str, object]: + client = AsyncOpenAI( + api_key="sk-capsem-standalone-python", + base_url=base_url, + max_retries=0, + timeout=5.0, + ) + tools = [ + { + "type": "function", + "function": { + "name": "fixture_lookup", + "description": "Read the deterministic fixture.", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + } + ] + chat = await client.chat.completions.create( + model="gpt-fixture", + messages=[{"role": "user", "content": "Use the fixture."}], + tools=tools, + ) + tool_call = chat.choices[0].message.tool_calls[0] + assert tool_call.function.name == "fixture_lookup" + assert json.loads(tool_call.function.arguments)["query"] == "Capsem ironbank poem" + assert chat.usage is not None and chat.usage.total_tokens == 456 + + chat_stream = await client.chat.completions.create( + model="gpt-fixture", + messages=[{"role": "user", "content": "Stream the fixture."}], + stream=True, + ) + streamed_chat = "" + async for chunk in chat_stream: + streamed_chat += chunk.choices[0].delta.content or "" + assert streamed_chat == "Capsem ironbank poem" + + response_tools = [ + { + "type": "function", + "name": "exec_command", + "description": "Run the deterministic fixture command.", + "parameters": { + "type": "object", + "properties": {"cmd": {"type": "string"}}, + "required": ["cmd"], + }, + } + ] + first = await client.responses.create( + model="gpt-fixture", + input="Call the fixture tool.", + tools=response_tools, + ) + function_call = next(item for item in first.output if item.type == "function_call") + assert function_call.name == "exec_command" + assert first.usage is not None and first.usage.total_tokens == 48 + + response_stream = await client.responses.create( + model="gpt-fixture", + input=[ + { + "type": "function_call_output", + "call_id": function_call.call_id, + "output": "Process exited with code 0", + } + ], + tools=response_tools, + stream=True, + ) + response_text = "" + response_usage = None + async for event in response_stream: + if event.type == "response.output_text.delta": + response_text += event.delta + elif event.type == "response.completed": + response_usage = event.response.usage + assert response_text == ( + "Capsem ironbank poem\nledgers count the sparks\nno secret crosses raw" + ) + assert response_usage is not None and response_usage.total_tokens == 12 + + pending = asyncio.create_task( + client.responses.create(model="capsem-sdk-cancel", input="wait") + ) + await asyncio.sleep(0.2) + pending.cancel() + try: + await pending + except asyncio.CancelledError: + cancelled = True + else: + cancelled = False + assert cancelled + + after_cancel = await client.chat.completions.create( + model="gpt-fixture", + messages=[{"role": "user", "content": "Still alive?"}], + ) + assert after_cancel.choices[0].message.content == ( + "Capsem ironbank poem\nledgers count the sparks\nno secret crosses raw" + ) + + try: + await client.responses.create( + model="capsem-sdk-upstream-error", + input="fail", + ) + except APIStatusError as error: + upstream_status = error.status_code + upstream_code = error.body["code"] + else: + raise AssertionError("upstream failure was returned as success") + assert upstream_status == 503 + assert upstream_code == "fixture_unavailable" + await client.close() + + return { + "client": "python", + "chat_stream": streamed_chat, + "chat_tool": tool_call.function.name, + "chat_total_tokens": chat.usage.total_tokens, + "responses_stream": response_text, + "responses_tool": function_call.name, + "responses_total_tokens": response_usage.total_tokens, + "cancelled": cancelled, + "upstream_status": upstream_status, + "upstream_code": upstream_code, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + args = parser.parse_args() + print(json.dumps(asyncio.run(probe(args.base_url)), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index 859d22d50..203e249f5 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -151,6 +151,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + [[package]] name = "attrs" version = "26.1.0" @@ -189,6 +202,7 @@ dev = [ { name = "build" }, { name = "editables" }, { name = "hatchling" }, + { name = "openai" }, { name = "pytest" }, { name = "pytest-cov" }, { name = "ruff" }, @@ -207,12 +221,22 @@ dev = [ { name = "build", specifier = ">=1.2" }, { name = "editables", specifier = "==0.5" }, { name = "hatchling", specifier = "==1.29.0" }, + { name = "openai", specifier = "==2.54.0" }, { name = "pytest", specifier = ">=9" }, { name = "pytest-cov", specifier = ">=6" }, { name = "ruff", specifier = ">=0.15.16" }, { name = "ty", specifier = ">=0.0.46" }, ] +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -341,6 +365,15 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11'" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + [[package]] name = "editables" version = "0.5" @@ -455,6 +488,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409, upload-time = "2025-10-06T05:38:16.721Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + [[package]] name = "hatchling" version = "1.29.0" @@ -470,6 +512,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d3/8a/44032265776062a89171285ede55a0bdaadc8ac00f27f0512a71a9e3e1c8/hatchling-1.29.0-py3-none-any.whl", hash = "sha256:50af9343281f34785fab12da82e445ed987a6efb34fd8c2fc0f6e6630dbcc1b0", size = 76356, upload-time = "2026-02-23T19:42:05.197Z" }, ] +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + [[package]] name = "idna" version = "3.19" @@ -488,6 +558,119 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jiter" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/1f/8176d92e001f86505424b41664032ae26a882bc9ca41a32c803f373f9195/jiter-0.17.0.tar.gz", hash = "sha256:03e432f226a453851079fb84cd17c6da9991eab723e28d716f14ae3d906e0c12", size = 229037, upload-time = "2026-09-12T15:14:14.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/50/17afdaffcc8af4bf4fddf2b6c26d066553aa2221983f2affcde435fc2532/jiter-0.17.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:cfafd7be8b16ceadd298db542cead37cddc211c4c49e04ad2596924df18625b1", size = 290699, upload-time = "2026-09-12T15:11:30.085Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e4/c185d32d5b3657ad84da26c84a9eb15f00aa1b39d6882fcc0052dba2d7c2/jiter-0.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8adca2e793288e5f1bb29279bb439d0d3cfbb50eddca7e7e6ffd42ff4f482406", size = 324246, upload-time = "2026-09-12T15:11:31.3Z" }, + { url = "https://files.pythonhosted.org/packages/24/7a/8b8903bfe91a90a8fa1ec9b45d9fda5b6287a386693b69d720a882d73f3c/jiter-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30c692d567ba206c7cca38c9d1d0ccc70c9786290173c184d871ca12e9981ed7", size = 343275, upload-time = "2026-09-12T15:11:32.758Z" }, + { url = "https://files.pythonhosted.org/packages/a2/5d/6821fae2abc71a3c3a84bef8598d31fc4f27d9edfb55bd8f6c08afb8ef93/jiter-0.17.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:81c83c0abe614446a283d994d2c07c4f58632dea2cdf66ba9e2921bb8ccd593e", size = 328034, upload-time = "2026-09-12T15:11:33.9Z" }, + { url = "https://files.pythonhosted.org/packages/f3/51/8e7a963b1c2dfdc01d6228b004f50a2a3d7c46f0549d7b096a5d15ef81d5/jiter-0.17.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:073dc68c1a700c8fc480e877864a6b6ffc887533e261f4380c08c16bf09d057a", size = 342457, upload-time = "2026-09-12T15:11:36.414Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/8b2a267e3bda45d9298331cacfe3521e761f0a2b05ad10a23c6548d08358/jiter-0.17.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:492f37230bbf9581ab2c17bcda862c249afb9ae2e3ab2dd6db59943bc4cc3153", size = 351271, upload-time = "2026-09-12T15:11:37.692Z" }, + { url = "https://files.pythonhosted.org/packages/4a/8f/5c74e5e142a6736833d7a991ab04d5c0738038dc44db05af0bb3cd2559e8/jiter-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5888fe5abc1ca2fa834a3e1b4c7ef0dcece286a7d7e95a609ef0934b777b9fc9", size = 346052, upload-time = "2026-09-12T15:11:39.722Z" }, + { url = "https://files.pythonhosted.org/packages/98/9c/f54920f06d1696e80b1be841d56412871c6856b1b1e3b541b1ed35346554/jiter-0.17.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:84ac78df457e1ee3f7e733bd114823302ae8c5ad5542d7e6647d92ffaa090a04", size = 328915, upload-time = "2026-09-12T15:11:41.065Z" }, + { url = "https://files.pythonhosted.org/packages/21/53/080f126863bceb055db9f1fd5431485eb493b35545a3c35e961ac18cd924/jiter-0.17.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7573e80232c5bcf80c24c038cf7e53a463f5c3b1dd1dd4109d66304f4dccc233", size = 337240, upload-time = "2026-09-12T15:11:42.356Z" }, + { url = "https://files.pythonhosted.org/packages/f0/76/3ab742823a0e0e70e143c6c90a482d9d90396ac2445e7b9483eb7245d3b5/jiter-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:11902505d401691720f5785c15b02204248526edee11b635cd6c40cd52b81599", size = 484917, upload-time = "2026-09-12T15:11:43.549Z" }, + { url = "https://files.pythonhosted.org/packages/14/e0/8ca71bc8b9cc9ed96c9da565863e00f3bb875a8fb82abcf03e975e067902/jiter-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:64846211a2debe7c071d2146d2283d2b0c1c93dc8fd5fb7794faac2ca6061b5c", size = 521868, upload-time = "2026-09-12T15:11:44.704Z" }, + { url = "https://files.pythonhosted.org/packages/c1/02/81f8719dcedb75713082a2048405376c81f6546b75af8167f3bba01a1ed0/jiter-0.17.0-cp311-cp311-win32.whl", hash = "sha256:c19b9357309b8cc6de8a48fca8e44a8c9c2feaaa2f5896d037fa505d48fcab80", size = 187844, upload-time = "2026-09-12T15:11:45.874Z" }, + { url = "https://files.pythonhosted.org/packages/3e/8c/59693f348488f01ed12d862e99ab8da14961152d3e9c39b9b1ef363f3572/jiter-0.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:e654b6b04e39c9cb19cb8b04c6ddf1f2db07751fa14156413969fd78bad0e5cb", size = 231402, upload-time = "2026-09-12T15:11:47.083Z" }, + { url = "https://files.pythonhosted.org/packages/fe/89/fb35e286463cb9f01edc2c4e47df6e5477ee36bca5096414e9ea87985588/jiter-0.17.0-cp311-cp311-win_arm64.whl", hash = "sha256:3ad556afc289f15d2b181b941982d01f06190863c07440185b9f354e1bd2def3", size = 187006, upload-time = "2026-09-12T15:11:48.245Z" }, + { url = "https://files.pythonhosted.org/packages/aa/f8/07bd8c3a23f7a8a6875e6a820bbffe1483a18f18f9398a91b5495123176e/jiter-0.17.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ebf918dfd6a74adc1b9ad71f63c4ab00902fcd3b7fd39f2e24d871db8d713b91", size = 291633, upload-time = "2026-09-12T15:11:49.431Z" }, + { url = "https://files.pythonhosted.org/packages/0e/5e/0de4c6f84ffefa6809ffc2d550b9a314365acf7e7ec9b6c7375d49047900/jiter-0.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:61aed66ee042b3b49ef85fdf75714234d055d89d8496ac1c6e47f89e7a30d5e4", size = 321695, upload-time = "2026-09-12T15:11:52.727Z" }, + { url = "https://files.pythonhosted.org/packages/20/ac/befe2e82065bee37a0252081666ed2f48c1ac5f5c6c318c2de8168ba393d/jiter-0.17.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76eb4a5c20e86f9f848286f167024890f2862258a965d254774deb7fc1545ca1", size = 341967, upload-time = "2026-09-12T15:11:54.231Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cd/9797c1e529746750ae589da7c1a8c24373f00d88e11a989f9e5eb1959079/jiter-0.17.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bcc064f99183a9cbe7f26ed648c352031a74145cd61ed75d34632c73eb46a5a8", size = 326546, upload-time = "2026-09-12T15:11:55.41Z" }, + { url = "https://files.pythonhosted.org/packages/d9/fd/e6914c38d6347bab4ebff2b1f0c0f191db276e7a1d5c376176757da42fe3/jiter-0.17.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73b64e69c4150748e020356d958af94bec33c70a0a93d665cfa8f6d580fe1a63", size = 340995, upload-time = "2026-09-12T15:11:58.211Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7d/611b3abf6f88945b5474da5cdc6d1a185e805ac9bf446bb7766dcda6ea87/jiter-0.17.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0bc7f684b65bcda9c20434267577db71bf9905ceddd32b60d1d93278d8c8d3a", size = 352188, upload-time = "2026-09-12T15:11:59.414Z" }, + { url = "https://files.pythonhosted.org/packages/52/f8/b6e513ecbdf3b3cebe587c2279281ecf775b729a58cf4cc7bdf898ded029/jiter-0.17.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c21265b251d99bbb40080d178a8953e35601d3a1564e05c4de4c0d2ca616797", size = 345025, upload-time = "2026-09-12T15:12:00.697Z" }, + { url = "https://files.pythonhosted.org/packages/28/a8/fe26d06c5a6c5a4cfe703c5154c8a140da1305671eb3681aba9422d4f393/jiter-0.17.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:f3d7f7b34114f7ddc6d72a8e882d49de636b35d9fd12b4d420d3c5729f6c9812", size = 329180, upload-time = "2026-09-12T15:12:01.831Z" }, + { url = "https://files.pythonhosted.org/packages/e1/58/e6d66a26af40a20e62486feb7e222fd50f6e7aaa4f107abd89675dcc835b/jiter-0.17.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5078ab00664307fab2019b522a93aeb191122789f085daf5fd9e362154021d4a", size = 335805, upload-time = "2026-09-12T15:12:03.056Z" }, + { url = "https://files.pythonhosted.org/packages/ef/3e/96520aa2fef5ef831d95483a902140bfab83dcac9eaa74f7df61b5e50a1b/jiter-0.17.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:470e1b1e4c42f1ead2189166a299691871a2df5056c976e7fb96feafaf5f9d44", size = 484121, upload-time = "2026-09-12T15:12:04.414Z" }, + { url = "https://files.pythonhosted.org/packages/6a/8f/5d9d92fe538bf36ff481a2278c48147e59c1cf8eb2f7be665260665febe5/jiter-0.17.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:6eb6aedeb7352b8f3b6af9cbd67983840165c00428e63f1b420a85885128ea31", size = 521310, upload-time = "2026-09-12T15:12:05.612Z" }, + { url = "https://files.pythonhosted.org/packages/50/06/a09f979b22e652afbc3de66c709b2ba92edcef555f7535ab937c86b4f21a/jiter-0.17.0-cp312-cp312-win32.whl", hash = "sha256:362bb47423886d45a9f705d2d9d4008c6eedd4e41eb1bab4e96fb6daa06b33fd", size = 185029, upload-time = "2026-09-12T15:12:06.994Z" }, + { url = "https://files.pythonhosted.org/packages/6c/d9/98265a005b2473ec2be5a84e2b64c2f65382c673879f1574845cd4bcd77c/jiter-0.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:9bd3caac219df476dd0cc3fe01d2f1581ed588906feac767abd9614c1c12f8b3", size = 227381, upload-time = "2026-09-12T15:12:08.823Z" }, + { url = "https://files.pythonhosted.org/packages/a8/11/2e05bf5a56e57a543ebb8f585074adf09383e99d7b062dac92eab1f4d57f/jiter-0.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:36ee6e69027396664e59995b9a635a947a5304ee9837279584a0bb8145c8f6b8", size = 183610, upload-time = "2026-09-12T15:12:10.374Z" }, + { url = "https://files.pythonhosted.org/packages/40/eb/2c4a8075ed5ea02b56911e9375d4c8d7784572ff4af32e5a99ae0d071044/jiter-0.17.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1b18434638228c0c184281609bf3d9459026a0f1ea48fb76c205e3ef72069caa", size = 290991, upload-time = "2026-09-12T15:12:11.641Z" }, + { url = "https://files.pythonhosted.org/packages/ca/b1/34bfa29599d420423baac6ff7cada6674fe63d5a7a2ccb3900b904678783/jiter-0.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ec89771f4272b989487a6364e519db6bbaba323e8bbf949ac89a45ea9c18b7a3", size = 321425, upload-time = "2026-09-12T15:12:13.855Z" }, + { url = "https://files.pythonhosted.org/packages/11/71/a5ac64a62a04aebd556afadab14a6b730001e16df87266ded943a100a1d9/jiter-0.17.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e3f052c671d5f425cca5ea5901cf11a831369fba4a55a3862cab93c323b4c3b", size = 343138, upload-time = "2026-09-12T15:12:15.046Z" }, + { url = "https://files.pythonhosted.org/packages/01/dd/f761e320ea473314cb68612bc6a435393464dbd198051399b36848b4ebf3/jiter-0.17.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:785a216bbaf8f15fc974e964ced7322cd3d774bb0e86949edd78c6bffd6ba35b", size = 325805, upload-time = "2026-09-12T15:12:16.506Z" }, + { url = "https://files.pythonhosted.org/packages/19/1a/27d8e40f0fb29bbc7a5adf30907144396a115dbe93d5d8976c054a6dfe96/jiter-0.17.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d85c558c9f8532bba287a990ac63767c7daf756f0d8c030219f62499b1fa228a", size = 340230, upload-time = "2026-09-12T15:12:17.682Z" }, + { url = "https://files.pythonhosted.org/packages/ac/c0/30bcde78a28155461f965d16b7aca4ffca6d17494d905f7a0bb072e6c64e/jiter-0.17.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c23849235d2142ce444b2b8c6eceee9f82f4cc0bd5c9081602e4155c6197807", size = 351343, upload-time = "2026-09-12T15:12:19.337Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/91420b156315ae22732f5ee1a7b5725a030aab9dc8fd7dcdacfb4aa588d3/jiter-0.17.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58df29268a95e910f17db7ec9178eb7f15aa8619aaca3575275c4e6b3f4fe4c5", size = 344990, upload-time = "2026-09-12T15:12:20.705Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a2/ae6d5672644cc11127970277c9aeb0fa6fae376845587f5b0a8e8828167c/jiter-0.17.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:a277f97eba7d66b1ee27eb5dab5b774ff46a10c78d89a1d3dcce04ce1357c8ca", size = 328624, upload-time = "2026-09-12T15:12:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/04/62/45cb1162f6aa586536e4a973fc339d72dc6b08cca030d70a838a307aa778/jiter-0.17.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe15ddf316f1f1f643347d3a474e74ce61880c79a11ec5dca53df20c071bd3e8", size = 334731, upload-time = "2026-09-12T15:12:25.229Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5f/45c1574b644da7deda0b7591c349520dcf83ce45b24d7ca19922dab1fc27/jiter-0.17.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:02adebb7ce6413c44d40af9ad59d1c1cd79630ccdcb6f7bdd2d461e48c03d8f9", size = 483649, upload-time = "2026-09-12T15:12:27.557Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d3/ebea1ecb5b241c519f192b30215c79a8e47f42f1621acbcd6f8830728416/jiter-0.17.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:55d0e0e613a3f9ad600cf436e0e2b8057d1b52bcf1d91b2d36ac53451231e6a8", size = 520758, upload-time = "2026-09-12T15:12:28.99Z" }, + { url = "https://files.pythonhosted.org/packages/64/e6/682b641ff0765ea9bdc349dbc7d223de5c8af8ec1abda0db3406992f92fe/jiter-0.17.0-cp313-cp313-win32.whl", hash = "sha256:2c45ad7c973ef33fe5114a953377b35a95240f4542c0724d9f781e47dc24bac7", size = 184334, upload-time = "2026-09-12T15:12:30.813Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d2/9a49aac2b27af4cc5015e368c0cc3588491a532f717a668ffce1f1ac57da/jiter-0.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:a3cebb1fe4a1abb00465f3f8a17e09112603e8b7c59e5c3adbcd9f7815a64acd", size = 226601, upload-time = "2026-09-12T15:12:32.096Z" }, + { url = "https://files.pythonhosted.org/packages/b4/ce/9a43e9f614608eafa78de22aedcff54cd21324467b5d442d5c9b00244145/jiter-0.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:96b8b0c6dc5d78682f54a450785e075aa929cde768304cad363cd4efba5a82ac", size = 183103, upload-time = "2026-09-12T15:12:34.396Z" }, + { url = "https://files.pythonhosted.org/packages/01/9e/23065f8e2c7a4c372c1b6f6622e4cfab4dc786cb5150052b1527e6a6a840/jiter-0.17.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:00d783a779c5664e16dbad5e3a3c3a75e128b07dd5f4765159658d9210a50ca5", size = 292210, upload-time = "2026-09-12T15:12:35.613Z" }, + { url = "https://files.pythonhosted.org/packages/ea/81/67b58647560bc82a4490d722caa8561d7a86a9f45d4fa620b7e5fe282c7a/jiter-0.17.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0619d806e260ecf0c2a64521942c94af5d547c9ec99b55ae4f51b538b5576a76", size = 321512, upload-time = "2026-09-12T15:12:36.907Z" }, + { url = "https://files.pythonhosted.org/packages/c7/07/6658359a25f55927f7f8bf0e16465dee2ccd0b2a1a5208acc0df8972e074/jiter-0.17.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc0288ce39190ee33fe6e4ec73161eed34e7e2da509b525546ca061778d62b64", size = 343897, upload-time = "2026-09-12T15:12:38.189Z" }, + { url = "https://files.pythonhosted.org/packages/46/04/5d50a9f0319cbdc37fd53c27f8c313d46afc34f1b048219ae6d8ea068da4/jiter-0.17.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5a52a430d04225ffde633e6840bf2381d34c019ff98526b5929755b9052fb199", size = 326519, upload-time = "2026-09-12T15:12:39.532Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c7/d02517832b29eb8275fdd0f4ce0f17b80f58cc4c3ebecd4d9ace990d633d/jiter-0.17.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37f33d327900bf2879613b3363fd48df97b4232d0c41f54bcf2e790c2fc40a71", size = 341369, upload-time = "2026-09-12T15:12:41.486Z" }, + { url = "https://files.pythonhosted.org/packages/3b/07/499b5f5603501cdd93a73a6a176dfad9c96555a3ae58ca9f8e3acba63dc9/jiter-0.17.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6cf564d43c4388149ca58ee571d0f5ccf875e20d1fd4662fd94cc0d1ea3b10ef", size = 352160, upload-time = "2026-09-12T15:12:42.721Z" }, + { url = "https://files.pythonhosted.org/packages/f5/75/b04013c7743269d4533ef4e746fc0ed678a143968dd7448658e3f51daad2/jiter-0.17.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:523c499235fb65add25d4bb01b1c4709ce695efdc7deb6c0a7bc515b5c44e0fb", size = 345018, upload-time = "2026-09-12T15:12:44.192Z" }, + { url = "https://files.pythonhosted.org/packages/1d/96/cbb6fd1e42a77c8412ec4643db95059b30cdfc635e387cc9193e098ce268/jiter-0.17.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:455e4ab35cb2a4a91a8404e08fd3c621bae433922e59bf1c494fe20a426b013b", size = 329244, upload-time = "2026-09-12T15:12:45.491Z" }, + { url = "https://files.pythonhosted.org/packages/15/67/d3be402f398566a379bf40ae65be5c3505b14d9e95e0802a597ddde7ddee/jiter-0.17.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6871973bfbd4408f7f1c632b30bbb5bbd9671c1bc8650af6823e24b7be13709b", size = 335693, upload-time = "2026-09-12T15:12:46.935Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8d/98e2c4130b93d64f1d67c89060b928d04102549bf05e64451c9e6024f9ca/jiter-0.17.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:77f6aac0137309b31448c1bdcda4c6c77077664a6d018ece8d94019c68a5a5b9", size = 484329, upload-time = "2026-09-12T15:12:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/78/5e/8da91e49f0fbca37c3489fb4cf3ad6676d4965f00ae5468bca3a2513737a/jiter-0.17.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:93946d89fa04d5ba64dd323a8dd8d901676cb8a3c81d99ae4f6c051a9b4c3f2f", size = 521358, upload-time = "2026-09-12T15:12:49.856Z" }, + { url = "https://files.pythonhosted.org/packages/be/21/5388684a5a38af3557cd9c2424b9827c71809cff24373c75ef9d0d3dfba9/jiter-0.17.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:70f19a2ca8429f91e82eeffb2f51cb87bc2d6e953b009b91a92d29c3a16ccb03", size = 110459, upload-time = "2026-09-12T15:12:51.747Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ad/58b3a93525d2ffca7f54d9dee441381990082bd1172fbeb8d6a3f72a4dc3/jiter-0.17.0-cp314-cp314-win32.whl", hash = "sha256:71dbd74314c5df52a1bccf7b8bca46d14e943af7a2012e73b23f49977ef194c8", size = 185043, upload-time = "2026-09-12T15:12:54.477Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4a/1aa520eb6c359b262c14ff995ca7283837208ddfb1202082ce9d73cf214d/jiter-0.17.0-cp314-cp314-win_amd64.whl", hash = "sha256:ac3c6ee3264d6f5c44c617f90bc7e8b9e1587e7d6708c9d8f811cb65582ee312", size = 227163, upload-time = "2026-09-12T15:12:55.931Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e4/5997f648794bd9b499491d0ff480b096cc9a9c65bdba29f57568e6aa1705/jiter-0.17.0-cp314-cp314-win_arm64.whl", hash = "sha256:6219adaf59711ba7063a52496e8ec6d3fa3e209d7827d83eee3b2abc780a1744", size = 183505, upload-time = "2026-09-12T15:12:58.196Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4a/84a5ec271d09f7590b6073af5ee4abb44eab4ccace453b7e2c5ce45234ca/jiter-0.17.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:59bddbe6f9ffecc68d641e1e2d619ce64cf8a9e9eeb74e5c518f74fc87abf1b0", size = 321527, upload-time = "2026-09-12T15:12:59.394Z" }, + { url = "https://files.pythonhosted.org/packages/39/71/9e1fd0045f5920b4c36be35c3f0f0dfd123668684f8ad352619d7aa44183/jiter-0.17.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cb41cd1432f1dc19a231cf70b54d42b2c9f05085155859263fce06fa4d41388", size = 340865, upload-time = "2026-09-12T15:13:00.756Z" }, + { url = "https://files.pythonhosted.org/packages/b7/2b/14627fd2bc377f3dd09491bcace6b90e34b4d7fea2f1f3295031ff91f528/jiter-0.17.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd7790aa79c8b518e512ebcdfce9f11d8ef5f30efd43720c8a19a548b39fa489", size = 325412, upload-time = "2026-09-12T15:13:02.152Z" }, + { url = "https://files.pythonhosted.org/packages/4c/f3/8d5808f7bf0f456bde79e6393587183a0cee5f83d179fe1f7f1eff2ba067/jiter-0.17.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dbbfe4e3c21c8166980cddc5bee1a315df082454f007947dfb6fb73800768165", size = 340473, upload-time = "2026-09-12T15:13:03.485Z" }, + { url = "https://files.pythonhosted.org/packages/4f/da/1d8c7c6c4ae6b2423b94a81b6b907d37b28f87664e077427b531bf1b5313/jiter-0.17.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c286860abfe8b100cac1c02e225e5776eb9216edd71ba17cdb237da4af32bc9", size = 350757, upload-time = "2026-09-12T15:13:04.828Z" }, + { url = "https://files.pythonhosted.org/packages/eb/96/c1813dcca15c5a370145a448aaea7d1f83f6f0228a5f1130e79340ee385f/jiter-0.17.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f753eb70b1474a29e635e7542ff7312e6d6b951e0b25e8a2e8c34eeb1ddcd478", size = 345203, upload-time = "2026-09-12T15:13:06.131Z" }, + { url = "https://files.pythonhosted.org/packages/d7/f7/fc61cbcf2992d169ede13648fc3fd8e2d3171a3669dde43cd4db556549ac/jiter-0.17.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:eae86b1f027031e39db2e0e9c4842221edb7b8cd474d23f87a79b3bd4b651768", size = 328322, upload-time = "2026-09-12T15:13:07.392Z" }, + { url = "https://files.pythonhosted.org/packages/8f/88/46418a3abbdffb7dc41b314200360f24f75faaeb35573e81c92de322cce9/jiter-0.17.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5bf350452a43173e69e1fc74847c57a60e3d7515807287f29849baa2a85d8718", size = 336570, upload-time = "2026-09-12T15:13:08.666Z" }, + { url = "https://files.pythonhosted.org/packages/f0/28/b8a55b949be6306df8888e365a8df05441de8a7b11289f6957004302e41e/jiter-0.17.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:da139721f4b7cafdbff580a4f511ea24cb91f4909330c6b926a1ca53836c0a59", size = 482879, upload-time = "2026-09-12T15:13:10.037Z" }, + { url = "https://files.pythonhosted.org/packages/75/3b/21d0afa53ba0680962c39f3eb95ed2946f8793369ed44b0c82b490723081/jiter-0.17.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:8079849db9a1371bfd90bad088458a8fb836261879df2233cc9632464ecf64e1", size = 520406, upload-time = "2026-09-12T15:13:11.456Z" }, + { url = "https://files.pythonhosted.org/packages/ef/03/bcbaf8b6b9ea23c2c074411f8ecfbb02d820abac5d0cb8f4e280209174a2/jiter-0.17.0-cp314-cp314t-win32.whl", hash = "sha256:8f770b0c77e5fac482e1ba03ca1a7e18286bfb213d749932a00a7e4cd5de5e06", size = 184434, upload-time = "2026-09-12T15:13:13.037Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b5/5d6ce2c93ef6fe1241b37a9005547f9b6d58db1f07f39fe95807d4b98f51/jiter-0.17.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c4289293e5278d9314b00f15c37f2120fa51d3d68565292e715524c750e775a9", size = 227392, upload-time = "2026-09-12T15:13:14.933Z" }, + { url = "https://files.pythonhosted.org/packages/f5/4b/1e52baf90187606e33a7b8cfa8f96f5829acd7f01870077eb01059ab76d0/jiter-0.17.0-cp314-cp314t-win_arm64.whl", hash = "sha256:4dfbfe5a6e1e80a7082af559f66386405025ec278833e0c649f69cbc6e1004cc", size = 182776, upload-time = "2026-09-12T15:13:16.239Z" }, + { url = "https://files.pythonhosted.org/packages/05/fc/efe3ac75564ab10f53517958f5ccdc231fc7334af66c76776cb554a88967/jiter-0.17.0-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:84963d3f395ef5e9a32ce47155e08a7962fa292c159a10cb98b931cef1416925", size = 292143, upload-time = "2026-09-12T15:13:17.502Z" }, + { url = "https://files.pythonhosted.org/packages/d1/4c/46982118d91f9ffe9714319d21ec4f98d9b7e0cfd9062826c524a54de24e/jiter-0.17.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:ffa0380ad091de7d3fc33e17a97ff479851ee18a0a2a3ee56ff3215cdc886656", size = 321341, upload-time = "2026-09-12T15:13:19.133Z" }, + { url = "https://files.pythonhosted.org/packages/e7/12/9b1ac6ecc6307049913db54839ddba1c11c1ef72c5a8bbb5514bc3b50d1b/jiter-0.17.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:755079792868ce5d4938e83b91a0939b34fb858a1ca65a104f2d771bea57faa1", size = 344383, upload-time = "2026-09-12T15:13:20.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/b6/527cc72af836d824e9d4d666e64f0a1ca7eafd662a8da9657b78592172ba/jiter-0.17.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3bf4dc2b84a464117fb097d15a25c58d100d2692888e3b0d92df5b48ed16b7c0", size = 326841, upload-time = "2026-09-12T15:13:21.83Z" }, + { url = "https://files.pythonhosted.org/packages/d1/41/567f98617e88005b249503b933803f633ec6ba2d427cf4cc35e5c832125c/jiter-0.17.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:02a360707033d8cef53f7f3480817a1489177a259ec6ec01e98c37e0b922ddca", size = 341354, upload-time = "2026-09-12T15:13:23.323Z" }, + { url = "https://files.pythonhosted.org/packages/40/da/b29cda895b785f7d426e224638a885b6145a08ce853b381f34afe3e88c5d/jiter-0.17.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:300ce01ab0215e3dea4d00090143c909aedc65c0f809b3c07983e1d038f291b9", size = 351985, upload-time = "2026-09-12T15:13:26.526Z" }, + { url = "https://files.pythonhosted.org/packages/f7/5c/8a73829e7389e72ea298a450f2b3cb58e71a3e464b45f6d8753740f1c4f5/jiter-0.17.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:746243a080b4ca790b8499af3d7cf9825d5f5987933950cd818e767ee353d826", size = 346052, upload-time = "2026-09-12T15:13:27.887Z" }, + { url = "https://files.pythonhosted.org/packages/1d/2f/98d6001026932c095ba440925570123043bed29f5ff56158dfe729a9e81b/jiter-0.17.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:b550585523339b71cb852b811aae49d08d7601ad8ffe9f5dc1562f4c3d22fd87", size = 329159, upload-time = "2026-09-12T15:13:31.569Z" }, + { url = "https://files.pythonhosted.org/packages/94/2e/708dc1d2678f092c31c12754e860cd8353e6a85ecbdb1010157edca0da9e/jiter-0.17.0-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0239520085cac678e77a606fd7e3f1c60c371d719790c5e3807388d3da4354c2", size = 336001, upload-time = "2026-09-12T15:13:32.846Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/75a5ae38862f4eaf0fe2f8a9fbf6484c4890df04c06dcdffc45e36bca61a/jiter-0.17.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:eb2295da7c3769f6719b227a237aa6a5cfa6550e478bc838001b592c57e16575", size = 484281, upload-time = "2026-09-12T15:13:35.333Z" }, + { url = "https://files.pythonhosted.org/packages/a0/32/6636fae811c27c7f93e1b11fb5800de6a5c9e4269a27cf718e0b31218ad1/jiter-0.17.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:e088612ff90ebc9247e1a43074b72835804261c47e6a6c01cb3ddcb55360d688", size = 521300, upload-time = "2026-09-12T15:13:37.101Z" }, + { url = "https://files.pythonhosted.org/packages/61/aa/12df7e0b0b1a2602e3d5a5a7104d7d9700f254b400f134a9b50955c4d231/jiter-0.17.0-cp315-cp315-win32.whl", hash = "sha256:0b52d52035b3907c5b1f6277857b29c1cbfc965e24e0f27330dbed83edb591ec", size = 185138, upload-time = "2026-09-12T15:13:38.901Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ec/3dd2e495032cddde05723c1f4c743b67a23e55d2af244692a7f58f0cdae3/jiter-0.17.0-cp315-cp315-win_amd64.whl", hash = "sha256:10f5558eed511b830488003449d942bd75829ad6257dc58cb9a03e596a7777b1", size = 226950, upload-time = "2026-09-12T15:13:40.17Z" }, + { url = "https://files.pythonhosted.org/packages/c3/c7/ef85704e0a57e9cadb2babc05f6d7c5df4a1c75da1a6ee31e1986b0099a5/jiter-0.17.0-cp315-cp315-win_arm64.whl", hash = "sha256:fa13acf1046f95df808c64b1310705e143fab87aee73ae00cc42d640867fd2c1", size = 183618, upload-time = "2026-09-12T15:13:41.432Z" }, + { url = "https://files.pythonhosted.org/packages/0e/9a/a4b348349de68762b58d6713973d363ad80a1c741d0bf8def7975f0ecb26/jiter-0.17.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:af2f7501580f274b63c4b2283bc425f5df7edf06ae5b171e5f87d912ff359a20", size = 321155, upload-time = "2026-09-12T15:13:42.716Z" }, + { url = "https://files.pythonhosted.org/packages/c1/70/aebd6d0b5f0677de3a3d0bdc4a05fac949b97c4ede454c8809f180ac7b17/jiter-0.17.0-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10c5349312e5cb02b7a21e123a57665afa895953f05bf252a9dd4c13a572b7ab", size = 340985, upload-time = "2026-09-12T15:13:44.115Z" }, + { url = "https://files.pythonhosted.org/packages/a7/82/4c3b49796b5eb62f3f5046f957683f4ba0135fe1a60957c11180512460df/jiter-0.17.0-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:86f3f9343a288eb85a81ef20a752b2f84564296636db54a9fff0b5c8deaf1df2", size = 325670, upload-time = "2026-09-12T15:13:45.901Z" }, + { url = "https://files.pythonhosted.org/packages/bc/43/f6341ecb4872202a4ef150486fcee0e1ace4aa3da39b71b82061452cdd3a/jiter-0.17.0-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4607ec7d93355fbc25b8dc5189153cf21d66063b9f9cd04dd2774e6e783f9b6a", size = 340339, upload-time = "2026-09-12T15:13:47.442Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c4/bc2c86e08fa065e03cb2fbc53b367c3640a7d257ef9d877b29118ea636b7/jiter-0.17.0-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10cd64a5720ad7f809ac5466ff1705813f1b6b510f195a73acafba0ac0e1f675", size = 350705, upload-time = "2026-09-12T15:13:48.848Z" }, + { url = "https://files.pythonhosted.org/packages/9d/67/91f12aa111cca6e3a197c3e36bf60a034bf9f122f6d41112a639e44217d8/jiter-0.17.0-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efe9f61bb30174d2f5c8396445c360c96c44e78164d0815dfe627ccf57849574", size = 345011, upload-time = "2026-09-12T15:13:50.215Z" }, + { url = "https://files.pythonhosted.org/packages/f5/cb/9f5556e8f6ec89755fb5a709d8eb8270c9a324e31079eda0dfbeca451b6e/jiter-0.17.0-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:370d8fe5bf201dc6925e8a84c81ac7291f74d9fd1778234fc79d517064a5c76b", size = 328268, upload-time = "2026-09-12T15:13:51.809Z" }, + { url = "https://files.pythonhosted.org/packages/22/98/153f20680fb75781a490fb849940e2b00f95035c7aa054df592f36ed33fc/jiter-0.17.0-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6b303d88e6a0bda789ec4b7801c7bad68e27230ba1fe4baffc756d1fbd32dc9d", size = 337024, upload-time = "2026-09-12T15:13:53.095Z" }, + { url = "https://files.pythonhosted.org/packages/af/59/b16c9be3a5035df4466cc72e888188c027562de90a723d290ab6814cb9d4/jiter-0.17.0-cp315-cp315t-musllinux_1_1_aarch64.whl", hash = "sha256:30793a24a31e968969757c9e08d830cbb15a2cd3c4959b4498b38f4b1c2258eb", size = 482766, upload-time = "2026-09-12T15:13:55.713Z" }, + { url = "https://files.pythonhosted.org/packages/d0/55/667dea313094024bef082175d6bfe8976f90d1c00c926af9df1d8e0eab48/jiter-0.17.0-cp315-cp315t-musllinux_1_1_x86_64.whl", hash = "sha256:686c93d86f2b426c803024b805bd161a6cd10e9627c23e901640eab646c0ad8a", size = 520367, upload-time = "2026-09-12T15:13:57.674Z" }, + { url = "https://files.pythonhosted.org/packages/21/e3/4b1a43501fb9ed17b01d137e380cb0e8fdcb39a254ce31aa2ab95bc861ac/jiter-0.17.0-cp315-cp315t-win32.whl", hash = "sha256:86d703d9faa1ffc8ae4e9de0fa007712ed2171b5c0d93811a8e2e105ac729b0d", size = 184603, upload-time = "2026-09-12T15:13:59.27Z" }, + { url = "https://files.pythonhosted.org/packages/f9/f2/b8ee0372b6ebdf1bde5cc44495d5291d17f961065f5b48f8616cc67cac2e/jiter-0.17.0-cp315-cp315t-win_amd64.whl", hash = "sha256:42b0260445251b1bc520a63baa94a32d88e0f931fba234f1764db7feb7c72174", size = 227936, upload-time = "2026-09-12T15:14:00.472Z" }, + { url = "https://files.pythonhosted.org/packages/a4/b4/923a1215daba959aed8355973315cb3f81f53e0d01c5b211870a27b41f45/jiter-0.17.0-cp315-cp315t-win_arm64.whl", hash = "sha256:d47687806f9c54c84ea38733507081337922beca90ce819c7d852dd485bc0f23", size = 182977, upload-time = "2026-09-12T15:14:01.799Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3b/05a917204413e2e09906dfa35240c1021227aeb56c7305abea9562c598b4/jiter-0.17.0-graalpy311-graalpy242_311_native-macosx_10_12_x86_64.whl", hash = "sha256:eaba834b72d573547b9d966465b3394b749d5e14208cc70acb63aca37619ab33", size = 288143, upload-time = "2026-09-12T15:14:02.998Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e1/a1cd3c0cf8f79945939e4f8caae9990529f67d7f77f671769f73956329b9/jiter-0.17.0-graalpy311-graalpy242_311_native-macosx_11_0_arm64.whl", hash = "sha256:51e1519d676a9f14dad9c2a411170d43b022ddb7989562df4e849b261ce127b2", size = 281847, upload-time = "2026-09-12T15:14:04.414Z" }, + { url = "https://files.pythonhosted.org/packages/72/b4/9b797679e09f4a46c32986aeb3670bd9bc562fc0b373c7a0ee5c5dce1206/jiter-0.17.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ce4feb52493e3513335b2accdcd75605652e4632772d3c8c2f7b86954d7f39", size = 304797, upload-time = "2026-09-12T15:14:05.733Z" }, + { url = "https://files.pythonhosted.org/packages/25/4a/0d77415b27a00d970e4e710f7c1de62e96a11c4cab3ed1add0015af04626/jiter-0.17.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29f49b325e0234e4ad9ecca5b861ffbd09b95ccac9bd46fa55841b6e56eea5fe", size = 307815, upload-time = "2026-09-12T15:14:07.105Z" }, + { url = "https://files.pythonhosted.org/packages/17/31/4bb27f54333d3b9ef1e5bd3312dc0b4bbe59c68bb0885fdb40583a6b1567/jiter-0.17.0-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:454c4997d73cc466c71fd565d91e603b0274e48ea0c6b0b7a7aee6967e4ceb7c", size = 288415, upload-time = "2026-09-12T15:14:08.455Z" }, + { url = "https://files.pythonhosted.org/packages/28/30/879570ecf82574eaea77c5eb10309f4b630dece5f2a556e9814a90ba3f2d/jiter-0.17.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:40d2c240f8f80b5b0f201b29f0ae129c81448c60c772227a41747b5e0026f6a2", size = 279113, upload-time = "2026-09-12T15:14:10.117Z" }, + { url = "https://files.pythonhosted.org/packages/77/7a/1f0b8a35fbd079a4f1752c31a15dc99cf277f863747c459be0af39e900e5/jiter-0.17.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e05f5adbf68c4bd11e1610f394034d984152988e84be6f8314235ce6f2139e5", size = 303708, upload-time = "2026-09-12T15:14:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/e1/8b/d76219ebdbcf3d4209d9d21a0810db4c8d0a6f88e3ee87d30bdea4e90d30/jiter-0.17.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2c0bf24c72fd0491405dce5d40194f2070e9021ce648c1a1d46234b93d848ff", size = 307147, upload-time = "2026-09-12T15:14:12.897Z" }, +] + [[package]] name = "multidict" version = "6.9.1" @@ -646,6 +829,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/59/e26cb779be4c591d1a910f59d29aca9fba4de70349840a833beba2652371/multidict-6.9.1-py3-none-any.whl", hash = "sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b", size = 19176, upload-time = "2026-09-21T17:59:03.501Z" }, ] +[[package]] +name = "openai" +version = "2.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/9a/8c75e8c8a5b407a0586faeb2afac91674ff955c191ecc1d6d3b6669f6788/openai-2.54.0.tar.gz", hash = "sha256:e3e6f8bc1ba30ddf381ace1a14340eed381cb984a1a59bd0f34b5be3b5d49cfa", size = 1100285, upload-time = "2026-08-11T18:46:59.035Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/a8/bb76c7356de8ad57f59d5ff993d434df0607f07f08bcc9c9a5c275e399c0/openai-2.54.0-py3-none-any.whl", hash = "sha256:89089789197ccdb87f173a03145ed1598d00795220c93e96cf712b1cbf5e5f2b", size = 1660351, upload-time = "2026-08-11T18:46:56.684Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -974,6 +1176,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/07/d781f8f8e1ac24bef9f3269cf62ffb1407ca24c3a8f12e5e22874f90528c/ruff-0.16.6-py3-none-win_arm64.whl", hash = "sha256:7a976c79b958f94e50a022a19f0f8c87387448020935ec14fc74331bd0a7f2c5", size = 10412850, upload-time = "2026-09-03T16:57:26.416Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "tomli" version = "2.4.1" @@ -1028,6 +1239,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] +[[package]] +name = "tqdm" +version = "4.70.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, +] + [[package]] name = "trove-classifiers" version = "2026.6.1.19" From 285a54fbde62c73edcf8de178ed8494bd20e174b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:34:37 +0000 Subject: [PATCH 191/464] test(typescript-sdk): probe standalone OpenAI endpoint --- sdk/typescript/package.json | 1 + sdk/typescript/pnpm-lock.yaml | 31 ++++ .../tests/standalone-openai-client.ts | 147 ++++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 sdk/typescript/tests/standalone-openai-client.ts diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 9148105e1..7777a6175 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -27,6 +27,7 @@ "@types/node": "^24.0.0", "@vitest/coverage-v8": "4.1.11", "eslint": "10.10.0", + "openai": "7.31.0", "typescript": "5.9.3", "typescript-eslint": "8.70.0", "vitest": "4.1.11" diff --git a/sdk/typescript/pnpm-lock.yaml b/sdk/typescript/pnpm-lock.yaml index b93ba07c1..5b10a4ca9 100644 --- a/sdk/typescript/pnpm-lock.yaml +++ b/sdk/typescript/pnpm-lock.yaml @@ -27,6 +27,9 @@ importers: eslint: specifier: 10.10.0 version: 10.10.0 + openai: + specifier: 7.31.0 + version: 7.31.0(zod@4.6.1) typescript: specifier: 5.9.3 version: 5.9.3 @@ -693,6 +696,30 @@ packages: resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} engines: {node: '>=12.20.0'} + openai@7.31.0: + resolution: {integrity: sha512-ZGAzauuYb0DdB53qVbIR+X8rdmNtWgiElrrMthbZCVreLcIsQJBwzAIWUbpB2FkQu2Cj6OJ9zJR1EKMUYn4/SQ==} + engines: {node: '>=22.0.0'} + peerDependencies: + '@aws-sdk/credential-provider-node': '>=3.972.0 <4' + '@smithy/hash-node': '>=4.3.0 <5' + '@smithy/signature-v4': '>=5.4.0 <6' + undici: '>=5 <9' + ws: ^8.21.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@aws-sdk/credential-provider-node': + optional: true + '@smithy/hash-node': + optional: true + '@smithy/signature-v4': + optional: true + undici: + optional: true + ws: + optional: true + zod: + optional: true + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -1526,6 +1553,10 @@ snapshots: obug@2.2.1: {} + openai@7.31.0(zod@4.6.1): + optionalDependencies: + zod: 4.6.1 + optionator@0.9.4: dependencies: deep-is: 0.1.4 diff --git a/sdk/typescript/tests/standalone-openai-client.ts b/sdk/typescript/tests/standalone-openai-client.ts new file mode 100644 index 000000000..2f47e91cf --- /dev/null +++ b/sdk/typescript/tests/standalone-openai-client.ts @@ -0,0 +1,147 @@ +/** Official TypeScript OpenAI client probe for the standalone Capsem endpoint. */ + +import OpenAI, {APIError, APIUserAbortError} from 'openai'; +import type {ChatCompletionTool} from 'openai/resources/chat/completions'; +import type {Tool} from 'openai/resources/responses/responses'; + +const baseURL = process.argv[2]; +if (!baseURL) throw new Error('usage: standalone-openai-client.mjs BASE_URL'); + +const client = new OpenAI({ + apiKey: 'sk-capsem-standalone-typescript', + baseURL, + maxRetries: 0, + timeout: 5_000, +}); + +const tools: ChatCompletionTool[] = [{ + type: 'function', + function: { + name: 'fixture_lookup', + description: 'Read the deterministic fixture.', + parameters: { + type: 'object', + properties: {query: {type: 'string'}}, + required: ['query'], + }, + }, +}]; +const chat = await client.chat.completions.create({ + model: 'gpt-fixture', + messages: [{role: 'user', content: 'Use the fixture.'}], + tools, +}); +const toolCall = chat.choices[0]?.message.tool_calls?.[0]; +if (!toolCall || toolCall.type !== 'function') throw new Error('chat function call missing'); +if (toolCall.function.name !== 'fixture_lookup') throw new Error('chat tool name changed'); +const chatArguments: unknown = JSON.parse(toolCall.function.arguments); +if ( + typeof chatArguments !== 'object' + || chatArguments === null + || !('query' in chatArguments) + || chatArguments.query !== 'Capsem ironbank poem' +) { + throw new Error('chat tool arguments changed'); +} +if (chat.usage?.total_tokens !== 456) throw new Error('chat usage changed'); + +const chatStream = await client.chat.completions.create({ + model: 'gpt-fixture', + messages: [{role: 'user', content: 'Stream the fixture.'}], + stream: true, +}); +let streamedChat = ''; +for await (const chunk of chatStream) streamedChat += chunk.choices[0]?.delta.content ?? ''; +if (streamedChat !== 'Capsem ironbank poem') throw new Error('chat stream changed'); + +const responseTools: Tool[] = [{ + type: 'function', + name: 'exec_command', + description: 'Run the deterministic fixture command.', + parameters: { + type: 'object', + properties: {cmd: {type: 'string'}}, + required: ['cmd'], + }, + strict: false, +}]; +const first = await client.responses.create({ + model: 'gpt-fixture', + input: 'Call the fixture tool.', + tools: responseTools, +}); +const functionCall = first.output.find(item => item.type === 'function_call'); +if (!functionCall || functionCall.name !== 'exec_command') throw new Error('response tool changed'); +if (first.usage?.total_tokens !== 48) throw new Error('response usage changed'); + +const responseStream = await client.responses.create({ + model: 'gpt-fixture', + input: [{ + type: 'function_call_output', + call_id: functionCall.call_id, + output: 'Process exited with code 0', + }], + tools: responseTools, + stream: true, +}); +let responseText = ''; +let responseUsage; +for await (const event of responseStream) { + if (event.type === 'response.output_text.delta') responseText += event.delta; + if (event.type === 'response.completed') responseUsage = event.response.usage; +} +if (responseText !== 'Capsem ironbank poem\nledgers count the sparks\nno secret crosses raw') { + throw new Error('response stream changed'); +} +if (responseUsage?.total_tokens !== 12) throw new Error('response stream usage changed'); + +const controller = new AbortController(); +const pending = client.responses.create( + {model: 'capsem-sdk-cancel', input: 'wait'}, + {signal: controller.signal}, +); +setTimeout(() => controller.abort(), 200); +let cancelled = false; +try { + await pending; +} catch (error) { + cancelled = error instanceof APIUserAbortError || (error instanceof Error && error.name === 'AbortError'); +} +if (!cancelled) throw new Error('request cancellation was not observed'); + +const afterCancel = await client.chat.completions.create({ + model: 'gpt-fixture', + messages: [{role: 'user', content: 'Still alive?'}], +}); +if (afterCancel.choices[0]?.message.content !== 'Capsem ironbank poem\nledgers count the sparks\nno secret crosses raw') { + throw new Error('proxy did not survive client cancellation'); +} + +let upstreamStatus: number | undefined; +let upstreamCode: string | null | undefined; +try { + await client.responses.create({model: 'capsem-sdk-upstream-error', input: 'fail'}); +} catch (error) { + if (error instanceof APIError) { + const status: unknown = error.status; + const code: unknown = error.code; + if (typeof status === 'number') upstreamStatus = status; + if (typeof code === 'string' || code === null || code === undefined) upstreamCode = code; + } +} +if (upstreamStatus !== 503 || upstreamCode !== 'fixture_unavailable') { + throw new Error(`upstream failure changed: ${upstreamStatus}/${upstreamCode}`); +} + +console.log(JSON.stringify({ + client: 'typescript', + chat_stream: streamedChat, + chat_tool: toolCall.function.name, + chat_total_tokens: chat.usage.total_tokens, + responses_stream: responseText, + responses_tool: functionCall.name, + responses_total_tokens: responseUsage.total_tokens, + cancelled, + upstream_status: upstreamStatus, + upstream_code: upstreamCode, +})); From 994f03abc3d51d6ef20e77b05b51b02eda73fc0f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:34:40 +0000 Subject: [PATCH 192/464] test(ironbank): prove standalone model proxy --- tests/ironbank/test_standalone_proxy.py | 274 ++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 tests/ironbank/test_standalone_proxy.py diff --git a/tests/ironbank/test_standalone_proxy.py b/tests/ironbank/test_standalone_proxy.py new file mode 100644 index 000000000..2f49122dd --- /dev/null +++ b/tests/ironbank/test_standalone_proxy.py @@ -0,0 +1,274 @@ +"""Black-box acceptance for the VM-free OpenAI-compatible proxy.""" + +from __future__ import annotations + +import json +import os +import selectors +import signal +import socket +import sqlite3 +import subprocess +import textwrap +import time +from contextlib import closing +from pathlib import Path + +from helpers.constants import BIN_DIR +from helpers.mock_server import start_mock_server, stop_process +from helpers.service import PROJECT_ROOT, ServiceInstance +from helpers.session_ledger import open_session_ledger + +PYTHON_SECRET = "sk-capsem-standalone-python" +TYPESCRIPT_SECRET = "sk-capsem-standalone-typescript" + + +def _read_cli_startup(proc: subprocess.Popen[bytes]) -> tuple[str, str]: + assert proc.stdout is not None + selector = selectors.DefaultSelector() + selector.register(proc.stdout, selectors.EVENT_READ) + deadline = time.monotonic() + 15 + received = bytearray() + while time.monotonic() < deadline: + if proc.poll() is not None: + raise AssertionError( + f"capsem proxy exited before startup: {proc.returncode}; output={received!r}" + ) + for key, _ in selector.select(timeout=0.2): + received.extend(os.read(key.fileobj.fileno(), 4096)) + lines = received.decode(errors="replace").splitlines() + session = next( + (line.removeprefix("Proxy session: ") for line in lines if line.startswith("Proxy session: ")), + None, + ) + base_url = next( + (line.removeprefix("Base URL: ") for line in lines if line.startswith("Base URL: ")), + None, + ) + if session and base_url: + return session, base_url + raise AssertionError(f"capsem proxy startup output timed out: {received!r}") + + +def _run_json(command: list[str]) -> dict[str, object]: + completed = subprocess.run( + command, + cwd=PROJECT_ROOT, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=45, + check=False, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + lines = [line for line in completed.stdout.splitlines() if line.strip()] + assert lines, completed.stdout + completed.stderr + return json.loads(lines[-1]) + + +def _records(path: Path) -> list[dict[str, object]]: + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + + +def _wait_for_model_rows(db_path: Path, minimum: int = 8) -> list[sqlite3.Row]: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if db_path.exists(): + with closing(open_session_ledger(db_path)) as conn: + conn.row_factory = sqlite3.Row + rows = conn.execute("SELECT * FROM model_calls ORDER BY id").fetchall() + if len(rows) >= minimum: + return rows + time.sleep(0.25) + raise AssertionError(f"standalone ledger did not reach {minimum} model rows: {db_path}") + + +def _raw_http(port: int, request: bytes) -> bytes: + with socket.create_connection(("127.0.0.1", port), timeout=3) as stream: + stream.sendall(request) + stream.shutdown(socket.SHUT_WR) + chunks: list[bytes] = [] + while chunk := stream.recv(65536): + chunks.append(chunk) + return b"".join(chunks) + + +def _assert_no_raw_secret(db_path: Path, session_dir: Path) -> None: + with closing(open_session_ledger(db_path)) as conn: + tables = [ + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name" + ) + ] + for table in tables: + columns = conn.execute(f"PRAGMA table_info({table})").fetchall() + text_columns = [row[1] for row in columns if str(row[2]).upper() in {"TEXT", ""}] + if not text_columns: + continue + selected = ", ".join(f'"{column}"' for column in text_columns) + for row in conn.execute(f'SELECT {selected} FROM "{table}"').fetchall(): + rendered = "\n".join(str(value) for value in row) + assert PYTHON_SECRET not in rendered + assert TYPESCRIPT_SECRET not in rendered + for path in session_dir.iterdir(): + if path.is_file() and path != db_path: + content = path.read_bytes() + assert PYTHON_SECRET.encode() not in content, path + assert TYPESCRIPT_SECRET.encode() not in content, path + + +def _assert_sdk_result(result: dict[str, object], client: str) -> None: + assert result == { + "cancelled": True, + "chat_stream": "Capsem ironbank poem", + "chat_tool": "fixture_lookup", + "chat_total_tokens": 456, + "client": client, + "responses_stream": "Capsem ironbank poem\nledgers count the sparks\nno secret crosses raw", + "responses_tool": "exec_command", + "responses_total_tokens": 12, + "upstream_code": "fixture_unavailable", + "upstream_status": 503, + } + + +def test_standalone_proxy_serves_official_sdks_without_a_vm(): + service = ServiceInstance(sign_binaries=False) + mock_proc = None + proxy_proc: subprocess.Popen[bytes] | None = None + previous_corp = os.environ.get("CAPSEM_CORP_CONFIG") + try: + transcript = service.tmp_dir / "standalone-upstream.jsonl" + mock_proc, ready = start_mock_server(request_log=transcript) + corp_path = service.tmp_dir / "corp.toml" + corp_path.write_text( + textwrap.dedent( + f""" + refresh_policy = "24h" + + [network.upstream_overrides."api.openai.com:443"] + dial = {json.dumps(ready["http_addr"])} + protocol = "http" + + [corp.rules.allow_standalone_openai_fixture] + name = "allow_standalone_openai_fixture" + action = "allow" + priority = -100 + detection_level = "informational" + reason = "Allow the hermetic standalone OpenAI acceptance fixture." + match = 'http.host == "api.openai.com" && tcp.port == "443"' + """ + ).strip() + + "\n", + encoding="utf-8", + ) + os.environ["CAPSEM_CORP_CONFIG"] = str(corp_path) + service.start() + client = service.client() + assert client.get("/vms/list")["sandboxes"] == [] + + proxy_env = os.environ.copy() + proxy_env.update( + { + "CAPSEM_HOME": str(service.home_dir), + "CAPSEM_RUN_DIR": str(service.tmp_dir), + "HOME": str(service.home_dir), + } + ) + proxy_proc = subprocess.Popen( + [str(BIN_DIR / "capsem"), "--uds-path", str(service.uds_path), "proxy"], + cwd=PROJECT_ROOT, + env=proxy_env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + bufsize=0, + ) + session_id, base_url = _read_cli_startup(proxy_proc) + assert session_id.startswith("proxy-") + assert base_url.endswith("/v1") + assert client.get("/vms/list")["sandboxes"] == [] + + python_result = _run_json( + [ + "uv", + "run", + "--project", + "sdk/python", + "--frozen", + "python", + "sdk/python/tests/standalone_openai_client.py", + "--base-url", + base_url, + ] + ) + typescript_result = _run_json( + ["node", "sdk/typescript/tests/standalone-openai-client.ts", base_url] + ) + _assert_sdk_result(python_result, "python") + _assert_sdk_result(typescript_result, "typescript") + + port = int(base_url.split(":", 2)[2].split("/", 1)[0]) + before_connect = len(_records(transcript)) + connect_response = _raw_http( + port, + b"CONNECT attacker.invalid:443 HTTP/1.1\r\nHost: attacker.invalid:443\r\n\r\n", + ) + assert not connect_response.startswith(b"HTTP/1.1 2") + assert len(_records(transcript)) == before_connect, ( + "standalone endpoint must reject CONNECT instead of forwarding it" + ) + + absolute_response = _raw_http( + port, + b"GET http://attacker.invalid/secret HTTP/1.1\r\n" + b"Host: attacker.invalid\r\nConnection: close\r\n\r\n", + ) + assert not absolute_response.startswith(b"HTTP/1.1 2") + assert len(_records(transcript)) == before_connect, ( + "standalone endpoint must reject absolute-form forward-proxy requests" + ) + + session_dir = service.tmp_dir / "sessions" / session_id + db_path = session_dir / "session.db" + model_rows = _wait_for_model_rows(db_path) + assert {row["path"] for row in model_rows} == { + "/v1/chat/completions", + "/v1/responses", + } + assert {row["provider"] for row in model_rows} == {"openai"} + assert {row["status_code"] for row in model_rows} >= {200, 503} + assert any(row["input_tokens"] == 66 and row["output_tokens"] == 390 for row in model_rows) + assert any(row["input_tokens"] == 7 and row["output_tokens"] == 5 for row in model_rows) + upstream = _records(transcript) + assert {row["path"] for row in upstream} >= { + "/v1/chat/completions", + "/v1/responses", + } + assert all(row["headers"].get("host") == "api.openai.com" for row in upstream) + _assert_no_raw_secret(db_path, session_dir) + + proxy_proc.send_signal(signal.SIGTERM) + assert proxy_proc.wait(timeout=15) == 0 + proxy_proc = None + deadline = time.monotonic() + 5 + while session_dir.exists() and time.monotonic() < deadline: + time.sleep(0.1) + assert not session_dir.exists() + with socket.socket() as probe: + probe.settimeout(1) + assert probe.connect_ex(("127.0.0.1", port)) != 0 + assert client.get("/vms/list")["sandboxes"] == [] + finally: + if proxy_proc is not None: + proxy_proc.kill() + proxy_proc.wait(timeout=5) + service.stop() + stop_process(mock_proc) + if previous_corp is None: + os.environ.pop("CAPSEM_CORP_CONFIG", None) + else: + os.environ["CAPSEM_CORP_CONFIG"] = previous_corp From 1e118cdc3315ec4a8a20fdcba67c0cbad1d33023 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:35:04 +0000 Subject: [PATCH 193/464] chore(surface): register standalone proxy API --- config/public-surface.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/config/public-surface.toml b/config/public-surface.toml index 9830cb5db..aae48c5e1 100644 --- a/config/public-surface.toml +++ b/config/public-surface.toml @@ -37,7 +37,7 @@ approved = [ ] [capsem_cli] -count = 41 +count = 42 approved = [ "assets ensure", "assets status", @@ -67,6 +67,7 @@ approved = [ "network list", "network logs", "persist", + "proxy", "purge", "restart", "resume", @@ -83,7 +84,7 @@ approved = [ ] [http] -count = 95 +count = 98 approved = [ # All nine connection/grant routes approved by the user on 2026-10-14. "GET /connections", @@ -155,6 +156,9 @@ approved = [ "POST /corp/reload", "POST /corp/validate", "POST /purge", + "POST /proxies", + "POST /proxies/{id}/heartbeat", + "POST /proxies/{id}/stop", "POST /restart", "POST /run", "POST /internal/vms/{id}/exposures/{exposure_id}/preview-admission", From 0b752d7eced956c2355e579d5e6b1c1289405072 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:37:15 +0000 Subject: [PATCH 194/464] test(ci): execute confined workers on both hosts --- .github/workflows/ci.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 906027e6e..0a1a39146 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -126,6 +126,12 @@ jobs: timeout-minutes: 45 run: just test-linux-rust + # These integration targets execute the real self-confined worker + # binaries. The unit coverage command uses --lib/--bins and therefore + # cannot prove Landlock/seccomp took effect before readiness. + - name: Confined worker subprocess tests + run: cargo test --locked -p capsem-ledger -p capsem-proxy --test subprocess + # cargo check still runs BLAKE3's native SIMD build. Both guest targets # belong on their native Linux C toolchain, already provisioned above. # Build and link the production feature set, not a pure-Rust substitute. @@ -284,6 +290,11 @@ jobs: cargo llvm-cov nextest --no-cfg-coverage --lib --bins --profile ci-unit --codecov --output-path cache/target/coverage/rust/unit.json -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-ledger -p capsem-proxy -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server cargo llvm-cov report --summary-only -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-ledger -p capsem-proxy -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server 2>&1 | tee cache/target/coverage/rust/summary.txt + # Run the production executables under the host's real Seatbelt policy. + # --lib/--bins above compiles but does not execute integration targets. + - name: Confined worker subprocess tests + run: cargo test --locked -p capsem-ledger -p capsem-proxy --test subprocess + # Integration tests (tests/ directory, cross-crate) - name: Integration tests with coverage run: | From b615f35db1a21abf20ef7e388e0472953b8b9cb1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:44:36 +0000 Subject: [PATCH 195/464] fix(mcp): sync linked SDK dependency lock --- mcp/typescript/pnpm-lock.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/mcp/typescript/pnpm-lock.yaml b/mcp/typescript/pnpm-lock.yaml index 8000fad70..66f21efbc 100644 --- a/mcp/typescript/pnpm-lock.yaml +++ b/mcp/typescript/pnpm-lock.yaml @@ -49,6 +49,9 @@ importers: eslint: specifier: 10.10.0 version: 10.10.0 + openai: + specifier: 7.31.0 + version: 7.31.0(zod@4.6.1) typescript: specifier: 5.9.3 version: 5.9.3 @@ -950,6 +953,30 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + openai@7.31.0: + resolution: {integrity: sha512-ZGAzauuYb0DdB53qVbIR+X8rdmNtWgiElrrMthbZCVreLcIsQJBwzAIWUbpB2FkQu2Cj6OJ9zJR1EKMUYn4/SQ==} + engines: {node: '>=22.0.0'} + peerDependencies: + '@aws-sdk/credential-provider-node': '>=3.972.0 <4' + '@smithy/hash-node': '>=4.3.0 <5' + '@smithy/signature-v4': '>=5.4.0 <6' + undici: '>=5 <9' + ws: ^8.21.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + '@aws-sdk/credential-provider-node': + optional: true + '@smithy/hash-node': + optional: true + '@smithy/signature-v4': + optional: true + undici: + optional: true + ws: + optional: true + zod: + optional: true + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -2131,6 +2158,10 @@ snapshots: dependencies: wrappy: 1.0.2 + openai@7.31.0(zod@4.6.1): + optionalDependencies: + zod: 4.6.1 + optionator@0.9.4: dependencies: deep-is: 0.1.4 From 9307327d6fa2b99e155b89f77ae4729536c4fabf Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 09:46:12 +0000 Subject: [PATCH 196/464] docs(architecture): document confined host workers --- .../docs/architecture/host-isolation.md | 270 ++++++++++++++++++ .../docs/architecture/mcp-aggregator.md | 21 +- .../content/docs/architecture/mcp-gateway.md | 50 ++-- .../content/docs/architecture/mitm-proxy.md | 19 +- .../docs/architecture/service-architecture.md | 76 +++-- .../docs/architecture/session-telemetry.md | 63 ++-- .../docs/security/network-isolation.md | 19 +- .../src/content/docs/security/overview.md | 25 +- .../content/docs/security/virtualization.md | 33 +-- web/docs/src/content/docs/usage/cli.md | 33 +++ 10 files changed, 510 insertions(+), 99 deletions(-) create mode 100644 web/docs/src/content/docs/architecture/host-isolation.md diff --git a/web/docs/src/content/docs/architecture/host-isolation.md b/web/docs/src/content/docs/architecture/host-isolation.md new file mode 100644 index 000000000..a0ce5a2ae --- /dev/null +++ b/web/docs/src/content/docs/architecture/host-isolation.md @@ -0,0 +1,270 @@ +--- +title: Host Process Isolation +description: Trusted coordination, confined workers, descriptor capabilities, and lifecycle revocation. +sidebar: + order: 5 +--- + +Capsem does not treat every process running as the desktop user as equally +trusted. The service is the trusted coordinator. It creates resources, selects +policy, and grants already-open descriptors to small workers whose operating +system sandbox limits what they can do after startup. + +This page describes the host boundary. [Virtualization Security](/security/virtualization/) +describes the guest boundary, and [Network Isolation](/security/network-isolation/) +describes packet paths inside a VM. + +## Ownership model + +```mermaid +flowchart TB + CLI["capsem CLI"] + UI["app and tray"] + HostMCP["host MCP client"] + Gateway["capsem-gateway
global confined API worker"] + Service["capsem-service
trusted coordinator"] + + subgraph Session["one VM session"] + Owner["capsem-process
VM owner"] + Proxy["capsem-proxy
HTTP, DNS, and MCP policy"] + Ledger["capsem-ledger
sole session.db owner"] + Relay["capsem-router --expose
published-port byte relay"] + Aggregator["capsem-mcp-aggregator
external MCP transport"] + MCPBridge["VM-owner MCP adapter
scoped tools and transport"] + VM["guest VM"] + end + + subgraph PrivateNetwork["one named private network"] + Switch["capsem-router --network
L2 switch"] + end + + UI -->|"authenticated control"| Gateway + HostMCP -->|"authenticated control"| Gateway + CLI -->|"local UDS control"| Service + Gateway -->|"granted service channels"| Service + Service -->|"typed lifecycle control"| Owner + Service -->|"generation-bound capabilities"| Proxy + Service -->|"role-bound ledger channels"| Ledger + Owner -->|"virtio and VSOCK"| VM + Owner -->|"connected stream pairs"| Relay + Proxy -->|"MCP requests after policy"| MCPBridge + Owner -->|"owns"| MCPBridge + MCPBridge --> Aggregator + Owner -->|"one cable per membership"| Switch + Proxy -->|"admit and flush only"| Ledger + Owner -->|"admit and flush only"| Ledger + Service -->|"read, retain, export, snapshot"| Ledger +``` + +There is one `capsem-process`, `capsem-proxy`, and `capsem-ledger` generation +per running VM session. A stopped retained session has no VM owner or proxy; +the service starts or reconnects its ledger worker when a read or maintenance +operation needs the retained ledger. Standalone model-proxy sessions have a +proxy and ledger but no VM owner. Each named private network has one switch +process. Each running VM with published ports has one publication relay. + +Libraries do not add process boundaries. `capsem-core`, `capsem-logger`, and +the Rust SDK hold shared types and behavior inside the binaries above. Tokio +tasks and worker threads share their process sandbox. + +## Three kinds of traffic + +Keeping the paths separate prevents a data connection from becoming control +authority. + +| Path | Examples | Authority | +|---|---|---| +| Control | client API, service-to-owner IPC, worker grant frames, policy reload | Typed and bounded messages; fresh process generation required | +| Private data plane | VSOCK, private-network frames, published-port byte streams | Connected descriptors for one session, network membership, or flow | +| Proxied traffic | HTTP, DNS, model API, framed guest MCP | A descriptor granted to the session proxy plus separate upstream, policy, credential, ledger, and telemetry capabilities | + +A worker does not receive a path and then reopen the resource. The coordinator +opens or accepts the resource, creates a connected channel, attaches the file +descriptor to a fixed-size control record, and waits for the current worker +generation to adopt it. Unknown, duplicated, stale, oversized, or out-of-order +grants fail closed. + +## Process authority + +| Process | Permitted after readiness | Denied or absent | +|---|---|---| +| `capsem-service` | Global lifecycle, settings and corp policy, credential store, session registry, resource creation and capability grants | It is part of the trusted computing base; clients reach it through the local UDS or authenticated gateway routes | +| `capsem-gateway` | Accept on listeners bound before confinement; use coordinator-granted service/owner channels; update its private readiness files | Ambient file reads, path-based UDS connects, outbound TCP, new binds, process execution | +| `capsem-process` | One VM and its exact session runtime paths; read exact boot assets and active policy; use coordinator-granted upstream and ledger channels; execute only the installed router helper | Other sessions, direct `session.db` access, ambient outbound connects/binds, unrelated host files and processes | +| `capsem-proxy` | Consume granted HTTP, DNS, MCP, upstream, credential, ledger, metrics, private-name, and policy channels | All filesystem access, socket creation, arbitrary dial/bind/listen, process execution, signals to the coordinator | +| `capsem-ledger` | Read and write one session directory; serve authenticated operations over granted connected channels | Other paths, all network, new Unix sockets, process execution, signals to the coordinator | +| `capsem-router --expose` | Copy bytes between connected host TCP and guest VSOCK descriptors | Listener ownership, destination selection, filesystem, arbitrary network, VM control | +| `capsem-router --network` | Switch frames among the descriptor-backed cables of one named network | Uplink, listener, outbound socket, another network's members, policy decisions | +| `capsem-mcp-aggregator` | Connect to configured external MCP servers and return protocol results | VM control, session files, ledger storage, service API; requests arrive only after the session policy boundary | + +The gateway, VM owner, proxy, and ledger install a platform sandbox and perform +negative self-checks before publishing readiness. A startup that can still read +an ambient host file, dial a path-owned control socket, execute an ungranted +program, or signal its coordinator is rejected. + +## Platform enforcement + +On macOS, workers install Seatbelt profiles. Proxy and ledger workers use +`deny default`; the proxy has no path parameters, and the ledger receives one +read-write session directory. Gateway and VM-owner profiles deny filesystem, +network, execution, and cross-sandbox signal operations, then add their exact +path grants. Only the gateway may accept inbound traffic on a listener it +already owns. + +On Linux, workers install a Landlock ABI 6 ruleset across every existing thread +and then a seccomp filter with thread synchronization. Landlock limits +filesystem and network scope; seccomp denies direct connect, bind, listen, +process inspection, namespace/mount operations, kernel attack surfaces, and +permission/ownership changes. Proxy and ledger workers also cannot create +sockets or send signals. Existing connected descriptors remain usable because +they are the capability. + +Both platforms clear inherited environment state where secrets could otherwise +leak and close descriptors that were not deliberately preserved. The sandbox +is additive to normal ownership and mode checks; mode `0600` sockets and mode +`0700` session directories remain in use. + +## Shared proxy engine + +VM interception and the standalone model API are transport adapters around the +same `ProxyEngine`. + +```mermaid +flowchart LR + Guest["VM HTTP, DNS, or MCP
over VSOCK"] --> VMAdapter["VM adapter"] + SDK["OpenAI-compatible SDK
plaintext local HTTP"] --> Standalone["standalone adapter
fixed configured provider"] + VMAdapter --> Engine["ProxyEngine
one policy snapshot per request"] + Standalone --> Engine + Engine --> Rules["CEL rules and plugins"] + Engine --> Credentials["credential broker"] + Engine --> Upstream["coordinator-brokered upstream"] + Engine --> Ledger["session ledger capability"] + Engine --> Metrics["metric capability"] + Engine --> MCP["scoped MCP capability"] +``` + +The engine owns request normalization, policy evaluation, preprocessing and +postprocessing, credential capture/substitution, model and tool parsing, +redaction, and ledger emission. A request keeps one immutable policy snapshot +through routing and completion. Policy reload atomically replaces the snapshot +used by later requests. + +The VM adapter accepts intercepted TLS/plain HTTP, DNS, and framed MCP traffic. +The standalone adapter accepts only origin-form OpenAI-compatible HTTP and pins +the host, port, scheme, and base path to the selected provider from effective +policy. It rejects `CONNECT` and absolute-form forward-proxy requests. + +MCP sits above the SDK/model protocol layer. Model-native tool calls are parsed +and logged by the shared engine. Guest MCP frames enter the same security-event +rail before a scoped request reaches the MCP aggregator. A model call may emit +many tool calls, and later model requests can carry their responses; the ledger +links them with `turn_id`, `model_call_id`, and `tool_call_id`. + +## Ledger ownership + +`capsem-ledger` is the only process that opens `session.db` and its body +archive. A connected channel carries a generation and a coordinator-assigned +role: + +| Role | Operations | +|---|---| +| VM owner, proxy, coordinator | Admit events and flush | +| Reader | Read and export | +| Maintainer | Read, retain, export, and take a coherent snapshot | +| Supervisor | Shutdown | + +The VM owner and proxy use `DbWriter` as a remote typed client; a diagnostic +path in that object does not grant filesystem authority. Service routes also +query through a ledger client, including after the VM stops. A missing table or +column is a schema error, and a flush is the read-after-write barrier. + +Raw observed credentials do not belong in ledger rows, body archives, worker +logs, or metrics. The proxy sends credential observations over its broker +capability and receives only the material needed for the selected upstream. + +## Revocation, failure, and recovery + +Every VM spawn, proxy worker, and ledger worker gets a fresh generation. A +session ID, PID, path, or numeric request ID is never enough to recover old +authority. Removing or replacing the registered generation cancels its grants; +closing the connected descriptor interrupts in-flight capability use. + +The service supervises children and bounds startup, adoption, shutdown, and +cleanup. Parent death closes control channels and triggers child exit. VM stop +revokes traffic before process teardown. A proxy or ledger crash makes the +dependent operation fail instead of falling back to direct host access. The +service can start a fresh ledger generation for later stopped-session reads. +A private-network switch failure affects only that network; current members are +replugged into a fresh switch, while departed members remain revoked. + +`capsem proxy` adds a 15-second lease renewed by the foreground CLI every five +seconds. Ctrl-C, SIGTERM, service shutdown, heartbeat failure, or lease expiry +revokes the listener and capabilities, stops both workers, and removes the +ephemeral proxy session directory. + +## Standalone OpenAI-compatible endpoint + +Start an endpoint for a provider key from effective built-in, user, and corp +policy: + +```sh +capsem proxy --provider openai +# Proxy session: proxy-... +# Base URL: http://127.0.0.1:49152/v1 +# Press Ctrl-C to stop. +``` + +Point an official SDK at the printed base URL. OpenAI SDKs require a nonempty +client key, but Capsem's credential broker controls the upstream credential and +redacts the observed client value. + +```python +import os +from openai import OpenAI + +client = OpenAI( + api_key="capsem-client-placeholder", + base_url=os.environ["CAPSEM_PROXY_BASE_URL"], +) +response = client.responses.create(model="gpt-5", input="Hello through Capsem") +print(response.output_text) +``` + +```ts +import OpenAI from "openai"; + +const client = new OpenAI({ + apiKey: "capsem-client-placeholder", + baseURL: process.env.CAPSEM_PROXY_BASE_URL, +}); +const response = await client.responses.create({ + model: "gpt-5", + input: "Hello through Capsem", +}); +console.log(response.output_text); +``` + +The printed listener has no application authentication. Its default loopback +bind limits access to local processes; selecting a non-loopback `--bind` makes +network access control the operator's responsibility. The endpoint governs +only requests directed to its base URL. It does not confine the calling host +process, intercept that process's other traffic, provide a general forward +proxy, execute tools, or create a VM. VM traffic remains the stronger choice +when untrusted code itself needs containment. + +## Verified limits + +- The trusted computing base includes `capsem-service`, the host kernel, the + hypervisor, and the package/runtime artifacts they load. +- OS sandbox rules reduce a compromised worker's ambient authority. They do + not protect against a compromised coordinator or kernel. +- Connected descriptors carry real authority until closed. The coordinator + therefore grants them to one fresh generation and retains revocation. +- The standalone endpoint supports the OpenAI Chat Completions and Responses + request shapes exercised by the official Python and TypeScript SDKs, + including streams, tool payloads, cancellation, usage, and upstream errors. + Provider-specific extensions remain subject to the configured provider and + shared parser behavior. +- Private-network traffic is switched end to end between member guests and + does not traverse the HTTP proxy. Membership and port-security checks are + its boundary. diff --git a/web/docs/src/content/docs/architecture/mcp-aggregator.md b/web/docs/src/content/docs/architecture/mcp-aggregator.md index 5d01e88d7..129564bb3 100644 --- a/web/docs/src/content/docs/architecture/mcp-aggregator.md +++ b/web/docs/src/content/docs/architecture/mcp-aggregator.md @@ -5,24 +5,35 @@ sidebar: order: 26 --- -The MCP aggregator (`capsem-mcp-aggregator`) is a low-privilege subprocess that manages connections to external MCP servers. It runs in an isolated process with only network access -- no VM, no session database, no filesystem, no service IPC. +The MCP aggregator (`capsem-mcp-aggregator`) is a low-privilege subprocess that +manages connections to external MCP servers. It has no VM, ledger storage, or +service API authority. Guest MCP requests reach it only through the confined +session proxy after policy evaluation. ## Why a separate process -External MCP servers require network access, broker-resolved auth material, and custom HTTP headers. The main per-VM process (`capsem-process`) has extensive privileges: VM control, session database, VirtioFS workspace, service IPC. Running external server connections inside capsem-process would expose all of those privileges to any vulnerability in an MCP server connection or the HTTP/SSE transport layer. +External MCP servers require network access, broker-resolved auth material, +and custom HTTP headers. The per-VM owner controls virtualization and exact +session runtime paths, while `capsem-ledger` owns storage and `capsem-proxy` +owns MCP policy. Running external server transports inside any of those +processes would combine unrelated authority with remote protocol parsing. The aggregator subprocess enforces a hard privilege boundary: | | capsem-process | capsem-mcp-aggregator | |---|---|---| | VM control (vsock) | Yes | No | -| Session database | Yes | No | +| Session database path | No | No | | VirtioFS workspace | Yes | No | -| Service IPC | Yes | No | +| Scoped service/owner control | Yes | No | | Network (external MCP servers) | No | Yes | | Broker-resolved auth material | No | Yes | -If the aggregator is compromised, the attacker has network access and short-lived MCP auth material resolved by the broker -- but cannot reach the VM, read telemetry, or modify files. +If the aggregator is compromised, its remaining authority is the configured +external MCP transport and broker-resolved material needed there. It cannot +control the VM, open a session ledger, or modify the workspace. See +[Host Process Isolation](/architecture/host-isolation/) for the complete +process matrix. ## Architecture diff --git a/web/docs/src/content/docs/architecture/mcp-gateway.md b/web/docs/src/content/docs/architecture/mcp-gateway.md index 00fb08f16..7ed46b3f4 100644 --- a/web/docs/src/content/docs/architecture/mcp-gateway.md +++ b/web/docs/src/content/docs/architecture/mcp-gateway.md @@ -19,9 +19,9 @@ graph TB GATEWAY -->|explicit routes| SERVICE["capsem-service"] GUEST_AGENT["Guest AI agent"] -->|stdio| RELAY["capsem-mcp-server"] - RELAY -->|framed vsock| ENDPOINT["VM-owned MCP endpoint"] - ENDPOINT -->|scoped owner tool| OWNER["VM owner Publisher"] - ENDPOINT -->|policy and telemetry| AGG["capsem-mcp-aggregator"] + RELAY -->|"framed vsock surrendered by VM owner"| ENDPOINT["confined capsem-proxy MCP endpoint"] + ENDPOINT -->|"scoped owner capability"| OWNER["VM owner MCP adapter"] + OWNER --> AGG["capsem-mcp-aggregator"] AGG --> BUILTIN["capsem-mcp-builtin"] AGG --> EXTERNAL["External MCP servers"] ``` @@ -64,7 +64,10 @@ sanitized category and optional HTTP status without gateway bodies or secrets. ## Guest MCP relay (capsem-mcp-server) -The guest MCP relay is a minimal stdio-to-framed-vsock bridge. It does not route or execute tools; the host MITM MCP endpoint owns parsing, policy, telemetry, and dispatch. +The guest MCP relay is a minimal stdio-to-framed-vsock bridge. It does not +route or execute tools. The VM owner surrenders the connected VSOCK descriptor +to the confined proxy, whose MCP endpoint owns parsing, policy, telemetry, and +dispatch through a separately granted MCP capability. ### Framed relay @@ -72,7 +75,7 @@ The guest MCP relay is a minimal stdio-to-framed-vsock bridge. It does not route sequenceDiagram participant Agent as Guest AI process participant Relay as capsem-mcp-server - participant EP as Host MITM MCP Endpoint + participant EP as Confined proxy MCP endpoint Relay->>EP: \0CAPSEM_META:claude\n (metadata) Agent->>Relay: {"jsonrpc":"2.0","method":"tools/list"}\n (stdin) @@ -155,19 +158,20 @@ telemetry bug. See [Session Telemetry](/architecture/session-telemetry/) for the full `tool_calls`, `tool_responses`, and security-rule ledger joins. -## Endpoint runtime state +## Endpoint capabilities -| Field | Type | Purpose | -|-------|------|---------| -| `aggregator` | `AggregatorClient` | Client handle for the isolated MCP aggregator subprocess | -| `db` | `Arc` | Async telemetry writer | -| `security_rules` | `RwLock>` | Hot-reloadable security-event rules | -| `plugin_policy` | `RwLock>` | Hot-reloadable plugin modes for security-event preprocessing/postprocessing | +| Capability | Purpose | +|-------|------| +| MCP channel | Bounded requests to the VM owner's scoped tools and isolated aggregator | +| Ledger channel | Admit tool, protocol, and security rows without a database path | +| Policy channel | Apply exact active-policy bytes and atomically replace the snapshot | +| Credential channel | Capture and substitute brokered material without store access | +| Traffic descriptor | One already-connected guest MCP stream | -The `AggregatorClient` is cloneable (`Arc`-wrapped mpsc channel) and shared -across endpoint sessions for a given VM. The rule set uses double-Arc style -atomic swap through the endpoint state. New frames read the current rules, so -reloads affect already-open guest MCP connections. +The proxy snapshots the current policy for each request. Reloads therefore +affect new frames on an existing stream without mixing revisions inside one +request. Closing the MCP or traffic descriptor revokes that part of the +endpoint immediately. ## Configuration files @@ -191,11 +195,13 @@ url = "https://mcp.example.com/github" enabled = true ``` -The service validates the merged configuration and writes it into each -session's `vm/active_policy.toml`; capsem-process reads that file and passes -the server list to the [MCP Aggregator](/architecture/mcp-aggregator/) -subprocess at spawn time. Credentials are broker-owned references, not raw -tokens in MCP config. +The service validates the merged configuration and materializes an exact +active-policy revision for the session. The VM owner builds the scoped MCP +transport and passes server definitions to the +[MCP Aggregator](/architecture/mcp-aggregator/) at spawn time; the proxy gets +policy bytes and a connected MCP capability, not the policy path or +aggregator process. Credentials are broker-owned references, not raw tokens in +MCP config. ## Key source files @@ -205,6 +211,8 @@ tokens in MCP config. | `capsem-agent/src/mcp_server.rs` | Guest relay: stdin/stdout <-> framed MCP over vsock:5002 | | `capsem-core/src/net/mitm_proxy/mcp_frame.rs` | Framed transport parser, stream lifecycle, and disconnect metrics | | `capsem-core/src/net/mitm_proxy/mcp_endpoint.rs` | Host endpoint: JSON-RPC dispatch, policy, telemetry | +| `capsem-proxy/src/main.rs` | Confined endpoint runtime and descriptor capability adoption | +| `capsem-process/src/proxy_mcp.rs` | Scoped bridge from the proxy to owner tools and aggregator | | `capsem-proto/src/mcp_aggregator.rs` | Aggregator protocol types and `AggregatorClient` | | `capsem-core/src/mcp/builtin_tools.rs` | Builtin HTTP tools (fetch_http, grep_http, http_headers) | | `capsem-mcp-aggregator/src/server_manager.rs` | External MCP server lifecycle and tool catalog | diff --git a/web/docs/src/content/docs/architecture/mitm-proxy.md b/web/docs/src/content/docs/architecture/mitm-proxy.md index 9507996ba..23cb8b359 100644 --- a/web/docs/src/content/docs/architecture/mitm-proxy.md +++ b/web/docs/src/content/docs/architecture/mitm-proxy.md @@ -5,10 +5,17 @@ sidebar: order: 15 --- -The MITM proxy is Capsem's HTTPS inspection layer. It terminates TLS from the -guest, normalizes protocol details into `SecurityEvent`, evaluates the shared -security rule rail, forwards allowed requests to the real upstream, and logs -telemetry plus matched rule rows to the session database. +The proxy engine is Capsem's HTTP, DNS, model, and MCP inspection layer. In VM +mode it terminates TLS from the guest. In standalone mode it accepts plaintext +OpenAI-compatible HTTP on an explicit listener. Both adapters normalize +protocol details into `SecurityEvent`, evaluate the same rule rail, forward +allowed requests through a brokered upstream capability, and emit telemetry +through the session ledger capability. + +The engine runs in a per-session `capsem-proxy` process with no filesystem or +arbitrary socket authority. The service and VM owner pass it connected +descriptors for traffic, upstream, credentials, policy, telemetry, private +names, MCP, and the ledger. See [Host Process Isolation](/architecture/host-isolation/#shared-proxy-engine). ## Connection pipeline @@ -40,7 +47,7 @@ The proxy uses hyper for HTTP parsing and tokio-rustls for TLS. Each vsock conne graph LR CA["CertAuthority
(static CA keypair)"] POL["Network mechanics
(hot-swappable via RwLock)"] - DB["DbWriter
(async telemetry)"] + DB["ProxyLedger
(typed ledger capability)"] TLS["Upstream TLS config
(webpki roots)"] PRICE["PricingTable
(embedded JSON)"] TRACE["TraceState
(multi-turn linking)"] @@ -57,7 +64,7 @@ graph LR |-------|------|---------| | `ca` | `Arc` | Static Capsem CA for leaf cert minting | | `policy` | `Arc>>` | Hot-swappable network mechanics such as body capture and upstream port handling | -| `db` | `Arc` | Async telemetry writer to session.db | +| ledger | `Arc` | Typed admit/flush capability to the session's ledger worker | | `upstream_tls` | `Arc` | Shared TLS config with webpki root CAs | | `pricing` | `PricingTable` | Embedded model pricing for cost estimation | | `trace_state` | `Mutex` | Links multi-turn tool-use conversations by trace_id | diff --git a/web/docs/src/content/docs/architecture/service-architecture.md b/web/docs/src/content/docs/architecture/service-architecture.md index c5da58730..d180b4bc9 100644 --- a/web/docs/src/content/docs/architecture/service-architecture.md +++ b/web/docs/src/content/docs/architecture/service-architecture.md @@ -5,11 +5,15 @@ sidebar: order: 0 --- -Capsem uses a service-oriented architecture with multiple cooperating binaries. Every VM operation flows through a single path: client -> service -> per-VM process -> guest. +Capsem uses a service-oriented architecture with multiple cooperating binaries. +Every VM operation flows through a single path: client -> service -> per-VM +owner -> guest. Security-sensitive parsing and storage run in separately +confined proxy and ledger workers. See [Host Process Isolation](/architecture/host-isolation/) +for the authority of each process and its failure behavior. ## Host binaries -Seven binaries run on the host machine. They are installed to +The native host binaries are installed to `~/.capsem/bin/` by the platform package or source install flow. | Binary | Role | Communication | @@ -17,6 +21,9 @@ Seven binaries run on the host machine. They are installed to | **capsem** | CLI client | HTTP over UDS to service | | **capsem-service** | Background daemon | Axum HTTP over UDS (`~/.capsem/run/service.sock`) | | **capsem-process** | Per-VM process | Spawned by service, bounded MessagePack over UDS (after a MessagePack Hello) | +| **capsem-proxy** | Per-session HTTP, DNS, model, and MCP policy worker | Generation-bound connected descriptor grants | +| **capsem-ledger** | Sole owner of one session ledger | Role- and generation-bound connected ledger channels | +| **capsem-router** | Per-VM published-port relay or per-network L2 switch | Connected descriptor grants; never chooses or opens destinations | | **capsem-mcp-aggregator** | External MCP server connections | Length-prefixed MessagePack frames over stdin/stdout, spawned by capsem-process | | **capsem-mcp-builtin** | Built-in HTTP tools | stdio MCP, spawned by the aggregator | | **capsem-gateway** | HTTP/WebSocket gateway | TCP port 19222, proxies to service UDS | @@ -64,10 +71,15 @@ graph TD SVC["capsem-service (daemon)"] - SVC -->|"bounded MessagePack/UDS"| PROC["capsem-process (per-VM)"] + SVC -->|"bounded MessagePack/UDS"| PROC["capsem-process (per-VM owner)"] + SVC -->|"generation-bound descriptors"| PROXY["capsem-proxy (per session)"] + SVC -->|"role-bound channels"| LEDGER["capsem-ledger (per session)"] PROC -->|"MessagePack frames/stdio"| AGG["capsem-mcp-aggregator"] AGG -->|"HTTP/SSE"| EXT["External MCP servers"] + PROC -->|"HTTP/DNS/MCP descriptors"| PROXY + PROC -->|"typed writer channel"| LEDGER + PROXY -->|"typed writer channel"| LEDGER subgraph "Linux VM (guest)" AGENT["capsem-pty-agent"] @@ -78,9 +90,9 @@ graph TD end PROC -->|"vsock:5000,5001,5005,5006"| AGENT - PROC -->|"vsock:5002"| NETPROXY - PROC -->|"vsock:5007"| DNSPROXY - PROC -->|"vsock:5002"| MCPGW + NETPROXY -->|"vsock:5002 via owner"| PROXY + DNSPROXY -->|"vsock:5007 via owner"| PROXY + MCPGW -->|"vsock:5002 via owner"| PROXY PROC -->|"vsock:5004"| SYSUTIL ``` @@ -138,16 +150,28 @@ The CLI (`capsem`) auto-launches the service if it's not running. On every servi 3. Fall back to direct spawn 4. Poll socket for up to 5 seconds -## Per-VM process isolation - -Each running VM gets its own `capsem-process` child. This provides security isolation: - -- **Minimal environment**: service uses `env_clear()` before spawn -- API keys and tokens from the user's shell never reach the process -- **Socket permissions 0600**: only the owning user can connect to per-VM sockets -- **Session directory 0700**: contains workspace, system, serial.log, session.db -- **No guest-triggered exit**: control channel errors cause loop exit, not `process::exit()` -- **VirtioFS boundary**: only `session_dir/guest/` is shared -- host-only files (session.db, serial.log, checkpoints) are outside the share -- **MCP aggregator isolation**: external MCP server connections run in a separate subprocess (`capsem-mcp-aggregator`) with only network access -- no VM, database, or filesystem access. See [MCP Aggregator](/architecture/mcp-aggregator/) for details. +## Per-session process isolation + +Each running VM gets its own VM owner, proxy worker, and ledger worker. The +service creates resources and hands each worker connected descriptors instead +of paths or destination names. The short summary is: + +- **VM owner**: exact session runtime paths, boot assets, hypervisor, and granted + control channels; no direct session-ledger path access. +- **Proxy**: descriptor-only policy engine with no filesystem or arbitrary + socket authority. +- **Ledger**: the only process that opens one session's `session.db` and body + archive; no network or execution authority. +- **Minimal inheritance**: child environments are cleared where secrets could + leak and unrelated descriptors are closed before readiness. +- **OS confinement**: Seatbelt on macOS and Landlock plus seccomp on Linux. + Workers prove important denials before announcing readiness. +- **MCP transport**: external connections stay in `capsem-mcp-aggregator`; + guest requests cross proxy policy before receiving that scoped transport. + +Socket permissions `0600`, session directories `0700`, a read-only guest +rootfs, and the VirtioFS share boundary still apply. They complement the +process sandbox rather than defining it. ## Service HTTP API @@ -204,6 +228,17 @@ connection. | GET | `/vms/{id}/history/counts` | History counters | | GET | `/vms/{id}/history/transcript` | Terminal transcript history | +### Standalone Proxy Runtime + +The CLI owns a short lease and presents only the printed data-plane URL to the +model SDK. The lease token stays on the local service control path. + +| Method | Path | Purpose | +|--------|------|---------| +| POST | `/proxies` | Start a VM-free provider-pinned proxy and return its base URL plus lease | +| POST | `/proxies/{id}/heartbeat` | Renew the current lease token | +| POST | `/proxies/{id}/stop` | Revoke the listener and worker capabilities, then remove the ephemeral session | + ### Ledger Runtime | Method | Path | Purpose | @@ -277,7 +312,7 @@ authority path. ``` ~/.capsem/ - bin/ capsem, capsem-service, capsem-process, capsem-mcp-aggregator, capsem-mcp-builtin, capsem-gateway, capsem-tray + bin/ capsem, service, gateway, process, proxy, ledger, router, MCP workers, tray assets/ manifest.json, manifest-metadata.json, vmlinuz-{hash16}, initrd-{hash16}.img, rootfs-{hash16}.erofs run/ service.sock, service.pid, gateway.token, gateway.port, instances/ settings.toml User policy (rules, plugins, MCP, AI providers) and app preferences @@ -309,8 +344,11 @@ from VM asset releases. |-------|------|------| | `capsem-core` | lib | All shared business logic (VM, network, policy, telemetry, config) | | `capsem-service` | bin | Daemon. Axum HTTP over UDS, spawns/manages capsem-process children | -| `capsem-process` | bin | Per-VM. Boots VM via capsem-core, bridges vsock, job store | -| `capsem` | bin | CLI. HTTP over UDS to service, direct UDS to process for shell | +| `capsem-process` | bin | Per-VM owner. Boots VM, bridges VSOCK, owns session runtime paths | +| `capsem-proxy` | bin | Confined per-session HTTP, DNS, model, and MCP policy worker | +| `capsem-ledger` | bin | Confined sole owner of one session ledger and body archive | +| `capsem-router` | bin | Confined publication relay and named-network switch modes | +| `capsem` | bin | CLI. HTTP over UDS to service; streams remain service-mediated | | `capsem-mcp-aggregator` | bin | Isolated subprocess. Manages external MCP server connections over length-prefixed MessagePack frames | | `capsem-mcp-builtin` | bin | Isolated built-in HTTP MCP tools | | `capsem-gateway` | bin | HTTP gateway. Axum on TCP:19222, Bearer auth, `/vms/{id}/stream` WebSocket tunnel to the service | diff --git a/web/docs/src/content/docs/architecture/session-telemetry.md b/web/docs/src/content/docs/architecture/session-telemetry.md index c4c05b0cd..37f2e3131 100644 --- a/web/docs/src/content/docs/architecture/session-telemetry.md +++ b/web/docs/src/content/docs/architecture/session-telemetry.md @@ -5,7 +5,19 @@ sidebar: order: 20 --- -Every Capsem VM gets its own session ledger: a SQLite database (`session.db`) that records network requests, DNS queries, AI model calls, MCP tool invocations, exec activity, kernel audit events, file changes, security rule matches, and credential substitutions, and beside it a body archive directory (`session.bodies/`) that holds the full request, response, tool, exec and security payloads those rows describe. The database and archive directory live in the session directory and follow the VM lifecycle; retained and forked VMs keep both for forensic review, and one without the other is not a ledger. +Every Capsem VM and standalone proxy session gets its own session ledger: a +SQLite database (`session.db`) that records network requests, DNS queries, AI +model calls, MCP tool invocations, exec activity, kernel audit events, file +changes, security rule matches, and credential substitutions, and beside it a +body archive (`session.bodies`) that holds the full request, response, tool, +exec and security payloads those rows describe. The two files live in the +session directory and follow its lifecycle; retained and forked VMs keep both +for forensic review, and one without the other is not a ledger. + +A supervised `capsem-ledger` process is the sole filesystem owner of those +files. VM owners, proxy workers, and service readers use authenticated, +generation-bound channels with role-specific operations. See +[Host Process Isolation](/architecture/host-isolation/#ledger-ownership). The session ledger has no migrations. A ledger written by an older build fails to open, by name, instead of being upgraded in place. @@ -635,28 +647,31 @@ graph LR FS["VirtioFS
(file watcher)"] end - subgraph "Writer Pipeline" - CH["tokio mpsc channel"] - WT["Dedicated writer thread
(capsem-db-writer)"] + subgraph "capsem-ledger (one confined process per session)" + IPC["authenticated ledger channel
generation + client role"] + CH["bounded write queue"] + WT["dedicated SQLite owner thread"] DB["session.db
(SQLite WAL)"] BODIES["session.bodies/
(archive generations)"] end - MITM -->|"WriteOp::NetEvent
WriteOp::ModelCall"| CH - MCP -->|"WriteOp::McpCall"| CH - DNS -->|"WriteOp::DnsEvent"| CH - EXEC -->|"WriteOp::ExecEvent
WriteOp::ExecEventComplete"| CH - AUDIT -->|"WriteOp::AuditEvent"| CH - FS -->|"WriteOp::FileEvent"| CH + MITM -->|"WriteOp::NetEvent
WriteOp::ModelCall"| IPC + MCP -->|"WriteOp::McpCall"| IPC + DNS -->|"WriteOp::DnsEvent"| IPC + EXEC -->|"WriteOp::ExecEvent
WriteOp::ExecEventComplete"| IPC + AUDIT -->|"WriteOp::AuditEvent"| IPC + FS -->|"WriteOp::FileEvent"| IPC + IPC --> CH CH --> WT WT --> DB WT -->|"bodies, one segment per flush"| BODIES ``` -The writer thread owns the database and archive directory. A body is fed to the -open block's compressor as its event is written; each disk flush appends what -the block produced since the last one to the selected generation file as a -segment, syncs it, and only then commits the body's index row and the block's grown extent in +The ledger worker owns both files; its writer thread owns the SQLite +connection and archive writer. A body is fed to the open block's +compressor as its event is written; each disk flush appends what the block +produced since the last one to `session.bodies` as a segment, syncs it, and +only then commits the body's index row and the block's grown extent in `body_blocks`. ## Body archive @@ -857,7 +872,9 @@ transport outcome at that boundary. ## Writer Architecture -The `DbWriter` spawns a dedicated thread that owns the SQLite connection: +Inside `capsem-ledger`, the ledger server gives `DbWriter` a dedicated thread +that owns the SQLite connection. Callers see a typed remote handle and never a +database file descriptor: 1. Async callers send `WriteOp` via `tx.send()` (non-blocking) 2. Writer thread blocks on `rx.blocking_recv()` for the first op @@ -900,13 +917,11 @@ For AI provider traffic, the response body is parsed inline to extract: Session totals are not computed by scanning the ledger. The writer keeps a counter snapshot (`capsem_logger::counters::LedgerCounters`) and commits it in the same transaction as the rows it counts, so a count can never include a row -that was rolled back or refused. Readers take it whole with -`DbHandle::ledger_counters()`, from the handle's memory: the handle's reader -thread reads the snapshot when SQLite's `data_version` says the writer -committed -- on `ready()`, and on its own every 250 ms -- so a polled route -(`/vms/list`, `/info`, `stats/summary`, the security status routes) never -touches the file or the reader thread. `ready()` is the read-after-write -barrier. +that was rolled back or refused. Readers take it whole through the typed +ledger API. The ledger-owned reader state observes SQLite's `data_version` +after commits, so a polled route (`/vms/list`, `/info`, `stats/summary`, the +security status routes) does not open or fingerprint the file. A flush request +is the read-after-write barrier. | Counter group | Holds | |---------------|-------| @@ -931,7 +946,9 @@ When a session stops, the host ledger (`~/.capsem/sessions/host.db`) records thi Capsem does not expose arbitrary SQL over HTTP, gateway, frontend, or MCP. `session.db` and `session.bodies` are the durable ledger and can be inspected directly by a developer when doing local forensics, but product routes use -typed logger/database APIs. +typed, role-authorized ledger operations. Stopped retained sessions use the +same worker boundary: the service starts or reconnects the session's ledger +worker rather than opening SQLite in a route handler. Any hot `mem`/disk split belongs inside the logger DB object, never in service route state. diff --git a/web/docs/src/content/docs/security/network-isolation.md b/web/docs/src/content/docs/security/network-isolation.md index 175f753f4..cd712b23f 100644 --- a/web/docs/src/content/docs/security/network-isolation.md +++ b/web/docs/src/content/docs/security/network-isolation.md @@ -120,6 +120,22 @@ ordinary layer-2 switch, and joining it plugs a cable into that switch. because the transport can deliver a socket shutdown ahead of bytes still in flight. +## Standalone model API + +`capsem proxy` creates a VM-free OpenAI-compatible endpoint. The service binds +the requested listener, selects one configured provider, and grants accepted +client sockets to the same confined `capsem-proxy` engine used for VM traffic. +The standalone adapter pins every request to that provider's configured +scheme, host, port, and base path. It rejects `CONNECT` and absolute-form +forward-proxy requests before upstream dispatch. + +This mode applies Capsem model parsing, policy, credential brokering, +redaction, telemetry, and a per-proxy session ledger to a cooperating host SDK. +It does not intercept the host process's other traffic or isolate its files and +syscalls. The listener is unauthenticated and binds to loopback by default. +See [Host Process Isolation](/architecture/host-isolation/#standalone-openai-compatible-endpoint) +for examples and the authority boundary. + ## MITM proxy overview The host MITM proxy receives each connection on vsock:5002 and runs a full inspection pipeline: @@ -215,7 +231,8 @@ See [Policy](/security/policy/) for the full rule reference. ## Telemetry -Every proxied request is logged to the per-VM `session.db`: +Every proxied request is logged to the session ledger (`session.db` for a VM +or standalone proxy session): | Column | Content | |--------|---------| diff --git a/web/docs/src/content/docs/security/overview.md b/web/docs/src/content/docs/security/overview.md index 0cfbb20c2..36d7001c2 100644 --- a/web/docs/src/content/docs/security/overview.md +++ b/web/docs/src/content/docs/security/overview.md @@ -5,13 +5,18 @@ sidebar: order: 10 --- -Capsem sandboxes AI agents inside Linux VMs. The security model treats the guest as fully untrusted and the host as the trusted computing base. +Capsem sandboxes AI agents inside Linux VMs and splits sensitive host work into +confined processes. The guest is fully untrusted. The trusted computing base is +the coordinator service, host kernel, hypervisor, and verified package/runtime +artifacts; a parser worker running as the desktop user does not automatically +receive the coordinator's authority. ## Threat Model | Party | Trust Level | Goal | |-------|------------|------| -| Host (Capsem binary, macOS/Linux kernel) | Trusted | Contain guest escape, protect host resources | +| Host kernel, hypervisor, and `capsem-service` coordinator | Trusted | Create resources, grant bounded authority, contain the guest and workers | +| Gateway, VM owner, proxy, ledger, router, and MCP workers | Confined | Perform one role with explicit paths or connected descriptors | | Guest (AI agent, user code, guest kernel) | Untrusted | May attempt sandbox escape, resource exhaustion, data exfiltration | | Network (external services) | Controlled | DNS and HTTPS pass through host policy boundaries before upstream dispatch | @@ -22,7 +27,8 @@ Capsem sandboxes AI agents inside Linux VMs. The security model treats the guest - Unaudited data exfiltration via HTTPS **What Capsem does not defend against:** -- Compromised host processes (they already have equivalent privileges) +- A compromised coordinator service, host kernel, or hypervisor +- Unrelated same-user host applications outside Capsem's sandbox and lifecycle - Hardware side-channel attacks (mitigated by OS/firmware, not Capsem) - Denial of service against the guest itself (the guest is disposable) @@ -34,6 +40,7 @@ Capsem sandboxes AI agents inside Linux VMs. The security model treats the guest | **Kernel hardening** | No modules, no debugfs, no IPv6, no swap, read-only rootfs | Reduces guest kernel attack surface | | **Network isolation** | Air-gapped NIC, DNS proxy, iptables, MITM proxy | DNS and HTTPS are funneled through audited host policy handlers | | **Filesystem sandboxing** | VirtioFS with path validation, resource limits | Guest confined to workspace directory | +| **Host worker confinement** | Seatbelt or Landlock + seccomp, descriptor grants, fresh generations | Parser/storage/relay compromise is limited to the worker's current capabilities | | **Build verification** | Code signing, notarization, SBOM, OBOM | Host binary and VM base-image integrity | ## Trust Boundaries @@ -62,7 +69,17 @@ separate: upstream may need real protocol bytes, while session DB, structured logs, routes, and UI stats receive only the ledger-safe event output produced by logging plugins. Per-session telemetry records every request and DNS query. -**Filesystem boundary (VirtioFS):** The host VirtioFS server validates all path components, canonicalizes symlinks, and rejects any path that resolves outside the shared workspace. Resource limits prevent guest-driven host exhaustion. +**Filesystem boundary (VirtioFS):** The host VirtioFS server validates every +component and performs guest-writable filesystem operations relative to an +already-open workspace descriptor with no-follow semantics. Symlinks cannot +redirect a later host open outside the share. Resource limits prevent +guest-driven host exhaustion. + +**Host-process boundary:** The service opens resources and grants connected +descriptors to generation-bound workers. The proxy has no filesystem or socket +creation authority; the ledger can access one session directory and no +network; the gateway and VM owner receive exact path and inherited-listener +grants. See [Host Process Isolation](/architecture/host-isolation/). ## Per-Layer Documentation diff --git a/web/docs/src/content/docs/security/virtualization.md b/web/docs/src/content/docs/security/virtualization.md index e9ec4b756..8ef4c7f70 100644 --- a/web/docs/src/content/docs/security/virtualization.md +++ b/web/docs/src/content/docs/security/virtualization.md @@ -15,7 +15,7 @@ VirtioFS exposes a POSIX-compatible shared mount between host and guest. The gue |-----------|-------|-------------| | Guest FUSE client | Untrusted | May send malformed requests, attempt path traversal, exhaust resources | | Host VirtioFS server | Trusted | Must validate all requests, enforce limits, never trust guest input | -| Shared directory | Host-controlled | Capsem creates it fresh per session; no external process modifies it | +| Shared directory | Guest-writable through VirtioFS | Every host access must remain beneath the already-open workspace descriptor | ## Path Traversal Protection @@ -26,25 +26,18 @@ Every FUSE LOOKUP resolves a single path component (filename) under a parent ino - `.` and `..` - Names containing `/` or `\0` -**Path canonicalization** (resolves symlinks, verifies containment): -``` -child_path = parent_path.join(name) -canonical = child_path.canonicalize() -assert canonical.starts_with(root_canonical) -``` - -If the guest creates a symlink inside the workspace pointing outside it (e.g., `ln -s /etc/passwd escape`), `canonicalize()` follows the symlink and the containment check rejects the resolved path. - -### TOCTOU Analysis - -There is a time-of-check-to-time-of-use window between `canonicalize()` and the subsequent filesystem operation. Exploiting this window requires a host-side process to replace a directory in the workspace with a symlink between the two syscalls. - -This is acceptable because: -1. The untrusted party is the **guest**, not host processes. The guest communicates via FUSE opcodes and cannot manipulate the host filesystem directly. -2. The shared directory is **Capsem-controlled** (`~/.capsem/sessions//workspace/`). No external process should modify it during a VM session. -3. A malicious host process already has the same user privileges as Capsem and could attack directly. - -If defense-in-depth against compromised host processes is ever needed, the implementation can migrate to fd-relative operations (`openat` with `O_NOFOLLOW` + `O_PATH`). +**Descriptor-relative access** opens the workspace once, descends one component +at a time with `openat`, and uses `O_NOFOLLOW` in the same syscall that opens +an entry. Directory descriptors, rather than reconstructed absolute paths, +anchor later reads, writes, links, renames, metadata operations, and deletion. +An entry swapped for a symlink between requests cannot redirect a later open. + +The FUSE inode table also canonicalizes identities needed for guest symlink +semantics and checkpoint restore, but canonicalization is not the authority for +host filesystem syscalls. Guest-writable filesystem access goes through +`capsem_foundation::unix::contained::ContainedDir` or the equivalent +descriptor-backed VirtioFS operation. A symlink to `/etc/passwd`, FIFO, socket, +or device is reported as such and is never followed as a regular file. ## Resource Exhaustion Defenses diff --git a/web/docs/src/content/docs/usage/cli.md b/web/docs/src/content/docs/usage/cli.md index a4a062d41..c74fb72db 100644 --- a/web/docs/src/content/docs/usage/cli.md +++ b/web/docs/src/content/docs/usage/cli.md @@ -34,6 +34,10 @@ graph TD STOP["stop"] end + subgraph "Model Proxy" + PROXY["proxy"] + end + subgraph "Misc Commands" UPDATE["update"] DOCTOR["doctor"] @@ -394,6 +398,35 @@ cable. `-f` keeps printing new events until Ctrl-C. The history is kept by network id, so it survives disconnecting every member and deleting the network; a new network under the same name starts empty. +## Model proxy + +### proxy + +Run a VM-free OpenAI-compatible endpoint through Capsem's policy, credential, +model parsing, telemetry, and session-ledger pipeline. + +```sh +capsem proxy # configured OpenAI provider, random loopback port +capsem proxy --provider openai # explicit provider key +capsem proxy --bind 127.0.0.1 --port 8080 +``` + +| Flag | Default | Description | +|---|---|---| +| `--provider ` | `openai` | Provider key from effective built-in, user, and corp policy | +| `--bind ` | `127.0.0.1` | Address for the unauthenticated model API listener | +| `--port ` | `0` | Listener port; zero asks the OS for an available port | + +The command prints a `Base URL` ending in the configured provider's base path, +keeps a short service lease alive while it runs, and stops the endpoint on +Ctrl-C or SIGTERM. Point the official SDK's `base_url`/`baseURL` at that value. +The endpoint accepts origin-form model API requests; it is not a `CONNECT` or +absolute-form forward proxy. A non-loopback bind exposes an unauthenticated +listener, so its network access control belongs to the deployment. + +See [Host Process Isolation](/architecture/host-isolation/#standalone-openai-compatible-endpoint) +for Python and TypeScript examples, lifecycle behavior, and limits. + ## Service commands The background service (`capsem-service`) runs as a daemon. It auto-starts on login via LaunchAgent (macOS) or systemd (Linux). From 72ea5a44e755cded0b20e210fe0d45f9c3aa74a7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:14:24 +0000 Subject: [PATCH 197/464] fix(foundation): gate legacy syscalls by architecture --- .../src/unix/worker_sandbox/linux.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs index 433afa177..b03031f3a 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs @@ -235,12 +235,9 @@ fn install_seccomp(role: Role) -> io::Result<()> { libc::SYS_init_module, libc::SYS_finit_module, libc::SYS_delete_module, - libc::SYS_chmod, libc::SYS_fchmod, libc::SYS_fchmodat, - libc::SYS_chown, libc::SYS_fchown, - libc::SYS_lchown, libc::SYS_fchownat, libc::SYS_setxattr, libc::SYS_lsetxattr, @@ -248,11 +245,17 @@ fn install_seccomp(role: Role) -> io::Result<()> { libc::SYS_removexattr, libc::SYS_lremovexattr, libc::SYS_fremovexattr, + libc::SYS_utimensat, + ]; + #[cfg(target_arch = "x86_64")] + denied.extend([ + libc::SYS_chmod, + libc::SYS_chown, + libc::SYS_lchown, libc::SYS_utime, libc::SYS_utimes, libc::SYS_futimesat, - libc::SYS_utimensat, - ]; + ]); if matches!(role, Role::Gateway | Role::Ledger | Role::Proxy) { denied.extend([libc::SYS_execve, libc::SYS_execveat]); } From 9461ee1186094b8adfad9b2dc11caadc344202b0 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:22:59 +0000 Subject: [PATCH 198/464] refactor(archive): isolate block codec implementations --- crates/capsem-archive/src/codec.rs | 176 ++++++++++++++++++++++ crates/capsem-archive/src/format.rs | 2 +- crates/capsem-archive/src/format/tests.rs | 10 +- crates/capsem-archive/src/lib.rs | 12 +- crates/capsem-archive/src/reader.rs | 98 ++++++------ crates/capsem-archive/src/tests/stream.rs | 82 +++++++++- crates/capsem-archive/src/writer.rs | 110 +++++++------- 7 files changed, 368 insertions(+), 122 deletions(-) create mode 100644 crates/capsem-archive/src/codec.rs diff --git a/crates/capsem-archive/src/codec.rs b/crates/capsem-archive/src/codec.rs new file mode 100644 index 000000000..b830e16b9 --- /dev/null +++ b/crates/capsem-archive/src/codec.rs @@ -0,0 +1,176 @@ +//! Compression streams used by archive blocks. +//! +//! The archive owns framing, bounds and hashes. A codec owns only one block's +//! streaming compression state. Keeping this interface in the pure-Rust +//! archive crate lets the confined ledger process provide a native codec +//! without linking that native library into the service or session process. + +use std::io; +use std::sync::Arc; + +use flate2::{Compress, Compression, FlushCompress, Status}; +use miniz_oxide::inflate::stream::{inflate, InflateState}; +use miniz_oxide::{DataFormat, MZFlush, MZStatus}; + +use crate::format::{CODEC_DEFLATE, DEFLATE_LEVEL, SYNC_FLUSH_TAIL}; +use crate::{ArchiveError, Result}; + +/// One block's streaming compressor. +pub trait BlockEncoder: Send { + fn write(&mut self, input: &[u8], output: &mut Vec) -> Result<()>; + fn flush(&mut self, output: &mut Vec, finish: bool) -> Result<()>; +} + +/// One block's streaming decompressor. +pub trait BlockDecoder: Send { + /// Decode exactly one framed segment while retaining the block's history. + fn decode_segment(&mut self, input: &[u8], raw_len: usize, finish: bool, at: u64) -> Result>; +} + +/// Factory for the encoder and decoder of one on-disk codec id. +pub trait BlockCodec: Send + Sync { + fn id(&self) -> u8; + fn encoder(&self) -> Result>; + fn decoder(&self) -> Result>; +} + +/// The codec used for new blocks and the codecs accepted while reading. +#[derive(Clone)] +pub struct ArchiveCodecs { + write: Arc, + read: Arc<[Arc]>, +} + +impl ArchiveCodecs { + /// The v3 compatibility configuration: write and read raw deflate. + #[must_use] + pub fn deflate() -> Self { + let codec: Arc = Arc::new(DeflateCodec); + Self { + write: Arc::clone(&codec), + read: vec![codec].into(), + } + } + + /// Write with `codec` while retaining v3 deflate read compatibility. + pub fn with_write_codec(codec: Arc) -> Result { + let id = codec.id(); + if id == 0 || id == CODEC_DEFLATE { + return Err(ArchiveError::InvalidCodecId(id)); + } + let deflate: Arc = Arc::new(DeflateCodec); + Ok(Self { + write: Arc::clone(&codec), + read: vec![deflate, codec].into(), + }) + } + + pub(crate) fn encoder(&self) -> Result<(u8, Box)> { + Ok((self.write.id(), self.write.encoder()?)) + } + + pub(crate) fn decoder(&self, id: u8, block_offset: u64) -> Result> { + self.read + .iter() + .find(|codec| codec.id() == id) + .ok_or(ArchiveError::UnsupportedCodec { + block_offset, + codec: id, + })? + .decoder() + } +} + +impl Default for ArchiveCodecs { + fn default() -> Self { + Self::deflate() + } +} + +struct DeflateCodec; + +impl BlockCodec for DeflateCodec { + fn id(&self) -> u8 { + CODEC_DEFLATE + } + + fn encoder(&self) -> Result> { + Ok(Box::new(DeflateEncoder(Compress::new( + Compression::new(DEFLATE_LEVEL), + false, + )))) + } + + fn decoder(&self) -> Result> { + Ok(Box::new(DeflateDecoder(InflateState::new_boxed(DataFormat::Raw)))) + } +} + +struct DeflateEncoder(Compress); + +impl BlockEncoder for DeflateEncoder { + fn write(&mut self, input: &[u8], output: &mut Vec) -> Result<()> { + deflate_into(&mut self.0, input, output, FlushCompress::None) + } + + fn flush(&mut self, output: &mut Vec, finish: bool) -> Result<()> { + let flush = if finish { + FlushCompress::Finish + } else { + FlushCompress::Sync + }; + deflate_into(&mut self.0, &[], output, flush) + } +} + +struct DeflateDecoder(Box); + +impl BlockDecoder for DeflateDecoder { + fn decode_segment(&mut self, input: &[u8], raw_len: usize, finish: bool, at: u64) -> Result> { + if !finish && !input.ends_with(&SYNC_FLUSH_TAIL) { + return Err(ArchiveError::BadSegment(at)); + } + let mut raw = vec![0u8; raw_len + 1]; + let (mut consumed, mut written) = (0, 0); + let mut ended = false; + loop { + let result = inflate(&mut self.0, &input[consumed..], &mut raw[written..], MZFlush::None); + consumed += result.bytes_consumed; + written += result.bytes_written; + match result.status { + Ok(MZStatus::StreamEnd) => { + ended = true; + break; + } + Ok(_) if consumed == input.len() || (result.bytes_consumed == 0 && result.bytes_written == 0) => break, + Ok(_) => {} + Err(miniz_oxide::MZError::Buf) => break, + Err(error) => return Err(ArchiveError::Inflate(at, format!("{error:?}"))), + } + } + raw.truncate(written.min(raw_len)); + if consumed != input.len() || written != raw_len || ended != finish { + return Err(ArchiveError::Integrity(at)); + } + Ok(raw) + } +} + +fn deflate_into(compressor: &mut Compress, input: &[u8], out: &mut Vec, flush: FlushCompress) -> Result<()> { + let mut consumed = 0; + loop { + out.reserve((input.len() - consumed) + (input.len() - consumed) / 1000 + 64 * 1024); + let before = compressor.total_in(); + let status = compressor + .compress_vec(&input[consumed..], out, flush) + .map_err(|error| ArchiveError::Io(io::Error::other(error)))?; + consumed += usize::try_from(compressor.total_in() - before).expect("bounded by the input"); + let done = match flush { + FlushCompress::Finish => status == Status::StreamEnd, + _ => consumed == input.len() && out.len() < out.capacity(), + }; + if done { + return Ok(()); + } + } +} diff --git a/crates/capsem-archive/src/format.rs b/crates/capsem-archive/src/format.rs index 707343c54..854c6672c 100644 --- a/crates/capsem-archive/src/format.rs +++ b/crates/capsem-archive/src/format.rs @@ -225,7 +225,7 @@ pub fn parse_block_header(bytes: &[u8; BLOCK_HEADER_BYTES], block_offset: u64) - return Err(ArchiveError::BadBlockHeader(block_offset)); } let codec = bytes[4]; - if codec != CODEC_DEFLATE || bytes[5] != 0 || bytes[6..8] != [0, 0] { + if bytes[5] != 0 || bytes[6..8] != [0, 0] { return Err(ArchiveError::UnsupportedCodec { block_offset, codec }); } Ok(codec) diff --git a/crates/capsem-archive/src/format/tests.rs b/crates/capsem-archive/src/format/tests.rs index dd2530796..ac4b920ea 100644 --- a/crates/capsem-archive/src/format/tests.rs +++ b/crates/capsem-archive/src/format/tests.rs @@ -126,15 +126,9 @@ fn block_header_round_trips_its_codec() { } #[test] -fn an_unknown_codec_is_refused_by_name() { +fn a_structurally_valid_codec_id_is_left_for_the_codec_registry() { let header = encode_block_header(2); - assert!(matches!( - parse_block_header(&header, AT), - Err(ArchiveError::UnsupportedCodec { - block_offset: AT, - codec: 2 - }) - )); + assert_eq!(parse_block_header(&header, AT).unwrap(), 2); } #[test] diff --git a/crates/capsem-archive/src/lib.rs b/crates/capsem-archive/src/lib.rs index 08c61f5e4..3fc4384bf 100644 --- a/crates/capsem-archive/src/lib.rs +++ b/crates/capsem-archive/src/lib.rs @@ -2,7 +2,7 @@ //! //! SQLite stays the index of every ledger row and selects one immutable //! generation by typed archive and generation identities. Bodies (HTTP, -//! model, tool, security payloads) are fed to one open deflate stream, a +//! model, tool, security payloads) are fed to one open compression stream, a //! block, and each disk flush appends what that stream produced as a segment. //! The block stays open across flushes, so every body compresses against the //! ones before it, until it reaches its target size. Consecutive bodies from @@ -13,19 +13,21 @@ //! header. Descriptor-based readers enforce the ledger's committed file and //! block extents, so appended or torn bytes cannot widen a selected snapshot. //! -//! Pure Rust by rule: the runtime links one C library (SQLite) and this -//! crate must not add a second in front of attacker-influenced bytes. -//! `tests/citadel/test_runtime_native_dependencies.py` holds it. +//! This crate remains pure Rust. The codec interface lets a separately +//! confined owner provide a reviewed native implementation without adding +//! that dependency to every process which uses the archive types. #[cfg(not(unix))] compile_error!("capsem-archive relies on O_NOFOLLOW and mode 0600"); +mod codec; pub mod format; pub mod reader; pub mod retain; pub mod warc; pub mod writer; +pub use codec::{ArchiveCodecs, BlockCodec, BlockDecoder, BlockEncoder}; pub use format::{ ArchiveId, BodyRef, FileHeader, GenerationId, BLOCK_HEADER_BYTES, FILE_HEADER_BYTES, MAX_BLOCK_RAW_BYTES, TARGET_BLOCK_BYTES, @@ -61,6 +63,8 @@ pub enum ArchiveError { /// later writer's bytes as something they are not. #[error("block at offset {block_offset} uses unsupported codec {codec}")] UnsupportedCodec { block_offset: u64, codec: u8 }, + #[error("codec id {0} is reserved or conflicts with built-in deflate")] + InvalidCodecId(u8), /// A segment header that is not one, or that fails its bounds: wrong /// magic or flags, a raw extent that does not continue the block, a /// length past the ceilings, or an extent that does not end on a segment. diff --git a/crates/capsem-archive/src/reader.rs b/crates/capsem-archive/src/reader.rs index 0c635d007..890658b19 100644 --- a/crates/capsem-archive/src/reader.rs +++ b/crates/capsem-archive/src/reader.rs @@ -1,4 +1,4 @@ -//! Resolve a `BodyRef` to bytes, inflating a block segment by segment. +//! Resolve a `BodyRef` to bytes, decoding a block segment by segment. //! //! The SQLite index is the only source of truth for what exists: a reader //! never scans the file. It seeks to the block an index row names and walks @@ -24,17 +24,14 @@ use std::io::{Read, Seek, SeekFrom}; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -use capsem_foundation::unix::contained::{ContainedDir, ContainedOpenOptions}; -use capsem_foundation::unix::fs as unix_fs; -use miniz_oxide::inflate::stream::{inflate, InflateState}; -use miniz_oxide::{DataFormat, MZFlush, MZStatus}; - use super::format::{ self, BodyRef, FileHeader, SegmentHeader, BLOCK_HEADER_BYTES, FILE_HEADER_BYTES, MAX_BLOCK_RAW_BYTES, - SEGMENT_HEADER_BYTES, SYNC_FLUSH_TAIL, + SEGMENT_HEADER_BYTES, }; use crate::writer::refuse_symlink; -use crate::{ArchiveError, Result}; +use crate::{ArchiveCodecs, ArchiveError, BlockDecoder, Result}; +use capsem_foundation::unix::contained::{ContainedDir, ContainedOpenOptions}; +use capsem_foundation::unix::fs as unix_fs; /// Reads bodies back out of one `session.bodies`. /// @@ -49,7 +46,8 @@ pub struct BodyLogReader { identity: Option, header: FileHeader, committed_end: u64, - /// How far into one block this reader has inflated. Bodies of one + codecs: ArchiveCodecs, + /// How far into one block this reader has decoded. Bodies of one /// exchange land in one block, and a UI walks rows in order, so the next /// read usually continues this cursor rather than starting over. cursor: RefCell>, @@ -57,15 +55,15 @@ pub struct BodyLogReader { segments: Cell, } -/// One block, inflated up to the end of some segment. +/// One block, decoded up to the end of some segment. struct Cursor { block_offset: u64, block_end: u64, raw_limit: u32, - /// The block's inflater, carrying the dictionary the next segment was + /// The block's decoder, carrying the dictionary the next segment was /// compressed against. Boxed: it is about 40 KiB. - inflater: Box, - /// Every raw byte of the block inflated so far. + decoder: Box, + /// Every raw byte of the block decoded so far. raw: Vec, /// File offset of the next segment header. next_segment_at: u64, @@ -99,11 +97,15 @@ impl FileIdentity { impl BodyLogReader { pub fn open(path: &Path) -> Result { + Self::open_with_codecs(path, ArchiveCodecs::default()) + } + + pub fn open_with_codecs(path: &Path, codecs: ArchiveCodecs) -> Result { refuse_symlink(path)?; let file = unix_fs::open_regular_file_no_follow(path)?; let identity = FileIdentity::of(&file.metadata()?); let committed_end = file.metadata()?.len(); - let mut reader = Self::from_descriptor_unchecked(file, committed_end)?; + let mut reader = Self::from_descriptor_unchecked(file, committed_end, codecs)?; reader.path = Some(path.to_path_buf()); reader.identity = Some(identity); // Transitional path-based callers follow an append-only live file. @@ -116,10 +118,19 @@ impl BodyLogReader { /// Open the exact generation selected by a SQLite snapshot beneath an /// already-open private directory. pub fn open_generation(directory: &ContainedDir, expected: FileHeader, committed_end: u64) -> Result { + Self::open_generation_with_codecs(directory, expected, committed_end, ArchiveCodecs::default()) + } + + pub fn open_generation_with_codecs( + directory: &ContainedDir, + expected: FileHeader, + committed_end: u64, + codecs: ArchiveCodecs, + ) -> Result { directory.validate_private()?; let name = expected.generation_id.file_name(); let file = directory.open_file(OsStr::new(&name), ContainedOpenOptions::read_only())?; - let reader = Self::from_descriptor(file, expected, committed_end)?; + let reader = Self::from_descriptor_with_codecs(file, expected, committed_end, codecs)?; reader .header .validate(expected.archive_id, expected.generation_id, &name)?; @@ -129,14 +140,23 @@ impl BodyLogReader { /// Adopt an independently-owned descriptor and validate it against the /// selected ledger identity and committed extent. pub fn from_descriptor(file: File, expected: FileHeader, committed_end: u64) -> Result { - let reader = Self::from_descriptor_unchecked(file, committed_end)?; + Self::from_descriptor_with_codecs(file, expected, committed_end, ArchiveCodecs::default()) + } + + pub fn from_descriptor_with_codecs( + file: File, + expected: FileHeader, + committed_end: u64, + codecs: ArchiveCodecs, + ) -> Result { + let reader = Self::from_descriptor_unchecked(file, committed_end, codecs)?; if reader.header != expected { return Err(ArchiveError::ArchiveIdentityMismatch); } Ok(reader) } - fn from_descriptor_unchecked(mut file: File, committed_end: u64) -> Result { + fn from_descriptor_unchecked(mut file: File, committed_end: u64, codecs: ArchiveCodecs) -> Result { if committed_end < FILE_HEADER_BYTES as u64 { return Err(ArchiveError::BadFileHeader); } @@ -153,6 +173,7 @@ impl BodyLogReader { identity: None, header, committed_end, + codecs, cursor: RefCell::new(None), blocks: Cell::new(0), segments: Cell::new(0), @@ -228,13 +249,14 @@ impl BodyLogReader { file.read_exact(&mut head) .map_err(|_| ArchiveError::BadBlockHeader(block_offset))?; } - format::parse_block_header(&head, block_offset)?; + let codec = format::parse_block_header(&head, block_offset)?; + let decoder = self.codecs.decoder(codec, block_offset)?; self.blocks.set(self.blocks.get() + 1); Ok(Cursor { block_offset, block_end, raw_limit, - inflater: InflateState::new_boxed(DataFormat::Raw), + decoder, raw: Vec::new(), next_segment_at: block_offset + BLOCK_HEADER_BYTES as u64, finished: false, @@ -334,45 +356,17 @@ impl BodyLogReader { /// to the header's. A non-final segment must also end on a sync flush, or the /// next one would not decode after it. fn inflate_segment(cursor: &mut Cursor, header: &SegmentHeader, comp: &[u8], at: u64) -> Result<()> { - let bad = |reason: &str| ArchiveError::Inflate(at, reason.to_string()); - if !header.last && !comp.ends_with(&SYNC_FLUSH_TAIL) { - return Err(ArchiveError::BadSegment(at)); - } let start = cursor.raw.len(); let want = header.raw_len as usize; - cursor.raw.resize(start + want + 1, 0); - let (mut consumed, mut written) = (0, 0); - let mut ended = false; - loop { - let result = inflate( - &mut cursor.inflater, - &comp[consumed..], - &mut cursor.raw[start + written..], - MZFlush::None, - ); - consumed += result.bytes_consumed; - written += result.bytes_written; - match result.status { - Ok(MZStatus::StreamEnd) => { - ended = true; - break; - } - // All input taken, or no progress possible (the slack byte is - // spent, or the stream wants input the segment does not have): - // the checks below say which. - Ok(_) if consumed == comp.len() || (result.bytes_consumed == 0 && result.bytes_written == 0) => break, - Ok(_) => {} - Err(miniz_oxide::MZError::Buf) => break, - Err(error) => return Err(bad(&format!("{error:?}"))), - } - } - cursor.raw.truncate(start + written.min(want)); - if consumed != comp.len() || written != want || ended != header.last { + let raw = cursor.decoder.decode_segment(comp, want, header.last, at)?; + if raw.len() != want { return Err(ArchiveError::Integrity(at)); } - if capsem_crypto::blake3::hash(&cursor.raw[start..]).as_bytes() != &header.hash { + if blake3::hash(&raw).as_bytes() != &header.hash { return Err(ArchiveError::Integrity(at)); } + cursor.raw.extend_from_slice(&raw); + debug_assert_eq!(cursor.raw.len(), start + want); Ok(()) } diff --git a/crates/capsem-archive/src/tests/stream.rs b/crates/capsem-archive/src/tests/stream.rs index 95abaac6a..10e90ae4f 100644 --- a/crates/capsem-archive/src/tests/stream.rs +++ b/crates/capsem-archive/src/tests/stream.rs @@ -12,7 +12,87 @@ use flate2::{Compress, Compression, FlushCompress}; use super::{archive, file_len, XorShift}; use crate::format::{BodyRef, FILE_HEADER_BYTES, MAX_BLOCK_RAW_BYTES}; -use crate::{BodyLogReader, BodyLogWriter}; +use std::sync::Arc; + +use crate::{ArchiveCodecs, BlockCodec, BlockDecoder, BlockEncoder, BodyLogReader, BodyLogWriter, Result}; + +const CHAIN_CODEC: u8 = 2; + +struct ChainCodec; + +struct ChainEncoder { + offset: usize, +} + +struct ChainDecoder { + offset: usize, +} + +impl BlockCodec for ChainCodec { + fn id(&self) -> u8 { + CHAIN_CODEC + } + + fn encoder(&self) -> Result> { + Ok(Box::new(ChainEncoder { offset: 0 })) + } + + fn decoder(&self) -> Result> { + Ok(Box::new(ChainDecoder { offset: 0 })) + } +} + +impl BlockEncoder for ChainEncoder { + fn write(&mut self, input: &[u8], output: &mut Vec) -> Result<()> { + output.extend( + input + .iter() + .enumerate() + .map(|(index, byte)| byte ^ ((self.offset + index) as u8)), + ); + self.offset += input.len(); + Ok(()) + } + + fn flush(&mut self, _output: &mut Vec, _finish: bool) -> Result<()> { + Ok(()) + } +} + +impl BlockDecoder for ChainDecoder { + fn decode_segment(&mut self, input: &[u8], raw_len: usize, _finish: bool, at: u64) -> Result> { + if input.len() != raw_len { + return Err(crate::ArchiveError::Integrity(at)); + } + let decoded = input + .iter() + .enumerate() + .map(|(index, byte)| byte ^ ((self.offset + index) as u8)) + .collect(); + self.offset += raw_len; + Ok(decoded) + } +} + +#[test] +fn a_registered_codec_keeps_stream_state_across_durable_segments() { + let dir = tempfile::tempdir().unwrap(); + let path = archive(&dir); + let codecs = ArchiveCodecs::with_write_codec(Arc::new(ChainCodec)).unwrap(); + let mut writer = BodyLogWriter::open_with_codecs(&path, codecs.clone()).unwrap(); + let first_body = b"first body with durable visibility"; + let second_body = b"second body depends on the prior stream position"; + let first = writer.stage(first_body).unwrap(); + writer.flush_segment().unwrap(); + let second = writer.stage(second_body).unwrap(); + writer.close_block().unwrap(); + + let reader = BodyLogReader::open_with_codecs(&path, codecs).unwrap(); + assert_eq!(reader.read(first).unwrap(), first_body); + assert_eq!(reader.read(second).unwrap(), second_body); + let encoded = std::fs::read(&path).unwrap(); + assert_eq!(encoded[FILE_HEADER_BYTES + 4], CHAIN_CODEC); +} /// Random bodies -- text, incompressible, empty -- with random flushes and /// closes. After every flush, a copy of the file cut at the writer's end must diff --git a/crates/capsem-archive/src/writer.rs b/crates/capsem-archive/src/writer.rs index de67a8a83..92f5daa0c 100644 --- a/crates/capsem-archive/src/writer.rs +++ b/crates/capsem-archive/src/writer.rs @@ -1,7 +1,7 @@ //! Stage bodies into an open block; flush a segment when asked, close the //! block when it is full. //! -//! A block is one deflate stream. `stage` feeds a body to the compressor and +//! A block is one compression stream. `stage` feeds a body to the compressor and //! hands back a reference naming the block's real offset; `flush_segment` //! sync-flushes the compressor and appends everything it produced since the //! last flush, behind a segment header, in one write. The block stays open, @@ -27,15 +27,13 @@ use std::os::unix::fs::PermissionsExt; use std::path::Path; use std::time::Instant; -use capsem_foundation::unix::contained::ContainedDir; -use capsem_foundation::unix::fs as unix_fs; -use flate2::{Compress, Compression, FlushCompress, Status}; - use super::format::{ - self, ArchiveId, BodyRef, FileHeader, GenerationId, SegmentHeader, BLOCK_HEADER_BYTES, CODEC_DEFLATE, - DEFLATE_LEVEL, FILE_HEADER_BYTES, MAX_BLOCK_RAW_BYTES, SEGMENT_HEADER_BYTES, TARGET_BLOCK_BYTES, + self, ArchiveId, BodyRef, FileHeader, GenerationId, SegmentHeader, BLOCK_HEADER_BYTES, FILE_HEADER_BYTES, + MAX_BLOCK_RAW_BYTES, SEGMENT_HEADER_BYTES, TARGET_BLOCK_BYTES, }; -use crate::{ArchiveError, BodyLogReader, Result}; +use crate::{ArchiveCodecs, ArchiveError, BlockEncoder, BodyLogReader, Result}; +use capsem_foundation::unix::contained::ContainedDir; +use capsem_foundation::unix::fs as unix_fs; /// What one flush or close put on disk. The owner records `raw_len` and /// `disk_len` against `block_offset` in the same transaction as the index @@ -54,7 +52,8 @@ pub struct SegmentWritten { /// The block being written: one compressor, and the segment in progress. struct OpenBlock { offset: u64, - compressor: Compress, + codec: u8, + encoder: Box, /// Raw bytes staged into the block, flushed or not. raw_len: u32, /// Raw bytes already inside a written segment. @@ -80,6 +79,7 @@ pub struct BodyLogWriter { header: FileHeader, end: u64, block: Option, + codecs: ArchiveCodecs, /// Set by a write that failed part-way through. See /// [`ArchiveError::Poisoned`]. poisoned: bool, @@ -114,6 +114,10 @@ impl BodyLogWriter { /// A reopened file must carry this version's header. It is appended after /// its current end, torn tail included: no index row points there. pub fn open(path: &Path) -> Result { + Self::open_with_codecs(path, ArchiveCodecs::default()) + } + + pub fn open_with_codecs(path: &Path, codecs: ArchiveCodecs) -> Result { refuse_symlink(path)?; let mut file = unix_fs::open_private_append_no_follow(path)?; let end = file.metadata()?.len(); @@ -123,14 +127,14 @@ impl BodyLogWriter { generation_id: GenerationId::generate(), }; file.write_all(&format::encode_file_header(header.archive_id, header.generation_id))?; - return Ok(Self::at(file, header, FILE_HEADER_BYTES as u64)); + return Ok(Self::at(file, header, FILE_HEADER_BYTES as u64, codecs)); } file.set_permissions(std::fs::Permissions::from_mode(0o600))?; let mut header = [0u8; FILE_HEADER_BYTES]; file.seek(SeekFrom::Start(0))?; file.read_exact(&mut header).map_err(|_| ArchiveError::BadFileHeader)?; let header = format::decode_file_header(&header)?; - Ok(Self::at(file, header, end)) + Ok(Self::at(file, header, end, codecs)) } /// Exclusively create a generation beneath an already-open private @@ -140,6 +144,15 @@ impl BodyLogWriter { directory: &ContainedDir, archive_id: ArchiveId, generation_id: GenerationId, + ) -> Result { + Self::create_generation_with_codecs(directory, archive_id, generation_id, ArchiveCodecs::default()) + } + + pub fn create_generation_with_codecs( + directory: &ContainedDir, + archive_id: ArchiveId, + generation_id: GenerationId, + codecs: ArchiveCodecs, ) -> Result { directory.validate_private()?; let name = generation_id.file_name(); @@ -149,13 +162,22 @@ impl BodyLogWriter { generation_id, }; file.write_all(&format::encode_file_header(archive_id, generation_id))?; - Ok(Self::at(file, header, FILE_HEADER_BYTES as u64)) + Ok(Self::at(file, header, FILE_HEADER_BYTES as u64, codecs)) } /// Reopen the exact generation selected by the authoritative SQLite row. /// Unreferenced bytes past `committed_end` are retained and a new block /// starts at actual EOF; an old compressor stream is never resumed. pub fn open_generation(directory: &ContainedDir, expected: FileHeader, committed_end: u64) -> Result { + Self::open_generation_with_codecs(directory, expected, committed_end, ArchiveCodecs::default()) + } + + pub fn open_generation_with_codecs( + directory: &ContainedDir, + expected: FileHeader, + committed_end: u64, + codecs: ArchiveCodecs, + ) -> Result { directory.validate_private()?; let name = expected.generation_id.file_name(); let mut file = directory.open_existing_private_append(OsStr::new(&name))?; @@ -168,7 +190,7 @@ impl BodyLogWriter { file.read_exact(&mut encoded).map_err(|_| ArchiveError::BadFileHeader)?; let header = format::decode_file_header(&encoded)?; header.validate(expected.archive_id, expected.generation_id, &name)?; - Ok(Self::at(file, header, end)) + Ok(Self::at(file, header, end, codecs)) } /// Copy one already-validated committed block extent verbatim into a new @@ -222,12 +244,13 @@ impl BodyLogWriter { Ok(()) } - fn at(file: File, header: FileHeader, end: u64) -> Self { + fn at(file: File, header: FileHeader, end: u64, codecs: ArchiveCodecs) -> Self { Self { file, header, end, block: None, + codecs, poisoned: false, #[cfg(test)] fail_write_after: None, @@ -271,17 +294,21 @@ impl BodyLogWriter { { return Err(ArchiveError::BlockFull); } - let end = self.end; - let block = self.block.get_or_insert_with(|| OpenBlock { - offset: end, - compressor: Compress::new(Compression::new(DEFLATE_LEVEL), false), - raw_len: 0, - flushed_raw: 0, - disk_len: 0, - hasher: capsem_crypto::blake3::Hasher::new(), - out: Vec::new(), - }); - if let Err(error) = deflate_into(&mut block.compressor, body, &mut block.out, FlushCompress::None) { + if self.block.is_none() { + let (codec, encoder) = self.codecs.encoder()?; + self.block = Some(OpenBlock { + offset: self.end, + codec, + encoder, + raw_len: 0, + flushed_raw: 0, + disk_len: 0, + hasher: blake3::Hasher::new(), + out: Vec::new(), + }); + } + let block = self.block.as_mut().expect("created above"); + if let Err(error) = block.encoder.write(body, &mut block.out) { // The compressor may have taken part of the body; nothing it // produces from here on can be vouched for. self.poisoned = true; @@ -366,12 +393,7 @@ impl BodyLogWriter { #[cfg(not(test))] let fail_after = None; let block = self.block.as_mut().expect("callers check a block is open"); - let flush = if last { - FlushCompress::Finish - } else { - FlushCompress::Sync - }; - if let Err(error) = deflate_into(&mut block.compressor, &[], &mut block.out, flush) { + if let Err(error) = block.encoder.flush(&mut block.out, last) { // The compressor's state is unknown; nothing it produces next can // be vouched for. self.poisoned = true; @@ -387,7 +409,7 @@ impl BodyLogWriter { }); let mut bytes = Vec::with_capacity(BLOCK_HEADER_BYTES + SEGMENT_HEADER_BYTES + block.out.len()); if block.disk_len == 0 { - bytes.extend_from_slice(&format::encode_block_header(CODEC_DEFLATE)); + bytes.extend_from_slice(&format::encode_block_header(block.codec)); } bytes.extend_from_slice(&header); bytes.extend_from_slice(&block.out); @@ -460,30 +482,6 @@ fn write_all(file: &mut File, bytes: &[u8], fail_after: Option) -> io::Re file.write_all(bytes) } -/// Run `input` through the compressor with `flush`, growing `out` until the -/// compressor has taken all of it and emitted everything the flush owes. -fn deflate_into(compressor: &mut Compress, input: &[u8], out: &mut Vec, flush: FlushCompress) -> Result<()> { - let mut consumed = 0; - loop { - // Room for the input at worst-case expansion plus the flush's own - // bytes, so a sync flush almost always completes in one call. - out.reserve((input.len() - consumed) + (input.len() - consumed) / 1000 + 64 * 1024); - let before = compressor.total_in(); - let status = compressor - .compress_vec(&input[consumed..], out, flush) - .map_err(|error| ArchiveError::Io(io::Error::other(error)))?; - consumed += usize::try_from(compressor.total_in() - before).expect("bounded by the input"); - let done = match flush { - FlushCompress::Finish => status == Status::StreamEnd, - // Output stopped short of the buffer's end: the flush is complete. - _ => consumed == input.len() && out.len() < out.capacity(), - }; - if done { - return Ok(()); - } - } -} - #[cfg(test)] mod tests; From 667a80b45d9f93aa0446f9f0a5453b58ac22cb79 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:26:13 +0000 Subject: [PATCH 199/464] refactor(logger): thread archive codec authority --- crates/capsem-logger/src/db.rs | 21 +++--- crates/capsem-logger/src/db/bodies.rs | 8 ++- .../src/db/handle_tests/bodies.rs | 12 +++- crates/capsem-logger/src/db/reader_worker.rs | 17 +++-- crates/capsem-logger/src/db/warc_export.rs | 13 ++-- crates/capsem-logger/src/ledger_server.rs | 7 +- crates/capsem-logger/src/writer.rs | 24 +++++-- crates/capsem-logger/src/writer/bodies.rs | 66 ++++++++++++++----- crates/capsem-logger/src/writer/retention.rs | 11 +++- .../capsem-logger/src/writer/tests/bodies.rs | 26 +++++++- 10 files changed, 154 insertions(+), 51 deletions(-) diff --git a/crates/capsem-logger/src/db.rs b/crates/capsem-logger/src/db.rs index 6cfe2e4d3..430542d25 100644 --- a/crates/capsem-logger/src/db.rs +++ b/crates/capsem-logger/src/db.rs @@ -4,6 +4,7 @@ use std::sync::{mpsc, Arc, Mutex}; use std::thread::JoinHandle; use std::time::Instant; +use capsem_archive::ArchiveCodecs; use capsem_telemetry::db::{ DB_QUERY_DURATION_MS, DB_QUERY_PARAMS_COUNT, DB_QUERY_RESULT_BYTES, DB_QUERY_RESULT_ROWS, DB_QUERY_TOTAL, }; @@ -254,13 +255,17 @@ impl DbHandle { /// reader can open the same DB, and starts a DB-owned reader worker. Route /// code receives a handle; it does not receive a connection. pub fn open(path: &Path) -> rusqlite::Result { + Self::open_with_codecs(path, ArchiveCodecs::default()) + } + + pub fn open_with_codecs(path: &Path, codecs: ArchiveCodecs) -> rusqlite::Result { let started = Instant::now(); - let writer = Arc::new(DbWriter::open(path, 1024)?); + let writer = Arc::new(DbWriter::open_with_codecs(path, 1024, codecs.clone())?); // Reads go to the file, like every other handle's: the writer's // memory holds only rows it has not flushed, and SQLite's // `data_version` tells the reader when a flush landed. DbReader::open(path)?; - let handle = Self::open_with_writer(path.to_path_buf(), writer)?; + let handle = Self::open_with_writer(path.to_path_buf(), writer, codecs)?; tracing::debug!( db_path = %path.display(), @@ -284,7 +289,7 @@ impl DbHandle { pub fn open_external_reader(path: &Path) -> rusqlite::Result { let started = Instant::now(); DbReader::open(path)?; - let handle = Self::open_reader(path.to_path_buf())?; + let handle = Self::open_reader(path.to_path_buf(), ArchiveCodecs::default())?; tracing::debug!( db_path = %path.display(), operation = "open_external_reader", @@ -294,14 +299,14 @@ impl DbHandle { Ok(handle) } - fn open_reader(db_path: PathBuf) -> rusqlite::Result { + fn open_reader(db_path: PathBuf, codecs: ArchiveCodecs) -> rusqlite::Result { let (reader_tx, reader_rx) = mpsc::channel(); let reader_path = db_path.clone(); let hot = Arc::new(HotCounters::default()); let reader_hot = Arc::clone(&hot); let reader_join = std::thread::Builder::new() .name("capsem-db-reader".into()) - .spawn(move || reader_loop(reader_path, reader_rx, reader_hot)) + .spawn(move || reader_loop(reader_path, reader_rx, reader_hot, codecs)) .expect("failed to spawn db reader thread"); Ok(Self { @@ -322,12 +327,12 @@ impl DbHandle { }) } - fn open_with_writer(db_path: PathBuf, writer: Arc) -> rusqlite::Result { + fn open_with_writer(db_path: PathBuf, writer: Arc, codecs: ArchiveCodecs) -> rusqlite::Result { // Owning the writer changes only whether `write` is accepted. Its // writes reach the file on the writer's flush, which the reader sees // through `data_version` like any other commit; a local write moving // the epoch is not the moment the rows become readable. - let handle = Self::open_reader(db_path)?; + let handle = Self::open_reader(db_path, codecs)?; let mut inner = Arc::try_unwrap(handle.inner).ok().expect("new handle is unique"); inner.writer = Some(writer); Ok(Self { inner: Arc::new(inner) }) @@ -337,7 +342,7 @@ impl DbHandle { pub(crate) fn open_existing_for_tests(path: &Path) -> rusqlite::Result { DbReader::open(path)?; let writer = Arc::new(DbWriter::open_in_memory(1)?); - Self::open_with_writer(path.to_path_buf(), writer) + Self::open_with_writer(path.to_path_buf(), writer, ArchiveCodecs::default()) } pub fn path(&self) -> &Path { diff --git a/crates/capsem-logger/src/db/bodies.rs b/crates/capsem-logger/src/db/bodies.rs index 76535554f..45522b45f 100644 --- a/crates/capsem-logger/src/db/bodies.rs +++ b/crates/capsem-logger/src/db/bodies.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant}; -use capsem_archive::{ArchiveError, BlockExtent, BodyLogReader, BodyRef}; +use capsem_archive::{ArchiveCodecs, ArchiveError, BlockExtent, BodyLogReader, BodyRef}; use capsem_foundation::unix::contained::ContainedDir; use capsem_foundation::unix::lock::{self, LockMode}; use rusqlite::{Connection, Row}; @@ -468,6 +468,7 @@ pub(super) fn capture_body_rows( db_path: &std::path::Path, queries: Vec, requested_ids: usize, + codecs: &ArchiveCodecs, ) -> DbResult { if requested_ids > INTERACTIVE_CAPTURE_MAX_IDS { return Err(format!( @@ -492,8 +493,9 @@ pub(super) fn capture_body_rows( Ok(directory) }) .map_err(|error| format!("open archive generation directory: {error}"))?; - let reader = BodyLogReader::open_generation(&directory, state.header, state.committed_end) - .map_err(|error| format!("open captured archive generation: {error}"))?; + let reader = + BodyLogReader::open_generation_with_codecs(&directory, state.header, state.committed_end, codecs.clone()) + .map_err(|error| format!("open captured archive generation: {error}"))?; drop(archive_lock); pause_archive_capture_for_tests(db_path); diff --git a/crates/capsem-logger/src/db/handle_tests/bodies.rs b/crates/capsem-logger/src/db/handle_tests/bodies.rs index 4efa211b2..8a6200ced 100644 --- a/crates/capsem-logger/src/db/handle_tests/bodies.rs +++ b/crates/capsem-logger/src/db/handle_tests/bodies.rs @@ -33,9 +33,15 @@ pub(super) async fn count(db: &DbHandle, sql: &str) -> i64 { #[test] fn interactive_capture_refuses_id_and_metadata_budgets_before_unbounded_work() { let conn = rusqlite::Connection::open_in_memory().unwrap(); - let error = crate::db::bodies::capture_body_rows(&conn, std::path::Path::new("unused.db"), Vec::new(), 10_001) - .err() - .expect("too many ids are rejected before archive acquisition"); + let error = crate::db::bodies::capture_body_rows( + &conn, + std::path::Path::new("unused.db"), + Vec::new(), + 10_001, + &capsem_archive::ArchiveCodecs::default(), + ) + .err() + .expect("too many ids are rejected before archive acquisition"); assert!(error.contains("limit is 10000"), "{error}"); let limit = 16 * 1024 * 1024; diff --git a/crates/capsem-logger/src/db/reader_worker.rs b/crates/capsem-logger/src/db/reader_worker.rs index 4006700b2..edb8b0e44 100644 --- a/crates/capsem-logger/src/db/reader_worker.rs +++ b/crates/capsem-logger/src/db/reader_worker.rs @@ -7,7 +7,12 @@ use super::hot_counters::HOT_REFRESH_INTERVAL; use super::*; -pub(super) fn reader_loop(path: PathBuf, rx: mpsc::Receiver, hot: Arc) { +pub(super) fn reader_loop( + path: PathBuf, + rx: mpsc::Receiver, + hot: Arc, + codecs: ArchiveCodecs, +) { let started = Instant::now(); // WAL makes the file readable while its writer commits, so this reader // queries `main` and holds no copy of it, whichever process writes. @@ -170,18 +175,18 @@ pub(super) fn reader_loop(path: PathBuf, rx: mpsc::Receiver, hot: A reply, } => { let result = observe_change(&reader).and_then(|observed| { - super::bodies::capture_body_rows(reader.connection(), &path, queries, requested_ids).map(|value| { - Observed { + super::bodies::capture_body_rows(reader.connection(), &path, queries, requested_ids, &codecs).map( + |value| Observed { changed: commit(&reader, observed), value, - } - }) + }, + ) }); let _ = reply.send(result); } ReadRequest::CaptureWarc { reply } => { let result = observe_change(&reader).and_then(|observed| { - super::warc_export::capture_warc(reader.connection(), &path).map(|value| Observed { + super::warc_export::capture_warc(reader.connection(), &path, &codecs).map(|value| Observed { changed: commit(&reader, observed), value, }) diff --git a/crates/capsem-logger/src/db/warc_export.rs b/crates/capsem-logger/src/db/warc_export.rs index 865eaae46..afe542947 100644 --- a/crates/capsem-logger/src/db/warc_export.rs +++ b/crates/capsem-logger/src/db/warc_export.rs @@ -58,7 +58,7 @@ use std::path::PathBuf; use std::sync::{LazyLock, Mutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use capsem_archive::{warc, BlockExtent, BodyLogReader, BodyRef, WarcRecord}; +use capsem_archive::{warc, ArchiveCodecs, BlockExtent, BodyLogReader, BodyRef, WarcRecord}; use capsem_foundation::unix::contained::ContainedDir; use capsem_foundation::unix::lock::{self, LockMode}; use rusqlite::{Connection, OptionalExtension}; @@ -396,7 +396,11 @@ fn warc_page_sql() -> String { ) } -pub(super) fn capture_warc(conn: &Connection, db_path: &std::path::Path) -> DbResult { +pub(super) fn capture_warc( + conn: &Connection, + db_path: &std::path::Path, + codecs: &ArchiveCodecs, +) -> DbResult { let deadline = std::time::Instant::now() + WARC_CAPTURE_DEADLINE; let archive_lock = lock::acquire_existing_until( &crate::writer::archive_lock_path_for_db(db_path), @@ -415,8 +419,9 @@ pub(super) fn capture_warc(conn: &Connection, db_path: &std::path::Path) -> DbRe Ok(directory) }) .map_err(|error| format!("open WARC generation directory: {error}"))?; - let reader = BodyLogReader::open_generation(&directory, state.header, state.committed_end) - .map_err(|error| format!("open WARC generation: {error}"))?; + let reader = + BodyLogReader::open_generation_with_codecs(&directory, state.header, state.committed_end, codecs.clone()) + .map_err(|error| format!("open WARC generation: {error}"))?; drop(archive_lock); super::bodies::pause_archive_capture_for_tests(db_path); diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index 41e3ae3de..4ab521734 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -5,6 +5,7 @@ use std::os::unix::net::UnixStream; use std::path::{Path, PathBuf}; use std::sync::Arc; +use capsem_archive::ArchiveCodecs; use capsem_foundation::ipc_channel; use capsem_proto::ledger::{ LedgerChannelGrant, LedgerFailure, LedgerFailureCode, LedgerRequest, LedgerResponse, LedgerWelcome, @@ -40,8 +41,12 @@ pub struct LedgerServer { impl LedgerServer { pub fn open(path: &Path) -> rusqlite::Result { + Self::open_with_codecs(path, ArchiveCodecs::default()) + } + + pub fn open_with_codecs(path: &Path, codecs: ArchiveCodecs) -> rusqlite::Result { Ok(Self { - db: Arc::new(DbHandle::open(path)?), + db: Arc::new(DbHandle::open_with_codecs(path, codecs)?), session_dir: path.parent().unwrap_or_else(|| Path::new(".")).to_path_buf(), admission: tokio::sync::Mutex::new(AdmissionState::default()), }) diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 2b76dac2d..8dcd59718 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -4,6 +4,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{mpsc, Arc}; use std::time::{Duration, Instant, SystemTime}; +use capsem_archive::ArchiveCodecs; use capsem_telemetry::db::{DB_MEMORY_UNFLUSHED_OPS, DB_SHUTDOWN_FLUSH_MS}; use rusqlite::{params, Connection, ErrorCode, OpenFlags, OptionalExtension}; use serde::{Deserialize, Serialize}; @@ -334,7 +335,11 @@ impl DbWriter { /// Spawn a dedicated writer thread that owns the DB connection. /// `capacity` controls the mpsc channel size (backpressure). pub fn open(path: &Path, capacity: usize) -> rusqlite::Result { - Self::open_with_clock(path, capacity, SystemTime::now) + Self::open_with_codecs(path, capacity, ArchiveCodecs::default()) + } + + pub fn open_with_codecs(path: &Path, capacity: usize, codecs: ArchiveCodecs) -> rusqlite::Result { + Self::open_with_clock_and_codecs(path, capacity, SystemTime::now, codecs) } /// `open`, with the archive index's timestamps read from `now`. @@ -342,13 +347,22 @@ impl DbWriter { /// Only a replay of an existing ledger has any business supplying one; see /// `LedgerClock`. pub fn open_with_clock(path: &Path, capacity: usize, now: LedgerClock) -> rusqlite::Result { + Self::open_with_clock_and_codecs(path, capacity, now, ArchiveCodecs::default()) + } + + fn open_with_clock_and_codecs( + path: &Path, + capacity: usize, + now: LedgerClock, + codecs: ArchiveCodecs, + ) -> rusqlite::Result { if let Some(parent) = path.parent() { let _ = std::fs::create_dir_all(parent); } let mut last_busy = None; for _ in 0..50 { - match Self::open_once(path, WriterSchedule::new(capacity), now) { + match Self::open_once(path, capacity, now, codecs.clone()) { Ok(writer) => return Ok(writer), Err(error) if is_sqlite_busy(&error) => { last_busy = Some(error); @@ -363,7 +377,7 @@ impl DbWriter { ))) } - fn open_once(path: &Path, schedule: WriterSchedule, now: LedgerClock) -> rusqlite::Result { + fn open_once(path: &Path, capacity: usize, now: LedgerClock, codecs: ArchiveCodecs) -> rusqlite::Result { let held = writer_lock::acquire(path)?; legacy::set_aside_pre_archive_ledger(path)?; let flags = OpenFlags::SQLITE_OPEN_READ_WRITE @@ -378,14 +392,14 @@ impl DbWriter { schema::record_sqlite_mmap_telemetry(&conn, path, "writer", "open"); let bodies = match schema::archive_schema_status(&conn)? { schema::ArchiveSchemaStatus::Fresh => { - let (bodies, archive_lock, header) = BodyArchive::prepare_new(path, now)?; + let (bodies, archive_lock, header) = BodyArchive::prepare_new(path, now, codecs)?; schema::create_tables_with_archive_header(&conn, Some(header))?; drop(archive_lock); bodies } schema::ArchiveSchemaStatus::Current(_) => { schema::create_tables(&conn)?; - BodyArchive::open_existing(path, now, &conn)? + BodyArchive::open_existing(path, now, &conn, codecs)? } // Moved aside above, under this same writer lock. schema::ArchiveSchemaStatus::PreArchive => { diff --git a/crates/capsem-logger/src/writer/bodies.rs b/crates/capsem-logger/src/writer/bodies.rs index c12f7d511..a7220af76 100644 --- a/crates/capsem-logger/src/writer/bodies.rs +++ b/crates/capsem-logger/src/writer/bodies.rs @@ -35,7 +35,9 @@ use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant, SystemTime}; -use capsem_archive::{ArchiveError, ArchiveId, BodyLogWriter, BodyRef, FileHeader, GenerationId, SegmentWritten}; +use capsem_archive::{ + ArchiveCodecs, ArchiveError, ArchiveId, BodyLogWriter, BodyRef, FileHeader, GenerationId, SegmentWritten, +}; use capsem_foundation::unix::contained::{ContainedDir, EntryKind}; use capsem_foundation::unix::fs::{durable_sync_directory, ensure_private_dir}; use capsem_foundation::unix::lock::{self, FileLock, LockAttempt, LockMode}; @@ -119,6 +121,7 @@ pub(super) struct BodyArchive { /// unprovable. Both stage nothing rather than writing an index nobody can /// resolve. writer: Option, + codecs: ArchiveCodecs, /// What `created_at`, `sealed_at` and a block's age are read from. Every /// other column in the index is content, so this is the only value in it /// that a replay of the same session cannot reproduce -- and the only @@ -221,7 +224,11 @@ fn archive_contract_error(message: impl Into) -> rusqlite::Error { impl BodyArchive { /// Prepare a fresh generation and stable acquisition lock before the /// schema transaction publishes their identities. - pub(super) fn prepare_new(db_path: &Path, now: LedgerClock) -> rusqlite::Result<(Self, FileLock, FileHeader)> { + pub(super) fn prepare_new( + db_path: &Path, + now: LedgerClock, + codecs: ArchiveCodecs, + ) -> rusqlite::Result<(Self, FileLock, FileHeader)> { let path = archive_path_for_db(db_path); match std::fs::symlink_metadata(&path) { Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} @@ -272,8 +279,13 @@ impl BodyArchive { archive_id: ArchiveId::generate(), generation_id: GenerationId::generate(), }; - let mut writer = BodyLogWriter::create_generation(&directory, header.archive_id, header.generation_id) - .map_err(|error| archive_contract_error(format!("create generation: {error}")))?; + let mut writer = BodyLogWriter::create_generation_with_codecs( + &directory, + header.archive_id, + header.generation_id, + codecs.clone(), + ) + .map_err(|error| archive_contract_error(format!("create generation: {error}")))?; writer .sync() .map_err(|error| archive_contract_error(format!("sync generation: {error}")))?; @@ -281,7 +293,7 @@ impl BodyArchive { .sync() .map_err(|error| archive_contract_error(format!("sync {}: {error}", path.display())))?; Ok(( - Self::with_writer(db_path.to_path_buf(), directory, writer, now), + Self::with_writer(db_path.to_path_buf(), directory, writer, now, codecs), archive_lock, header, )) @@ -290,7 +302,12 @@ impl BodyArchive { /// Recover and reopen the exact generation selected by SQLite. The /// archive lock stays exclusive through the real revision durability /// fence; no path scan can elect another file. - pub(super) fn open_existing(db_path: &Path, now: LedgerClock, conn: &Connection) -> rusqlite::Result { + pub(super) fn open_existing( + db_path: &Path, + now: LedgerClock, + conn: &Connection, + codecs: ArchiveCodecs, + ) -> rusqlite::Result { let path = archive_path_for_db(db_path); let metadata = std::fs::symlink_metadata(&path) .map_err(|error| archive_contract_error(format!("inspect {}: {error}", path.display())))?; @@ -311,8 +328,9 @@ impl BodyArchive { lock::acquire_existing_until(&lock_path, LockMode::Exclusive, Instant::now() + Duration::from_secs(5)) .map_err(|error| archive_contract_error(format!("lock {}: {error}", lock_path.display())))?; let state = crate::schema::archive_state(conn)?; - let writer = BodyLogWriter::open_generation(&directory, state.header, state.committed_end) - .map_err(|error| archive_contract_error(format!("open active generation: {error}")))?; + let writer = + BodyLogWriter::open_generation_with_codecs(&directory, state.header, state.committed_end, codecs.clone()) + .map_err(|error| archive_contract_error(format!("open active generation: {error}")))?; let next_revision = state .revision .checked_add(1) @@ -326,7 +344,7 @@ impl BodyArchive { return Err(archive_contract_error("archive_state changed during writer recovery")); } Self::gc_unreferenced_generations(&directory, state.header.generation_id); - Ok(Self::with_writer(db_path.to_path_buf(), directory, writer, now)) + Ok(Self::with_writer(db_path.to_path_buf(), directory, writer, now, codecs)) } /// Retry deletion of exact managed generation names after the elected @@ -370,7 +388,7 @@ impl BodyArchive { } pub(super) fn disabled(now: LedgerClock) -> Self { - Self::new(None, None, None, now) + Self::new(None, None, None, now, ArchiveCodecs::default()) } #[cfg(test)] @@ -380,7 +398,7 @@ impl BodyArchive { }; let path = archive_path_for_db(db_path); let opened = if path.exists() { - Self::open_existing(db_path, now, conn) + Self::open_existing(db_path, now, conn, ArchiveCodecs::default()) } else { let state = crate::schema::archive_state(conn); state.and_then(|state| { @@ -403,17 +421,29 @@ impl BodyArchive { directory .sync() .map_err(|error| archive_contract_error(error.to_string()))?; - Ok(Self::with_writer(db_path.to_path_buf(), directory, writer, now)) + Ok(Self::with_writer( + db_path.to_path_buf(), + directory, + writer, + now, + ArchiveCodecs::default(), + )) }) }; opened.unwrap_or_else(|error| { warn!(error = %error, "test archive could not be opened"); - Self::new(Some(db_path.to_path_buf()), None, None, now) + Self::new(Some(db_path.to_path_buf()), None, None, now, ArchiveCodecs::default()) }) } - fn with_writer(db_path: PathBuf, directory: ContainedDir, writer: BodyLogWriter, now: LedgerClock) -> Self { - Self::new(Some(db_path), Some(directory), Some(writer), now) + fn with_writer( + db_path: PathBuf, + directory: ContainedDir, + writer: BodyLogWriter, + now: LedgerClock, + codecs: ArchiveCodecs, + ) -> Self { + Self::new(Some(db_path), Some(directory), Some(writer), now, codecs) } fn new( @@ -421,11 +451,13 @@ impl BodyArchive { directory: Option, writer: Option, now: LedgerClock, + codecs: ArchiveCodecs, ) -> Self { Self { db_path, directory, writer, + codecs, now, block_opened_at: None, staged: Vec::new(), @@ -440,6 +472,10 @@ impl BodyArchive { } } + pub(super) fn codecs(&self) -> ArchiveCodecs { + self.codecs.clone() + } + /// Stage one body: its bytes into the open block, its index row beside /// them. Empty and absent bodies produce neither. pub(super) fn stage(&mut self, conn: &Connection, blob: EventBodyBlob<'_>) { diff --git a/crates/capsem-logger/src/writer/retention.rs b/crates/capsem-logger/src/writer/retention.rs index e7d4751f9..e891d6634 100644 --- a/crates/capsem-logger/src/writer/retention.rs +++ b/crates/capsem-logger/src/writer/retention.rs @@ -118,9 +118,14 @@ pub(super) fn retain_bodies( return Ok(RetainOutcome::default()); } - let generation_id = GenerationId::generate(); - let mut candidate = BodyLogWriter::create_generation(&directory, state.header.archive_id, generation_id) - .map_err(|error| format!("create retained generation: {error}"))?; + let generation_id = GenerationId::new_v4(); + let mut candidate = BodyLogWriter::create_generation_with_codecs( + &directory, + state.header.archive_id, + generation_id, + bodies.codecs(), + ) + .map_err(|error| format!("create retained generation: {error}"))?; let copied = match copy_survivors(conn, cutoff, &mut source, &mut candidate) { Ok(copied) => copied, Err(cause) => { diff --git a/crates/capsem-logger/src/writer/tests/bodies.rs b/crates/capsem-logger/src/writer/tests/bodies.rs index 72acedf77..a1479776a 100644 --- a/crates/capsem-logger/src/writer/tests/bodies.rs +++ b/crates/capsem-logger/src/writer/tests/bodies.rs @@ -43,7 +43,15 @@ fn recovered_writer_fences_then_collects_only_owned_orphan_generations() { drop(directory.create_new_private_file(OsStr::new("operator-note")).unwrap()); directory.sync().unwrap(); - drop(BodyArchive::open_existing(&db_path, SystemTime::now, &conn).unwrap()); + drop( + BodyArchive::open_existing( + &db_path, + SystemTime::now, + &conn, + capsem_archive::ArchiveCodecs::default(), + ) + .unwrap(), + ); let after = crate::schema::archive_state(&conn).unwrap(); assert_eq!( @@ -732,7 +740,13 @@ fn publication_crash_boundaries_recover_and_append_to_the_elected_generation() { let writer = DbWriter::open(&db_path, 1).unwrap(); writer.shutdown_blocking(); let conn = rusqlite::Connection::open(&db_path).unwrap(); - let mut archive = BodyArchive::open_existing(&db_path, SystemTime::now, &conn).unwrap(); + let mut archive = BodyArchive::open_existing( + &db_path, + SystemTime::now, + &conn, + capsem_archive::ArchiveCodecs::default(), + ) + .unwrap(); archive.stage(&conn, body_blob("aa0000000001", "old bytes")); archive.close_block(); commit(&mut archive, &conn); @@ -762,7 +776,13 @@ fn publication_crash_boundaries_recover_and_append_to_the_elected_generation() { published, "{case}" ); - let mut archive = BodyArchive::open_existing(&db_path, SystemTime::now, &conn).unwrap(); + let mut archive = BodyArchive::open_existing( + &db_path, + SystemTime::now, + &conn, + capsem_archive::ArchiveCodecs::default(), + ) + .unwrap(); let fenced = crate::schema::archive_state(&conn).unwrap(); assert_eq!(fenced.revision, elected.revision + 1, "{case} recovery fence"); From 925dd8010e43d0b724bae6889dacdc786ff3002c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:27:00 +0000 Subject: [PATCH 200/464] feat(archive): assign the zstd block codec id --- crates/capsem-archive/src/format.rs | 3 +++ crates/capsem-archive/src/lib.rs | 4 ++-- crates/capsem-archive/src/tests/stream.rs | 8 +++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/capsem-archive/src/format.rs b/crates/capsem-archive/src/format.rs index 854c6672c..5ed50fe3b 100644 --- a/crates/capsem-archive/src/format.rs +++ b/crates/capsem-archive/src/format.rs @@ -47,6 +47,9 @@ pub const SEGMENT_HEADER_BYTES: usize = 52; /// Raw deflate (RFC 1951), no zlib or gzip framing; every segment but the /// last ends in a sync flush, the last in the stream's end. pub const CODEC_DEFLATE: u8 = 1; +/// Zstandard frame kept open across segments with `ZSTD_e_flush`; native +/// implementation is supplied only by the confined ledger executable. +pub const CODEC_ZSTD: u8 = 2; /// Flag bit 0 of a segment: this segment ends its block's stream. pub const SEGMENT_FINAL: u8 = 0x01; /// The four bytes a sync flush ends with: an empty stored block's length and diff --git a/crates/capsem-archive/src/lib.rs b/crates/capsem-archive/src/lib.rs index 3fc4384bf..8dc1cd9f6 100644 --- a/crates/capsem-archive/src/lib.rs +++ b/crates/capsem-archive/src/lib.rs @@ -29,8 +29,8 @@ pub mod writer; pub use codec::{ArchiveCodecs, BlockCodec, BlockDecoder, BlockEncoder}; pub use format::{ - ArchiveId, BodyRef, FileHeader, GenerationId, BLOCK_HEADER_BYTES, FILE_HEADER_BYTES, MAX_BLOCK_RAW_BYTES, - TARGET_BLOCK_BYTES, + ArchiveId, BodyRef, FileHeader, GenerationId, BLOCK_HEADER_BYTES, CODEC_ZSTD, FILE_HEADER_BYTES, + MAX_BLOCK_RAW_BYTES, TARGET_BLOCK_BYTES, }; pub use reader::{BlockExtent, BodyLogReader}; pub use retain::{commit_retained, stage_retained_blocks, validate_block_extent, RetainedStaging}; diff --git a/crates/capsem-archive/src/tests/stream.rs b/crates/capsem-archive/src/tests/stream.rs index 10e90ae4f..7bfca4647 100644 --- a/crates/capsem-archive/src/tests/stream.rs +++ b/crates/capsem-archive/src/tests/stream.rs @@ -11,13 +11,11 @@ use flate2::{Compress, Compression, FlushCompress}; use super::{archive, file_len, XorShift}; -use crate::format::{BodyRef, FILE_HEADER_BYTES, MAX_BLOCK_RAW_BYTES}; +use crate::format::{BodyRef, CODEC_ZSTD, FILE_HEADER_BYTES, MAX_BLOCK_RAW_BYTES}; use std::sync::Arc; use crate::{ArchiveCodecs, BlockCodec, BlockDecoder, BlockEncoder, BodyLogReader, BodyLogWriter, Result}; -const CHAIN_CODEC: u8 = 2; - struct ChainCodec; struct ChainEncoder { @@ -30,7 +28,7 @@ struct ChainDecoder { impl BlockCodec for ChainCodec { fn id(&self) -> u8 { - CHAIN_CODEC + CODEC_ZSTD } fn encoder(&self) -> Result> { @@ -91,7 +89,7 @@ fn a_registered_codec_keeps_stream_state_across_durable_segments() { assert_eq!(reader.read(first).unwrap(), first_body); assert_eq!(reader.read(second).unwrap(), second_body); let encoded = std::fs::read(&path).unwrap(); - assert_eq!(encoded[FILE_HEADER_BYTES + 4], CHAIN_CODEC); + assert_eq!(encoded[FILE_HEADER_BYTES + 4], CODEC_ZSTD); } /// Random bodies -- text, incompressible, empty -- with random flushes and From 2de90c73e0a2f6c9ed52696935120df5fc470e34 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:29:38 +0000 Subject: [PATCH 201/464] security(ledger): confine streaming zstd archives --- CHANGELOG.md | 4 + Cargo.lock | 2 + crates/capsem-ledger/Cargo.toml | 2 + crates/capsem-ledger/src/codec.rs | 115 +++++++++++++++++++++++ crates/capsem-ledger/src/codec/tests.rs | 56 +++++++++++ crates/capsem-ledger/src/main.rs | 5 +- crates/capsem-ledger/tests/subprocess.rs | 107 ++++++++++++++++++++- 7 files changed, 287 insertions(+), 4 deletions(-) create mode 100644 crates/capsem-ledger/src/codec.rs create mode 100644 crates/capsem-ledger/src/codec/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d477aaef..ff346bb5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 signals and ambient files; erase inherited environment bytes and reject symlinked session grants before serving clients. +- Compress new session body blocks with streaming zstd only inside the + confined ledger owner, retaining durable cross-flush history and legacy + deflate reads without linking the native decoder into privileged workers. + - Confine the Linux API gateway before readiness to its inherited listeners and coordinator-minted service and VM-owner channels, denying filesystem access, process execution and direct network or control-socket dialing. diff --git a/Cargo.lock b/Cargo.lock index dadd8ca89..50f8564b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1043,6 +1043,7 @@ name = "capsem-ledger" version = "0.7.0" dependencies = [ "anyhow", + "capsem-archive", "capsem-foundation", "capsem-guard", "capsem-logger", @@ -1051,6 +1052,7 @@ dependencies = [ "tempfile", "tokio", "tracing", + "zstd", ] [[package]] diff --git a/crates/capsem-ledger/Cargo.toml b/crates/capsem-ledger/Cargo.toml index bc424e334..bac1c9db7 100644 --- a/crates/capsem-ledger/Cargo.toml +++ b/crates/capsem-ledger/Cargo.toml @@ -11,6 +11,7 @@ authors.workspace = true [dependencies] anyhow.workspace = true +capsem-archive = { path = "../capsem-archive" } capsem-foundation = { path = "../capsem-foundation" } capsem-guard = { path = "../capsem-guard" } capsem-logger = { path = "../capsem-logger" } @@ -18,6 +19,7 @@ capsem-proto = { path = "../capsem-proto" } clap = { workspace = true, features = ["derive"] } tokio.workspace = true tracing.workspace = true +zstd = "0.13" [dev-dependencies] tempfile = "3" diff --git a/crates/capsem-ledger/src/codec.rs b/crates/capsem-ledger/src/codec.rs new file mode 100644 index 000000000..8b3771e14 --- /dev/null +++ b/crates/capsem-ledger/src/codec.rs @@ -0,0 +1,115 @@ +//! Native archive codec owned exclusively by the confined ledger executable. + +use std::io; +use std::sync::Arc; + +use capsem_archive::{ + ArchiveCodecs, ArchiveError, BlockCodec, BlockDecoder, BlockEncoder, Result as ArchiveResult, CODEC_ZSTD, +}; +use zstd::stream::raw::{Decoder, Encoder, InBuffer, Operation, OutBuffer}; + +const ZSTD_LEVEL: i32 = 3; +const OUTPUT_CHUNK: usize = 128 * 1024; + +pub(crate) fn archive_codecs() -> ArchiveResult { + ArchiveCodecs::with_write_codec(Arc::new(ZstdCodec)) +} + +struct ZstdCodec; + +impl BlockCodec for ZstdCodec { + fn id(&self) -> u8 { + CODEC_ZSTD + } + + fn encoder(&self) -> ArchiveResult> { + Encoder::new(ZSTD_LEVEL) + .map(|encoder| Box::new(ZstdEncoder(encoder)) as Box) + .map_err(ArchiveError::Io) + } + + fn decoder(&self) -> ArchiveResult> { + Decoder::new() + .map(|decoder| Box::new(ZstdDecoder(decoder)) as Box) + .map_err(ArchiveError::Io) + } +} + +struct ZstdEncoder(Encoder<'static>); + +impl BlockEncoder for ZstdEncoder { + fn write(&mut self, input: &[u8], output: &mut Vec) -> ArchiveResult<()> { + let mut input = InBuffer::around(input); + while input.pos() < input.src.len() { + let before = input.pos(); + with_output_capacity(output, |out| self.0.run(&mut input, out)).map_err(ArchiveError::Io)?; + if input.pos() == before { + return Err(ArchiveError::Io(io::Error::other( + "zstd encoder made no input progress", + ))); + } + } + Ok(()) + } + + fn flush(&mut self, output: &mut Vec, finish: bool) -> ArchiveResult<()> { + loop { + let remaining = with_output_capacity(output, |out| { + if finish { + self.0.finish(out, true) + } else { + self.0.flush(out) + } + }) + .map_err(ArchiveError::Io)?; + if remaining == 0 { + return Ok(()); + } + } + } +} + +struct ZstdDecoder(Decoder<'static>); + +impl BlockDecoder for ZstdDecoder { + fn decode_segment(&mut self, input: &[u8], raw_len: usize, finish: bool, at: u64) -> ArchiveResult> { + let mut raw = Vec::with_capacity(raw_len + 1); + let (consumed, written, ended) = { + let mut input = InBuffer::around(input); + let mut output = OutBuffer::around(&mut raw); + let mut ended = false; + loop { + let before = (input.pos(), output.pos()); + let remaining = self + .0 + .run(&mut input, &mut output) + .map_err(|error| ArchiveError::Inflate(at, error.to_string()))?; + if remaining == 0 { + ended = true; + break; + } + if input.pos() == input.src.len() || before == (input.pos(), output.pos()) { + break; + } + } + (input.pos(), output.pos(), ended) + }; + if consumed != input.len() || written != raw_len || ended != finish { + return Err(ArchiveError::Integrity(at)); + } + Ok(raw) + } +} + +fn with_output_capacity( + output: &mut Vec, + operation: impl FnOnce(&mut OutBuffer<'_, Vec>) -> io::Result, +) -> io::Result { + output.reserve(OUTPUT_CHUNK); + let position = output.len(); + let mut buffer = OutBuffer::around_pos(output, position); + operation(&mut buffer) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-ledger/src/codec/tests.rs b/crates/capsem-ledger/src/codec/tests.rs new file mode 100644 index 000000000..e019430fc --- /dev/null +++ b/crates/capsem-ledger/src/codec/tests.rs @@ -0,0 +1,56 @@ +use capsem_archive::format::{BLOCK_HEADER_BYTES, FILE_HEADER_BYTES, SEGMENT_HEADER_BYTES}; +use capsem_archive::{BodyLogReader, BodyLogWriter, CODEC_ZSTD}; + +use super::archive_codecs; + +#[test] +fn zstd_keeps_history_across_durable_segments_and_deflate_stays_readable() { + let dir = tempfile::tempdir().unwrap(); + let zstd_path = dir.path().join("zstd.bodies"); + let codecs = archive_codecs().unwrap(); + let mut writer = BodyLogWriter::open_with_codecs(&zstd_path, codecs.clone()).unwrap(); + let body: Vec = (0_usize..8192) + .map(|index| (index.wrapping_mul(37) & 0xff) as u8) + .collect(); + let first = writer.stage(&body).unwrap(); + writer.flush_segment().unwrap(); + let before = writer.end(); + let second = writer.stage(&body).unwrap(); + writer.close_block().unwrap(); + assert!( + writer.end() - before < 256, + "the second segment did not reuse zstd history" + ); + drop(writer); + + let encoded = std::fs::read(&zstd_path).unwrap(); + assert_eq!(encoded[FILE_HEADER_BYTES + 4], CODEC_ZSTD); + let reader = BodyLogReader::open_with_codecs(&zstd_path, codecs.clone()).unwrap(); + assert_eq!(reader.read(first).unwrap(), body); + assert_eq!(reader.read(second).unwrap(), body); + + let deflate_path = dir.path().join("deflate.bodies"); + let mut deflate = BodyLogWriter::open(&deflate_path).unwrap(); + let reference = deflate.stage(b"an archive from before zstd").unwrap(); + deflate.close_block().unwrap(); + let compatible = BodyLogReader::open_with_codecs(&deflate_path, codecs).unwrap(); + assert_eq!(compatible.read(reference).unwrap(), b"an archive from before zstd"); +} + +#[test] +fn zstd_corruption_is_refused_before_bytes_are_returned() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("corrupt.bodies"); + let codecs = archive_codecs().unwrap(); + let mut writer = BodyLogWriter::open_with_codecs(&path, codecs.clone()).unwrap(); + let reference = writer.stage(b"authenticated zstd bytes").unwrap(); + writer.close_block().unwrap(); + drop(writer); + + let mut encoded = std::fs::read(&path).unwrap(); + let payload = FILE_HEADER_BYTES + BLOCK_HEADER_BYTES + SEGMENT_HEADER_BYTES; + encoded[payload + 1] ^= 0x80; + std::fs::write(&path, encoded).unwrap(); + let reader = BodyLogReader::open_with_codecs(&path, codecs).unwrap(); + assert!(reader.read(reference).is_err()); +} diff --git a/crates/capsem-ledger/src/main.rs b/crates/capsem-ledger/src/main.rs index f202ee3fd..96952201b 100644 --- a/crates/capsem-ledger/src/main.rs +++ b/crates/capsem-ledger/src/main.rs @@ -20,6 +20,8 @@ use capsem_proto::ledger_control::{ use clap::Parser; use tokio::task::{JoinError, JoinSet}; +mod codec; + const CLIENT_LIMIT: usize = 64; type ControlSender = router_channel::DescriptorSender; @@ -56,7 +58,8 @@ fn main() -> Result<()> { let control = UnixStream::from(fd::duplicate(stdin.as_fd())?); let session_dir = session_directory(&args.database)?; let denied_file = std::env::current_exe().context("resolve ledger executable before confinement")?; - let server = Arc::new(LedgerServer::open(&args.database).context("open session ledger")?); + let codecs = codec::archive_codecs().context("initialize confined archive codecs")?; + let server = Arc::new(LedgerServer::open_with_codecs(&args.database, codecs).context("open session ledger")?); let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) .enable_all() diff --git a/crates/capsem-ledger/tests/subprocess.rs b/crates/capsem-ledger/tests/subprocess.rs index 63eef51c8..cf621e21b 100644 --- a/crates/capsem-ledger/tests/subprocess.rs +++ b/crates/capsem-ledger/tests/subprocess.rs @@ -1,10 +1,15 @@ -use std::os::fd::OwnedFd; +use std::os::fd::{AsRawFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::process::{Child, Command, Stdio}; -use std::time::Duration; +use std::time::{Duration, SystemTime}; +use capsem_foundation::ipc_channel; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; -use capsem_proto::ledger::LedgerGeneration; +use capsem_logger::ledger_protocol::{LedgerClientMessage, LedgerCommand, LedgerReply, LedgerServerMessage}; +use capsem_logger::{Decision, NetEvent, WriteOp}; +use capsem_proto::ledger::{ + LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerHello, LedgerOutcome, LedgerRequest, +}; use capsem_proto::ledger_control::{ decode_ledger_control_event, encode_ledger_control_request, LedgerControlEvent, LedgerControlRequest, LEDGER_CONTROL_FRAME_SIZE, LEDGER_CONTROL_MAX_FDS, @@ -41,6 +46,37 @@ async fn event(receiver: &ControlReceiver) -> LedgerControlEvent { decode_ledger_control_event(&frame.bytes).expect("valid ledger control event") } +fn body_event() -> WriteOp { + WriteOp::NetEvent(NetEvent { + event_id: Some("0123456789ab".into()), + timestamp: SystemTime::UNIX_EPOCH, + domain: "confined-zstd.test".into(), + port: 443, + decision: Decision::Allowed, + process_name: None, + pid: None, + method: Some("POST".into()), + path: Some("/archive".into()), + query: None, + status_code: Some(200), + bytes_sent: 0, + bytes_received: 0, + duration_ms: 1, + matched_rule: None, + request_headers: Some("content-type: application/json".into()), + response_headers: None, + request_body: Some(br#"{"body":"written after confinement"}"#.to_vec()), + response_body: None, + conn_type: Some("https".into()), + policy_mode: None, + policy_action: None, + policy_rule: None, + policy_reason: None, + trace_id: None, + credential_ref: None, + }) +} + #[tokio::test] async fn executable_opens_one_ledger_and_reports_durable_stop() { let dir = tempfile::tempdir().unwrap(); @@ -74,6 +110,59 @@ async fn executable_opens_one_ledger_and_reports_durable_stop() { .windows(b"CAPSEM_LEDGER_SECRET".len()) .any(|value| value == b"CAPSEM_LEDGER_SECRET")); } + let grant = LedgerChannelGrant::new(GENERATION, 1, LedgerClientRole::VmOwner).unwrap(); + let (client, ledger_end) = UnixStream::pair().unwrap(); + requests + .send( + &encode_ledger_control_request(LedgerControlRequest::Attach(grant)), + &[ledger_end.as_raw_fd()], + ) + .await + .unwrap(); + assert_eq!( + event(&events).await, + LedgerControlEvent::Adopted { + generation: GENERATION, + client_id: 1, + } + ); + drop(ledger_end); + let (sender, receiver) = ipc_channel::channel_from_std::(client).unwrap(); + sender + .send(LedgerClientMessage::Hello { + hello: LedgerHello::for_grant(&grant), + }) + .await + .unwrap(); + assert!(matches!( + receiver.recv().await.unwrap(), + LedgerServerMessage::Welcome { .. } + )); + for (request_id, operation) in [ + ( + 1, + LedgerCommand::Admit { + event: Box::new(body_event()), + }, + ), + (2, LedgerCommand::Flush), + ] { + sender + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(request_id, operation).unwrap(), + }) + .await + .unwrap(); + let LedgerServerMessage::Response { response } = receiver.recv().await.unwrap() else { + panic!("expected ledger response") + }; + assert!(matches!( + response.outcome(), + LedgerOutcome::Success { + reply: LedgerReply::Accepted { .. } | LedgerReply::Durable { .. } + } + )); + } requests .send( &encode_ledger_control_request(LedgerControlRequest::Shutdown { generation: GENERATION }), @@ -97,6 +186,18 @@ async fn executable_opens_one_ledger_and_reports_durable_stop() { String::from_utf8_lossy(&output.stderr) ); assert!(database.is_file()); + let archive_dir = database.with_extension("bodies"); + let generation = std::fs::read_dir(&archive_dir) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().and_then(|extension| extension.to_str()) == Some("cbl")) + .expect("one archive generation"); + let archive = std::fs::read(generation).unwrap(); + assert_eq!( + archive[capsem_archive::FILE_HEADER_BYTES + 4], + capsem_archive::CODEC_ZSTD, + "body admitted after worker readiness must use the confined zstd codec" + ); } #[tokio::test] From d76bf75955b8e02893fcc7f0fe01c1c578973455 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:30:07 +0000 Subject: [PATCH 202/464] test(citadel): pin confined zstd ownership --- .../test_runtime_native_dependencies.py | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/citadel/test_runtime_native_dependencies.py b/tests/citadel/test_runtime_native_dependencies.py index 00fe76580..36fd6d1b9 100644 --- a/tests/citadel/test_runtime_native_dependencies.py +++ b/tests/citadel/test_runtime_native_dependencies.py @@ -1,9 +1,10 @@ -"""Citadel guard: runtime crates link no native codec, and keep two write rails. +"""Citadel guard: native codecs stay in the confined ledger, and keep two write rails. -The one C library the runtime ships is SQLite through bundled rusqlite. The -ledger is parsed from attacker-influenced bytes, so every additional native -decoder is attack surface the Rust type system does not cover. Pure-Rust -codecs (miniz_oxide, lz4_flex, ruzstd) are allowed; their *-sys twins are not. +Privileged runtime processes link only SQLite through bundled rusqlite. The +dedicated ledger worker may additionally link the reviewed zstd closure: it is +confined to one session before it decodes bodies, and no service, VM owner, +gateway or proxy reaches that native parser. Pure-Rust codecs remain allowed +everywhere; other *-sys twins are not. The second rule here is about storage rather than parsing, and it catches what the first cannot: sled, redb and fjall are pure Rust, so nothing about them @@ -23,9 +24,11 @@ WORKSPACE = PROJECT_ROOT / "Cargo.toml" NATIVE_DEPENDENCY_RATIONALE = """\ -Native compression/storage dependency in a runtime crate. +Native compression/storage dependency outside the confined ledger exception. -Runtime crates ship exactly one C library: SQLite via bundled rusqlite. +Privileged runtime crates ship exactly one C library: SQLite via bundled +rusqlite. The confined capsem-ledger worker alone may add the exact zstd, +zstd-safe and zstd-sys closure. zstd-sys, lz4-sys, librocksdb-sys, libsqlite3-sys outside rusqlite, snappy, brotli-sys and friends add a native decoder in front of attacker-influenced ledger bytes. Use a pure-Rust codec (miniz_oxide is the house choice) or move @@ -69,6 +72,9 @@ "libdeflate-sys", } +# The exact native codec closure reviewed for the session-confined ledger. +CONFINED_NATIVE_CODECS = {"capsem-ledger": {"zstd", "zstd-safe", "zstd-sys"}} + # Tooling crates that never ship in the runtime and may use native codecs. TOOLING_CRATES = {"capsem-admin", "capsem-bench", "capsem-mock-server"} @@ -170,7 +176,8 @@ def offenders(lock_text: str, member_names: Iterable[str]) -> list[str]: for name in member_names: if name in TOOLING_CRATES: continue - hit = sorted(FORBIDDEN & _closure(packages, name)) + allowed = CONFINED_NATIVE_CODECS.get(name, set()) + hit = sorted((FORBIDDEN & _closure(packages, name)) - allowed) if hit: result.append(f"{name}: {', '.join(hit)}") return result @@ -182,6 +189,12 @@ def test_runtime_crates_link_no_native_compression_or_storage() -> None: assert not found, "\n".join(found) + "\n" + NATIVE_DEPENDENCY_RATIONALE +def test_confined_ledger_owns_the_exact_reviewed_zstd_closure() -> None: + packages = _lock_packages(LOCKFILE.read_text()) + native = FORBIDDEN & _closure(packages, "capsem-ledger") + assert native == CONFINED_NATIVE_CODECS["capsem-ledger"] + + def test_every_workspace_member_resolves_to_a_lockfile_package() -> None: member_names = _member_package_names(WORKSPACE.read_text()).values() packages = _lock_packages(LOCKFILE.read_text()) @@ -241,3 +254,8 @@ def test_offenders_flags_a_synthetic_native_dependency() -> None: injected = _inject_dependency(lock_text, "capsem-logger", "zstd 0.13.3") found = offenders(injected, ["capsem-logger"]) assert any(item.startswith("capsem-logger:") for item in found), found + + +def test_confined_ledger_exception_rejects_another_native_codec() -> None: + lock_text = _inject_dependency(LOCKFILE.read_text(), "capsem-ledger", "lz4-sys 1.11.1") + assert offenders(lock_text, ["capsem-ledger"]) == ["capsem-ledger: lz4-sys"] From a61bd9c6a9cde04f2b593917224eb7afc9928031 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:32:15 +0000 Subject: [PATCH 203/464] test(ledger): measure archive codec tradeoffs --- crates/capsem-ledger/src/codec/tests.rs | 109 ++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/crates/capsem-ledger/src/codec/tests.rs b/crates/capsem-ledger/src/codec/tests.rs index e019430fc..d3fae2ee8 100644 --- a/crates/capsem-ledger/src/codec/tests.rs +++ b/crates/capsem-ledger/src/codec/tests.rs @@ -1,5 +1,6 @@ use capsem_archive::format::{BLOCK_HEADER_BYTES, FILE_HEADER_BYTES, SEGMENT_HEADER_BYTES}; use capsem_archive::{BodyLogReader, BodyLogWriter, CODEC_ZSTD}; +use capsem_logger::{DbHandle, Decision, NetEvent, WriteOp}; use super::archive_codecs; @@ -54,3 +55,111 @@ fn zstd_corruption_is_refused_before_bytes_are_returned() { let reader = BodyLogReader::open_with_codecs(&path, codecs).unwrap(); assert!(reader.read(reference).is_err()); } + +#[tokio::test] +#[ignore = "repeatable manual codec measurement"] +async fn measure_identical_ledger_codec() { + const EXCHANGES: usize = 1_000; + const FLUSH_EVERY: usize = 20; + let codec = std::env::var("CAPSEM_CODEC_BENCH").expect("CAPSEM_CODEC_BENCH=deflate|zstd"); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("session.db"); + let db = match codec.as_str() { + "deflate" => DbHandle::open(&path).unwrap(), + "zstd" => DbHandle::open_with_codecs(&path, archive_codecs().unwrap()).unwrap(), + other => panic!("unsupported codec {other}"), + }; + let baseline_rss = capsem_foundation::proctable::resident_bytes(std::process::id()).unwrap(); + let mut peak_rss = baseline_rss; + let mut raw_bytes = 0usize; + let started = std::time::Instant::now(); + for index in 0..EXCHANGES { + let (event, bytes) = benchmark_event(index); + raw_bytes += bytes; + db.write(event).await.unwrap(); + if index % FLUSH_EVERY == FLUSH_EVERY - 1 { + db.flush().await.unwrap(); + peak_rss = peak_rss.max(capsem_foundation::proctable::resident_bytes(std::process::id()).unwrap()); + } + } + db.flush().await.unwrap(); + let elapsed = started.elapsed(); + drop(db); + let archive_bytes: u64 = std::fs::read_dir(path.with_extension("bodies")) + .unwrap() + .map(|entry| entry.unwrap().metadata().unwrap().len()) + .sum(); + let seconds = elapsed.as_secs_f64(); + eprintln!( + "codec={codec} exchanges={EXCHANGES} raw_bytes={raw_bytes} archive_bytes={archive_bytes} ratio={:.4} elapsed_ms={:.3} throughput_mib_s={:.3} peak_rss_delta_bytes={}", + archive_bytes as f64 / raw_bytes as f64, + seconds * 1000.0, + raw_bytes as f64 / (1024.0 * 1024.0) / seconds, + peak_rss.saturating_sub(baseline_rss), + ); +} + +fn benchmark_event(index: usize) -> (WriteOp, usize) { + let request = benchmark_text(index, 2048); + let response = benchmark_text(index.wrapping_mul(31) + 7, 4096); + let bytes = request.len() + response.len(); + ( + WriteOp::NetEvent(NetEvent { + event_id: Some(format!("{index:012x}")), + timestamp: std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(index as u64), + domain: "api.bench.example".into(), + port: 443, + decision: Decision::Allowed, + process_name: Some("codec-bench".into()), + pid: Some(1), + method: Some("POST".into()), + path: Some("/v1/messages".into()), + query: None, + status_code: Some(200), + bytes_sent: request.len() as u64, + bytes_received: response.len() as u64, + duration_ms: 1, + matched_rule: None, + request_headers: Some("content-type: application/json".into()), + response_headers: Some("content-type: application/json".into()), + request_body: Some(request.into_bytes()), + response_body: Some(response.into_bytes()), + conn_type: Some("https".into()), + policy_mode: None, + policy_action: None, + policy_rule: None, + policy_reason: None, + trace_id: Some(format!("{index:016x}")), + credential_ref: None, + }), + bytes, + ) +} + +fn benchmark_text(seed: usize, len: usize) -> String { + const WORDS: [&str; 12] = [ + "assistant", + "content", + "stream", + "delta", + "tool", + "result", + "input", + "output", + "usage", + "message", + "role", + "index", + ]; + let mut state = (seed as u64).wrapping_add(1).wrapping_mul(0x9e37_79b9_7f4a_7c15); + let mut output = String::with_capacity(len + 16); + while output.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + output.push_str(WORDS[(state % WORDS.len() as u64) as usize]); + output.push(if state.is_multiple_of(7) { ',' } else { ' ' }); + } + output.truncate(len); + output +} From 8ddda89b36cf74bbbf4549bee478d79d11fc3f32 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:37:03 +0000 Subject: [PATCH 204/464] feat(proto): define producer ledger commitments --- crates/capsem-proto/build.rs | 5 +- crates/capsem-proto/src/ledger_commitment.rs | 199 ++++++++++++++++++ .../src/ledger_commitment/tests.rs | 42 ++++ crates/capsem-proto/src/lib.rs | 1 + 4 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 crates/capsem-proto/src/ledger_commitment.rs create mode 100644 crates/capsem-proto/src/ledger_commitment/tests.rs diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index cd0679e86..6b9bdeba9 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -23,7 +23,10 @@ impl VisitMut for StripDocs { fn main() { let files = ["lib.rs", "ipc.rs", "handshake.rs", "router.rs", "exec_stream.rs"]; write_hash("schema_hash.txt", &files); - write_hash("ledger_schema_hash.txt", &["ledger.rs", "ledger_control.rs"]); + write_hash( + "ledger_schema_hash.txt", + &["ledger.rs", "ledger_commitment.rs", "ledger_control.rs"], + ); write_hash("proxy_schema_hash.txt", &["proxy_control.rs", "proxy_policy.rs"]); } diff --git a/crates/capsem-proto/src/ledger_commitment.rs b/crates/capsem-proto/src/ledger_commitment.rs new file mode 100644 index 000000000..2691fa041 --- /dev/null +++ b/crates/capsem-proto/src/ledger_commitment.rs @@ -0,0 +1,199 @@ +//! Producer-authored commitments anchored by the trusted ledger coordinator. + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerHello, LedgerWelcome}; + +pub const MAX_COMMITMENT_EVENT_KIND_BYTES: usize = 64; +pub const MAX_COMMITMENTS_PER_CHECKPOINT: usize = 4096; +pub const ZERO_COMMITMENT_HASH: [u8; 32] = [0; 32]; + +/// Identity, order and content commitment for one producer record. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerCommitment { + generation: LedgerGeneration, + client_id: u64, + role: LedgerClientRole, + producer_sequence: u64, + global_sequence: u64, + event_kind: String, + event_hash: [u8; 32], + previous_hash: [u8; 32], + commitment_hash: [u8; 32], +} + +impl LedgerCommitment { + pub fn new( + grant: LedgerChannelGrant, + producer_sequence: u64, + global_sequence: u64, + event_kind: impl Into, + event_hash: [u8; 32], + previous_hash: [u8; 32], + ) -> Result { + let mut commitment = Self { + generation: grant.generation(), + client_id: grant.client_id(), + role: grant.role(), + producer_sequence, + global_sequence, + event_kind: event_kind.into(), + event_hash, + previous_hash, + commitment_hash: ZERO_COMMITMENT_HASH, + }; + commitment.validate_fields()?; + commitment.commitment_hash = commitment.calculate_hash(); + Ok(commitment) + } + + pub fn validate(&self) -> Result<(), LedgerCommitmentError> { + self.validate_fields()?; + if self.calculate_hash() != self.commitment_hash { + return Err(LedgerCommitmentError::HashMismatch); + } + Ok(()) + } + + pub fn validate_grant(&self, grant: LedgerChannelGrant) -> Result<(), LedgerCommitmentError> { + self.validate()?; + if self.generation != grant.generation() || self.client_id != grant.client_id() || self.role != grant.role() { + return Err(LedgerCommitmentError::AuthorityMismatch); + } + Ok(()) + } + + pub const fn generation(&self) -> LedgerGeneration { + self.generation + } + + pub const fn client_id(&self) -> u64 { + self.client_id + } + + pub const fn role(&self) -> LedgerClientRole { + self.role + } + + pub const fn producer_sequence(&self) -> u64 { + self.producer_sequence + } + + pub const fn global_sequence(&self) -> u64 { + self.global_sequence + } + + pub fn event_kind(&self) -> &str { + &self.event_kind + } + + pub const fn event_hash(&self) -> [u8; 32] { + self.event_hash + } + + pub const fn previous_hash(&self) -> [u8; 32] { + self.previous_hash + } + + pub const fn commitment_hash(&self) -> [u8; 32] { + self.commitment_hash + } + + fn validate_fields(&self) -> Result<(), LedgerCommitmentError> { + if self.client_id == 0 || self.producer_sequence == 0 || self.global_sequence == 0 { + return Err(LedgerCommitmentError::ZeroSequence); + } + if self.role.producer_name().is_none() { + return Err(LedgerCommitmentError::NonProducer); + } + if self.event_kind.is_empty() || self.event_kind.len() > MAX_COMMITMENT_EVENT_KIND_BYTES { + return Err(LedgerCommitmentError::InvalidEventKind); + } + Ok(()) + } + + fn calculate_hash(&self) -> [u8; 32] { + let mut hash = blake3::Hasher::new(); + hash.update(b"capsem-ledger-commitment-v1\0"); + hash.update(&self.generation.as_bytes()); + hash.update(&self.client_id.to_be_bytes()); + hash.update(&[role_tag(self.role)]); + hash.update(&self.producer_sequence.to_be_bytes()); + hash.update(&self.global_sequence.to_be_bytes()); + hash.update(&(self.event_kind.len() as u64).to_be_bytes()); + hash.update(self.event_kind.as_bytes()); + hash.update(&self.event_hash); + hash.update(&self.previous_hash); + *hash.finalize().as_bytes() + } +} + +fn role_tag(role: LedgerClientRole) -> u8 { + match role { + LedgerClientRole::VmOwner => 1, + LedgerClientRole::Proxy => 2, + LedgerClientRole::Coordinator => 3, + LedgerClientRole::Reader => 4, + LedgerClientRole::Maintainer => 5, + LedgerClientRole::Supervisor => 6, + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum CommitmentCommand { + Reserve { + producer_sequence: u64, + event_kind: String, + event_hash: [u8; 32], + previous_hash: [u8; 32], + }, + Cancel { + global_sequence: u64, + }, + Anchor { + commitments: Vec, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum CommitmentReply { + Reserved { global_sequence: u64 }, + Canceled, + Anchored { checkpoint_sequence: u64 }, + Failed { message: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum CommitmentClientMessage { + Hello { + hello: LedgerHello, + }, + Request { + request_id: u64, + command: CommitmentCommand, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum CommitmentServerMessage { + Welcome { welcome: LedgerWelcome }, + Response { request_id: u64, reply: CommitmentReply }, +} + +#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)] +pub enum LedgerCommitmentError { + #[error("ledger commitment sequence must not be zero")] + ZeroSequence, + #[error("ledger commitment role is not a producer")] + NonProducer, + #[error("ledger commitment event kind is invalid")] + InvalidEventKind, + #[error("ledger commitment hash does not match its fields")] + HashMismatch, + #[error("ledger commitment authority does not match its channel")] + AuthorityMismatch, +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/ledger_commitment/tests.rs b/crates/capsem-proto/src/ledger_commitment/tests.rs new file mode 100644 index 000000000..d9261ee23 --- /dev/null +++ b/crates/capsem-proto/src/ledger_commitment/tests.rs @@ -0,0 +1,42 @@ +use super::*; + +fn grant(role: LedgerClientRole) -> LedgerChannelGrant { + LedgerChannelGrant::new(LedgerGeneration::new([7; 16]), 11, role).unwrap() +} + +#[test] +fn commitment_binds_authority_order_kind_content_and_chain() { + let commitment = + LedgerCommitment::new(grant(LedgerClientRole::Proxy), 2, 19, "net_event", [3; 32], [5; 32]).unwrap(); + commitment.validate_grant(grant(LedgerClientRole::Proxy)).unwrap(); + assert_ne!(commitment.commitment_hash(), ZERO_COMMITMENT_HASH); + + let mut altered = commitment.clone(); + altered.event_hash[0] ^= 1; + assert_eq!(altered.validate(), Err(LedgerCommitmentError::HashMismatch)); + + let stale = LedgerChannelGrant::new(LedgerGeneration::new([8; 16]), 11, LedgerClientRole::Proxy).unwrap(); + assert_eq!( + commitment.validate_grant(stale), + Err(LedgerCommitmentError::AuthorityMismatch) + ); +} + +#[test] +fn commitments_reject_reader_and_unbounded_identity() { + assert_eq!( + LedgerCommitment::new(grant(LedgerClientRole::Reader), 1, 1, "net_event", [1; 32], [0; 32]), + Err(LedgerCommitmentError::NonProducer) + ); + assert_eq!( + LedgerCommitment::new( + grant(LedgerClientRole::VmOwner), + 1, + 1, + "x".repeat(MAX_COMMITMENT_EVENT_KIND_BYTES + 1), + [1; 32], + [0; 32] + ), + Err(LedgerCommitmentError::InvalidEventKind) + ); +} diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index c7213d05d..1a7d0a0c4 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -19,6 +19,7 @@ pub mod handshake; pub mod host_session; pub mod ipc; pub mod ledger; +pub mod ledger_commitment; pub mod ledger_control; pub mod ledger_counters; pub mod mcp; From 87b92e9d2c8eaeb7df93f1478e6256ca6ae154bc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:40:53 +0000 Subject: [PATCH 205/464] feat(proto): grant trusted ledger checkpoint channels --- crates/capsem-proto/src/proxy_control.rs | 14 ++++++++++++-- crates/capsem-proto/src/proxy_control/tests.rs | 17 +++++++++++++++++ crates/capsem-proto/src/upstream_grant.rs | 12 +++++++----- crates/capsem-proto/src/upstream_grant/tests.rs | 2 +- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/crates/capsem-proto/src/proxy_control.rs b/crates/capsem-proto/src/proxy_control.rs index 0fcd0322c..66d45259c 100644 --- a/crates/capsem-proto/src/proxy_control.rs +++ b/crates/capsem-proto/src/proxy_control.rs @@ -5,8 +5,8 @@ use thiserror::Error; use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerProtocolError}; pub const PROXY_CONTROL_FRAME_SIZE: usize = 56; -pub const PROXY_CONTROL_MAX_FDS: usize = 1; -pub const PROXY_CONTROL_VERSION: u16 = 4; +pub const PROXY_CONTROL_MAX_FDS: usize = 2; +pub const PROXY_CONTROL_VERSION: u16 = 5; const MAGIC: [u8; 2] = *b"PX"; const VERSION_RANGE: std::ops::Range = 2..4; @@ -134,6 +134,16 @@ impl ProxyChannelGrant { pub const fn ledger_grant(self) -> Option { self.ledger_grant } + + /// Ledger writers receive the storage channel and its independent trusted + /// checkpoint channel. Every other capability owns one descriptor. + pub const fn expected_descriptor_count(self) -> usize { + if matches!(self.capability, ProxyCapability::Ledger) { + 2 + } else { + 1 + } + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/crates/capsem-proto/src/proxy_control/tests.rs b/crates/capsem-proto/src/proxy_control/tests.rs index 3723dfad1..ad4933a00 100644 --- a/crates/capsem-proto/src/proxy_control/tests.rs +++ b/crates/capsem-proto/src/proxy_control/tests.rs @@ -44,6 +44,23 @@ fn ledger_grants_round_trip_exact_worker_authority() { assert_eq!(grant.ledger_grant(), Some(ledger)); } +#[test] +fn only_ledger_grants_require_two_descriptors() { + let ledger = LedgerChannelGrant::new(LedgerGeneration::new([3; 16]), 9, LedgerClientRole::Proxy).unwrap(); + assert_eq!( + ProxyChannelGrant::with_ledger(GENERATION, 7, ledger) + .unwrap() + .expected_descriptor_count(), + 2 + ); + assert_eq!( + ProxyChannelGrant::new(GENERATION, 8, ProxyCapability::Credential) + .unwrap() + .expected_descriptor_count(), + 1 + ); +} + #[test] fn ledger_capability_without_exact_authority_is_rejected() { assert_eq!( diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index 6a1642293..9614b53e0 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -14,8 +14,8 @@ use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; /// Exact record size carried by the bounded SCM_RIGHTS channel. pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 4480; -/// A response grants at most one connected upstream or ledger descriptor. -pub const UPSTREAM_GRANT_MAX_FDS: usize = 1; +/// A ledger response grants its storage and trusted-checkpoint channels. +pub const UPSTREAM_GRANT_MAX_FDS: usize = 2; /// DNS names are at most 253 wire-text bytes without a root dot. pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; /// Maximum relative path carried by a guest-share metadata request. @@ -207,7 +207,7 @@ pub enum UpstreamGrantRequest { }, } -/// Coordinator-to-worker record. Successful descriptor grants carry one fd. +/// Coordinator-to-worker record. Ledger grants carry two independent fds. #[derive(Clone, Debug, Eq, PartialEq)] pub enum UpstreamGrantResponse { TcpResolved { @@ -253,10 +253,12 @@ impl UpstreamGrantRequest { } impl UpstreamGrantResponse { - /// Successful connected-socket and ledger grants carry one descriptor. + /// Successful connected sockets carry one descriptor; ledger grants carry + /// the direct worker channel and the trusted checkpoint channel. pub const fn expected_descriptor_count(&self) -> usize { match self { - Self::DescriptorGranted { .. } | Self::LedgerGranted { .. } => 1, + Self::DescriptorGranted { .. } => 1, + Self::LedgerGranted { .. } => 2, Self::TcpResolved { .. } | Self::Denied { .. } | Self::GuestModeSet { .. } diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index 976cdefa5..037215844 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -180,7 +180,7 @@ fn descriptor_expectations_are_explicit() { grant: LedgerChannelGrant::new(LedgerGeneration::new([1; 16]), 3, LedgerClientRole::VmOwner).unwrap(), } .expected_descriptor_count(), - 1 + 2 ); assert_eq!( UpstreamGrantResponse::DescriptorGranted { From 58b2e20a251b834aebdd70634f19efc80fa617bf Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:54:04 +0000 Subject: [PATCH 206/464] security(logger): bind records to producer commitments --- crates/capsem-logger/src/db.rs | 16 + .../capsem-logger/src/ledger_client/tests.rs | 3 +- crates/capsem-logger/src/ledger_protocol.rs | 22 +- .../src/ledger_protocol/tests.rs | 81 ++-- crates/capsem-logger/src/ledger_server.rs | 26 +- .../src/ledger_server/queries.rs | 11 + .../capsem-logger/src/ledger_server/tests.rs | 13 +- crates/capsem-logger/src/schema/columns.rs | 14 + crates/capsem-logger/src/schema/ddl.rs | 17 + .../capsem-logger/src/schema/memory_sync.rs | 1 + crates/capsem-logger/src/writer.rs | 33 +- crates/capsem-logger/src/writer/barriers.rs | 4 +- crates/capsem-logger/src/writer/batch.rs | 57 ++- crates/capsem-logger/src/writer/operation.rs | 8 + crates/capsem-logger/src/writer/producer.rs | 16 + crates/capsem-logger/src/writer/remote.rs | 387 ++++++++++++++++-- .../capsem-logger/src/writer/remote/tests.rs | 15 +- .../src/writer/tests/counters.rs | 14 +- 18 files changed, 636 insertions(+), 102 deletions(-) diff --git a/crates/capsem-logger/src/db.rs b/crates/capsem-logger/src/db.rs index 430542d25..f70c596b2 100644 --- a/crates/capsem-logger/src/db.rs +++ b/crates/capsem-logger/src/db.rs @@ -757,6 +757,22 @@ impl DbHandle { Ok(()) } + pub(crate) async fn write_committed( + &self, + op: WriteOp, + commitment: capsem_proto::ledger_commitment::LedgerCommitment, + ) -> DbResult<()> { + let affects_session_summary = !matches!(&op, WriteOp::PolicyMutationEvent(_) | WriteOp::HostEvent(_)); + let writer = self + .inner + .writer + .as_ref() + .ok_or_else(|| "db handle is read-only; no writer is available".to_string())?; + writer.write_committed(op, commitment).await?; + self.invalidate_after_write(affects_session_summary); + Ok(()) + } + /// Flush accepted writes through the DB-owned writer path. /// /// Tests and read-after-write callers use this as the visibility barrier. diff --git a/crates/capsem-logger/src/ledger_client/tests.rs b/crates/capsem-logger/src/ledger_client/tests.rs index ad6b19384..498fe2966 100644 --- a/crates/capsem-logger/src/ledger_client/tests.rs +++ b/crates/capsem-logger/src/ledger_client/tests.rs @@ -27,11 +27,12 @@ impl Fixture { let producer_grant = LedgerChannelGrant::new(GENERATION, 1, LedgerClientRole::VmOwner).unwrap(); let (producer, producer_server) = std::os::unix::net::UnixStream::pair().unwrap(); + let (commitment, _commitment_task) = crate::writer::test_commitment_channel(producer_grant); let producer_owner = Arc::clone(&server); let producer_task = tokio::spawn(async move { producer_owner.serve_client(producer_server, producer_grant).await }); let writer = tokio::task::spawn_blocking(move || { - crate::DbWriter::from_ledger_channel(producer, producer_grant, &path, 16) + crate::DbWriter::from_ledger_channel(producer, commitment, producer_grant, &path, 16) }) .await .unwrap() diff --git a/crates/capsem-logger/src/ledger_protocol.rs b/crates/capsem-logger/src/ledger_protocol.rs index fa60c887c..5f7a41dd7 100644 --- a/crates/capsem-logger/src/ledger_protocol.rs +++ b/crates/capsem-logger/src/ledger_protocol.rs @@ -6,6 +6,7 @@ use capsem_foundation::ipc_channel::MAX_IPC_FRAME_SIZE; use capsem_proto::ledger::{ LedgerCapability, LedgerHello, LedgerOperation, LedgerProtocolError, LedgerRequest, LedgerWelcome, }; +use capsem_proto::ledger_commitment::LedgerCommitment; use capsem_proto::ledger_counters::LedgerCounters; use serde::{Deserialize, Serialize}; @@ -48,6 +49,7 @@ pub enum LedgerServerMessage { pub enum LedgerCommand { Admit { event: Box, + commitment: LedgerCommitment, }, /// Make every accepted event through the returned admission id durable. Flush, @@ -85,6 +87,18 @@ impl LedgerOperation for LedgerCommand { fn validate(&self) -> Result<(), LedgerProtocolError> { match self { + Self::Admit { event, commitment } => { + commitment + .validate() + .map_err(|_| LedgerProtocolError::InvalidOperation)?; + if commitment.event_kind() != event.kind() + || commitment.event_hash() + != crate::writer::commitment_event_hash(event) + .map_err(|_| LedgerProtocolError::InvalidOperation)? + { + return Err(LedgerProtocolError::InvalidOperation); + } + } Self::Query { query } => query.validate()?, Self::ReadBodies { event_id } if !valid_event_id(event_id) => { return Err(LedgerProtocolError::InvalidOperation) @@ -139,6 +153,10 @@ pub enum LedgerQuery { Triage { limit: u16, }, + Commitments { + after_global_sequence: u64, + limit: u16, + }, } impl LedgerQuery { @@ -146,7 +164,9 @@ impl LedgerQuery { let valid_limit = |limit: u16| limit > 0 && usize::from(limit) <= MAX_LEDGER_QUERY_ROWS; let bounded_text = |value: &str| value.len() <= MAX_QUERY_TEXT_BYTES; let valid = match self { - Self::SecurityLatest { limit, .. } | Self::Triage { limit } => valid_limit(*limit), + Self::SecurityLatest { limit, .. } | Self::Triage { limit } | Self::Commitments { limit, .. } => { + valid_limit(*limit) + } Self::Timeline { layers, cutoff, diff --git a/crates/capsem-logger/src/ledger_protocol/tests.rs b/crates/capsem-logger/src/ledger_protocol/tests.rs index d268c297c..93b066012 100644 --- a/crates/capsem-logger/src/ledger_protocol/tests.rs +++ b/crates/capsem-logger/src/ledger_protocol/tests.rs @@ -3,6 +3,7 @@ use std::time::SystemTime; use capsem_foundation::ipc_channel; use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerRequest}; +use capsem_proto::ledger_commitment::{LedgerCommitment, ZERO_COMMITMENT_HASH}; use super::*; use crate::{ExecEventComplete, FileAction, FileEvent, FileKind, WriteOp}; @@ -12,20 +13,34 @@ fn grant(role: LedgerClientRole) -> LedgerChannelGrant { } fn file_write() -> LedgerCommand { + let event = WriteOp::FileEvent(FileEvent { + event_id: None, + timestamp: SystemTime::UNIX_EPOCH, + action: FileAction::Modified, + path: "/root/report.txt".into(), + size: Some(12), + kind: FileKind::File, + trace_id: None, + credential_ref: None, + }); LedgerCommand::Admit { - event: Box::new(WriteOp::FileEvent(FileEvent { - event_id: None, - timestamp: SystemTime::UNIX_EPOCH, - action: FileAction::Modified, - path: "/root/report.txt".into(), - size: Some(12), - kind: FileKind::File, - trace_id: None, - credential_ref: None, - })), + commitment: commitment(&event), + event: Box::new(event), } } +fn commitment(event: &WriteOp) -> LedgerCommitment { + LedgerCommitment::new( + grant(LedgerClientRole::VmOwner), + 1, + 1, + event.kind(), + crate::writer::commitment_event_hash(event).unwrap(), + ZERO_COMMITMENT_HASH, + ) + .unwrap() +} + #[tokio::test] async fn request_round_trips_over_the_landed_bounded_channel() { let request = LedgerRequest::new(1, file_write()).unwrap(); @@ -39,7 +54,7 @@ async fn request_round_trips_over_the_landed_bounded_channel() { assert!(matches!( decoded.operation(), LedgerCommand::Admit { - event + event, .. } if matches!(event.as_ref(), WriteOp::FileEvent(event) if event.path == "/root/report.txt") )); } @@ -80,17 +95,19 @@ fn snapshot_ids_must_be_nonzero_fixed_width_values() { #[test] fn oversized_event_and_malformed_query_are_rejected_before_dispatch() { + let huge_event = WriteOp::ExecEventComplete(ExecEventComplete { + exec_id: 1, + exit_code: 0, + duration_ms: 1, + stdout: vec![b'x'; MAX_LEDGER_OPERATION_BYTES], + stderr: Vec::new(), + stdout_bytes: MAX_LEDGER_OPERATION_BYTES as u64, + stderr_bytes: 0, + pid: None, + }); let huge = LedgerCommand::Admit { - event: Box::new(WriteOp::ExecEventComplete(ExecEventComplete { - exec_id: 1, - exit_code: 0, - duration_ms: 1, - stdout: vec![b'x'; MAX_LEDGER_OPERATION_BYTES], - stderr: Vec::new(), - stdout_bytes: MAX_LEDGER_OPERATION_BYTES as u64, - stderr_bytes: 0, - pid: None, - })), + commitment: commitment(&huge_event), + event: Box::new(huge_event), }; assert_eq!( LedgerRequest::new(1, huge) @@ -118,17 +135,19 @@ fn oversized_event_and_malformed_query_are_rejected_before_dispatch() { #[test] fn event_bytes_use_messagepack_binary_inside_the_request() { + let event = WriteOp::ExecEventComplete(ExecEventComplete { + exec_id: 1, + exit_code: 0, + duration_ms: 1, + stdout: vec![0xa5; 1024 * 1024], + stderr: Vec::new(), + stdout_bytes: 1024 * 1024, + stderr_bytes: 0, + pid: None, + }); let command = LedgerCommand::Admit { - event: Box::new(WriteOp::ExecEventComplete(ExecEventComplete { - exec_id: 1, - exit_code: 0, - duration_ms: 1, - stdout: vec![0xa5; 1024 * 1024], - stderr: Vec::new(), - stdout_bytes: 1024 * 1024, - stderr_bytes: 0, - pid: None, - })), + commitment: commitment(&event), + event: Box::new(event), }; let encoded = rmp_serde::to_vec_named(&LedgerRequest::new(1, command).unwrap()).unwrap(); assert!( diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index 4ab521734..a48a52b4d 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -93,27 +93,33 @@ impl LedgerServer { send_failure(&sender, request_id, code, error.to_string()).await?; continue; } - if self.dispatch(&sender, request).await? { + if self.dispatch(&sender, request, grant).await? { return Ok(LedgerClientExit::ShutdownRequested); } } } - async fn dispatch(&self, sender: &ServerSender, request: LedgerRequest) -> Result { + async fn dispatch( + &self, + sender: &ServerSender, + request: LedgerRequest, + grant: LedgerChannelGrant, + ) -> Result { let request_id = request.request_id(); let result = match request.into_operation() { - LedgerCommand::Admit { event } => { + LedgerCommand::Admit { event, commitment } => { let admission_id = { let mut admission = self.admission.lock().await; - match self.db.write(*event).await { - Ok(()) => { - admission.last_accepted = admission + match commitment.validate_grant(grant) { + Ok(()) => match self.db.write_committed(*event, commitment).await { + Ok(()) => admission .last_accepted .checked_add(1) - .ok_or_else(|| "ledger admission id exhausted".to_string())?; - Ok(admission.last_accepted) - } - Err(error) => Err(storage(error)), + .ok_or_else(|| "ledger admission id exhausted".to_string()) + .inspect(|next| admission.last_accepted = *next), + Err(error) => Err(storage(error)), + }, + Err(error) => Err(error.to_string()), } }; match admission_id { diff --git a/crates/capsem-logger/src/ledger_server/queries.rs b/crates/capsem-logger/src/ledger_server/queries.rs index 36b7350b8..7c7c5caa4 100644 --- a/crates/capsem-logger/src/ledger_server/queries.rs +++ b/crates/capsem-logger/src/ledger_server/queries.rs @@ -62,6 +62,17 @@ fn statements(query: LedgerQuery) -> Result, String> { } => history_statements(&layers, search, limit, offset), LedgerQuery::StatsDetail => Ok(stats_detail_statements()), LedgerQuery::Triage { limit } => Ok(triage_statements(limit)), + LedgerQuery::Commitments { + after_global_sequence, + limit, + } => Ok(vec![( + "SELECT global_sequence, lower(hex(generation)) AS generation, client_id, producer_role, \ + producer_sequence, event_kind, lower(hex(event_hash)) AS event_hash, \ + lower(hex(previous_hash)) AS previous_hash, lower(hex(commitment_hash)) AS commitment_hash \ + FROM ledger_commitments WHERE global_sequence > ?1 ORDER BY global_sequence ASC LIMIT ?2" + .into(), + vec![json!(after_global_sequence), json!(limit)], + )]), } } diff --git a/crates/capsem-logger/src/ledger_server/tests.rs b/crates/capsem-logger/src/ledger_server/tests.rs index 0007149f4..0eeace18f 100644 --- a/crates/capsem-logger/src/ledger_server/tests.rs +++ b/crates/capsem-logger/src/ledger_server/tests.rs @@ -99,13 +99,24 @@ async fn accepted_durable_query_body_retention_and_warc_run_through_one_owner() let producer_grant = grant(LedgerClientRole::VmOwner); let (producer, producer_replies, producer_task) = connect(Arc::clone(&server), producer_grant).await; let body = vec![b'z'; 300_000]; + let event = body_event(body.clone()); + let event_hash = crate::writer::commitment_event_hash(&event).unwrap(); assert!(matches!( request( &producer, &producer_replies, 1, LedgerCommand::Admit { - event: Box::new(body_event(body.clone())), + commitment: capsem_proto::ledger_commitment::LedgerCommitment::new( + producer_grant, + 1, + 1, + event.kind(), + event_hash, + capsem_proto::ledger_commitment::ZERO_COMMITMENT_HASH, + ) + .unwrap(), + event: Box::new(event), }, ) .await, diff --git a/crates/capsem-logger/src/schema/columns.rs b/crates/capsem-logger/src/schema/columns.rs index 5e8ca7f09..bf176b007 100644 --- a/crates/capsem-logger/src/schema/columns.rs +++ b/crates/capsem-logger/src/schema/columns.rs @@ -147,6 +147,20 @@ pub(crate) const READY_SCHEMA_COLUMNS: &[(&str, &[&str])] = &[ ), ("body_blocks", &["block_offset", "raw_len", "disk_len", "sealed_at"]), ("ledger_counters", &["singleton", "counters"]), + ( + "ledger_commitments", + &[ + "global_sequence", + "generation", + "client_id", + "producer_role", + "producer_sequence", + "event_kind", + "event_hash", + "previous_hash", + "commitment_hash", + ], + ), ( "fs_events", &[ diff --git a/crates/capsem-logger/src/schema/ddl.rs b/crates/capsem-logger/src/schema/ddl.rs index b473ad097..21f6f8adb 100644 --- a/crates/capsem-logger/src/schema/ddl.rs +++ b/crates/capsem-logger/src/schema/ddl.rs @@ -103,6 +103,23 @@ pub const CREATE_SCHEMA: &str = " counters BLOB NOT NULL CHECK (typeof(counters) = 'blob') ); + -- Producer-authored identity/content chains. The trusted coordinator + -- checkpoints these rows outside the ledger directory only after the + -- corresponding event flush is durable. + CREATE TABLE IF NOT EXISTS ledger_commitments ( + global_sequence INTEGER PRIMARY KEY CHECK (global_sequence > 0), + generation BLOB NOT NULL CHECK (typeof(generation) = 'blob' AND length(generation) = 16), + client_id INTEGER NOT NULL CHECK (client_id > 0), + producer_role TEXT NOT NULL CHECK (producer_role IN ('vm_owner', 'proxy', 'coordinator')), + producer_sequence INTEGER NOT NULL CHECK (producer_sequence > 0), + event_kind TEXT NOT NULL CHECK (length(event_kind) BETWEEN 1 AND 64), + event_hash BLOB NOT NULL CHECK (typeof(event_hash) = 'blob' AND length(event_hash) = 32), + previous_hash BLOB NOT NULL CHECK (typeof(previous_hash) = 'blob' AND length(previous_hash) = 32), + commitment_hash BLOB NOT NULL CHECK (typeof(commitment_hash) = 'blob' AND length(commitment_hash) = 32), + UNIQUE(generation, client_id, producer_sequence), + UNIQUE(commitment_hash) + ); + -- One block of an archive generation. The bytes live in the archive file; -- SQLite records where each block landed so a reader never scans. A block -- stays open across disk flushes and grows by one segment per flush, so diff --git a/crates/capsem-logger/src/schema/memory_sync.rs b/crates/capsem-logger/src/schema/memory_sync.rs index 6e5e31d73..0b5605b95 100644 --- a/crates/capsem-logger/src/schema/memory_sync.rs +++ b/crates/capsem-logger/src/schema/memory_sync.rs @@ -41,6 +41,7 @@ pub(super) const COPY_SECURITY_DECISION_OCCURRENCES: &str = "INSERT OR REPLACE I const DISK_ONLY_TABLES: &[&str] = &[ "archive_state", "ledger_counters", + "ledger_commitments", "event_body_blobs", "body_blocks", "transport_schema", diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 8dcd59718..423c801b3 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -255,7 +255,7 @@ type RetainReply = tokio::sync::oneshot::Sender>; #[allow(clippy::large_enum_variant)] #[derive(Debug)] enum WriterMessage { - Write(WriteOp), + Write(WriteEnvelope), /// Commit everything queued before this and report whether the disk flush /// happened. Flush(tokio::sync::oneshot::Sender), @@ -269,7 +269,14 @@ enum WriterMessage { impl WriterMessage { fn write(op: WriteOp) -> Self { - Self::Write(op) + Self::Write(WriteEnvelope { op, commitment: None }) + } + + fn committed(op: WriteOp, commitment: capsem_proto::ledger_commitment::LedgerCommitment) -> Self { + Self::Write(WriteEnvelope { + op, + commitment: Some(commitment), + }) } fn flush(reply: tokio::sync::oneshot::Sender) -> Self { @@ -277,6 +284,22 @@ impl WriterMessage { } } +#[derive(Debug)] +struct WriteEnvelope { + op: WriteOp, + commitment: Option, +} + +impl WriteEnvelope { + #[cfg(test)] + fn plain(op: WriteOp) -> Self { + Self { + op, + commitment: None, + } + } +} + /// What the argument above assumes. Not a budget anyone chose -- it is the /// size at which "moving this is cheaper than boxing it" was weighed, and a /// message that outgrows it by a lot deserves the question asked again. @@ -291,8 +314,11 @@ fn writer_channel(capacity: usize) -> (WriterSender, mpsc::Receiver) -> bool { + pub(super) fn accept(&mut self, message: WriterMessage, batch: &mut Vec) -> bool { match message { WriterMessage::Write(op) => { batch.push(op); diff --git a/crates/capsem-logger/src/writer/batch.rs b/crates/capsem-logger/src/writer/batch.rs index 229ddc4b0..cb3a9b1d8 100644 --- a/crates/capsem-logger/src/writer/batch.rs +++ b/crates/capsem-logger/src/writer/batch.rs @@ -25,23 +25,24 @@ pub(super) struct BatchWriteOutcome { /// second pass. Nothing here runs when the batch commits. pub(super) fn retry_batch_ops_individually( conn: &Connection, - batch: &[WriteOp], + batch: &[WriteEnvelope], bodies: &mut BodyArchive, exec_floor: i64, tally: &mut LedgerTally, ) -> BatchWriteOutcome { - let expected_writes = batch.iter().filter(|op| write_op_affects_storage(op)).count(); + let expected_writes = batch.iter().filter(|write| affects_storage(write)).count(); let mut salvaged = BatchWriteOutcome { tables: BTreeSet::new(), written: 0, rejected: 0, }; - for op in batch { - if !write_op_affects_storage(op) { + for write in batch { + let op = &write.op; + if !affects_storage(write) { continue; } let op_kind = op.kind(); - match execute_memory_batch(conn, std::slice::from_ref(op), bodies, exec_floor, tally) { + match execute_memory_batch(conn, std::slice::from_ref(write), bodies, exec_floor, tally) { Ok(outcome) => { salvaged.tables.extend(outcome.tables); salvaged.written += outcome.written; @@ -73,12 +74,12 @@ pub(super) fn retry_batch_ops_individually( pub(super) fn execute_memory_batch( conn: &Connection, - batch: &[WriteOp], + batch: &[WriteEnvelope], bodies: &mut BodyArchive, exec_floor: i64, tally: &mut LedgerTally, ) -> rusqlite::Result { - let stored_ops = batch.iter().filter(|op| write_op_affects_storage(op)).count(); + let stored_ops = batch.iter().filter(|write| affects_storage(write)).count(); if stored_ops == 0 { return Ok(BatchWriteOutcome { tables: BTreeSet::new(), @@ -157,15 +158,16 @@ fn counted_on_disk_now( fn insert_batch_ops<'a>( tx: &rusqlite::Transaction<'_>, - batch: &'a [WriteOp], + batch: &'a [WriteEnvelope], bodies: &mut BodyArchive, exec_floor: i64, affected_tables: &mut BTreeSet<&'static str>, op_counts: &mut std::collections::BTreeMap<&'static str, usize>, effects: &mut Vec<(&'a WriteOp, StoredEffect)>, ) -> rusqlite::Result<()> { - for op in batch { - if !write_op_affects_storage(op) { + for write in batch { + let op = &write.op; + if !affects_storage(write) { continue; } *op_counts.entry(op.kind()).or_default() += 1; @@ -203,7 +205,42 @@ fn insert_batch_ops<'a>( } } } + if let Some(commitment) = &write.commitment { + insert_commitment(tx, commitment)?; + } effects.push((op, effect)); } Ok(()) } + +fn affects_storage(write: &WriteEnvelope) -> bool { + write.commitment.is_some() || write_op_affects_storage(&write.op) +} + +fn insert_commitment( + tx: &rusqlite::Transaction<'_>, + commitment: &capsem_proto::ledger_commitment::LedgerCommitment, +) -> rusqlite::Result<()> { + let role = commitment + .role() + .producer_name() + .ok_or_else(|| rusqlite::Error::InvalidParameterName("non-producer ledger commitment".into()))?; + tx.execute( + "INSERT INTO main.ledger_commitments ( + global_sequence, generation, client_id, producer_role, producer_sequence, + event_kind, event_hash, previous_hash, commitment_hash + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + rusqlite::params![ + commitment.global_sequence(), + commitment.generation().as_bytes().as_slice(), + commitment.client_id(), + role, + commitment.producer_sequence(), + commitment.event_kind(), + commitment.event_hash().as_slice(), + commitment.previous_hash().as_slice(), + commitment.commitment_hash().as_slice(), + ], + )?; + Ok(()) +} diff --git a/crates/capsem-logger/src/writer/operation.rs b/crates/capsem-logger/src/writer/operation.rs index 636278972..c19f74180 100644 --- a/crates/capsem-logger/src/writer/operation.rs +++ b/crates/capsem-logger/src/writer/operation.rs @@ -76,6 +76,14 @@ impl WriteOp { } } +pub(crate) fn commitment_event_hash(op: &WriteOp) -> Result<[u8; 32], String> { + let encoded = rmp_serde::to_vec_named(op).map_err(|error| format!("encode ledger event commitment: {error}"))?; + let mut hash = blake3::Hasher::new(); + hash.update(b"capsem-ledger-event-v1\0"); + hash.update(&encoded); + Ok(*hash.finalize().as_bytes()) +} + fn ensure_option_event_id(event_id: &mut Option) -> Option { if event_id.is_none() { *event_id = Some(new_event_id()); diff --git a/crates/capsem-logger/src/writer/producer.rs b/crates/capsem-logger/src/writer/producer.rs index 92065ca3a..aad8c701d 100644 --- a/crates/capsem-logger/src/writer/producer.rs +++ b/crates/capsem-logger/src/writer/producer.rs @@ -42,6 +42,22 @@ impl DbWriter { Ok(()) } + pub(crate) async fn write_committed( + &self, + op: WriteOp, + commitment: capsem_proto::ledger_commitment::LedgerCommitment, + ) -> Result<(), String> { + if self.remote.is_some() { + return Err("a remote ledger writer cannot accept a prebuilt commitment".into()); + } + let Some(tx) = self.clone_sender() else { + return Err("db writer sender missing".to_string()); + }; + send_with_backpressure(&tx, WriterMessage::committed(op, commitment)) + .await + .map_err(|error| format!("db writer channel closed: {error}")) + } + /// Try to enqueue without blocking. Returns false when the queue is full or closed. pub fn try_write(&self, op: WriteOp) -> bool { let span = tracing::debug_span!( diff --git a/crates/capsem-logger/src/writer/remote.rs b/crates/capsem-logger/src/writer/remote.rs index 0dc0b6f2a..c8f2377c9 100644 --- a/crates/capsem-logger/src/writer/remote.rs +++ b/crates/capsem-logger/src/writer/remote.rs @@ -7,6 +7,10 @@ use std::time::Duration; use capsem_foundation::ipc_channel; use capsem_proto::ledger::{LedgerChannelGrant, LedgerHello, LedgerOutcome, LedgerRequest}; +use capsem_proto::ledger_commitment::{ + CommitmentClientMessage, CommitmentCommand, CommitmentReply, CommitmentServerMessage, LedgerCommitment, + ZERO_COMMITMENT_HASH, +}; use tokio::sync::oneshot; use crate::ledger_protocol::{LedgerClientMessage, LedgerCommand, LedgerReply, LedgerServerMessage}; @@ -21,6 +25,8 @@ const BACKPRESSURE_MAX_WAIT: Duration = Duration::from_millis(5); type ClientSender = ipc_channel::Sender; type ClientReceiver = ipc_channel::Receiver; +type CommitmentSender = ipc_channel::Sender; +type CommitmentReceiver = ipc_channel::Receiver; enum AdmitReply { Async(oneshot::Sender>), @@ -60,13 +66,18 @@ pub(super) struct RemoteWriter { } impl RemoteWriter { - pub(super) fn start(stream: UnixStream, grant: LedgerChannelGrant, capacity: usize) -> io::Result { + pub(super) fn start( + stream: UnixStream, + commitment_stream: UnixStream, + grant: LedgerChannelGrant, + capacity: usize, + ) -> io::Result { let abort = stream.try_clone()?; let (tx, rx) = mpsc::sync_channel(capacity.max(1)); let (startup_tx, startup_rx) = mpsc::sync_channel(1); let join = std::thread::Builder::new() .name("capsem-ledger-client".into()) - .spawn(move || run(stream, grant, rx, startup_tx))?; + .spawn(move || run(stream, commitment_stream, grant, rx, startup_tx))?; match startup_rx.recv_timeout(STARTUP_TIMEOUT) { Ok(Ok(())) => Ok(Self { tx: Mutex::new(Some(tx)), @@ -192,6 +203,7 @@ impl RemoteWriter { fn run( stream: UnixStream, + commitment_stream: UnixStream, grant: LedgerChannelGrant, messages: mpsc::Receiver, startup: mpsc::SyncSender>, @@ -214,10 +226,24 @@ fn run( return; } }; + let commitment_channel = { + let _entered = runtime.enter(); + ipc_channel::channel_from_std::(commitment_stream) + }; + let (commitment_sender, commitment_receiver) = match commitment_channel { + Ok(channel) => channel, + Err(error) => { + let _ = startup.send(Err(format!("open ledger commitment channel: {error}"))); + return; + } + }; let handshake = runtime.block_on(async { - tokio::time::timeout(OPERATION_TIMEOUT, handshake(&sender, &receiver, grant)) - .await - .map_err(|_| "ledger client handshake timed out".to_string())? + tokio::time::timeout(OPERATION_TIMEOUT, async { + handshake(&sender, &receiver, grant).await?; + commitment_handshake(&commitment_sender, &commitment_receiver, grant).await + }) + .await + .map_err(|_| "ledger client handshake timed out".to_string())? }); if let Err(error) = handshake { let _ = startup.send(Err(error)); @@ -228,11 +254,15 @@ fn run( } let mut request_id = 1_u64; + let mut commitments = CommitmentState::new(commitment_sender, commitment_receiver, grant); while let Ok(message) = messages.recv() { let result = runtime.block_on(async { - tokio::time::timeout(OPERATION_TIMEOUT, dispatch(&sender, &receiver, request_id, &message)) - .await - .map_err(|_| "ledger client operation timed out".to_string())? + tokio::time::timeout( + OPERATION_TIMEOUT, + dispatch(&sender, &receiver, request_id, &message, &mut commitments), + ) + .await + .map_err(|_| "ledger client operation timed out".to_string())? }); finish(message, result.clone()); if let Err(error) = result { @@ -246,6 +276,19 @@ fn run( }; request_id = next; } + if !commitments.pending.is_empty() { + let result = runtime.block_on(async { + tokio::time::timeout( + OPERATION_TIMEOUT, + flush_and_anchor(&sender, &receiver, request_id, &mut commitments), + ) + .await + .map_err(|_| "ledger client shutdown flush timed out".to_string())? + }); + if let Err(error) = result { + tracing::error!(%error, "ledger client shutdown left an unanchored tail"); + } + } } async fn handshake(sender: &ClientSender, receiver: &ClientReceiver, grant: LedgerChannelGrant) -> Result<(), String> { @@ -265,42 +308,240 @@ async fn handshake(sender: &ClientSender, receiver: &ClientReceiver, grant: Ledg grant.validate_welcome(&welcome).map_err(|error| error.to_string()) } +async fn commitment_handshake( + sender: &CommitmentSender, + receiver: &CommitmentReceiver, + grant: LedgerChannelGrant, +) -> Result<(), String> { + sender + .send(CommitmentClientMessage::Hello { + hello: LedgerHello::for_grant(&grant), + }) + .await + .map_err(|error| format!("send ledger commitment hello: {error}"))?; + let CommitmentServerMessage::Welcome { welcome } = receiver + .recv() + .await + .map_err(|error| format!("receive ledger commitment welcome: {error}"))? + else { + return Err("ledger commitment coordinator sent a response before welcome".into()); + }; + grant.validate_welcome(&welcome).map_err(|error| error.to_string()) +} + +struct CommitmentState { + sender: CommitmentSender, + receiver: CommitmentReceiver, + grant: LedgerChannelGrant, + request_id: u64, + producer_sequence: u64, + previous_hash: [u8; 32], + pending: Vec<(u64, LedgerCommitment)>, +} + +impl CommitmentState { + fn new(sender: CommitmentSender, receiver: CommitmentReceiver, grant: LedgerChannelGrant) -> Self { + Self { + sender, + receiver, + grant, + request_id: 1, + producer_sequence: 0, + previous_hash: ZERO_COMMITMENT_HASH, + pending: Vec::new(), + } + } + + async fn request(&mut self, command: CommitmentCommand) -> Result { + let request_id = self.request_id; + self.sender + .send(CommitmentClientMessage::Request { request_id, command }) + .await + .map_err(|error| format!("send ledger commitment request: {error}"))?; + let CommitmentServerMessage::Response { + request_id: response_id, + reply, + } = self + .receiver + .recv() + .await + .map_err(|error| format!("receive ledger commitment response: {error}"))? + else { + return Err("ledger commitment coordinator repeated welcome".into()); + }; + if response_id != request_id { + return Err("ledger commitment response id mismatch".into()); + } + self.request_id = request_id + .checked_add(1) + .ok_or_else(|| "ledger commitment request id exhausted".to_string())?; + match reply { + CommitmentReply::Failed { message } => Err(format!("ledger commitment coordinator: {message}")), + reply => Ok(reply), + } + } + + async fn reserve(&mut self, event: &WriteOp) -> Result { + let producer_sequence = self + .producer_sequence + .checked_add(1) + .ok_or_else(|| "ledger producer sequence exhausted".to_string())?; + let event_hash = super::operation::commitment_event_hash(event)?; + let reply = self + .request(CommitmentCommand::Reserve { + producer_sequence, + event_kind: event.kind().to_string(), + event_hash, + previous_hash: self.previous_hash, + }) + .await?; + let CommitmentReply::Reserved { global_sequence } = reply else { + return Err(format!( + "ledger commitment coordinator returned {reply:?} for reservation" + )); + }; + let commitment = LedgerCommitment::new( + self.grant, + producer_sequence, + global_sequence, + event.kind(), + event_hash, + self.previous_hash, + ) + .map_err(|error| error.to_string())?; + self.producer_sequence = producer_sequence; + self.previous_hash = commitment.commitment_hash(); + Ok(commitment) + } + + async fn cancel(&mut self, commitment: &LedgerCommitment) -> Result<(), String> { + let reply = self + .request(CommitmentCommand::Cancel { + global_sequence: commitment.global_sequence(), + }) + .await?; + if !matches!(reply, CommitmentReply::Canceled) { + return Err(format!( + "ledger commitment coordinator returned {reply:?} for cancellation" + )); + } + self.producer_sequence = self.producer_sequence.saturating_sub(1); + self.previous_hash = commitment.previous_hash(); + Ok(()) + } + + async fn anchor_through(&mut self, through_admission_id: u64) -> Result<(), String> { + let count = self + .pending + .iter() + .take_while(|(admission_id, _)| *admission_id <= through_admission_id) + .count(); + if count == 0 { + return Ok(()); + } + let commitments = self.pending[..count] + .iter() + .map(|(_, commitment)| commitment.clone()) + .collect(); + let reply = self.request(CommitmentCommand::Anchor { commitments }).await?; + if !matches!(reply, CommitmentReply::Anchored { .. }) { + return Err(format!( + "ledger commitment coordinator returned {reply:?} for checkpoint" + )); + } + self.pending.drain(..count); + Ok(()) + } +} + async fn dispatch( sender: &ClientSender, receiver: &ClientReceiver, request_id: u64, message: &RemoteMessage, + commitments: &mut CommitmentState, ) -> Result { + let reserved = match message { + RemoteMessage::Admit { event, .. } => Some(commitments.reserve(event).await?), + _ => None, + }; let command = match message { - RemoteMessage::Admit { event, .. } => LedgerCommand::Admit { event: event.clone() }, + RemoteMessage::Admit { event, .. } => LedgerCommand::Admit { + event: event.clone(), + commitment: reserved.clone().expect("admission reserved above"), + }, RemoteMessage::Flush { .. } => LedgerCommand::Flush, RemoteMessage::Retain { cutoff, .. } => LedgerCommand::Retain { cutoff: cutoff.clone() }, }; - sender - .send(LedgerClientMessage::Request { - request: LedgerRequest::new(request_id, command).map_err(|error| error.to_string())?, - }) - .await - .map_err(|error| format!("send ledger request: {error}"))?; - let message = receiver - .recv() - .await - .map_err(|error| format!("receive ledger response: {error}"))?; - let LedgerServerMessage::Response { response } = message else { - return Err("ledger worker repeated welcome".to_string()); - }; - if response.request_id() != request_id { - return Err(format!( - "ledger response id {} does not match {request_id}", - response.request_id() - )); + let response = async { + sender + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(request_id, command).map_err(|error| error.to_string())?, + }) + .await + .map_err(|error| format!("send ledger request: {error}"))?; + let message = receiver + .recv() + .await + .map_err(|error| format!("receive ledger response: {error}"))?; + let LedgerServerMessage::Response { response } = message else { + return Err("ledger worker repeated welcome".to_string()); + }; + if response.request_id() != request_id { + return Err(format!( + "ledger response id {} does not match {request_id}", + response.request_id() + )); + } + match response.outcome() { + LedgerOutcome::Success { reply } => { + reply.validate().map_err(|error| error.to_string())?; + Ok(reply.clone()) + } + LedgerOutcome::Failure { failure } => Err(format!("ledger {:?}: {}", failure.code(), failure.message())), + } } - match response.outcome() { - LedgerOutcome::Success { reply } => { - reply.validate().map_err(|error| error.to_string())?; - Ok(reply.clone()) + .await; + match (response, reserved) { + (Ok(LedgerReply::Accepted { admission_id }), Some(commitment)) => { + commitments.pending.push((admission_id, commitment)); + Ok(LedgerReply::Accepted { admission_id }) + } + (Ok(LedgerReply::Durable { through_admission_id }), None) => { + commitments.anchor_through(through_admission_id).await?; + Ok(LedgerReply::Durable { through_admission_id }) + } + (Err(error), Some(commitment)) => { + let cancellation = commitments.cancel(&commitment).await; + Err(match cancellation { + Ok(()) => error, + Err(cancel) => format!("{error}; commitment cancellation failed: {cancel}"), + }) } - LedgerOutcome::Failure { failure } => Err(format!("ledger {:?}: {}", failure.code(), failure.message())), + (result, _) => result, + } +} + +async fn flush_and_anchor( + sender: &ClientSender, + receiver: &ClientReceiver, + request_id: u64, + commitments: &mut CommitmentState, +) -> Result<(), String> { + let reply = dispatch( + sender, + receiver, + request_id, + &RemoteMessage::Flush { + reply: oneshot::channel().0, + }, + commitments, + ) + .await?; + if matches!(reply, LedgerReply::Durable { .. }) { + Ok(()) + } else { + Err(format!("ledger returned {reply:?} for shutdown flush")) } } @@ -338,5 +579,87 @@ fn fail_pending(messages: &mpsc::Receiver, reason: &str) { } } +#[cfg(test)] +pub(crate) fn test_commitment_channel( + grant: LedgerChannelGrant, +) -> (UnixStream, tokio::task::JoinHandle>) { + let (client, server) = UnixStream::pair().unwrap(); + let task = tokio::spawn(async move { + let (sender, receiver) = + ipc_channel::channel_from_std::(server) + .map_err(|error| error.to_string())?; + let CommitmentClientMessage::Hello { hello } = receiver.recv().await.map_err(|error| error.to_string())? else { + return Err("test commitment client omitted hello".into()); + }; + grant.validate_hello(&hello).map_err(|error| error.to_string())?; + sender + .send(CommitmentServerMessage::Welcome { + welcome: capsem_proto::ledger::LedgerWelcome::for_grant(&grant), + }) + .await + .map_err(|error| error.to_string())?; + let mut global_sequence = 0_u64; + let mut pending = Vec::::new(); + loop { + let CommitmentClientMessage::Request { request_id, command } = + receiver.recv().await.map_err(|error| error.to_string())? + else { + return Err("test commitment client repeated hello".into()); + }; + let reply = match command { + CommitmentCommand::Reserve { + producer_sequence, + event_kind, + event_hash, + previous_hash, + } => { + global_sequence += 1; + match LedgerCommitment::new( + grant, + producer_sequence, + global_sequence, + event_kind, + event_hash, + previous_hash, + ) { + Ok(commitment) => { + pending.push(commitment); + CommitmentReply::Reserved { global_sequence } + } + Err(error) => CommitmentReply::Failed { + message: error.to_string(), + }, + } + } + CommitmentCommand::Cancel { global_sequence } => { + if pending.last().map(LedgerCommitment::global_sequence) == Some(global_sequence) { + pending.pop(); + CommitmentReply::Canceled + } else { + CommitmentReply::Failed { + message: "cancellation did not name pending tail".into(), + } + } + } + CommitmentCommand::Anchor { commitments } => { + if commitments.len() <= pending.len() && pending[..commitments.len()] == commitments { + pending.drain(..commitments.len()); + CommitmentReply::Anchored { checkpoint_sequence: 1 } + } else { + CommitmentReply::Failed { + message: "checkpoint did not match pending prefix".into(), + } + } + } + }; + sender + .send(CommitmentServerMessage::Response { request_id, reply }) + .await + .map_err(|error| error.to_string())?; + } + }); + (client, task) +} + #[cfg(test)] mod tests; diff --git a/crates/capsem-logger/src/writer/remote/tests.rs b/crates/capsem-logger/src/writer/remote/tests.rs index 4d644af06..d184459d4 100644 --- a/crates/capsem-logger/src/writer/remote/tests.rs +++ b/crates/capsem-logger/src/writer/remote/tests.rs @@ -56,10 +56,11 @@ async fn remote_writer( let generation = LedgerGeneration::new([5; 16]); let grant = LedgerChannelGrant::new(generation, 9, role).unwrap(); let (client, worker) = UnixStream::pair().unwrap(); + let (commitment, _commitment_task) = super::test_commitment_channel(grant); let task = tokio::spawn(async move { server.serve_client(worker, grant).await }); let writer = tokio::task::spawn_blocking({ let path = path.clone(); - move || crate::DbWriter::from_ledger_channel(client, grant, &path, 8) + move || crate::DbWriter::from_ledger_channel(client, commitment, grant, &path, 8) }) .await .unwrap() @@ -96,7 +97,7 @@ async fn descriptor_writer_reports_role_refusal_to_the_producer() { let (_directory, writer, task) = remote_writer(LedgerClientRole::Reader).await; let error = writer.write_checked(event("denied.example")).await.unwrap_err(); assert!( - error.contains("cannot perform") || error.contains("unauthorized"), + error.contains("cannot perform") || error.contains("unauthorized") || error.contains("not a producer"), "{error}" ); tokio::task::spawn_blocking(move || writer.shutdown_blocking()) @@ -148,6 +149,7 @@ async fn a_stalled_owner_fails_the_pending_operation_and_bounded_shutdown() { let path = directory.path().join("session.db"); let grant = LedgerChannelGrant::new(LedgerGeneration::new([6; 16]), 10, LedgerClientRole::VmOwner).unwrap(); let (client, worker) = UnixStream::pair().unwrap(); + let (commitment, _commitment_task) = super::test_commitment_channel(grant); let server = tokio::spawn(async move { let (sender, receiver) = capsem_foundation::ipc_channel::channel_from_std::(worker) @@ -165,10 +167,11 @@ async fn a_stalled_owner_fails_the_pending_operation_and_bounded_shutdown() { let _request = receiver.recv().await.unwrap(); std::future::pending::<()>().await; }); - let writer = tokio::task::spawn_blocking(move || crate::DbWriter::from_ledger_channel(client, grant, &path, 2)) - .await - .unwrap() - .unwrap(); + let writer = + tokio::task::spawn_blocking(move || crate::DbWriter::from_ledger_channel(client, commitment, grant, &path, 2)) + .await + .unwrap() + .unwrap(); let started = std::time::Instant::now(); let error = writer.write_checked(event("stalled.example")).await.unwrap_err(); assert!(error.contains("timed out"), "{error}"); diff --git a/crates/capsem-logger/src/writer/tests/counters.rs b/crates/capsem-logger/src/writer/tests/counters.rs index 49b85daa4..e928ae97d 100644 --- a/crates/capsem-logger/src/writer/tests/counters.rs +++ b/crates/capsem-logger/src/writer/tests/counters.rs @@ -29,7 +29,7 @@ fn completing_a_flushed_exec_commits_its_count_with_the_row() { let path = dir.path().join("session.db"); let (conn, mut bodies, mut tally) = ledger(&path); - execute_memory_batch(&conn, &[exec(3)], &mut bodies, 0, &mut tally).unwrap(); + execute_memory_batch(&conn, &[WriteEnvelope::plain(exec(3))], &mut bodies, 0, &mut tally).unwrap(); flush_dirty_tables_to_disk( &conn, &mut BTreeSet::from(["exec_events"]), @@ -43,7 +43,7 @@ fn completing_a_flushed_exec_commits_its_count_with_the_row() { // The start row is on disk now, so the completion updates it there, and // that update is durable the moment this batch commits -- no flush follows. - execute_memory_batch(&conn, &[exec_done(3)], &mut bodies, 0, &mut tally).unwrap(); + execute_memory_batch(&conn, &[WriteEnvelope::plain(exec_done(3))], &mut bodies, 0, &mut tally).unwrap(); let (snapshot, completed_rows) = on_disk(&path); assert_eq!(completed_rows, 1); assert_eq!( @@ -53,7 +53,7 @@ fn completing_a_flushed_exec_commits_its_count_with_the_row() { assert_eq!(snapshot, *tally.counters()); // A duplicate completion rewrites the row and counts nothing. - execute_memory_batch(&conn, &[exec_done(3)], &mut bodies, 0, &mut tally).unwrap(); + execute_memory_batch(&conn, &[WriteEnvelope::plain(exec_done(3))], &mut bodies, 0, &mut tally).unwrap(); assert_eq!(tally.counters().exec.completed, 1); } @@ -69,7 +69,11 @@ fn a_rejected_op_is_not_counted_and_its_batch_neighbours_are() { None, 1_700_000_000.0, ); - let batch = [exec(1), refused, exec(2)]; + let batch = [ + WriteEnvelope::plain(exec(1)), + WriteEnvelope::plain(refused), + WriteEnvelope::plain(exec(2)), + ]; assert!(execute_memory_batch(&conn, &batch, &mut bodies, 0, &mut tally).is_err()); assert_eq!( tally.counters(), @@ -118,7 +122,7 @@ fn mcp_protocol_only_event_does_not_claim_tool_storage() { let outcome = metrics::with_local_recorder(&recorder, || { execute_memory_batch( &conn, - &[event], + &[WriteEnvelope::plain(event)], &mut bodies, 0, &mut crate::counters::LedgerTally::default(), From 1f733de73950f2d56b43b5315ec92801b123939e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:55:45 +0000 Subject: [PATCH 207/464] security(service): anchor producer commitments externally --- CHANGELOG.md | 5 + Cargo.lock | 1 + crates/capsem-service/Cargo.toml | 1 + .../capsem-service/src/ledger_commitment.rs | 505 ++++++++++++++++++ .../src/ledger_commitment/tests.rs | 249 +++++++++ crates/capsem-service/src/ledger_worker.rs | 80 ++- .../capsem-service/src/ledger_worker/tests.rs | 9 +- crates/capsem-service/src/main.rs | 2 + crates/capsem-service/src/proxy_worker.rs | 42 +- .../capsem-service/src/proxy_worker/tests.rs | 9 +- crates/capsem-service/src/service_runtime.rs | 5 +- .../capsem-service/src/session_db_handles.rs | 6 +- crates/capsem-service/src/tests.rs | 7 +- crates/capsem-service/src/upstream_broker.rs | 30 +- .../src/upstream_broker/tests.rs | 11 +- crates/capsem-service/src/vm_files/fork.rs | 3 +- 16 files changed, 913 insertions(+), 52 deletions(-) create mode 100644 crates/capsem-service/src/ledger_commitment.rs create mode 100644 crates/capsem-service/src/ledger_commitment/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ff346bb5e..4ba7f3ea8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ledger channel and route its existing event writer through the supervised session owner, eliminating the VM process's direct SQLite open. +- Bind every producer record to a producer-authored content and order + commitment, and acknowledge a flush only after the trusted service syncs + the matching checkpoint outside the ledger directory. Startup verification + now rejects altered, substituted, reordered, or omitted checkpointed rows. + - Pass the packaged ledger and proxy worker paths explicitly to installed and direct services, refuse incomplete confined-worker cohorts, and retire both children during reinstall or uninstall. diff --git a/Cargo.lock b/Cargo.lock index 50f8564b2..e7bba1c01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1292,6 +1292,7 @@ dependencies = [ "opentelemetry-http", "opentelemetry_sdk", "reqwest 0.12.28", + "rmp-serde", "rusqlite", "serde", "serde_json", diff --git a/crates/capsem-service/Cargo.toml b/crates/capsem-service/Cargo.toml index a47346bf3..2ed89ef82 100644 --- a/crates/capsem-service/Cargo.toml +++ b/crates/capsem-service/Cargo.toml @@ -28,6 +28,7 @@ tokio-util.workspace = true tracing.workspace = true serde.workspace = true serde_json.workspace = true +rmp-serde.workspace = true toml.workspace = true humantime.workspace = true clap.workspace = true diff --git a/crates/capsem-service/src/ledger_commitment.rs b/crates/capsem-service/src/ledger_commitment.rs new file mode 100644 index 000000000..8b95294a5 --- /dev/null +++ b/crates/capsem-service/src/ledger_commitment.rs @@ -0,0 +1,505 @@ +//! Trusted producer ordering and durable checkpoints outside the ledger worker. + +use std::collections::{HashMap, HashSet}; +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{bail, Context, Result}; +use capsem_foundation::ipc_channel; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerWelcome}; +use capsem_proto::ledger_commitment::{ + CommitmentClientMessage, CommitmentCommand, CommitmentReply, CommitmentServerMessage, LedgerCommitment, + MAX_COMMITMENTS_PER_CHECKPOINT, +}; +#[cfg(test)] +use capsem_proto::ledger_commitment::ZERO_COMMITMENT_HASH; +use serde::{Deserialize, Serialize}; + +use capsem_logger::ledger_protocol::{LedgerQuery, LedgerValue}; + +const CHECKPOINT_MAGIC: [u8; 4] = *b"CSCP"; +const MAX_CHECKPOINT_FRAME_BYTES: usize = 16 * 1024 * 1024; + +type Sender = ipc_channel::Sender; +type Receiver = ipc_channel::Receiver; + +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +struct ProducerKey { + generation: [u8; 16], + client_id: u64, +} + +impl ProducerKey { + fn from_grant(grant: LedgerChannelGrant) -> Self { + Self { + generation: grant.generation().as_bytes(), + client_id: grant.client_id(), + } + } + + fn from_commitment(commitment: &LedgerCommitment) -> Self { + Self { + generation: commitment.generation().as_bytes(), + client_id: commitment.client_id(), + } + } +} + +#[derive(Clone, Copy, Default)] +struct ProducerTail { + sequence: u64, + hash: [u8; 32], +} + +#[derive(Default)] +struct State { + next_global_sequence: u64, + next_checkpoint_sequence: u64, + anchored: HashMap, + pending: HashMap>, + global_sequences: HashSet, +} + +#[derive(Debug, Serialize, Deserialize)] +struct CheckpointRecord { + checkpoint_sequence: u64, + commitments: Vec, +} + +pub(crate) struct CommitmentAuthority { + path: PathBuf, + state: tokio::sync::Mutex, +} + +impl CommitmentAuthority { + pub(crate) async fn open(root: &Path, session_id: &str) -> Result> { + let session_key = blake3::hash(session_id.as_bytes()).to_hex().to_string(); + let path = root.join(session_key).join("checkpoints.log"); + let loaded_path = path.clone(); + let state = tokio::task::spawn_blocking(move || load(&loaded_path)) + .await + .context("join ledger commitment checkpoint load")??; + Ok(Arc::new(Self { + path, + state: tokio::sync::Mutex::new(state), + })) + } + + async fn reserve( + &self, + grant: LedgerChannelGrant, + producer_sequence: u64, + event_kind: String, + event_hash: [u8; 32], + previous_hash: [u8; 32], + ) -> Result { + let key = ProducerKey::from_grant(grant); + let mut state = self.state.lock().await; + let tail = state.pending.get(&key).and_then(|pending| pending.last()).map_or_else( + || state.anchored.get(&key).copied().unwrap_or_default(), + |commitment| ProducerTail { + sequence: commitment.producer_sequence(), + hash: commitment.commitment_hash(), + }, + ); + if producer_sequence != tail.sequence.checked_add(1).context("producer sequence exhausted")? + || previous_hash != tail.hash + { + bail!("producer commitment does not extend its trusted tail"); + } + let global_sequence = state + .next_global_sequence + .checked_add(1) + .context("global commitment sequence exhausted")?; + let commitment = LedgerCommitment::new( + grant, + producer_sequence, + global_sequence, + event_kind, + event_hash, + previous_hash, + )?; + state.next_global_sequence = global_sequence; + state.global_sequences.insert(global_sequence); + state.pending.entry(key).or_default().push(commitment); + drop(state); + Ok(global_sequence) + } + + async fn cancel(&self, grant: LedgerChannelGrant, global_sequence: u64) -> Result<()> { + let key = ProducerKey::from_grant(grant); + let mut state = self.state.lock().await; + let pending = state + .pending + .get_mut(&key) + .context("unknown producer commitment reservation")?; + if pending.last().map(LedgerCommitment::global_sequence) != Some(global_sequence) { + bail!("only the producer's unanchored tail may be canceled"); + } + pending.pop(); + if pending.is_empty() { + state.pending.remove(&key); + } + state.global_sequences.remove(&global_sequence); + drop(state); + Ok(()) + } + + async fn anchor(&self, grant: LedgerChannelGrant, commitments: Vec) -> Result { + if commitments.is_empty() || commitments.len() > MAX_COMMITMENTS_PER_CHECKPOINT { + bail!("commitment checkpoint batch is empty or exceeds its bound"); + } + let key = ProducerKey::from_grant(grant); + let mut state = self.state.lock().await; + let pending = state + .pending + .get(&key) + .context("producer has no reserved commitments")?; + if commitments.len() > pending.len() || pending[..commitments.len()] != commitments { + bail!("checkpoint does not match the producer's reserved prefix"); + } + for commitment in &commitments { + commitment.validate_grant(grant)?; + } + let checkpoint_sequence = state + .next_checkpoint_sequence + .checked_add(1) + .context("checkpoint sequence exhausted")?; + let record = CheckpointRecord { + checkpoint_sequence, + commitments: commitments.clone(), + }; + let path = self.path.clone(); + tokio::task::spawn_blocking(move || append(&path, &record)) + .await + .context("join ledger commitment checkpoint append")??; + + let tail = commitments.last().expect("nonempty commitments checked"); + state.anchored.insert( + key, + ProducerTail { + sequence: tail.producer_sequence(), + hash: tail.commitment_hash(), + }, + ); + let pending = state.pending.get_mut(&key).expect("pending prefix checked"); + pending.drain(..commitments.len()); + if pending.is_empty() { + state.pending.remove(&key); + } + state.next_checkpoint_sequence = checkpoint_sequence; + drop(state); + Ok(checkpoint_sequence) + } + + async fn abandon(&self, grant: LedgerChannelGrant) { + let key = ProducerKey::from_grant(grant); + let mut state = self.state.lock().await; + if let Some(pending) = state.pending.remove(&key) { + for commitment in pending { + state.global_sequences.remove(&commitment.global_sequence()); + } + } + } + + #[cfg(test)] + async fn anchored(&self) -> Vec { + let path = self.path.clone(); + tokio::task::spawn_blocking(move || read_records(&path).unwrap()) + .await + .unwrap() + .into_iter() + .flat_map(|record| record.commitments) + .collect() + } + + async fn checkpointed(&self) -> Result> { + let path = self.path.clone(); + Ok(tokio::task::spawn_blocking(move || read_records(&path)) + .await + .context("join ledger commitment checkpoint read")?? + .into_iter() + .flat_map(|record| record.commitments) + .collect()) + } + + pub(crate) async fn verify_ledger(&self, client: &capsem_logger::ledger_client::LedgerClient) -> Result<()> { + let expected = self.checkpointed().await?; + let mut actual = HashMap::new(); + let mut after = 0_u64; + loop { + let sets = client + .query(LedgerQuery::Commitments { + after_global_sequence: after, + limit: 2_000, + }) + .await + .map_err(anyhow::Error::msg)?; + let rows = sets + .into_iter() + .next() + .context("ledger commitment query returned no result set")?; + let count = rows.rows.len(); + for row in rows.rows { + let commitment = decode_commitment_row(&row)?; + if commitment.global_sequence() <= after { + bail!("ledger commitment rows are not in trusted global order"); + } + after = commitment.global_sequence(); + if actual.insert(after, commitment).is_some() { + bail!("ledger repeats a commitment global sequence"); + } + } + if count < 2_000 { + break; + } + } + for commitment in expected { + match actual.get(&commitment.global_sequence()) { + Some(stored) if stored == &commitment => {} + Some(_) => bail!( + "ledger commitment {} was substituted or altered", + commitment.global_sequence() + ), + None => bail!( + "ledger omitted anchored commitment {}", + commitment.global_sequence() + ), + } + } + Ok(()) + } +} + +fn decode_commitment_row(row: &[LedgerValue]) -> Result { + let [ + LedgerValue::Integer(global_sequence), + LedgerValue::Text(generation), + LedgerValue::Integer(client_id), + LedgerValue::Text(role), + LedgerValue::Integer(producer_sequence), + LedgerValue::Text(event_kind), + LedgerValue::Text(event_hash), + LedgerValue::Text(previous_hash), + LedgerValue::Text(commitment_hash), + ] = row + else { + bail!("ledger commitment query returned an invalid typed row"); + }; + let generation = hex_array::<16>(generation)?; + let role = match role.as_str() { + "vm_owner" => capsem_proto::ledger::LedgerClientRole::VmOwner, + "proxy" => capsem_proto::ledger::LedgerClientRole::Proxy, + "coordinator" => capsem_proto::ledger::LedgerClientRole::Coordinator, + _ => bail!("ledger commitment row has an invalid producer role"), + }; + let grant = LedgerChannelGrant::new( + capsem_proto::ledger::LedgerGeneration::new(generation), + u64::try_from(*client_id).context("ledger commitment client id is negative")?, + role, + )?; + let commitment = LedgerCommitment::new( + grant, + u64::try_from(*producer_sequence).context("ledger producer sequence is negative")?, + u64::try_from(*global_sequence).context("ledger global sequence is negative")?, + event_kind.clone(), + hex_array(event_hash)?, + hex_array(previous_hash)?, + )?; + if commitment.commitment_hash() != hex_array::<32>(commitment_hash)? { + bail!("ledger commitment row hash does not match its fields"); + } + Ok(commitment) +} + +fn hex_array(value: &str) -> Result<[u8; N]> { + if value.len() != N * 2 { + bail!("ledger commitment hex field has the wrong length"); + } + let mut bytes = [0_u8; N]; + for (index, byte) in bytes.iter_mut().enumerate() { + *byte = u8::from_str_radix(&value[index * 2..index * 2 + 2], 16) + .context("ledger commitment hex field is invalid")?; + } + Ok(bytes) +} + +pub(crate) async fn serve( + authority: Arc, + stream: UnixStream, + grant: LedgerChannelGrant, +) -> Result<()> { + let (sender, receiver) = ipc_channel::channel_from_std::(stream)?; + let first = receiver.recv().await?; + let CommitmentClientMessage::Hello { hello } = first else { + bail!("commitment client sent a request before hello"); + }; + grant.validate_hello(&hello)?; + sender + .send(CommitmentServerMessage::Welcome { + welcome: LedgerWelcome::for_grant(&grant), + }) + .await?; + + let outcome = serve_requests(&authority, &sender, &receiver, grant).await; + authority.abandon(grant).await; + outcome +} + +async fn serve_requests( + authority: &CommitmentAuthority, + sender: &Sender, + receiver: &Receiver, + grant: LedgerChannelGrant, +) -> Result<()> { + loop { + let CommitmentClientMessage::Request { request_id, command } = receiver.recv().await? else { + bail!("commitment client repeated hello"); + }; + if request_id == 0 { + bail!("commitment request id must not be zero"); + } + let result = match command { + CommitmentCommand::Reserve { + producer_sequence, + event_kind, + event_hash, + previous_hash, + } => authority + .reserve(grant, producer_sequence, event_kind, event_hash, previous_hash) + .await + .map(|global_sequence| CommitmentReply::Reserved { global_sequence }), + CommitmentCommand::Cancel { global_sequence } => authority + .cancel(grant, global_sequence) + .await + .map(|()| CommitmentReply::Canceled), + CommitmentCommand::Anchor { commitments } => authority + .anchor(grant, commitments) + .await + .map(|checkpoint_sequence| CommitmentReply::Anchored { checkpoint_sequence }), + }; + let reply = result.unwrap_or_else(|error| CommitmentReply::Failed { + message: format!("{error:#}"), + }); + sender + .send(CommitmentServerMessage::Response { request_id, reply }) + .await?; + } +} + +fn load(path: &Path) -> Result { + initialize(path)?; + let records = read_records(path)?; + let mut state = State::default(); + for record in records { + if record.checkpoint_sequence != state.next_checkpoint_sequence + 1 { + bail!("ledger commitment checkpoint sequence is not contiguous"); + } + if record.commitments.is_empty() || record.commitments.len() > MAX_COMMITMENTS_PER_CHECKPOINT { + bail!("ledger commitment checkpoint has an invalid batch size"); + } + for commitment in record.commitments { + commitment.validate()?; + let key = ProducerKey::from_commitment(&commitment); + let tail = state.anchored.get(&key).copied().unwrap_or_default(); + if commitment.producer_sequence() != tail.sequence + 1 || commitment.previous_hash() != tail.hash { + bail!("ledger commitment checkpoint breaks a producer chain"); + } + if !state.global_sequences.insert(commitment.global_sequence()) { + bail!("ledger commitment checkpoint repeats a global sequence"); + } + state.next_global_sequence = state.next_global_sequence.max(commitment.global_sequence()); + state.anchored.insert( + key, + ProducerTail { + sequence: commitment.producer_sequence(), + hash: commitment.commitment_hash(), + }, + ); + } + state.next_checkpoint_sequence = record.checkpoint_sequence; + } + Ok(state) +} + +fn initialize(path: &Path) -> Result<()> { + let parent = path.parent().context("commitment checkpoint path has no parent")?; + std::fs::create_dir_all(parent).with_context(|| format!("create {}", parent.display()))?; + let file = OpenOptions::new() + .create(true) + .append(true) + .read(true) + .open(path) + .with_context(|| format!("open {}", path.display()))?; + file.sync_all().with_context(|| format!("sync {}", path.display()))?; + File::open(parent) + .and_then(|directory| directory.sync_all()) + .with_context(|| format!("sync {}", parent.display())) +} + +fn append(path: &Path, record: &CheckpointRecord) -> Result<()> { + let payload = rmp_serde::to_vec_named(record).context("encode commitment checkpoint")?; + if payload.len() > MAX_CHECKPOINT_FRAME_BYTES { + bail!("encoded commitment checkpoint exceeds its bound"); + } + let mut file = OpenOptions::new().append(true).open(path)?; + file.write_all(&CHECKPOINT_MAGIC)?; + file.write_all(&(payload.len() as u32).to_be_bytes())?; + file.write_all(&payload)?; + file.write_all(blake3::hash(&payload).as_bytes())?; + file.sync_data()?; + Ok(()) +} + +fn read_records(path: &Path) -> Result> { + let mut file = OpenOptions::new().read(true).write(true).open(path)?; + let mut records = Vec::new(); + let mut valid_end = 0_u64; + loop { + let mut header = [0_u8; 8]; + match file.read(&mut header[..1])? { + 0 => break, + 1 => {} + _ => unreachable!(), + } + if let Err(error) = file.read_exact(&mut header[1..]) { + if error.kind() == io::ErrorKind::UnexpectedEof { + file.set_len(valid_end)?; + break; + } + return Err(error.into()); + } + if header[..4] != CHECKPOINT_MAGIC { + bail!("ledger commitment checkpoint magic is corrupt"); + } + let length = u32::from_be_bytes(header[4..].try_into().expect("four-byte length")) as usize; + if length == 0 || length > MAX_CHECKPOINT_FRAME_BYTES { + bail!("ledger commitment checkpoint frame length is invalid"); + } + let mut payload = vec![0; length]; + let mut checksum = [0_u8; 32]; + if let Err(error) = file + .read_exact(&mut payload) + .and_then(|()| file.read_exact(&mut checksum)) + { + if error.kind() == io::ErrorKind::UnexpectedEof { + file.set_len(valid_end)?; + break; + } + return Err(error.into()); + } + if checksum != *blake3::hash(&payload).as_bytes() { + bail!("ledger commitment checkpoint checksum is corrupt"); + } + records.push(rmp_serde::from_slice(&payload).context("decode commitment checkpoint")?); + valid_end = file.stream_position()?; + } + file.seek(SeekFrom::End(0))?; + Ok(records) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-service/src/ledger_commitment/tests.rs b/crates/capsem-service/src/ledger_commitment/tests.rs new file mode 100644 index 000000000..e4d8277a2 --- /dev/null +++ b/crates/capsem-service/src/ledger_commitment/tests.rs @@ -0,0 +1,249 @@ +use super::*; +use capsem_proto::ledger::{LedgerClientRole, LedgerGeneration}; +use capsem_proto::ledger::{LedgerHello, LedgerRequest}; +use std::sync::Arc; + +use capsem_logger::ledger_protocol::{LedgerClientMessage, LedgerCommand, LedgerServerMessage}; +use capsem_logger::{Decision, NetEvent, WriteOp}; + +fn grant(generation: u8, client_id: u64) -> LedgerChannelGrant { + LedgerChannelGrant::new( + LedgerGeneration::new([generation; 16]), + client_id, + LedgerClientRole::Proxy, + ) + .unwrap() +} + +fn event(domain: &str) -> WriteOp { + WriteOp::NetEvent(NetEvent { + event_id: Some("0123456789ab".into()), + timestamp: std::time::SystemTime::UNIX_EPOCH, + domain: domain.into(), + port: 443, + decision: Decision::Allowed, + process_name: None, + pid: None, + method: Some("GET".into()), + path: Some("/".into()), + query: None, + status_code: Some(200), + bytes_sent: 0, + bytes_received: 0, + duration_ms: 1, + matched_rule: None, + request_headers: None, + response_headers: None, + request_body: None, + response_body: None, + conn_type: Some("https".into()), + policy_mode: None, + policy_action: None, + policy_rule: None, + policy_reason: None, + trace_id: None, + credential_ref: None, + }) +} + +fn event_hash(event: &WriteOp) -> [u8; 32] { + let encoded = rmp_serde::to_vec_named(event).unwrap(); + let mut hash = blake3::Hasher::new(); + hash.update(b"capsem-ledger-event-v1\0"); + hash.update(&encoded); + *hash.finalize().as_bytes() +} + +async fn write_committed( + server: Arc, + grant: LedgerChannelGrant, + event: WriteOp, + commitment: LedgerCommitment, +) { + let (client, worker) = UnixStream::pair().unwrap(); + let task = tokio::spawn(async move { server.serve_client(worker, grant).await }); + let (sender, receiver) = capsem_foundation::ipc_channel::channel_from_std::< + LedgerClientMessage, + LedgerServerMessage, + >(client) + .unwrap(); + sender + .send(LedgerClientMessage::Hello { + hello: LedgerHello::for_grant(&grant), + }) + .await + .unwrap(); + assert!(matches!( + receiver.recv().await.unwrap(), + LedgerServerMessage::Welcome { .. } + )); + for (request_id, command) in [ + ( + 1, + LedgerCommand::Admit { + event: Box::new(event), + commitment, + }, + ), + (2, LedgerCommand::Flush), + ] { + sender + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(request_id, command).unwrap(), + }) + .await + .unwrap(); + assert!(matches!( + receiver.recv().await.unwrap(), + LedgerServerMessage::Response { .. } + )); + } + drop(sender); + drop(receiver); + task.await.unwrap().unwrap(); +} + +async fn reader( + server: Arc, + path: &Path, +) -> capsem_logger::ledger_client::LedgerClient { + let grant = LedgerChannelGrant::new(LedgerGeneration::new([9; 16]), 99, LedgerClientRole::Reader).unwrap(); + let (client, worker) = UnixStream::pair().unwrap(); + tokio::spawn(async move { server.serve_client(worker, grant).await }); + capsem_logger::ledger_client::LedgerClient::connect(client, grant, path.to_path_buf()) + .await + .unwrap() +} + +#[tokio::test] +async fn durable_checkpoints_survive_restart_and_continue_global_order() { + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "session-a").await.unwrap(); + let first = grant(1, 7); + let global = authority + .reserve(first, 1, "net_event".into(), [3; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(); + let commitment = LedgerCommitment::new(first, 1, global, "net_event", [3; 32], ZERO_COMMITMENT_HASH).unwrap(); + authority.anchor(first, vec![commitment.clone()]).await.unwrap(); + drop(authority); + + let restarted = CommitmentAuthority::open(root.path(), "session-a").await.unwrap(); + assert_eq!(restarted.anchored().await, vec![commitment]); + let second = grant(2, 1); + assert_eq!( + restarted + .reserve(second, 1, "dns_event".into(), [4; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(), + global + 1 + ); +} + +#[tokio::test] +async fn substitution_reorder_and_stale_authority_are_refused() { + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "session-b").await.unwrap(); + let owner = grant(3, 4); + let global = authority + .reserve(owner, 1, "net_event".into(), [8; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(); + let expected = LedgerCommitment::new(owner, 1, global, "net_event", [8; 32], ZERO_COMMITMENT_HASH).unwrap(); + + let stale = LedgerCommitment::new(grant(2, 4), 1, global, "net_event", [8; 32], ZERO_COMMITMENT_HASH).unwrap(); + assert!(authority.anchor(owner, vec![stale]).await.is_err()); + let substituted = LedgerCommitment::new(owner, 1, global, "dns_event", [8; 32], ZERO_COMMITMENT_HASH).unwrap(); + assert!(authority.anchor(owner, vec![substituted]).await.is_err()); + assert!(authority + .reserve(owner, 3, "net_event".into(), [9; 32], expected.commitment_hash()) + .await + .is_err()); + authority.anchor(owner, vec![expected]).await.unwrap(); +} + +#[tokio::test] +async fn incomplete_unacknowledged_tail_is_recovered_but_corruption_is_loud() { + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "session-c").await.unwrap(); + let path = authority.path.clone(); + drop(authority); + std::fs::write(&path, [CHECKPOINT_MAGIC.as_slice(), &[0, 0]].concat()).unwrap(); + CommitmentAuthority::open(root.path(), "session-c").await.unwrap(); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 0); + + std::fs::write(&path, [b"NOPE".as_slice(), &[0, 0, 0, 1], &[0], &[0; 32]].concat()).unwrap(); + assert!(CommitmentAuthority::open(root.path(), "session-c").await.is_err()); +} + +#[tokio::test] +async fn typed_verification_accepts_exact_rows_and_rejects_anchored_omission_and_substitution() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("session.db"); + let server = Arc::new(capsem_logger::ledger_server::LedgerServer::open(&path).unwrap()); + let authority = CommitmentAuthority::open(&root.path().join("outside"), "session-d") + .await + .unwrap(); + let producer = grant(4, 5); + let expected_event = event("expected.example"); + let expected_hash = event_hash(&expected_event); + let global = authority + .reserve(producer, 1, expected_event.kind().into(), expected_hash, ZERO_COMMITMENT_HASH) + .await + .unwrap(); + let expected = LedgerCommitment::new( + producer, + 1, + global, + expected_event.kind(), + expected_hash, + ZERO_COMMITMENT_HASH, + ) + .unwrap(); + authority.anchor(producer, vec![expected.clone()]).await.unwrap(); + + let client = reader(Arc::clone(&server), &path).await; + assert!(authority + .verify_ledger(&client) + .await + .unwrap_err() + .to_string() + .contains("omitted anchored")); + drop(client); + + let substituted_event = event("substituted.example"); + let substituted = LedgerCommitment::new( + producer, + 1, + global, + substituted_event.kind(), + event_hash(&substituted_event), + ZERO_COMMITMENT_HASH, + ) + .unwrap(); + write_committed(Arc::clone(&server), producer, substituted_event, substituted).await; + let client = reader(Arc::clone(&server), &path).await; + assert!(authority + .verify_ledger(&client) + .await + .unwrap_err() + .to_string() + .contains("substituted or altered")); + drop(client); + + let exact_root = tempfile::tempdir().unwrap(); + let exact_path = exact_root.path().join("session.db"); + let exact_server = Arc::new(capsem_logger::ledger_server::LedgerServer::open(&exact_path).unwrap()); + write_committed(Arc::clone(&exact_server), producer, expected_event, expected.clone()).await; + let exact_authority = CommitmentAuthority::open(&exact_root.path().join("outside"), "session-e") + .await + .unwrap(); + let reserved = exact_authority + .reserve(producer, 1, expected.event_kind().into(), expected.event_hash(), ZERO_COMMITMENT_HASH) + .await + .unwrap(); + assert_eq!(reserved, expected.global_sequence()); + exact_authority.anchor(producer, vec![expected]).await.unwrap(); + let client = reader(exact_server, &exact_path).await; + exact_authority.verify_ledger(&client).await.unwrap(); +} diff --git a/crates/capsem-service/src/ledger_worker.rs b/crates/capsem-service/src/ledger_worker.rs index f8709f56e..dcfc5e4f1 100644 --- a/crates/capsem-service/src/ledger_worker.rs +++ b/crates/capsem-service/src/ledger_worker.rs @@ -48,18 +48,21 @@ type ControlFrame = capsem_foundation::unix::router_channel::DescriptorFrame, } /// Service-owned table that serializes each session's ledger lifetime. pub(crate) struct LedgerWorkers { binary: PathBuf, + commitment_root: PathBuf, slots: tokio::sync::Mutex>, } impl LedgerWorkers { - pub(crate) fn new(binary: PathBuf) -> Self { + pub(crate) fn new(binary: PathBuf, commitment_root: PathBuf) -> Self { Self { binary, + commitment_root, slots: tokio::sync::Mutex::new(std::collections::HashMap::new()), } } @@ -74,7 +77,7 @@ impl LedgerWorkers { log_path: &Path, role: LedgerClientRole, ) -> Result { - let worker = { + let (worker, commitments) = { let mut slots = self.slots.lock().await; let mut watcher = None; if slots @@ -83,7 +86,7 @@ impl LedgerWorkers { { slots.remove(session_id); } - let worker = if let Some(slot) = slots.get(session_id) { + let selected = if let Some(slot) = slots.get(session_id) { if slot.database != database { bail!( "ledger worker slot for {session_id} owns {}, not {}", @@ -91,18 +94,26 @@ impl LedgerWorkers { database.display() ); } - slot.worker.clone() + (slot.worker.clone(), Arc::clone(&slot.commitments)) } else { let worker = self.spawn(database, log_path).await?; + let commitments = + crate::ledger_commitment::CommitmentAuthority::open(&self.commitment_root, session_id).await?; + #[cfg(not(test))] + if let Err(error) = verify_commitments(&worker, database, &commitments).await { + let _ = worker.clone().shutdown().await; + return Err(error); + } slots.insert( session_id.to_string(), LedgerSlot { database: database.to_path_buf(), worker: worker.clone(), + commitments: Arc::clone(&commitments), }, ); watcher = Some((worker.generation(), worker.stop_receiver())); - worker + (worker, commitments) }; drop(slots); if let Some((generation, mut stopped)) = watcher { @@ -121,9 +132,20 @@ impl LedgerWorkers { } }); } - worker + selected }; - worker.connect(role).await + let mut client = worker.connect(role).await?; + if role.producer_name().is_some() { + let (producer, coordinator) = UnixStream::pair().context("create ledger commitment channel")?; + let grant = client.grant; + tokio::spawn(async move { + if let Err(error) = crate::ledger_commitment::serve(commitments, coordinator, grant).await { + tracing::warn!(client_id = grant.client_id(), %error, "ledger commitment channel stopped"); + } + }); + client.commitment = Some(producer); + } + Ok(client) } /// Stop and reap the current generation before vacating its slot. @@ -165,9 +187,30 @@ impl LedgerWorkers { } } +#[cfg(not(test))] +async fn verify_commitments( + worker: &LedgerWorker, + database: &Path, + commitments: &crate::ledger_commitment::CommitmentAuthority, +) -> Result<()> { + let channel = worker.connect(LedgerClientRole::Reader).await?; + let (stream, commitment, grant) = channel.into_parts(); + if commitment.is_some() { + bail!("reader received a producer commitment channel"); + } + let client = capsem_logger::ledger_client::LedgerClient::connect(stream, grant, database.to_path_buf()) + .await + .map_err(anyhow::Error::msg)?; + commitments + .verify_ledger(&client) + .await + .context("verify session ledger against trusted producer checkpoints") +} + #[derive(Debug)] pub(crate) struct LedgerClient { stream: UnixStream, + commitment: Option, grant: LedgerChannelGrant, } @@ -176,14 +219,23 @@ impl LedgerClient { self.grant } - pub(crate) fn into_parts(self) -> (UnixStream, LedgerChannelGrant) { - (self.stream, self.grant) + pub(crate) fn into_parts(self) -> (UnixStream, Option, LedgerChannelGrant) { + (self.stream, self.commitment, self.grant) } #[cfg(test)] - pub(crate) fn test_pair(grant: LedgerChannelGrant) -> io::Result<(Self, UnixStream)> { + pub(crate) fn test_pair(grant: LedgerChannelGrant) -> io::Result<(Self, UnixStream, UnixStream)> { let (stream, worker) = UnixStream::pair()?; - Ok((Self { stream, grant }, worker)) + let (commitment, commitment_worker) = UnixStream::pair()?; + Ok(( + Self { + stream, + commitment: Some(commitment), + grant, + }, + worker, + commitment_worker, + )) } } @@ -359,7 +411,11 @@ impl WorkerProcess { generation, client_id: adopted, } if generation == self.generation && adopted == client_id => { - return Ok(LedgerClient { stream: client, grant }); + return Ok(LedgerClient { + stream: client, + commitment: None, + grant, + }); } LedgerControlEvent::Rejected { generation, diff --git a/crates/capsem-service/src/ledger_worker/tests.rs b/crates/capsem-service/src/ledger_worker/tests.rs index b1a16f47d..7fc9df81d 100644 --- a/crates/capsem-service/src/ledger_worker/tests.rs +++ b/crates/capsem-service/src/ledger_worker/tests.rs @@ -164,7 +164,8 @@ async fn supervisor_mints_unique_role_bound_channels_and_reaps() { assert_eq!(reader.grant().role(), LedgerClientRole::Reader); assert_eq!(producer.grant().role(), LedgerClientRole::VmOwner); assert_ne!(reader.grant().client_id(), producer.grant().client_id()); - let (reader_stream, reader_grant) = reader.into_parts(); + let (reader_stream, commitment, reader_grant) = reader.into_parts(); + assert!(commitment.is_none()); assert_eq!(reader_grant.role(), LedgerClientRole::Reader); drop(reader_stream); drop(producer); @@ -229,7 +230,7 @@ async fn stalled_shutdown_is_killed_and_reaped_within_the_bound() { async fn lifecycle_slot_serializes_concurrent_leases_and_replacement() { let directory = tempfile::tempdir().unwrap(); let binary = fake_binary(directory.path(), "normal"); - let workers = Arc::new(LedgerWorkers::new(binary)); + let workers = Arc::new(LedgerWorkers::new(binary, directory.path().join("commitments"))); let (database, log) = session_paths(directory.path(), "session-a"); let (reader, maintainer) = tokio::join!( @@ -266,7 +267,7 @@ async fn lifecycle_slot_serializes_concurrent_leases_and_replacement() { async fn stopping_one_session_leaves_an_unrelated_session_generation_alive() { let directory = tempfile::tempdir().unwrap(); let binary = fake_binary(directory.path(), "normal"); - let workers = Arc::new(LedgerWorkers::new(binary)); + let workers = Arc::new(LedgerWorkers::new(binary, directory.path().join("commitments"))); let (database_a, log_a) = session_paths(directory.path(), "session-a"); let (database_b, log_b) = session_paths(directory.path(), "session-b"); let client_a = workers @@ -296,7 +297,7 @@ async fn stopping_one_session_leaves_an_unrelated_session_generation_alive() { async fn crashed_slot_restarts_only_after_reap_with_a_fresh_generation() { let directory = tempfile::tempdir().unwrap(); let binary = fake_binary(directory.path(), "die_with_attach_pending"); - let workers = Arc::new(LedgerWorkers::new(binary.clone())); + let workers = Arc::new(LedgerWorkers::new(binary.clone(), directory.path().join("commitments"))); let (database, log) = session_paths(directory.path(), "stopped-session"); let error = workers diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 3d42a9625..602131e1e 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -43,6 +43,8 @@ mod owner_connection; mod owner_handoff; mod policy_mutation; use policy_mutation::{apply_policy_mutation, MutationRoute, PolicyMutation}; +mod credential_routes; +mod ledger_commitment; mod ledger_worker; mod mcp_routes; mod plugin_routes; diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index 0ad60c393..ecd6c0103 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -94,21 +94,26 @@ impl ProxyWorker { } pub(crate) async fn grant(&self, capability: ProxyCapability, descriptor: OwnedFd) -> Result<()> { - self.grant_capability(GrantedCapability::Ordinary(capability), descriptor) + self.grant_capability(GrantedCapability::Ordinary(capability), vec![descriptor]) .await } - pub(crate) async fn grant_ledger(&self, stream: UnixStream, grant: LedgerChannelGrant) -> Result<()> { - self.grant_capability(GrantedCapability::Ledger(grant), stream.into()) + pub(crate) async fn grant_ledger( + &self, + stream: UnixStream, + commitment: UnixStream, + grant: LedgerChannelGrant, + ) -> Result<()> { + self.grant_capability(GrantedCapability::Ledger(grant), vec![stream.into(), commitment.into()]) .await } - async fn grant_capability(&self, capability: GrantedCapability, descriptor: OwnedFd) -> Result<()> { + async fn grant_capability(&self, capability: GrantedCapability, descriptors: Vec) -> Result<()> { let (completed, result) = oneshot::channel(); self.commands .send(CommandRequest::Grant { capability, - descriptor, + descriptors, completed, }) .await @@ -210,7 +215,7 @@ pub(crate) fn spawn_for_session(binary: &Path, active_policy: Vec, log_path: enum CommandRequest { Grant { capability: GrantedCapability, - descriptor: OwnedFd, + descriptors: Vec, completed: oneshot::Sender>, }, ApplyPolicy { @@ -384,17 +389,21 @@ impl WorkerProcess { Ok(process) } - async fn grant(&mut self, capability: GrantedCapability, descriptor: OwnedFd) -> Result<()> { + async fn grant(&mut self, capability: GrantedCapability, descriptors: Vec) -> Result<()> { let grant_id = self.allocate_grant_id()?; let grant = capability.channel_grant(self.generation, grant_id)?; + if descriptors.len() != grant.expected_descriptor_count() { + bail!("proxy capability descriptor count does not match its grant"); + } + let raw = descriptors + .iter() + .map(|descriptor| descriptor.as_raw_fd()) + .collect::>(); self.control_tx - .send( - &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), - &[descriptor.as_raw_fd()], - ) + .send(&encode_proxy_control_request(ProxyControlRequest::Attach(grant)), &raw) .await .context("send proxy descriptor grant")?; - drop(descriptor); + drop(descriptors); match self.recv_event().await? { ProxyControlEvent::Adopted { generation, @@ -643,10 +652,10 @@ async fn supervise( match request { CommandRequest::Grant { capability, - descriptor, + descriptors, completed, } => { - let result = process.grant(capability, descriptor).await; + let result = process.grant(capability, descriptors).await; let failed = result.is_err(); let diagnostic = result.as_ref().err().map(|error| format!("{error:#}")); let _ = completed.send(result); @@ -784,9 +793,10 @@ impl ServiceState { ) .await .context("acquire proxy ledger channel")?; - let (stream, grant) = client.into_parts(); + let (stream, commitment, grant) = client.into_parts(); + let commitment = commitment.context("proxy ledger grant omitted its commitment channel")?; worker - .grant_ledger(stream, grant) + .grant_ledger(stream, commitment, grant) .await .context("grant proxy ledger channel") } diff --git a/crates/capsem-service/src/proxy_worker/tests.rs b/crates/capsem-service/src/proxy_worker/tests.rs index 0829e6f7c..0610a421b 100644 --- a/crates/capsem-service/src/proxy_worker/tests.rs +++ b/crates/capsem-service/src/proxy_worker/tests.rs @@ -120,7 +120,7 @@ async fn fake_proxy_worker_child() { assert_eq!(grant.capability(), ProxyCapability::Ledger); assert_eq!(grant.ledger_grant(), Some(proxy_ledger_grant())); } - held.push(UnixStream::from(frame.fds.into_iter().next().unwrap())); + held.extend(frame.fds.into_iter().map(UnixStream::from)); control_tx.send( &encode_proxy_control_event(ProxyControlEvent::Adopted { generation, @@ -219,8 +219,13 @@ async fn ordinary_traffic_disconnect_does_not_terminate_the_proxy_generation() { async fn ledger_descriptor_carries_its_exact_proxy_authority() { let worker = fake("ledger").await.unwrap(); let (peer, granted) = UnixStream::pair().unwrap(); - worker.grant_ledger(granted, proxy_ledger_grant()).await.unwrap(); + let (commitment_peer, commitment) = UnixStream::pair().unwrap(); + worker + .grant_ledger(granted, commitment, proxy_ledger_grant()) + .await + .unwrap(); drop(peer); + drop(commitment_peer); worker.shutdown().await.unwrap(); } diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index c6652d9f0..8b0b98208 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -264,7 +264,10 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { instances: Mutex::new(HashMap::new()), proxy_workers: Mutex::new(HashMap::new()), standalone_proxies: tokio::sync::Mutex::new(HashMap::new()), - ledger_workers: Arc::new(ledger_worker::LedgerWorkers::new(ledger_binary)), + ledger_workers: Arc::new(ledger_worker::LedgerWorkers::new( + ledger_binary, + capsem_foundation::paths::capsem_home().join("ledger-commitments"), + )), retirements: Default::default(), session_db_handles: Mutex::new(HashMap::new()), persistent_registry: SharedRegistry::new(persistent_registry), diff --git a/crates/capsem-service/src/session_db_handles.rs b/crates/capsem-service/src/session_db_handles.rs index 9464312a2..58f9f23b3 100644 --- a/crates/capsem-service/src/session_db_handles.rs +++ b/crates/capsem-service/src/session_db_handles.rs @@ -199,7 +199,8 @@ impl RemoteLedger { .await { Ok(channel) => { - let (stream, grant) = channel.into_parts(); + let (stream, commitment, grant) = channel.into_parts(); + debug_assert!(commitment.is_none()); return capsem_logger::ledger_client::LedgerClient::connect(stream, grant, database).await; } Err(error) => { @@ -298,7 +299,8 @@ impl ServiceState { .await .map_err(|error| error.to_string()) .map(|channel| { - let (stream, grant) = channel.into_parts(); + let (stream, commitment, grant) = channel.into_parts(); + debug_assert!(commitment.is_none()); (stream, grant) }); #[cfg(not(test))] diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 9f779dca8..0701fc4c3 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -63,9 +63,10 @@ pub(crate) fn test_state( instances: Mutex::new(HashMap::new()), proxy_workers: Mutex::new(HashMap::new()), standalone_proxies: tokio::sync::Mutex::new(HashMap::new()), - ledger_workers: Arc::new(ledger_worker::LedgerWorkers::new(PathBuf::from( - "/nonexistent/capsem-ledger", - ))), + ledger_workers: Arc::new(ledger_worker::LedgerWorkers::new( + PathBuf::from("/nonexistent/capsem-ledger"), + run_dir.join("ledger-commitments"), + )), retirements: Default::default(), session_db_handles: Mutex::new(HashMap::new()), persistent_registry: SharedRegistry::new( diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 8dc42631f..82d9b5e27 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -263,7 +263,7 @@ async fn run( let mut next_resource_id = 1_u64; let mut awaiting_adoption = None; let ledger_configured = ledger.is_some(); - let mut pending_ledger: Option<(LedgerChannelGrant, OwnedFd)> = None; + let mut pending_ledger: Option<(LedgerChannelGrant, Vec)> = None; let result = async { 'requests: loop { let frame = tokio::select! { @@ -518,13 +518,13 @@ async fn run( } UpstreamGrantRequest::Adopted { grant_id } => { if awaiting_adoption == Some(AwaitingAdoption::Ledger(grant_id)) { - let Some((grant, descriptor)) = pending_ledger.take() else { + let Some((grant, descriptors)) = pending_ledger.take() else { break Err(format!("ledger grant {grant_id} disappeared before adoption")); }; if grant.client_id() != grant_id { break Err(format!("ledger grant {grant_id} changed before adoption")); } - drop(descriptor); + drop(descriptors); awaiting_adoption = None; continue; } @@ -579,15 +579,18 @@ async fn run( continue; }; let client = source.acquire().await?; - let (stream, grant) = client.into_parts(); - let descriptor = OwnedFd::from(stream); - send_response( + let (stream, commitment, grant) = client.into_parts(); + let commitment = + commitment.ok_or_else(|| "ledger producer omitted commitment channel".to_string())?; + let descriptors = vec![OwnedFd::from(stream), OwnedFd::from(commitment)]; + let descriptor_refs = descriptors.iter().collect::>(); + send_response_descriptors( &responses, &UpstreamGrantResponse::LedgerGranted { request_id, grant }, - Some(&descriptor), + &descriptor_refs, ) .await?; - pending_ledger = Some((grant, descriptor)); + pending_ledger = Some((grant, descriptors)); awaiting_adoption = Some(AwaitingAdoption::Ledger(grant.client_id())); } UpstreamGrantRequest::AttachProxyTraffic { request_id, service } => { @@ -686,10 +689,19 @@ async fn send_response( sender: &WireSender, response: &UpstreamGrantResponse, descriptor: Option<&OwnedFd>, +) -> Result<(), String> { + let descriptors = descriptor.into_iter().collect::>(); + send_response_descriptors(sender, response, &descriptors).await +} + +async fn send_response_descriptors( + sender: &WireSender, + response: &UpstreamGrantResponse, + descriptors: &[&OwnedFd], ) -> Result<(), String> { let bytes = encode_upstream_grant_response(response).map_err(|error| format!("encode grant response: {error:#}"))?; - let descriptors = descriptor.map_or_else(Vec::new, |fd| vec![fd.as_raw_fd()]); + let descriptors = descriptors.iter().map(|fd| fd.as_raw_fd()).collect::>(); tokio::time::timeout(RESPONSE_TIMEOUT, sender.send(&bytes, &descriptors)) .await .map_err(|_| "send upstream grant response timed out".to_string())? diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index a9e40cb29..b86d6c7f7 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -213,13 +213,15 @@ fn start_broker_for_session( #[tokio::test] async fn ledger_grant_is_exact_single_use_and_releases_the_broker_copy_after_adoption() { let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x4d; 16]), 77, LedgerClientRole::VmOwner).unwrap(); - let (ledger, worker_peer) = LedgerClient::test_pair(grant).unwrap(); + let (ledger, worker_peer, commitment_peer) = LedgerClient::test_pair(grant).unwrap(); let (client, _publisher, authority, task) = start_broker_with_ledger(test_policy("policy-a", None, vec![]), ledger); let (response, mut fds) = client .request(&UpstreamGrantRequest::OpenLedger { request_id: 1 }) .await; assert_eq!(response, UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }); + assert_eq!(fds.len(), 2); + let granted_commitment = UnixStream::from(fds.pop().unwrap()); let granted = UnixStream::from(fds.pop().unwrap()); client .send(&UpstreamGrantRequest::Adopted { @@ -240,6 +242,8 @@ async fn ledger_grant_is_exact_single_use_and_releases_the_broker_copy_after_ado ); drop(granted); + drop(granted_commitment); + drop(commitment_peer); worker_peer.set_nonblocking(true).unwrap(); let mut worker_peer = tokio::net::UnixStream::from_std(worker_peer).unwrap(); let mut byte = [0_u8; 1]; @@ -273,7 +277,7 @@ async fn ledger_request_without_a_coordinator_grant_is_denied() { #[tokio::test] async fn policy_updates_do_not_revoke_a_pending_ledger_grant() { let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x37; 16]), 91, LedgerClientRole::VmOwner).unwrap(); - let (ledger, worker_peer) = LedgerClient::test_pair(grant).unwrap(); + let (ledger, worker_peer, commitment_peer) = LedgerClient::test_pair(grant).unwrap(); let (client, publisher, authority, task) = start_broker_with_ledger(test_policy("policy-a", None, vec![]), ledger); let (response, mut fds) = client .request(&UpstreamGrantRequest::OpenLedger { request_id: 1 }) @@ -287,7 +291,10 @@ async fn policy_updates_do_not_revoke_a_pending_ledger_grant() { }) .await; + assert_eq!(fds.len(), 2); drop(fds.pop().unwrap()); + drop(fds.pop().unwrap()); + drop(commitment_peer); worker_peer.set_nonblocking(true).unwrap(); let mut worker_peer = tokio::net::UnixStream::from_std(worker_peer).unwrap(); let mut byte = [0_u8; 1]; diff --git a/crates/capsem-service/src/vm_files/fork.rs b/crates/capsem-service/src/vm_files/fork.rs index 116a6b534..dfa611402 100644 --- a/crates/capsem-service/src/vm_files/fork.rs +++ b/crates/capsem-service/src/vm_files/fork.rs @@ -275,7 +275,8 @@ async fn clone_ledger( ) .await .map_err(|error| format!("acquire source ledger owner: {error:#}"))?; - let (stream, grant) = channel.into_parts(); + let (stream, commitment, grant) = channel.into_parts(); + debug_assert!(commitment.is_none()); let client = capsem_logger::ledger_client::LedgerClient::connect(stream, grant, database).await?; let snapshot = client.snapshot(*uuid::Uuid::new_v4().as_bytes()).await?; let snapshot_copy = snapshot.clone(); From 01fe7624578a9cf50e70472f8d66d868efb890f3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:57:23 +0000 Subject: [PATCH 208/464] feat(core): adopt ledger checkpoint channels --- crates/capsem-core/src/net/upstream_grant.rs | 15 ++++++++++----- .../src/net/upstream_grant/tests.rs | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/crates/capsem-core/src/net/upstream_grant.rs b/crates/capsem-core/src/net/upstream_grant.rs index c69f71101..403dff0e3 100644 --- a/crates/capsem-core/src/net/upstream_grant.rs +++ b/crates/capsem-core/src/net/upstream_grant.rs @@ -39,7 +39,7 @@ enum Command { reply: oneshot::Sender>, }, OpenLedger { - reply: oneshot::Sender>, + reply: oneshot::Sender>, }, SetGuestMode { relative_path: Vec, @@ -100,7 +100,7 @@ impl UpstreamGrantClient { self.stopped.clone() } - pub async fn open_ledger(&self) -> Result<(UnixStream, LedgerChannelGrant)> { + pub async fn open_ledger(&self) -> Result<(UnixStream, UnixStream, LedgerChannelGrant)> { let (reply, result) = oneshot::channel(); self.commands .send(Command::OpenLedger { reply }) @@ -385,7 +385,7 @@ async fn open_ledger( sender: &WireSender, receiver: &WireReceiver, request_id: u64, -) -> Result, String> { +) -> Result, String> { send_request(sender, &UpstreamGrantRequest::OpenLedger { request_id }).await?; let (response, mut fds) = receive_response(receiver).await?; match response { @@ -397,7 +397,8 @@ async fn open_ledger( request_id: response_id, grant, } if response_id == request_id => { - let descriptor = fds.pop().ok_or("ledger grant omitted its descriptor")?; + let commitment = fds.pop().ok_or("ledger grant omitted its commitment descriptor")?; + let descriptor = fds.pop().ok_or("ledger grant omitted its storage descriptor")?; send_request( sender, &UpstreamGrantRequest::Adopted { @@ -406,10 +407,14 @@ async fn open_ledger( ) .await?; let stream = UnixStream::from(descriptor); + let commitment = UnixStream::from(commitment); stream .peer_addr() .map_err(|error| format!("adopt granted ledger channel: {error}"))?; - Ok(Ok((stream, grant))) + commitment + .peer_addr() + .map_err(|error| format!("adopt granted commitment channel: {error}"))?; + Ok(Ok((stream, commitment, grant))) } response => Err(format!( "unexpected session ledger response for request {request_id}: {response:?}" diff --git a/crates/capsem-core/src/net/upstream_grant/tests.rs b/crates/capsem-core/src/net/upstream_grant/tests.rs index d5973c01a..682ab2882 100644 --- a/crates/capsem-core/src/net/upstream_grant/tests.rs +++ b/crates/capsem-core/src/net/upstream_grant/tests.rs @@ -24,15 +24,16 @@ async fn ledger_channel_preserves_grant_and_is_adopted_before_use() { let client = UpstreamGrantClient::start(worker).unwrap(); let (requests, responses) = channels(broker); let (ledger, mut ledger_peer) = UnixStream::pair().unwrap(); + let (commitment, mut commitment_peer) = UnixStream::pair().unwrap(); let broker_task = tokio::spawn(async move { assert_eq!( receive_request(&requests).await, UpstreamGrantRequest::OpenLedger { request_id: 1 } ); - send_response( + send_response_many( &responses, &UpstreamGrantResponse::LedgerGranted { request_id: 1, grant }, - Some(ledger.as_raw_fd()), + &[ledger.as_raw_fd(), commitment.as_raw_fd()], ) .await; assert_eq!( @@ -43,12 +44,15 @@ async fn ledger_channel_preserves_grant_and_is_adopted_before_use() { ); }); - let (mut stream, received) = client.open_ledger().await.unwrap(); + let (mut stream, mut commitment_stream, received) = client.open_ledger().await.unwrap(); assert_eq!(received, grant); stream.write_all(b"hello").unwrap(); let mut message = [0_u8; 5]; ledger_peer.read_exact(&mut message).unwrap(); assert_eq!(&message, b"hello"); + commitment_stream.write_all(b"proof").unwrap(); + commitment_peer.read_exact(&mut message).unwrap(); + assert_eq!(&message, b"proof"); broker_task.await.unwrap(); } @@ -183,9 +187,13 @@ async fn receive_request(receiver: &WireReceiver) -> UpstreamGrantRequest { } async fn send_response(sender: &WireSender, response: &UpstreamGrantResponse, descriptor: Option) { - let bytes = encode_upstream_grant_response(response).unwrap(); let fds = descriptor.map_or_else(Vec::new, |descriptor| vec![descriptor]); - sender.send(&bytes, &fds).await.unwrap(); + send_response_many(sender, response, &fds).await; +} + +async fn send_response_many(sender: &WireSender, response: &UpstreamGrantResponse, fds: &[RawFd]) { + let bytes = encode_upstream_grant_response(response).unwrap(); + sender.send(&bytes, fds).await.unwrap(); } fn query() -> Vec { From 9900764878b3a87092e6a9dbee3f1112d5b1960c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:58:03 +0000 Subject: [PATCH 209/464] feat(process): attach trusted ledger checkpoints --- crates/capsem-process/src/main.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 1b808e35f..aa68003c1 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -673,7 +673,7 @@ async fn run_async_main_loop( ) -> Result<()> { let terminal_output = Arc::new(capsem_core::TerminalOutputQueue::new()); - let (ledger_stream, ledger_grant) = upstream_grants + let (ledger_stream, commitment_stream, ledger_grant) = upstream_grants .open_ledger() .await .context("acquire supervised session ledger")?; @@ -685,7 +685,13 @@ async fn run_async_main_loop( // the worker that exclusively owns and opens the database. let db = Arc::new( tokio::task::spawn_blocking(move || { - capsem_logger::DbWriter::from_ledger_channel(ledger_stream, ledger_grant, &ledger_path, 1024) + capsem_logger::DbWriter::from_ledger_channel( + ledger_stream, + commitment_stream, + ledger_grant, + &ledger_path, + 1024, + ) }) .await .context("join supervised session ledger handshake")??, From 52be6b4415ae7e5482788b546cc0605d7d3b3cd8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 10:58:43 +0000 Subject: [PATCH 210/464] feat(proxy): attach trusted ledger checkpoints --- crates/capsem-proxy/src/main.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index ea0bb7823..310e3aed5 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -164,8 +164,8 @@ async fn run_control( } match request { ProxyControlRequest::Attach(grant) => { - if frame.fds.len() != 1 { - bail!("proxy capability attach requires exactly one descriptor"); + if frame.fds.len() != grant.expected_descriptor_count() { + bail!("proxy capability attach descriptor count does not match its grant"); } let capability = grant.capability(); let grant_id = grant.grant_id(); @@ -200,7 +200,8 @@ async fn run_control( .await?; continue; } - let descriptor = frame.fds.into_iter().next().expect("one checked descriptor"); + let mut descriptors = frame.fds.into_iter(); + let descriptor = descriptors.next().expect("at least one checked descriptor"); grants.entry(capability).or_default().push(grant_id); grant_ids.insert(grant_id); if capability == ProxyCapability::Policy { @@ -212,12 +213,16 @@ async fn run_control( }); } else if capability == ProxyCapability::Ledger { let stream = UnixStream::from(descriptor); + let commitment = UnixStream::from( + descriptors.next().expect("ledger commitment descriptor was checked"), + ); let ledger_grant = grant .ledger_grant() .expect("ledger capability was decoded with exact authority"); let writer = tokio::task::spawn_blocking(move || { capsem_logger::DbWriter::from_ledger_channel( stream, + commitment, ledger_grant, Path::new("capability:session-ledger"), 1024, From 908df60422b0834b5961c71b7f30c05d8585714b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:00:36 +0000 Subject: [PATCH 211/464] feat(logger): expose canonical commitment hashing --- crates/capsem-logger/src/lib.rs | 5 ++++- crates/capsem-logger/src/writer.rs | 7 ++----- crates/capsem-logger/src/writer/operation.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/capsem-logger/src/lib.rs b/crates/capsem-logger/src/lib.rs index 4d72e3de0..e1bee2054 100644 --- a/crates/capsem-logger/src/lib.rs +++ b/crates/capsem-logger/src/lib.rs @@ -29,4 +29,7 @@ pub use reader::{ SecurityRuleDetectionLevelCount, SecurityRuleEventTypeCount, SecurityRuleStats, SecurityRuleStatsByRule, }; pub use session_types::{epoch_to_iso, generate_session_id, is_valid_session_id, now_iso}; -pub use writer::{format_ledger_timestamp, output_preview, DbWriter, RetainOutcome, WriteOp, MAX_BODY_BLOB_BYTES}; +pub use writer::{ + commitment_event_hash, format_ledger_timestamp, output_preview, DbWriter, RetainOutcome, WriteOp, + MAX_BODY_BLOB_BYTES, +}; diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 423c801b3..7486e5fdf 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -293,10 +293,7 @@ struct WriteEnvelope { impl WriteEnvelope { #[cfg(test)] fn plain(op: WriteOp) -> Self { - Self { - op, - commitment: None, - } + Self { op, commitment: None } } } @@ -314,7 +311,7 @@ fn writer_channel(capacity: usize) -> (WriterSender, mpsc::Receiver Result<[u8; 32], String> { +pub fn commitment_event_hash(op: &WriteOp) -> Result<[u8; 32], String> { let encoded = rmp_serde::to_vec_named(op).map_err(|error| format!("encode ledger event commitment: {error}"))?; let mut hash = blake3::Hasher::new(); hash.update(b"capsem-ledger-event-v1\0"); From 1b8cb831b1dda2ba7753e9f424e449072cbd710c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:00:39 +0000 Subject: [PATCH 212/464] test(service): reuse canonical commitment hashing --- .../src/ledger_commitment/tests.rs | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/crates/capsem-service/src/ledger_commitment/tests.rs b/crates/capsem-service/src/ledger_commitment/tests.rs index e4d8277a2..ba66d9678 100644 --- a/crates/capsem-service/src/ledger_commitment/tests.rs +++ b/crates/capsem-service/src/ledger_commitment/tests.rs @@ -46,14 +46,6 @@ fn event(domain: &str) -> WriteOp { }) } -fn event_hash(event: &WriteOp) -> [u8; 32] { - let encoded = rmp_serde::to_vec_named(event).unwrap(); - let mut hash = blake3::Hasher::new(); - hash.update(b"capsem-ledger-event-v1\0"); - hash.update(&encoded); - *hash.finalize().as_bytes() -} - async fn write_committed( server: Arc, grant: LedgerChannelGrant, @@ -62,11 +54,8 @@ async fn write_committed( ) { let (client, worker) = UnixStream::pair().unwrap(); let task = tokio::spawn(async move { server.serve_client(worker, grant).await }); - let (sender, receiver) = capsem_foundation::ipc_channel::channel_from_std::< - LedgerClientMessage, - LedgerServerMessage, - >(client) - .unwrap(); + let (sender, receiver) = + capsem_foundation::ipc_channel::channel_from_std::(client).unwrap(); sender .send(LedgerClientMessage::Hello { hello: LedgerHello::for_grant(&grant), @@ -186,9 +175,15 @@ async fn typed_verification_accepts_exact_rows_and_rejects_anchored_omission_and .unwrap(); let producer = grant(4, 5); let expected_event = event("expected.example"); - let expected_hash = event_hash(&expected_event); + let expected_hash = capsem_logger::commitment_event_hash(&expected_event).unwrap(); let global = authority - .reserve(producer, 1, expected_event.kind().into(), expected_hash, ZERO_COMMITMENT_HASH) + .reserve( + producer, + 1, + expected_event.kind().into(), + expected_hash, + ZERO_COMMITMENT_HASH, + ) .await .unwrap(); let expected = LedgerCommitment::new( @@ -217,7 +212,7 @@ async fn typed_verification_accepts_exact_rows_and_rejects_anchored_omission_and 1, global, substituted_event.kind(), - event_hash(&substituted_event), + capsem_logger::commitment_event_hash(&substituted_event).unwrap(), ZERO_COMMITMENT_HASH, ) .unwrap(); @@ -239,7 +234,13 @@ async fn typed_verification_accepts_exact_rows_and_rejects_anchored_omission_and .await .unwrap(); let reserved = exact_authority - .reserve(producer, 1, expected.event_kind().into(), expected.event_hash(), ZERO_COMMITMENT_HASH) + .reserve( + producer, + 1, + expected.event_kind().into(), + expected.event_hash(), + ZERO_COMMITMENT_HASH, + ) .await .unwrap(); assert_eq!(reserved, expected.global_sequence()); From 508d539215dbb5482853a2f4e5cc2cdb35318821 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:00:42 +0000 Subject: [PATCH 213/464] test(ledger): exercise committed producer records --- crates/capsem-ledger/src/tests.rs | 17 ++++++++++++++++- crates/capsem-ledger/tests/subprocess.rs | 17 ++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/crates/capsem-ledger/src/tests.rs b/crates/capsem-ledger/src/tests.rs index 681a3764f..bc35ee55d 100644 --- a/crates/capsem-ledger/src/tests.rs +++ b/crates/capsem-ledger/src/tests.rs @@ -5,6 +5,7 @@ use capsem_foundation::ipc_channel; use capsem_logger::ledger_protocol::{LedgerClientMessage, LedgerCommand, LedgerReply, LedgerServerMessage}; use capsem_logger::{Decision, NetEvent, WriteOp}; use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerHello, LedgerOutcome, LedgerRequest}; +use capsem_proto::ledger_commitment::{LedgerCommitment, ZERO_COMMITMENT_HASH}; use capsem_proto::ledger_control::{ decode_ledger_control_event, encode_ledger_control_request, LedgerControlEvent, LedgerControlRequest, }; @@ -44,6 +45,18 @@ fn event() -> WriteOp { }) } +fn commitment(grant: LedgerChannelGrant, event: &WriteOp) -> LedgerCommitment { + LedgerCommitment::new( + grant, + 1, + 1, + event.kind(), + capsem_logger::commitment_event_hash(event).unwrap(), + ZERO_COMMITMENT_HASH, + ) + .unwrap() +} + async fn control_event(receiver: &ControlReceiver) -> LedgerControlEvent { let frame = receiver.recv().await.unwrap(); assert!(frame.fds.is_empty()); @@ -93,11 +106,13 @@ async fn worker_adopts_one_client_flushes_and_stops() { receiver.recv().await.unwrap(), LedgerServerMessage::Welcome { .. } )); + let admitted = event(); for (request_id, operation) in [ ( 1, LedgerCommand::Admit { - event: Box::new(event()), + commitment: commitment(grant, &admitted), + event: Box::new(admitted), }, ), (2, LedgerCommand::Flush), diff --git a/crates/capsem-ledger/tests/subprocess.rs b/crates/capsem-ledger/tests/subprocess.rs index cf621e21b..8c84a564b 100644 --- a/crates/capsem-ledger/tests/subprocess.rs +++ b/crates/capsem-ledger/tests/subprocess.rs @@ -10,6 +10,7 @@ use capsem_logger::{Decision, NetEvent, WriteOp}; use capsem_proto::ledger::{ LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerHello, LedgerOutcome, LedgerRequest, }; +use capsem_proto::ledger_commitment::{LedgerCommitment, ZERO_COMMITMENT_HASH}; use capsem_proto::ledger_control::{ decode_ledger_control_event, encode_ledger_control_request, LedgerControlEvent, LedgerControlRequest, LEDGER_CONTROL_FRAME_SIZE, LEDGER_CONTROL_MAX_FDS, @@ -77,6 +78,18 @@ fn body_event() -> WriteOp { }) } +fn commitment(grant: LedgerChannelGrant, event: &WriteOp) -> LedgerCommitment { + LedgerCommitment::new( + grant, + 1, + 1, + event.kind(), + capsem_logger::commitment_event_hash(event).unwrap(), + ZERO_COMMITMENT_HASH, + ) + .unwrap() +} + #[tokio::test] async fn executable_opens_one_ledger_and_reports_durable_stop() { let dir = tempfile::tempdir().unwrap(); @@ -138,11 +151,13 @@ async fn executable_opens_one_ledger_and_reports_durable_stop() { receiver.recv().await.unwrap(), LedgerServerMessage::Welcome { .. } )); + let admitted = body_event(); for (request_id, operation) in [ ( 1, LedgerCommand::Admit { - event: Box::new(body_event()), + commitment: commitment(grant, &admitted), + event: Box::new(admitted), }, ), (2, LedgerCommand::Flush), From 543870aeadbd54944e6181610ae6acb4778b9724 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:00:51 +0000 Subject: [PATCH 214/464] style(service): format commitment verification --- .../capsem-service/src/ledger_commitment.rs | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/crates/capsem-service/src/ledger_commitment.rs b/crates/capsem-service/src/ledger_commitment.rs index 8b95294a5..b57ef231f 100644 --- a/crates/capsem-service/src/ledger_commitment.rs +++ b/crates/capsem-service/src/ledger_commitment.rs @@ -10,12 +10,12 @@ use std::sync::Arc; use anyhow::{bail, Context, Result}; use capsem_foundation::ipc_channel; use capsem_proto::ledger::{LedgerChannelGrant, LedgerWelcome}; +#[cfg(test)] +use capsem_proto::ledger_commitment::ZERO_COMMITMENT_HASH; use capsem_proto::ledger_commitment::{ CommitmentClientMessage, CommitmentCommand, CommitmentReply, CommitmentServerMessage, LedgerCommitment, MAX_COMMITMENTS_PER_CHECKPOINT, }; -#[cfg(test)] -use capsem_proto::ledger_commitment::ZERO_COMMITMENT_HASH; use serde::{Deserialize, Serialize}; use capsem_logger::ledger_protocol::{LedgerQuery, LedgerValue}; @@ -264,10 +264,7 @@ impl CommitmentAuthority { "ledger commitment {} was substituted or altered", commitment.global_sequence() ), - None => bail!( - "ledger omitted anchored commitment {}", - commitment.global_sequence() - ), + None => bail!("ledger omitted anchored commitment {}", commitment.global_sequence()), } } Ok(()) @@ -275,17 +272,8 @@ impl CommitmentAuthority { } fn decode_commitment_row(row: &[LedgerValue]) -> Result { - let [ - LedgerValue::Integer(global_sequence), - LedgerValue::Text(generation), - LedgerValue::Integer(client_id), - LedgerValue::Text(role), - LedgerValue::Integer(producer_sequence), - LedgerValue::Text(event_kind), - LedgerValue::Text(event_hash), - LedgerValue::Text(previous_hash), - LedgerValue::Text(commitment_hash), - ] = row + let [LedgerValue::Integer(global_sequence), LedgerValue::Text(generation), LedgerValue::Integer(client_id), LedgerValue::Text(role), LedgerValue::Integer(producer_sequence), LedgerValue::Text(event_kind), LedgerValue::Text(event_hash), LedgerValue::Text(previous_hash), LedgerValue::Text(commitment_hash)] = + row else { bail!("ledger commitment query returned an invalid typed row"); }; From 3394cd9f4875fbdc29bc4d5dffb204c124cf7068 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:04:25 +0000 Subject: [PATCH 215/464] test(ledger): preserve commitments across retention --- crates/capsem-logger/src/ledger_server/tests.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/crates/capsem-logger/src/ledger_server/tests.rs b/crates/capsem-logger/src/ledger_server/tests.rs index 0eeace18f..e6b313272 100644 --- a/crates/capsem-logger/src/ledger_server/tests.rs +++ b/crates/capsem-logger/src/ledger_server/tests.rs @@ -212,6 +212,19 @@ async fn accepted_durable_query_body_retention_and_warc_run_through_one_owner() .await, LedgerReply::Retained { .. } )); + let commitments = request( + &reader, + &reader_replies, + 7, + LedgerCommand::Query { + query: LedgerQuery::Commitments { + after_global_sequence: 0, + limit: 10, + }, + }, + ) + .await; + assert!(matches!(commitments, LedgerReply::Query { ref sets } if sets.len() == 1 && sets[0].rows.len() == 1)); drop(producer); drop(reader); From d4b0150903636978e853b2df593877f2751a72d3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:04:27 +0000 Subject: [PATCH 216/464] test(service): define unanchored commitment loss --- .../src/ledger_commitment/tests.rs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/capsem-service/src/ledger_commitment/tests.rs b/crates/capsem-service/src/ledger_commitment/tests.rs index ba66d9678..cfe652e92 100644 --- a/crates/capsem-service/src/ledger_commitment/tests.rs +++ b/crates/capsem-service/src/ledger_commitment/tests.rs @@ -129,6 +129,39 @@ async fn durable_checkpoints_survive_restart_and_continue_global_order() { ); } +#[tokio::test] +async fn multiple_producers_share_global_order_and_unanchored_tail_is_not_a_checkpoint() { + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "session-tail").await.unwrap(); + let first = grant(1, 7); + let second = grant(1, 8); + + let first_global = authority + .reserve(first, 1, "net_event".into(), [1; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(); + let second_global = authority + .reserve(second, 1, "dns_event".into(), [2; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(); + assert_eq!(second_global, first_global + 1); + + let first_commitment = + LedgerCommitment::new(first, 1, first_global, "net_event", [1; 32], ZERO_COMMITMENT_HASH).unwrap(); + authority.anchor(first, vec![first_commitment.clone()]).await.unwrap(); + drop(authority); + + let restarted = CommitmentAuthority::open(root.path(), "session-tail").await.unwrap(); + assert_eq!(restarted.anchored().await, vec![first_commitment]); + assert_eq!( + restarted + .reserve(second, 1, "dns_event".into(), [2; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(), + first_global + 1 + ); +} + #[tokio::test] async fn substitution_reorder_and_stale_authority_are_refused() { let root = tempfile::tempdir().unwrap(); From 7f5da5f271d2dedce10618d0235e77c9201447c2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:06:07 +0000 Subject: [PATCH 217/464] docs(architecture): define ledger commitment proof --- .../docs/architecture/host-isolation.md | 29 +++++- .../docs/architecture/service-architecture.md | 7 +- .../docs/architecture/session-telemetry.md | 98 +++++++++++++++---- 3 files changed, 110 insertions(+), 24 deletions(-) diff --git a/web/docs/src/content/docs/architecture/host-isolation.md b/web/docs/src/content/docs/architecture/host-isolation.md index a0ce5a2ae..88a56b73f 100644 --- a/web/docs/src/content/docs/architecture/host-isolation.md +++ b/web/docs/src/content/docs/architecture/host-isolation.md @@ -23,6 +23,7 @@ flowchart TB HostMCP["host MCP client"] Gateway["capsem-gateway
global confined API worker"] Service["capsem-service
trusted coordinator"] + Checkpoints["producer checkpoints
outside session ledger directory"] subgraph Session["one VM session"] Owner["capsem-process
VM owner"] @@ -45,14 +46,17 @@ flowchart TB Service -->|"typed lifecycle control"| Owner Service -->|"generation-bound capabilities"| Proxy Service -->|"role-bound ledger channels"| Ledger + Service -->|"orders and syncs"| Checkpoints Owner -->|"virtio and VSOCK"| VM Owner -->|"connected stream pairs"| Relay Proxy -->|"MCP requests after policy"| MCPBridge Owner -->|"owns"| MCPBridge MCPBridge --> Aggregator Owner -->|"one cable per membership"| Switch - Proxy -->|"admit and flush only"| Ledger - Owner -->|"admit and flush only"| Ledger + Proxy -->|"committed admit and flush"| Ledger + Owner -->|"committed admit and flush"| Ledger + Proxy -->|"reserve and anchor"| Service + Owner -->|"reserve and anchor"| Service Service -->|"read, retain, export, snapshot"| Ledger ``` @@ -88,7 +92,7 @@ grants fail closed. | Process | Permitted after readiness | Denied or absent | |---|---|---| -| `capsem-service` | Global lifecycle, settings and corp policy, credential store, session registry, resource creation and capability grants | It is part of the trusted computing base; clients reach it through the local UDS or authenticated gateway routes | +| `capsem-service` | Global lifecycle, settings and corp policy, credential store, session registry, resource creation and capability grants; producer ordering and external commitment checkpoints | It is part of the trusted computing base; clients reach it through the local UDS or authenticated gateway routes | | `capsem-gateway` | Accept on listeners bound before confinement; use coordinator-granted service/owner channels; update its private readiness files | Ambient file reads, path-based UDS connects, outbound TCP, new binds, process execution | | `capsem-process` | One VM and its exact session runtime paths; read exact boot assets and active policy; use coordinator-granted upstream and ledger channels; execute only the installed router helper | Other sessions, direct `session.db` access, ambient outbound connects/binds, unrelated host files and processes | | `capsem-proxy` | Consume granted HTTP, DNS, MCP, upstream, credential, ledger, metrics, private-name, and policy channels | All filesystem access, socket creation, arbitrary dial/bind/listen, process execution, signals to the coordinator | @@ -178,6 +182,21 @@ path in that object does not grant filesystem authority. Service routes also query through a ledger client, including after the VM stops. A missing table or column is a schema error, and a flush is the read-after-write barrier. +Producers also receive a separate connected channel to the service's trusted +commitment authority. Before admission, that authority assigns one global +order across all producers. The producer commits its generation, client role, +producer-local order, event kind and canonical content hash into a BLAKE3 +chain. `capsem-ledger` stores the record and commitment atomically. After its +ledger flush succeeds, the service appends and syncs the matching prefix under +`~/.capsem/ledger-commitments/`, which is outside the path available to the +ledger worker. The producer sees flush success only after both durable writes. + +When the service starts a fresh ledger generation, it uses a typed reader +channel to compare SQLite commitments with the external checkpoint before it +grants producer channels. An altered or substituted row, anchored omission, +stale generation, or reordered commitment prevents startup. Body retention +preserves these rows and checkpoints. + Raw observed credentials do not belong in ledger rows, body archives, worker logs, or metrics. The proxy sends credential observations over its broker capability and receives only the material needed for the selected upstream. @@ -260,6 +279,10 @@ when untrusted code itself needs containment. not protect against a compromised coordinator or kernel. - Connected descriptors carry real authority until closed. The coordinator therefore grants them to one fresh generation and retains revocation. +- External checkpoints prove exact anchored producer prefixes. An unanchored + tail can be lost or replayed after a crash, and a compromised producer can + commit a false observation. The service, checkpoint storage, and host kernel + remain trusted. - The standalone endpoint supports the OpenAI Chat Completions and Responses request shapes exercised by the official Python and TypeScript SDKs, including streams, tool payloads, cancellation, usage, and upstream errors. diff --git a/web/docs/src/content/docs/architecture/service-architecture.md b/web/docs/src/content/docs/architecture/service-architecture.md index d180b4bc9..5ef836b57 100644 --- a/web/docs/src/content/docs/architecture/service-architecture.md +++ b/web/docs/src/content/docs/architecture/service-architecture.md @@ -19,7 +19,7 @@ The native host binaries are installed to | Binary | Role | Communication | |--------|------|---------------| | **capsem** | CLI client | HTTP over UDS to service | -| **capsem-service** | Background daemon | Axum HTTP over UDS (`~/.capsem/run/service.sock`) | +| **capsem-service** | Background daemon and trusted producer-checkpoint authority | Axum HTTP over UDS (`~/.capsem/run/service.sock`) plus connected worker capabilities | | **capsem-process** | Per-VM process | Spawned by service, bounded MessagePack over UDS (after a MessagePack Hello) | | **capsem-proxy** | Per-session HTTP, DNS, model, and MCP policy worker | Generation-bound connected descriptor grants | | **capsem-ledger** | Sole owner of one session ledger | Role- and generation-bound connected ledger channels | @@ -80,6 +80,8 @@ graph TD PROC -->|"HTTP/DNS/MCP descriptors"| PROXY PROC -->|"typed writer channel"| LEDGER PROXY -->|"typed writer channel"| LEDGER + PROC -->|"commitment channel"| SVC + PROXY -->|"commitment channel"| SVC subgraph "Linux VM (guest)" AGENT["capsem-pty-agent"] @@ -162,6 +164,9 @@ of paths or destination names. The short summary is: socket authority. - **Ledger**: the only process that opens one session's `session.db` and body archive; no network or execution authority. +- **Commitment authority**: the service globally orders producer commitments, + syncs their durable prefixes outside the ledger directory, and verifies them + through typed reads before a fresh ledger generation accepts producers. - **Minimal inheritance**: child environments are cleared where secrets could leak and unrelated descriptors are closed before readiness. - **OS confinement**: Seatbelt on macOS and Landlock plus seccomp on Linux. diff --git a/web/docs/src/content/docs/architecture/session-telemetry.md b/web/docs/src/content/docs/architecture/session-telemetry.md index 37f2e3131..9771533d6 100644 --- a/web/docs/src/content/docs/architecture/session-telemetry.md +++ b/web/docs/src/content/docs/architecture/session-telemetry.md @@ -1,6 +1,6 @@ --- title: Session Telemetry -description: Per-VM SQLite database schema, data flow, and query patterns. +description: Per-session storage, producer commitments, data flow, and query patterns. sidebar: order: 20 --- @@ -18,6 +18,8 @@ A supervised `capsem-ledger` process is the sole filesystem owner of those files. VM owners, proxy workers, and service readers use authenticated, generation-bound channels with role-specific operations. See [Host Process Isolation](/architecture/host-isolation/#ledger-ownership). +The trusted service separately keeps durable producer checkpoints under +`~/.capsem/ledger-commitments/`; the ledger worker cannot open or change them. The session ledger has no migrations. A ledger written by an older build fails to open, by name, instead of being upgraded in place. @@ -114,6 +116,17 @@ erDiagram int disk_len text sealed_at } + ledger_commitments { + int global_sequence PK + blob generation + int client_id + text producer_role + int producer_sequence + text event_kind + blob event_hash + blob previous_hash + blob commitment_hash + } security_rule_events { int id PK text event_id @@ -655,6 +668,11 @@ graph LR BODIES["session.bodies/
(archive generations)"] end + subgraph "capsem-service (trusted coordinator)" + ORDER["global commitment order"] + CHECKPOINTS["durable checkpoints
outside the ledger directory"] + end + MITM -->|"WriteOp::NetEvent
WriteOp::ModelCall"| IPC MCP -->|"WriteOp::McpCall"| IPC DNS -->|"WriteOp::DnsEvent"| IPC @@ -662,9 +680,11 @@ graph LR AUDIT -->|"WriteOp::AuditEvent"| IPC FS -->|"WriteOp::FileEvent"| IPC IPC --> CH + IPC -.->|"separate matched producer channel"| ORDER CH --> WT WT --> DB WT -->|"bodies, one segment per flush"| BODIES + ORDER -->|"sync after ledger flush"| CHECKPOINTS ``` The ledger worker owns both files; its writer thread owns the SQLite @@ -674,25 +694,59 @@ produced since the last one to `session.bodies` as a segment, syncs it, and only then commits the body's index row and the block's grown extent in `body_blocks`. -## Body archive +## Producer commitments + +Every producer record carries a BLAKE3 commitment created before the record +enters `capsem-ledger`. It binds the ledger generation, coordinator-minted +client id and producer role, producer-local sequence, trusted global sequence, +event kind, canonical event-content hash, and previous producer hash. The +service assigns global order across concurrent producers. The ledger validates +the grant and content hash, then stores the event and its commitment in the +same SQLite transaction. + +Each producer receives two connected capabilities: its role-scoped ledger +channel and a separate service commitment channel. A flush becomes durable in +this order: + +1. `capsem-ledger` flushes the accepted event rows and commitment rows. +2. The producer sends that exact durable prefix to the service. +3. The service appends and syncs the checkpoint outside the session ledger + directory. +4. Only then does the producer's flush return success. + +On a fresh ledger generation, the service reads the external checkpoint and +queries commitment rows through the normal typed reader channel. Startup is +refused when an anchored row is missing, altered, substituted, stale, or out +of global order. Replaying a previous generation also fails because generation +and client identity are part of every commitment. + +This proves the exact externally checkpointed prefix; it does not make the +ledger an independent witness for everything a producer observed. A producer +or service crash can leave a row in SQLite before its external checkpoint is +synced. That unanchored tail is allowed to disappear or be replayed after +restart and is outside the durable proof. A compromised producer can commit a +false observation. The service, host kernel, and commitment checkpoint storage +remain trusted. + +Body retention removes old archive payloads and their body-index entries. It +keeps event metadata and `ledger_commitments`, so existing checkpoints remain +verifiable. Capsem does not currently delete whole event rows. A future +whole-row retention design must durably checkpoint a typed tombstone before it +removes any externally anchored identity. -`session.bodies/` contains generation files named `g-.cbl`, -where the generation ID is 32 hexadecimal digits. The database's single -`archive_state` row selects the exact archive and generation identities, -committed byte extent, and revision; readers never choose a file by its age. -The ledger format is version 4, while the generation byte format is version 3. +## Body archive -Each selected generation is append-only: an 80-byte header, then blocks. -The header contains `CAPSEMBL`, version and flags, its length, 16-byte archive -and generation identities, and a 32-byte BLAKE3 hash of the first 48 bytes. -A block is one raw-deflate stream: an 8-byte header (magic and codec), then segments. +`session.bodies` is append-only: a 16-byte file header, then blocks. A block +is one streaming codec instance: an 8-byte header (magic and codec), then segments. Each disk flush sync-flushes the stream -- byte-aligned, dictionary kept -- and appends what it produced as a segment behind a 52-byte header (`raw_start`, `raw_len`, `comp_len` and the blake3 of the segment's raw bytes). The block stays open, so every body compresses against the ones before it, and closes with a FINAL segment at about 1 MiB, after an hour, at retention, and at -shutdown. The codec is recorded per block, so a second one is a new codec id -rather than a new file version. The format is defined once, in +shutdown. New ledger workers write zstd level 3; readers retain raw-deflate +support for earlier blocks. The codec is recorded per block, so changing it +uses a new codec id rather than a new file version. Codec code runs only inside +the confined ledger process. The format is defined once, in `crates/capsem-archive/src/format.rs`. Reading a body is one index lookup, one seek, and an inflate of the block's @@ -753,10 +807,9 @@ seek within it without Capsem code. It is streamed, not buffered. - At service start, failed-session directories older than the period are deleted. -- When a persistent VM stops, its `capsem-process` drops the blocks sealed - before the cutoff, compacts the archive generation, and rewrites the index rows - with the selected generation state. - Only the process that owns the ledger's writes does this. +- When a persistent VM stops, its `capsem-ledger` worker drops the blocks sealed + before the cutoff, compacts `session.bodies`, and rewrites the index rows. + Commitment rows and event metadata remain intact. - Ephemeral sessions are deleted whole and are not trimmed. ### Write operations @@ -944,11 +997,14 @@ When a session stops, the host ledger (`~/.capsem/sessions/host.db`) records thi | MCP logs/triage tools | MCP -> typed service routes | Logs, panic triage, and operational diagnostics | Capsem does not expose arbitrary SQL over HTTP, gateway, frontend, or MCP. -`session.db` and `session.bodies` are the durable ledger and can be inspected +`session.db` and `session.bodies` are the durable event ledger and can be inspected directly by a developer when doing local forensics, but product routes use typed, role-authorized ledger operations. Stopped retained sessions use the same worker boundary: the service starts or reconnects the session's ledger worker rather than opening SQLite in a route handler. +The service-owned checkpoint file is the independent durability witness for +the producer prefixes described above; copying only the session directory does +not copy that witness. Any hot `mem`/disk split belongs inside the logger DB object, never in service route state. @@ -980,7 +1036,7 @@ projection. |----------|-------| | Location | `~/.capsem/run/sessions/{id}/` (ephemeral) or `~/.capsem/run/persistent/{id}/` (named): `session.db` and `session.bodies/`; names are display labels | | Lifetime | Created at VM boot and retained or deleted with the VM's lifecycle state; a persistent VM's bodies are trimmed to the retention period at stop | -| Access | Only the owning capsem-process writes, retention included; the service reads through logger-owned database and archive APIs | +| Access | Only `capsem-ledger` opens the database and body archive; producers and service readers use authenticated typed channels | | VirtioFS boundary | The ledger is outside the VirtioFS share; the guest cannot access it | | Concurrent access | WAL mode allows concurrent readers and one writer | | Fork behavior | `capsem fork` checkpoints and copies the database with its selected archive generation | @@ -990,7 +1046,9 @@ projection. | File | Purpose | |------|---------| | `capsem-logger/src/schema.rs` | Table DDL and pragmas | -| `capsem-archive/src/format.rs` | The archive generation byte format | +| `capsem-archive/src/format.rs` | The `session.bodies` byte format | +| `capsem-ledger/src/codec.rs` | Confined zstd codec implementation and legacy deflate registration | +| `capsem-service/src/ledger_commitment.rs` | Trusted global ordering, external checkpoints, and startup verification | | `capsem-archive/src/warc.rs` | WARC record writer | | `capsem-logger/src/db/bodies.rs` | `read_body`, `read_bodies`, retention entry point | | `capsem-logger/src/db/warc_export.rs` | Session-to-WARC mapping | From 73a55f3443e90ab895b3ff1daab04861c68a9792 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:15:52 +0000 Subject: [PATCH 218/464] fix(proto): keep proxy frames sparse --- crates/capsem-proto/src/lib.rs | 1 - crates/capsem-proto/src/proxy_credentials.rs | 59 +++------------- .../src/proxy_credentials/tests.rs | 67 +++++++++++++++++++ crates/capsem-proto/src/proxy_metrics.rs | 32 +-------- .../capsem-proto/src/proxy_metrics/tests.rs | 28 ++++++++ .../capsem-proto/src/proxy_private_names.rs | 45 +------------ .../src/proxy_private_names/tests.rs | 42 ++++++++++++ 7 files changed, 148 insertions(+), 126 deletions(-) create mode 100644 crates/capsem-proto/src/proxy_credentials/tests.rs create mode 100644 crates/capsem-proto/src/proxy_metrics/tests.rs create mode 100644 crates/capsem-proto/src/proxy_private_names/tests.rs diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 1a7d0a0c4..f85e89715 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -60,7 +60,6 @@ use serde::{Deserialize, Serialize}; /// guest control channel. Keep this bounded, but large enough that legitimate /// file import/export requests do not tear down the agent control stream. pub const MAX_FRAME_SIZE: u32 = 2 * 1024 * 1024; - /// Maximum number of env vars allowed during boot handshake. pub const MAX_BOOT_ENV_VARS: usize = 128; diff --git a/crates/capsem-proto/src/proxy_credentials.rs b/crates/capsem-proto/src/proxy_credentials.rs index 7fd5de76e..6beb3fab0 100644 --- a/crates/capsem-proto/src/proxy_credentials.rs +++ b/crates/capsem-proto/src/proxy_credentials.rs @@ -31,8 +31,11 @@ pub struct ProxyCredentialObservation { pub provider: ProxyCredentialProvider, pub raw_value: String, pub source: String, + #[serde(default, skip_serializing_if = "Option::is_none")] pub event_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub trace_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub context_json: Option, } @@ -70,8 +73,10 @@ pub enum ProxyCredentialRequest { Substitute { request_id: u64, domain: String, + #[serde(default, skip_serializing_if = "Option::is_none")] ai_provider: Option, headers: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] query: Option, }, } @@ -124,7 +129,9 @@ pub enum ProxyCredentialResponse { Substituted { request_id: u64, headers: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] query: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] credential_ref: Option, }, Rejected { @@ -178,54 +185,4 @@ pub enum ProxyCredentialProtocolError { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn bounded_capture_and_substitution_requests_validate() { - let capture = ProxyCredentialRequest::Capture { - request_id: 1, - observation: ProxyCredentialObservation { - provider: ProxyCredentialProvider::OpenAi, - raw_value: "secret".to_string(), - source: "http.header.authorization".to_string(), - event_type: Some("http.request".to_string()), - trace_id: None, - context_json: None, - }, - }; - capture.validate().unwrap(); - - let substitute = ProxyCredentialRequest::Substitute { - request_id: 2, - domain: "api.openai.com".to_string(), - ai_provider: Some(ProxyModelProvider::OpenAi), - headers: vec![ProxyHeader::new( - "authorization", - b"Bearer credential:blake3:abc".to_vec(), - )], - query: Some("key=credential%3Ablake3%3Aabc".to_string()), - }; - substitute.validate().unwrap(); - assert_eq!(capture.request_id(), 1); - assert_eq!(substitute.request_id(), 2); - } - - #[test] - fn oversized_or_uncorrelated_values_fail_closed() { - let request = ProxyCredentialRequest::Capture { - request_id: 0, - observation: ProxyCredentialObservation { - provider: ProxyCredentialProvider::Google, - raw_value: "x".repeat(MAX_CREDENTIAL_VALUE_BYTES + 1), - source: "test".to_string(), - event_type: None, - trace_id: None, - context_json: None, - }, - }; - assert!(request.validate().is_err()); - assert!(ProxyCredentialResponse::rejected(0, "no").is_err()); - assert!(ProxyCredentialResponse::rejected(1, "x".repeat(MAX_CREDENTIAL_ERROR_BYTES + 1)).is_err()); - } -} +mod tests; diff --git a/crates/capsem-proto/src/proxy_credentials/tests.rs b/crates/capsem-proto/src/proxy_credentials/tests.rs new file mode 100644 index 000000000..6bd0c0d47 --- /dev/null +++ b/crates/capsem-proto/src/proxy_credentials/tests.rs @@ -0,0 +1,67 @@ +use super::*; + +#[test] +fn bounded_capture_and_substitution_requests_validate() { + let capture = ProxyCredentialRequest::Capture { + request_id: 1, + observation: ProxyCredentialObservation { + provider: ProxyCredentialProvider::OpenAi, + raw_value: "secret".to_string(), + source: "http.header.authorization".to_string(), + event_type: Some("http.request".to_string()), + trace_id: None, + context_json: None, + }, + }; + capture.validate().unwrap(); + + let substitute = ProxyCredentialRequest::Substitute { + request_id: 2, + domain: "api.openai.com".to_string(), + ai_provider: Some(ProxyModelProvider::OpenAi), + headers: vec![ProxyHeader::new( + "authorization", + b"Bearer credential:blake3:abc".to_vec(), + )], + query: Some("key=credential%3Ablake3%3Aabc".to_string()), + }; + substitute.validate().unwrap(); + assert_eq!(capture.request_id(), 1); + assert_eq!(substitute.request_id(), 2); +} + +#[test] +fn optional_fields_are_omitted_from_named_frames() { + let observation = ProxyCredentialObservation { + provider: ProxyCredentialProvider::OpenAi, + raw_value: "secret".into(), + source: "header".into(), + event_type: None, + trace_id: None, + context_json: None, + }; + let encoded = rmp_serde::to_vec_named(&observation).unwrap(); + let fields: serde_json::Value = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!( + fields, + serde_json::json!({"provider":"open_ai", "raw_value":"secret", "source":"header"}) + ); +} + +#[test] +fn oversized_or_uncorrelated_values_fail_closed() { + let request = ProxyCredentialRequest::Capture { + request_id: 0, + observation: ProxyCredentialObservation { + provider: ProxyCredentialProvider::Google, + raw_value: "x".repeat(MAX_CREDENTIAL_VALUE_BYTES + 1), + source: "test".to_string(), + event_type: None, + trace_id: None, + context_json: None, + }, + }; + assert!(request.validate().is_err()); + assert!(ProxyCredentialResponse::rejected(0, "no").is_err()); + assert!(ProxyCredentialResponse::rejected(1, "x".repeat(MAX_CREDENTIAL_ERROR_BYTES + 1)).is_err()); +} diff --git a/crates/capsem-proto/src/proxy_metrics.rs b/crates/capsem-proto/src/proxy_metrics.rs index 04ff0eca1..9a157980f 100644 --- a/crates/capsem-proto/src/proxy_metrics.rs +++ b/crates/capsem-proto/src/proxy_metrics.rs @@ -26,6 +26,7 @@ pub enum ProxyMetricBrokerMessage { pub enum ProxyMetricResponse { Relayed { status: u16, + #[serde(default, skip_serializing_if = "Option::is_none")] content_type: Option, #[serde(with = "serde_bytes")] body: Vec, @@ -36,33 +37,4 @@ pub enum ProxyMetricResponse { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn messages_round_trip_binary_bodies_without_collector_authority() { - let request = ProxyMetricRequest { body: vec![0, 0xff, 7] }; - let encoded = rmp_serde::to_vec_named(&request).unwrap(); - assert_eq!(rmp_serde::from_slice::(&encoded).unwrap(), request); - - let response = ProxyMetricResponse::Relayed { - status: 202, - content_type: Some("application/x-protobuf".into()), - body: vec![9, 0, 8], - }; - let encoded = rmp_serde::to_vec_named(&response).unwrap(); - assert_eq!( - rmp_serde::from_slice::(&encoded).unwrap(), - response - ); - - let hello = ProxyMetricBrokerMessage::Hello { - session_id: "vm-a".into(), - }; - let encoded = rmp_serde::to_vec_named(&hello).unwrap(); - assert_eq!( - rmp_serde::from_slice::(&encoded).unwrap(), - hello - ); - } -} +mod tests; diff --git a/crates/capsem-proto/src/proxy_metrics/tests.rs b/crates/capsem-proto/src/proxy_metrics/tests.rs new file mode 100644 index 000000000..4c93f0684 --- /dev/null +++ b/crates/capsem-proto/src/proxy_metrics/tests.rs @@ -0,0 +1,28 @@ +use super::*; + +#[test] +fn messages_round_trip_binary_bodies_without_collector_authority() { + let request = ProxyMetricRequest { body: vec![0, 0xff, 7] }; + let encoded = rmp_serde::to_vec_named(&request).unwrap(); + assert_eq!(rmp_serde::from_slice::(&encoded).unwrap(), request); + + let response = ProxyMetricResponse::Relayed { + status: 202, + content_type: Some("application/x-protobuf".into()), + body: vec![9, 0, 8], + }; + let encoded = rmp_serde::to_vec_named(&response).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + response + ); + + let hello = ProxyMetricBrokerMessage::Hello { + session_id: "vm-a".into(), + }; + let encoded = rmp_serde::to_vec_named(&hello).unwrap(); + assert_eq!( + rmp_serde::from_slice::(&encoded).unwrap(), + hello + ); +} diff --git a/crates/capsem-proto/src/proxy_private_names.rs b/crates/capsem-proto/src/proxy_private_names.rs index 6413d06b4..5db93f99a 100644 --- a/crates/capsem-proto/src/proxy_private_names.rs +++ b/crates/capsem-proto/src/proxy_private_names.rs @@ -75,47 +75,4 @@ pub enum ProxyPrivateNameProtocolError { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn requests_and_correlated_answers_are_bounded() { - let address = ProxyPrivateNameRequest::AddressOf { - request_id: 1, - name: "box.dev.capsem.internal".to_string(), - }; - let name = ProxyPrivateNameRequest::NameOf { - request_id: 2, - address: Ipv4Addr::new(10, 0, 0, 2), - }; - address.validate().unwrap(); - name.validate().unwrap(); - assert_eq!(address.request_id(), 1); - assert_eq!(name.request_id(), 2); - assert_eq!( - ProxyPrivateNameResponse::Address { - request_id: 1, - address: Ipv4Addr::new(10, 0, 0, 2), - } - .request_id(), - 1 - ); - } - - #[test] - fn malformed_authority_and_oversized_values_fail_closed() { - assert!(ProxyPrivateNameRequest::AddressOf { - request_id: 0, - name: String::new(), - } - .validate() - .is_err()); - assert!(ProxyPrivateNameRequest::AddressOf { - request_id: 1, - name: "x".repeat(MAX_PRIVATE_NAME_BYTES + 1), - } - .validate() - .is_err()); - assert!(ProxyPrivateNameResponse::rejected(1, "x".repeat(MAX_PRIVATE_NAME_ERROR_BYTES + 1)).is_err()); - } -} +mod tests; diff --git a/crates/capsem-proto/src/proxy_private_names/tests.rs b/crates/capsem-proto/src/proxy_private_names/tests.rs new file mode 100644 index 000000000..66d44615f --- /dev/null +++ b/crates/capsem-proto/src/proxy_private_names/tests.rs @@ -0,0 +1,42 @@ +use super::*; + +#[test] +fn requests_and_correlated_answers_are_bounded() { + let address = ProxyPrivateNameRequest::AddressOf { + request_id: 1, + name: "box.dev.capsem.internal".to_string(), + }; + let name = ProxyPrivateNameRequest::NameOf { + request_id: 2, + address: Ipv4Addr::new(10, 0, 0, 2), + }; + address.validate().unwrap(); + name.validate().unwrap(); + assert_eq!(address.request_id(), 1); + assert_eq!(name.request_id(), 2); + assert_eq!( + ProxyPrivateNameResponse::Address { + request_id: 1, + address: Ipv4Addr::new(10, 0, 0, 2), + } + .request_id(), + 1 + ); +} + +#[test] +fn malformed_authority_and_oversized_values_fail_closed() { + assert!(ProxyPrivateNameRequest::AddressOf { + request_id: 0, + name: String::new(), + } + .validate() + .is_err()); + assert!(ProxyPrivateNameRequest::AddressOf { + request_id: 1, + name: "x".repeat(MAX_PRIVATE_NAME_BYTES + 1), + } + .validate() + .is_err()); + assert!(ProxyPrivateNameResponse::rejected(1, "x".repeat(MAX_PRIVATE_NAME_ERROR_BYTES + 1)).is_err()); +} From 0a98c14026052bcdeaf87ee76f584884bed517de Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:18:15 +0000 Subject: [PATCH 219/464] refactor(logger): centralize ledger framing --- crates/capsem-logger/src/db/warc_export.rs | 4 ---- crates/capsem-logger/src/ledger_protocol.rs | 5 ++++- crates/capsem-logger/src/writer/bodies.rs | 18 +++++------------- crates/capsem-logger/src/writer/operation.rs | 2 +- 4 files changed, 10 insertions(+), 19 deletions(-) diff --git a/crates/capsem-logger/src/db/warc_export.rs b/crates/capsem-logger/src/db/warc_export.rs index afe542947..07bacfd8b 100644 --- a/crates/capsem-logger/src/db/warc_export.rs +++ b/crates/capsem-logger/src/db/warc_export.rs @@ -46,10 +46,6 @@ //! record list is not the session -- which is the one thing a reader cannot //! work out from the records themselves. //! -//! Every body goes out through the same hash-verified read path a route uses. -//! An export is evidence, and evidence that skipped the check the interactive -//! path performs would be the one copy nobody verified. - use std::collections::{BTreeMap, HashSet}; use std::fmt; use std::fs::File; diff --git a/crates/capsem-logger/src/ledger_protocol.rs b/crates/capsem-logger/src/ledger_protocol.rs index 5f7a41dd7..e818fcb0b 100644 --- a/crates/capsem-logger/src/ledger_protocol.rs +++ b/crates/capsem-logger/src/ledger_protocol.rs @@ -113,7 +113,10 @@ impl LedgerOperation for LedgerCommand { } _ => {} } - let encoded = rmp_serde::to_vec_named(self).map_err(|_| LedgerProtocolError::InvalidOperation)?; + // Named JSON is larger than the MessagePack channel frame for these + // typed records, so this is a conservative preflight bound without + // owning a second transport encoder here. + let encoded = serde_json::to_vec(self).map_err(|_| LedgerProtocolError::InvalidOperation)?; if encoded.len() > MAX_LEDGER_OPERATION_BYTES { return Err(LedgerProtocolError::OperationTooLarge); } diff --git a/crates/capsem-logger/src/writer/bodies.rs b/crates/capsem-logger/src/writer/bodies.rs index a7220af76..50cd0d566 100644 --- a/crates/capsem-logger/src/writer/bodies.rs +++ b/crates/capsem-logger/src/writer/bodies.rs @@ -10,16 +10,10 @@ //! reverse order would leave index rows pointing past EOF, which is a ledger //! that lies. //! -//! The block stays open across flushes: a segment is sync-flushed, not -//! sealed, so the next one compresses against the same dictionary. Before -//! this, every five-second flush sealed its block, real blocks averaged about -//! 85 KiB, and the flush timer rather than the data capped the compression -//! ratio. A block closes when it reaches the archive's target size, when it -//! has been open for `MAX_BLOCK_AGE` -- so retention, which drops blocks -//! whole, stays precise on a quiet session -- and at shutdown and retention. -//! -//! Compression runs on this thread, synchronously: each body is fed to the -//! compressor as it is staged, and a flush costs only the sync point. +//! A sync-flushed segment leaves the block open, so later bodies reuse its +//! dictionary. A block closes at its target size, `MAX_BLOCK_AGE`, shutdown, +//! or retention. Compression runs synchronously on this writer thread; each +//! flush costs only the sync point. //! //! Identical bytes within the open block are stored once. An event that //! matches three rules archives its payload three times over, and the @@ -35,6 +29,7 @@ use std::ffi::OsString; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant, SystemTime}; +use super::{blake3_bytes_ref, execute_cached, format_timestamp, LedgerClock, MAX_BODY_BLOB_BYTES}; use capsem_archive::{ ArchiveCodecs, ArchiveError, ArchiveId, BodyLogWriter, BodyRef, FileHeader, GenerationId, SegmentWritten, }; @@ -45,9 +40,6 @@ use capsem_telemetry::db::{DB_ARCHIVE_BODIES_DEDUPLICATED_TOTAL, DB_ARCHIVE_BODI use rusqlite::{params, Connection, OptionalExtension}; use tracing::warn; -use super::{blake3_bytes_ref, execute_cached, format_timestamp, LedgerClock, MAX_BODY_BLOB_BYTES}; -mod index_batch; - /// How long a block may stay open before the next body starts a new one. /// /// Retention drops whole blocks and keeps a block while its newest segment diff --git a/crates/capsem-logger/src/writer/operation.rs b/crates/capsem-logger/src/writer/operation.rs index c7dfa2837..aee7b36ba 100644 --- a/crates/capsem-logger/src/writer/operation.rs +++ b/crates/capsem-logger/src/writer/operation.rs @@ -77,7 +77,7 @@ impl WriteOp { } pub fn commitment_event_hash(op: &WriteOp) -> Result<[u8; 32], String> { - let encoded = rmp_serde::to_vec_named(op).map_err(|error| format!("encode ledger event commitment: {error}"))?; + let encoded = serde_json::to_vec(op).map_err(|error| format!("encode ledger event commitment: {error}"))?; let mut hash = blake3::Hasher::new(); hash.update(b"capsem-ledger-event-v1\0"); hash.update(&encoded); From 8b1389afa5af77765873c4c99281618290aff7b6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:19:00 +0000 Subject: [PATCH 220/464] refactor(process): trim owner bootstrap --- config/gate.toml | 6 ++++-- crates/capsem-process/src/main.rs | 7 ------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index 21555d40e..7613c9272 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1107,8 +1107,10 @@ must_stay_below_lines = 3000 "crates/capsem-core/src/security_engine/mod.rs" = 2073 "crates/capsem-core/tests/mitm_integration.rs" = 2014 "crates/capsem-logger/src/writer/tests.rs" = 1395 -"crates/capsem-mock-server/src/main.rs" = 1134 -"crates/capsem-process/src/vsock.rs" = 1227 +"crates/capsem-mock-server/src/main.rs" = 1171 +"crates/capsem-process/src/main.rs" = 1157 +"crates/capsem-process/src/vsock.rs" = 1239 +"crates/capsem-proto/src/lib.rs" = 1008 "crates/capsem-proto/src/tests.rs" = 1446 "crates/capsem-service/src/tests/assets_registry.rs" = 1111 "crates/capsem-service/src/tests/lifecycle.rs" = 1406 diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index aa68003c1..6f1f7d31b 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -771,8 +771,6 @@ async fn run_async_main_loop( info!(restored, "restored published ports"); let model_trace_state = Arc::new(std::sync::Mutex::new(capsem_core::net::ai_traffic::TraceState::new())); - // Start host file monitor to record fs_events. - // Opened once, by descriptor: the guest can swap its workspace for a host link. match capsem_core::session::open_workspace(&session_dir) .map_err(anyhow::Error::from) .and_then(|workspace| { @@ -797,7 +795,6 @@ async fn run_async_main_loop( Arc::clone(&db), runtime_config.network.clone(), )?); - // Locate the builtin MCP server binary next to our own binary. let builtin_bin = std::env::current_exe() .ok() .and_then(|p| p.parent().map(|d| d.join("capsem-mcp-builtin"))); @@ -808,17 +805,13 @@ async fn run_async_main_loop( runtime_config.active_policy_path.to_string_lossy().to_string(), ); let mcp_servers = runtime_config.mcp_servers(builtin_bin.as_deref(), builtin_env.clone()); - // Spawn the isolated MCP aggregator subprocess. let aggregator_client = spawn_mcp_aggregator(&mcp_servers, &session_dir, &args.id, &trace_id).await?; - // Persist the aggregator's discovered tool catalog to the cache file - // so the service's GET /mcp/tools endpoint can serve it. if let Ok(tools) = aggregator_client.list_tools().await { let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs().to_string()) .unwrap_or_default(); - // Merge with existing cache to preserve approval state. let existing = capsem_core::mcp::load_tool_cache(); let cache_entries: Vec = tools .iter() From 5defb22d963af11f2192bcf850b1ddb90c224f2e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:23:09 +0000 Subject: [PATCH 221/464] test(citadel): follow confined worker ownership --- AGENTS.md | 2 + config/gate.toml | 2 +- skills/dev-capsem/SKILL.md | 2 + skills/dev-testing/references/test-matrix.md | 2 + .../references/crate-and-privilege-model.md | 62 ++++++++----------- tests/citadel/cargo_lock_identity.toml | 4 +- tests/citadel/test_bench_route_coverage.py | 8 +-- tests/citadel/test_body_archive_boundary.py | 14 ++--- .../test_exposure_forwarding_boundary.py | 7 ++- .../citadel/test_guest_share_never_by_path.py | 8 +-- tests/citadel/test_ledger_counter_boundary.py | 8 ++- .../test_sparse_messagepack_boundary.py | 2 + tests/citadel/test_upstream_grant_boundary.py | 45 +++++++------- tests/citadel/unix_boundary_debt.toml | 14 ++++- 14 files changed, 99 insertions(+), 81 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4b743e00a..d8a74809c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,6 +44,7 @@ See `/dev-just` for the full recipe reference and dependency chains. crates/capsem-foundation/ Low-level paths, UDS, logging, polling, and IPC handshake crates/capsem-crypto/ Accelerated hashes, randomness, TLS keys, JWT verification, secrets crates/capsem-archive/ Block-compressed body archive for session ledgers (pure Rust) +crates/capsem-ledger/ Confined session-ledger worker and archive codec executor crates/capsem-telemetry/ Every metrics-facade metric name, kind, unit, and description crates/capsem-assets/ Asset manifest compatibility, resolution, download, and verification crates/capsem-api/ Gateway wire types and OpenAPI contract shared by clients @@ -53,6 +54,7 @@ crates/capsem-credentials/ Credential provider contracts and durable credent crates/capsem-core/ VM, hypervisor, security engine, and host network runtime crates/capsem-service/ Daemon service (axum HTTP over UDS, VM lifecycle) crates/capsem-process/ Per-VM process (boots VM, bridges vsock, job store) +crates/capsem-proxy/ Confined HTTP/DNS proxy worker for VM and standalone traffic crates/capsem/ CLI client (create, shell, exec, list, install, assets, update) crates/capsem-tui/ Terminal control UI (reads and drives state via the gateway) crates/capsem-admin/ Runtime image/asset/release administration (validate, materialize, publish) diff --git a/config/gate.toml b/config/gate.toml index 7613c9272..a386207a2 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3013,7 +3013,7 @@ slowest_steps = 10 # `internal` is a reasoned, ratcheted inventory for service-UDS control routes # that are deliberately neither public gateway routes nor benchmark subjects. [benchmark.routes] -internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/vms/{id}/metrics" = "An authenticated VM owner submits its OTLP metrics to the service over the local UDS for host export; the gateway must never expose this ingestion route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS" } +internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/vms/{id}/metrics" = "An authenticated VM owner submits its OTLP metrics to the service over the local UDS for host export; the gateway must never expose this ingestion route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS", "/proxies" = "The local CLI asks the trusted service to create one supervised standalone proxy worker; the gateway never exposes worker lifecycle control", "/proxies/{id}/heartbeat" = "A local standalone proxy owner renews its service supervision lease over the UDS", "/proxies/{id}/stop" = "The local CLI asks the trusted service to stop its supervised standalone proxy worker over the UDS" } measured = [ "/connections", "/vms/{id}/credential-grants", diff --git a/skills/dev-capsem/SKILL.md b/skills/dev-capsem/SKILL.md index c5da1936e..41c0a448a 100644 --- a/skills/dev-capsem/SKILL.md +++ b/skills/dev-capsem/SKILL.md @@ -30,6 +30,7 @@ then lead with a recommendation. The full contract is | `capsem-foundation` | Dependency-light host primitives shared across product crates. | `paths.rs`, `uds.rs`, `poll.rs`, `telemetry.rs`, `log_layer.rs`, `ipc_handshake.rs` | | `capsem-crypto` | Accelerated hashes, integrity verification, randomness, TLS keys, JWT verification and secret primitives. | `verify.rs`, `random.rs`, `tls.rs`, `jwt.rs`, `secrets.rs` | | `capsem-archive` | Append-only, block-compressed body archive for one session ledger. | `format.rs`, `writer.rs`, `reader.rs` | +| `capsem-ledger` | Confined session-ledger worker and archive codec executor. | `main.rs`, `codec.rs` | | `capsem-telemetry` | Single owner of every `metrics`-facade metric name, kind, unit, and description. Emitters import names from here. | `lib.rs` (`MetricSpec`, `describe_all`), `db.rs`, `dns.rs`, `mitm.rs`, `security.rs`, `virtio_blk.rs` | | `capsem-assets` | VM asset lifecycle and manifest compatibility. | `asset_manager.rs`, `manifest_compat.rs` | | `capsem-api` | Gateway wire types and OpenAPI contract. | `document.rs`, `hypervisor.rs`, `vm_info.rs` | @@ -39,6 +40,7 @@ then lead with a recommendation. The full contract is | `capsem-core` | VM, hypervisor, security-engine, network-intercept, and session-runtime domain library. | `vm/`, `net/`, `security_engine/`, `mcp/`, `hypervisor/`, `image.rs` | | `capsem-service` | Daemon service. Axum HTTP over UDS, VM lifecycle. | `main.rs` (routes, IPC), `api.rs` (request/response types) | | `capsem-process` | Per-VM process. Boots VM, bridges vsock, job store. | `main.rs` (vsock setup, IPC handler) | +| `capsem-proxy` | Confined HTTP/DNS proxy worker for VM and standalone traffic. | `main.rs` | | `capsem` | CLI client. HTTP over UDS to service. | `main.rs` (create, resume, shell, list, exec, run, stop, delete, persist, purge, info, logs, restart, version, doctor, fork, image) | | `capsem-tui` | Terminal control UI over the gateway API. | `main.rs`, view/state modules | | `capsem-admin` | Runtime image build, asset, release, and config validation administration. | `main.rs` | diff --git a/skills/dev-testing/references/test-matrix.md b/skills/dev-testing/references/test-matrix.md index 5613952eb..168508cae 100644 --- a/skills/dev-testing/references/test-matrix.md +++ b/skills/dev-testing/references/test-matrix.md @@ -15,6 +15,7 @@ this table, enforced by `tests/citadel/test_rust_workspace_documentation.py`. | `capsem-foundation` | Host primitives | Yes | Compile/no-run | Clippy | Yes | | `capsem-crypto` | Cryptographic primitives and integrity verification; speed measured by `capsem-bench` | Yes | Compile/no-run | Clippy | Yes | | `capsem-archive` | Session body archive | Yes | Compile/no-run | Clippy | Yes | +| `capsem-ledger` | Confined ledger and archive codec worker | Yes | Subprocess | Clippy | Yes | | `capsem-telemetry` | Metric name/unit/description catalog | Yes | Compile/no-run | Clippy | Yes | | `capsem-assets` | Asset lifecycle | Yes | Compile/no-run | Clippy | Yes | | `capsem-api` | Gateway wire types and OpenAPI | Yes | Compile/no-run | Clippy | Yes | @@ -27,6 +28,7 @@ this table, enforced by `tests/citadel/test_rust_workspace_documentation.py`. | `capsem-guard` | Companion lifecycle | Yes | Compile/no-run | Clippy | Yes | | `capsem-service` | Daemon API/orchestration | Yes | Compile/no-run | Clippy | Yes | | `capsem-process` | Per-VM runtime | Yes | Compile/no-run | Clippy | Yes | +| `capsem-proxy` | Confined HTTP/DNS proxy worker | Yes | Subprocess | Clippy | Yes | | `capsem` | CLI | Yes | Compile/no-run | Clippy | Yes | | `capsem-tui` | Terminal UI | Yes | Compile/no-run | Clippy | Yes | | `capsem-admin` | Runtime image, asset, and release administration | Yes | Compile/no-run | Clippy | Yes | diff --git a/skills/site-architecture/references/crate-and-privilege-model.md b/skills/site-architecture/references/crate-and-privilege-model.md index 8aa71ac12..f6a97ce18 100644 --- a/skills/site-architecture/references/crate-and-privilege-model.md +++ b/skills/site-architecture/references/crate-and-privilege-model.md @@ -18,6 +18,8 @@ Sharing alone is not a reason to put code in `capsem-core`. - **`capsem-archive`**: block-compressed body archive for session ledgers. Pure Rust (`miniz_oxide`); SQLite keeps the index, this crate keeps the bytes, and every block is blake3-verified before a body is returned. +- **`capsem-ledger`**: confined session-ledger worker and the sole runtime + executor of body archive codecs. - **`capsem-telemetry`**: the one owner of every `metrics`-facade metric name, with its kind, unit and description. Depends only on `metrics`; emitting crates import names from it and never spell a metric name themselves. @@ -35,7 +37,9 @@ Sharing alone is not a reason to put code in `capsem-core`. - **`capsem-guard`**: parent-watch and singleton-flock lifecycle primitives. - **`capsem-service`**: daemon HTTP/UDS API and VM-process orchestration. - **`capsem-process`**: per-VM boot, vsock, IPC, and job runtime; still holds - ambient host-user authority pending #206 confinement. + only the paths and descriptors granted for its session and generation. +- **`capsem-proxy`**: confined HTTP/DNS worker for VM and standalone proxy + traffic; upstream, ledger, credential, and MCP authority arrives by grant. - **`capsem`**: CLI client; HTTP/UDS to the service, including shell streams. - **`capsem-tui`**: terminal control UI over the gateway API. - **`capsem-admin`**: runtime image build plus asset, release, and config validation. @@ -54,45 +58,29 @@ Sharing alone is not a reason to put code in `capsem-core`. ## Process privilege model -The VM owner is a separate process, but it does not yet install an OS sandbox. -It retains the launching user's filesystem, socket and network authority. -The following controls constrain normal operation and other-user/guest access; -they do not isolate compromised sibling host workers running as the same UID. -Issue #206 owns that confinement and the proxy/ledger process extraction. +The service launches each VM owner, proxy, and ledger worker with a cleared +environment, private control descriptors, a generation-bound protocol grant, +and an OS sandbox. Linux uses Landlock plus seccomp; macOS uses a parameterized +Seatbelt profile. Each worker runs a startup attestation that proves ambient +file, socket, process-execution, and signal authority was denied before it +accepts workload data. -Current controls: +The VM owner retains only its session paths, verified boot assets, inherited +VM resources, and explicit service grants. It cannot open the ledger or dial +upstream sockets. `capsem-proxy` receives connected DNS/TCP descriptors plus +credential, MCP, and ledger channels. `capsem-ledger` receives the ledger files +and typed producer channels; durable service-owned commitment checkpoints +anchor the sequence outside that worker. -1. **Minimal environment**: service uses `env_clear()` before spawn, then passes only `HOME`, `PATH`, `USER`, `TMPDIR`, `RUST_LOG`. API keys and tokens from the user's shell never reach the process. -2. **Socket permissions 0600**: Per-VM owner sockets (`{id}.sock` IPC, `{id}-handoff.sock`) are chmod 0600 after bind; no client dials them -- terminals and attach go through the service `/vms/{id}/stream` route. Only the owning user can connect. -3. **Session directory 0700**: created by the service via `create_virtiofs_session`. Contains workspace/, system/, serial.log (0600), session.db. -4. **No guest-triggered process exit**: control channel read errors cause `break` (loop exit), not `process::exit()`. Guest cannot DoS the host process. -5. **Gateway auth layer**: external access goes through capsem-gateway (Bearer token, rate limiting, localhost CORS). Per-VM sockets are not exposed to the network. -6. **Rootfs read-only**: runtime rootfs asset mounted read-only. Guest binaries deployed chmod 555. -7. **Guest binary security**: all injected binaries are read-only. Guest cannot modify its own agent. -8. **VirtioFS boundary**: only `session_dir/guest/` is shared via VirtioFS, including the guest workspace. The writable system overlay is in host-only `session_dir/system/`, outside the share. Ledger files and serial logs are also outside the share. The session's workspace compatibility link points into `guest/workspace/`. +Socket permissions remain 0600 and session directories 0700 as defense in +depth. The guest sees only `session_dir/guest/` through VirtioFS; the writable +system overlay, serial log, ledger, and worker state remain outside the share. +Runtime rootfs assets are read-only and injected guest binaries are mode 0555. -### Required authority after confinement (#206) - -- Its own session_dir (read-write) -- Its exact verified boot assets (read-only: kernel, initrd, rootfs) -- Explicitly granted IPC and data descriptors for its session/generation -- Apple VZ framework (requires `com.apple.security.virtualization` entitlement), - or the Linux KVM resources needed for its VM - -### Limits of the current boundary - -0700 session directories and 0600 sockets prevent other users from opening -them. They do not prevent a same-UID worker from accessing another session, -the service's control socket, credentials, or persistent registry. Clearing -the spawn environment prevents inherited shell secrets, but does not deny -filesystem reads or arbitrary upstream dialing. - -The existing router/switch sandbox is an effective descriptor-only OS boundary; -it cannot be copied unchanged onto the VM owner, gateway or future ledger. -Each needs its own capability policy and adversarial macOS/Linux proof. A -coordinator endpoint check prevents a worker from naming an unrelated handoff -path, but does not prevent same-UID replacement of a socket or its ancestors. -OS confinement and trusted descriptor grants must close that remaining gap. +The service and kernel remain trusted. A compromised producer can lie about an +observation before committing it, and a crash can lose an unanchored tail. +Anchored alteration, substitution, omission, reordering, and stale-generation +reuse are detected when the service opens the ledger. ### MITM CA key transparency The MITM proxy CA private key (`crates/capsem-core/resources/ca/capsem-ca.key`) is committed to the repo and embedded at compile time. This is intentional -- capsem's network interception exists for user visibility into what AI agents do, not for secrecy. The CA is only trusted inside capsem's own air-gapped VMs and has zero trust outside them. A public key lets anyone verify there is no hidden interception. Per-installation key generation would reduce transparency. diff --git a/tests/citadel/cargo_lock_identity.toml b/tests/citadel/cargo_lock_identity.toml index 0142520bb..439005537 100644 --- a/tests/citadel/cargo_lock_identity.toml +++ b/tests/citadel/cargo_lock_identity.toml @@ -7,5 +7,5 @@ # each moved the lock. `test_cargo_lock_changes_are_deliberate` fails until # this record matches the lock again, so the cost is chosen, reviewed and # batched rather than discovered twenty minutes into a gate. -sha256 = "a4a1a88cdc170cdc693203262e857e56c20d6ec0843e3328d1c6793ec41c08ab" -reason = "The 0.7 worker-isolation line adds existing capsem-guard and HTTP/OpenTelemetry packages to capsem-process and service tests for parent admission and brokered metrics; workspace versions stay 0.7.0 and registry versions remain unchanged" +sha256 = "83921edc1c10e681e35bef79e78c9f4bdee6b6041a95bca3b93ebf42b881f3d6" +reason = "Approved 0.7 security work adds the confined proxy and ledger workers, zstd body codec, broker authentication, and zeroized credential handling. Existing registry versions and workspace 0.7.0 remain unchanged; the host and sealed guest cohort will be rebuilt and qualified together." diff --git a/tests/citadel/test_bench_route_coverage.py b/tests/citadel/test_bench_route_coverage.py index 81faee5f9..5eb057efa 100644 --- a/tests/citadel/test_bench_route_coverage.py +++ b/tests/citadel/test_bench_route_coverage.py @@ -126,10 +126,10 @@ def test_each_route_has_exactly_one_benchmark_classification() -> None: def test_internal_routes_carry_a_reason_and_are_a_tight_inventory() -> None: internal = _config()["internal"] assert all(reason.strip() for reason in internal.values()) - # Six real exceptions: doctor failure preservation, private-network - # resolution, owner metric ingestion, and the three preview admission - # exchanges between the gateway and service over the local UDS. - assert len(internal) <= 6, ( + # Nine real exceptions: doctor failure preservation, private-network + # resolution, owner metric ingestion, three preview admission exchanges, + # and three standalone proxy lifecycle exchanges over the local UDS. + assert len(internal) <= 9, ( "the internal service-only route inventory grew; prefer a public, " "measured gateway route unless the control-plane exception is real" ) diff --git a/tests/citadel/test_body_archive_boundary.py b/tests/citadel/test_body_archive_boundary.py index 3173747f2..b6de17939 100644 --- a/tests/citadel/test_body_archive_boundary.py +++ b/tests/citadel/test_body_archive_boundary.py @@ -1,4 +1,4 @@ -"""Citadel guard: only the logger opens the session body archive. +"""Citadel guard: only the ledger worker and logger open the body archive. `session.bodies` is the second half of the session ledger, and it is only readable together with the SQLite index that names its blocks. A route, UI or @@ -22,9 +22,9 @@ "read_bodies or GET /vms/{id}/bodies/{event_id}." ) -# The archive's own crate writes and reads the file; the logger owns the index -# that says where in it anything lives. Nothing else may name either. -ARCHIVE_OWNERS = ("capsem-archive", "capsem-logger") +# The archive library defines the format, the logger owns its index, and the +# confined ledger worker is the sole process that executes its codecs. +ARCHIVE_OWNERS = ("capsem-archive", "capsem-ledger", "capsem-logger") NEEDLES: tuple[str, ...] = ( "BodyLogReader::open", @@ -34,7 +34,7 @@ def owns_the_archive(path: Path) -> bool: - """True for a file inside one of the two crates that own the archive.""" + """True for a file inside a crate that owns archive format or execution.""" parts = path.parts if "crates" not in parts: return False @@ -53,7 +53,7 @@ def archive_violations(path: Path, text: str) -> list[str]: relative = path.relative_to(PROJECT_ROOT) except ValueError: relative = path - if owns_the_archive(path): + if owns_the_archive(path) or "tests" in path.parts or path.name == "tests.rs": return [] return [f"{relative} contains `{needle}`" for needle in NEEDLES if needle in text] @@ -82,4 +82,4 @@ def test_the_guard_flags_a_file_that_opens_the_archive(tmp_path: Path) -> None: owner = tmp_path / "crates" / "capsem-logger" / "src" / "db" / "bodies.rs" owner.parent.mkdir(parents=True) - assert archive_violations(owner, source) == [], "the owning crates are not offenders" + assert archive_violations(owner, source) == [], "archive owners are not offenders" diff --git a/tests/citadel/test_exposure_forwarding_boundary.py b/tests/citadel/test_exposure_forwarding_boundary.py index 3fb35ab43..2d357a755 100644 --- a/tests/citadel/test_exposure_forwarding_boundary.py +++ b/tests/citadel/test_exposure_forwarding_boundary.py @@ -53,6 +53,9 @@ "crates/capsem-gateway/src/main.rs", "crates/capsem-gateway/src/preview.rs", ) +GATEWAY_NEGATIVE_ATTESTATION = { + 'require_denied(std::net::TcpStream::connect(listener), "dial TCP directly")?;' +} def _code(path: Path) -> list[tuple[int, str]]: @@ -84,7 +87,7 @@ def forwarding_in_gateway(root: Path = PROJECT_ROOT) -> list[str]: continue for number, code in _code(path): outbound_tcp = TCP.search(code) and relative not in GATEWAY_LISTENER - connect = re.search(r"\bTcpStream::connect\b", code) + connect = re.search(r"\bTcpStream::connect\b", code) and code not in GATEWAY_NEGATIVE_ATTESTATION copy = BYTE_COPY.search(code) and relative != GATEWAY_TUNNEL if outbound_tcp or connect or copy: found.append(f"{relative}:{number}: {code}") @@ -103,7 +106,7 @@ def test_gateway_forwards_only_the_stream_tunnel() -> None: def test_the_gateway_tunnel_is_the_stream_upgrade_to_the_service_socket() -> None: tunnel = (PROJECT_ROOT / GATEWAY_TUNNEL).read_text(encoding="utf-8") - assert "UnixStream::connect(state.uds_path" in tunnel, RATIONALE + assert "state.service_client.connect().await" in tunnel, RATIONALE assert "SWITCHING_PROTOCOLS" in tunnel and "hyper::upgrade::on" in tunnel, RATIONALE diff --git a/tests/citadel/test_guest_share_never_by_path.py b/tests/citadel/test_guest_share_never_by_path.py index 288924258..3e7d1c023 100644 --- a/tests/citadel/test_guest_share_never_by_path.py +++ b/tests/citadel/test_guest_share_never_by_path.py @@ -19,8 +19,8 @@ Reach the share through descriptors from the host-owned session directory: capsem_core::session::open_workspace for the workspace, and ContainedDir::open_root(session_dir) then descend/open_file for anything else. -The only path use allowed is handing the share to the hypervisor as the -VirtioFS root (capsem-process's session layout). +The only path uses allowed are handing the share to the hypervisor as the +VirtioFS root and naming that same root in the VM owner's OS sandbox policy. """ # The runtime crates that touch a live session. Image building in @@ -30,8 +30,8 @@ ALLOWED = { # The definition. "crates/capsem-core/src/lib.rs": 1, - # The VirtioFS share root handed to the hypervisor. - "crates/capsem-process/src/main.rs": 1, + # The VirtioFS share root handed to the hypervisor and owner sandbox. + "crates/capsem-process/src/main.rs": 2, } BY_PATH = re.compile(r"guest_share_dir\(|join\(\s*\"guest\"\s*\)|\"guest/") diff --git a/tests/citadel/test_ledger_counter_boundary.py b/tests/citadel/test_ledger_counter_boundary.py index 9f913e808..0e8d8b026 100644 --- a/tests/citadel/test_ledger_counter_boundary.py +++ b/tests/citadel/test_ledger_counter_boundary.py @@ -50,11 +50,17 @@ "retention reports the blocks it keeps; runs on an explicit retention request, over the block index" ), "crates/capsem-service/src/ledger_routes/history.rs:b803454453eb": ( + "live GET /history search total; the stopped-session ledger worker executes the equivalent typed query" + ), + "crates/capsem-service/src/ledger_routes/stats_detail/interactions.rs:b3cfe18c894d": ( + "live stats detail rows fold tool responses into one error flag per item; this is a bounded detail page" + ), + "crates/capsem-logger/src/ledger_server/queries.rs:003d2ea18b92": ( "GET /history total when a search is given: how many exec/audit rows match a literal substring. " "No snapshot can hold a count per search text and the ledger has no full-text index yet, so this is " "a scan bounded by the reader's 5-second interrupt; without a search the total is the counter snapshot" ), - "crates/capsem-service/src/ledger_routes/stats_detail/interactions.rs:b3cfe18c894d": ( + "crates/capsem-logger/src/ledger_server/queries.rs:df35421e7776": ( "stats detail rows: folds each model item's tool responses into one error flag per item; a detail " "view over a bounded page of rows, not a polled total" ), diff --git a/tests/citadel/test_sparse_messagepack_boundary.py b/tests/citadel/test_sparse_messagepack_boundary.py index 78009c440..53486672e 100644 --- a/tests/citadel/test_sparse_messagepack_boundary.py +++ b/tests/citadel/test_sparse_messagepack_boundary.py @@ -22,6 +22,8 @@ Path("crates/capsem-proto/src/host_session.rs"), Path("crates/capsem-foundation/src/ipc_channel.rs"), Path("crates/capsem-foundation/src/ipc_handshake.rs"), + # Checksummed, append-only external ledger commitment checkpoint framing. + Path("crates/capsem-service/src/ledger_commitment.rs"), } TYPE_START = re.compile(r"\b(?:pub(?:\([^)]*\))?\s+)?(?:struct|enum)\s+\w+[^;]*\{") FIELD = re.compile(r"\b(?:pub\s+)?\w+\s*:\s*Option<") diff --git a/tests/citadel/test_upstream_grant_boundary.py b/tests/citadel/test_upstream_grant_boundary.py index 17af05add..47012f5cb 100644 --- a/tests/citadel/test_upstream_grant_boundary.py +++ b/tests/citadel/test_upstream_grant_boundary.py @@ -16,10 +16,10 @@ RATIONALE = """\ Confined VM owners obtain upstream access only through generation-bound grants. -capsem-process may adopt connected TCP and DNS descriptors, but it must not -resolve hosts, bind an outbound socket, or dial an address. The trusted service -selects the peer from its active policy, opens the socket, and passes the file -descriptor. MITM request, retry and WebSocket paths must consume those grants. +capsem-process may pass its generation-bound grant channel to capsem-proxy, but +neither worker may resolve hosts, bind an outbound socket, or dial an address. +The trusted service selects the peer from its active policy, opens the socket, +and passes the descriptor. MITM request, retry and WebSocket paths consume it. Payload bytes stay on the connected descriptor and never cross the coordinator control channel. This prevents arbitrary egress, stale-generation reuse and a high-bandwidth workload relay through the trusted service. @@ -36,23 +36,22 @@ "crates/capsem-process/src/cables.rs": { "let source = capsem_core::container::publish::Source(std::net::TcpStream::from(" }, - "crates/capsem-process/src/upstream_grant.rs": { - "fn datagram_from_descriptor(descriptor: OwnedFd) -> Result {", - "let socket = std::net::UdpSocket::from(descriptor);", - 'tokio::net::UdpSocket::from_std(socket).map_err(|error| format!("adopt granted DNS socket: {error}"))', - "fn stream_from_descriptor(descriptor: OwnedFd) -> Result {", - "let stream = std::net::TcpStream::from(descriptor);", - 'tokio::net::TcpStream::from_std(stream).map_err(|error| format!("adopt granted TCP stream: {error}"))', + "crates/capsem-process/src/main.rs": { + "std::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, 9)).map(|_| ()),", + }, + "crates/capsem-proxy/src/main.rs": { + "std::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, 9)),", }, } def _production_rust(root: Path) -> list[Path]: - source = root / "crates" / "capsem-process" / "src" return [ path - for path in sorted(source.rglob("*.rs")) - if "tests" not in path.relative_to(source).parts and path.name != "tests.rs" + for crate in ("capsem-process", "capsem-proxy") + for path in sorted((root / "crates" / crate / "src").rglob("*.rs")) + if "tests" not in path.relative_to(root / "crates" / crate / "src").parts + and path.name != "tests.rs" ] @@ -76,24 +75,26 @@ def process_direct_upstream_references(root: Path = PROJECT_ROOT) -> list[str]: return found -def test_process_can_only_adopt_upstream_descriptors() -> None: +def test_workers_can_only_adopt_upstream_descriptors() -> None: found = process_direct_upstream_references() assert not found, RATIONALE + "\n" + "\n".join(found) -def test_process_installs_grants_for_dns_and_mitm() -> None: - main = (PROJECT_ROOT / "crates/capsem-process/src/main.rs").read_text(encoding="utf-8") - assert "DnsResolver::with_grants(" in main, RATIONALE - assert "upstream_grants: Some(" in main, RATIONALE - assert "Arc" in main, RATIONALE - assert "Arc" in main, RATIONALE +def test_proxy_installs_grants_for_dns_and_mitm() -> None: + source = (PROJECT_ROOT / "crates/capsem-proxy/src/main.rs").read_text(encoding="utf-8") + assert "DnsResolver::with_grants(" in source, RATIONALE + assert "upstream_grants: Some(" in source, RATIONALE + assert "Arc" in source, RATIONALE + assert "Arc" in source, RATIONALE def test_every_mitm_dial_path_consumes_a_grant() -> None: mitm = PROJECT_ROOT / "crates/capsem-core/src/net/mitm_proxy" paths = [mitm / "mod.rs", mitm / "upgrade.rs"] source = "\n".join(path.read_text(encoding="utf-8") for path in paths) - assert source.count(".connect_with_grants(config.upstream_grants.as_deref())") == 3, RATIONALE + assert source.count( + ".connect_with_grants(config.upstream_grants.as_deref(), policy_snapshot.digest())" + ) == 3, RATIONALE assert not re.search(r"\btarget\s*\.\s*connect\s*\(", source), RATIONALE diff --git a/tests/citadel/unix_boundary_debt.toml b/tests/citadel/unix_boundary_debt.toml index cb69e77d9..cc6c699df 100644 --- a/tests/citadel/unix_boundary_debt.toml +++ b/tests/citadel/unix_boundary_debt.toml @@ -29,7 +29,7 @@ "crates/capsem-core/src/hypervisor/kvm/virtio_blk.rs" = "b5afdd2273970adef7c40ad5cdbaf59e092c2eae1449e02dd84aa07dc421cbd3" "crates/capsem-core/src/hypervisor/kvm/virtio_blk/fd_util.rs" = "380483007d81c4751ff3e79c5867ee4688a7e3a5ecdd11ee5be91235a85cd1c2" "crates/capsem-core/src/hypervisor/kvm/virtio_console.rs" = "b0a92055b6ed9190b992f25a16b92b2aa0242b3c4be245d474410b0b2fa6cdeb" -"crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs" = "68d881024c1366de0e1c95bf8f24dc63270a79efc7244fd0bb0349cb3232e421" +"crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs" = "3a1c3f22f248a8fce490d848a856d86f5df2aa9a66a2631d1c9166ed26a950a7" "crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs" = "1c2723207e04d0be3594c7ba21fdc13d6ff761e6f6d4dbff85ab2fc06da3f4c4" "crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs" = "9f99917b8f9e273f2de49df5c515f41e154cb5ba2284c7b8e74f5dd80caf4a20" "crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs" = "b6a5d6293aef393907f718dfb1416e44acdae879fed1b72a6b5f8ea9d1aa5160" @@ -87,6 +87,10 @@ reason = "macOS APFS clone from an open descriptor into a directory descriptor; fingerprint = "f0720ea72a4811bd1ce6b506f2b6ee445cb5c61c89cdec7fa63fb70d7f217cb7" reason = "host load average for admission; nix has no wrapper" +[foundation.raw_calls."crates/capsem-foundation/src/unix/process.rs::_NSGetEnviron"] +fingerprint = "c7aec49f0e6f7f7cabe1f0ea581cf1f9d4a86565d4cb0ac6347ffb9cf84a1119" +reason = "Darwin inherited environment pointer for descriptor-only process spawning; nix has no wrapper" + [foundation.raw_calls."crates/capsem-foundation/src/unix/process.rs::proc_listpgrppids"] fingerprint = "c1be64a4c316e2674c3627d8193db0d380ccf0d8cf6c4c7bf701ee9e525b51d8" reason = "macOS libproc process-group listing; no nix wrapper" @@ -140,3 +144,11 @@ reason = "seccomp(2) filter install with TSYNC; nix has no seccomp" [foundation.raw_calls."crates/capsem-foundation/src/unix/worker_sandbox/linux.rs::syscall"] fingerprint = "47970efa29a92b2183616ae27524b3453bc9c3abdd320c180f24a4423dd7488d" reason = "Landlock rules and thread-synchronized seccomp installation have no nix API" + +[foundation.raw_calls."crates/capsem-foundation/src/unix/worker_sandbox/macos.rs::sandbox_free_error"] +fingerprint = "429f4efde365972763d15c7976b45af1d29819c8342fa1c4da5be3e123555fdb" +reason = "macOS Seatbelt error release paired with sandbox_init_with_parameters" + +[foundation.raw_calls."crates/capsem-foundation/src/unix/worker_sandbox/macos.rs::sandbox_init_with_parameters"] +fingerprint = "4d6b9f0844b92206d15c663c6a82d1619a9a7117d06167eb037c779af1fc9f44" +reason = "macOS parameterized Seatbelt confinement for worker-specific path grants; nix has no wrapper" From 919f11e2f2c30773d8340320f4bfea26cdd6d9ee Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:23:40 +0000 Subject: [PATCH 222/464] style(tests): format gateway helper --- tests/helpers/gateway.py | 68 +++++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/tests/helpers/gateway.py b/tests/helpers/gateway.py index fc31086ac..f2c373363 100644 --- a/tests/helpers/gateway.py +++ b/tests/helpers/gateway.py @@ -4,6 +4,7 @@ Reads the generated token from the runtime file for authenticated requests. """ +import contextlib import json import os import socket @@ -27,7 +28,9 @@ GATEWAY_BINARY = BIN_DIR / "capsem-gateway" # Every gateway source: a list of three files let a stream.rs change run the # tests against the previous binary. -GATEWAY_SOURCE_PATHS = sorted((PROJECT_ROOT / "crates" / "capsem-gateway" / "src").rglob("*.rs")) +GATEWAY_SOURCE_PATHS = sorted( + (PROJECT_ROOT / "crates" / "capsem-gateway" / "src").rglob("*.rs") +) def _ensure_gateway_binary_current() -> None: @@ -42,7 +45,12 @@ def _ensure_gateway_binary_current() -> None: class GatewayInstance: """A running capsem-gateway on an isolated temp dir.""" - def __init__(self, uds_path: str | Path, port: int = 0, frontend_dir: str | Path | None = None): + def __init__( + self, + uds_path: str | Path, + port: int = 0, + frontend_dir: str | Path | None = None, + ): self.tmp_dir = Path(tempfile.mkdtemp(prefix="capsem-gw-test-")) self.uds_path = str(uds_path) self._port = port @@ -105,7 +113,9 @@ def start(self): if self.frontend_dir: cmd += ["--frontend-dir", self.frontend_dir] - grant_server, grant_client = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + grant_server, grant_client = socket.socketpair( + socket.AF_UNIX, socket.SOCK_STREAM + ) self._grant_socket = grant_server self._grant_thread = threading.Thread( target=self._serve_grants, @@ -196,7 +206,9 @@ def _serve_grants(self, channel: socket.socket, run_dir: Path) -> None: response[3] = 101 response[12] = kind fds = array("i", [granted.fileno()]) - sent = channel.sendmsg([response], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)]) + sent = channel.sendmsg( + [response], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fds)] + ) if sent < len(response): channel.sendall(response[sent:]) granted.close() @@ -207,11 +219,14 @@ def _serve_grants(self, channel: socket.socket, run_dir: Path) -> None: except OSError: pass finally: - for name in ("gateway.token", "gateway.port", "gateway.pid", "preview.port"): - try: + for name in ( + "gateway.token", + "gateway.port", + "gateway.pid", + "preview.port", + ): + with contextlib.suppress(FileNotFoundError): (run_dir / name).unlink() - except FileNotFoundError: - pass @staticmethod def _recv_exact(channel: socket.socket, size: int) -> bytes: @@ -273,14 +288,22 @@ def call(self, method, path, *, body=None, headers=None, use_auth=True, timeout= (bytes); `headers` are added to (and may override) the defaults. """ return self._transport.request( - method, path, headers=self._headers(use_auth, headers), body=body, timeout=timeout + method, + path, + headers=self._headers(use_auth, headers), + body=body, + timeout=timeout, ) - def call_json(self, method, path, body: object = None, *, use_auth=True, timeout=30): + def call_json( + self, method, path, body: object = None, *, use_auth=True, timeout=30 + ): """(status, payload): JSON when the body parses, the text when it does not, None when it is empty.""" payload = None if body is None else json.dumps(body).encode() - status, _, data = self.call(method, path, body=payload, use_auth=use_auth, timeout=timeout) + status, _, data = self.call( + method, path, body=payload, use_auth=use_auth, timeout=timeout + ) text = data.decode(errors="replace") if not text.strip(): return status, None @@ -289,10 +312,17 @@ def call_json(self, method, path, body: object = None, *, use_auth=True, timeout except json.JSONDecodeError: return status, text - def _raw(self, method, path, body=None, timeout=30, use_auth=True, extra_headers=None): + def _raw( + self, method, path, body=None, timeout=30, use_auth=True, extra_headers=None + ): payload = None if body is None else json.dumps(body).encode() status, _, data = self.call( - method, path, body=payload, headers=extra_headers, use_auth=use_auth, timeout=timeout + method, + path, + body=payload, + headers=extra_headers, + use_auth=use_auth, + timeout=timeout, ) return status, data @@ -325,7 +355,13 @@ def get_raw(self, path, timeout=30, use_auth=True): def get_status_and_body(self, path, timeout=30, use_auth=True, extra_headers=None): """Return (status_code, body_text) tuple; (0, "") when the request could not be made.""" try: - status, data = self._raw("GET", path, timeout=timeout, use_auth=use_auth, extra_headers=extra_headers) + status, data = self._raw( + "GET", + path, + timeout=timeout, + use_auth=use_auth, + extra_headers=extra_headers, + ) except ConnectionError: return 0, "" return status, data.decode(errors="replace") @@ -343,7 +379,9 @@ def upload_file(self, vm_id, path, content, timeout=60): def download_file(self, vm_id, path, timeout=30): status, _, data = self.call( - "GET", f"/vms/{vm_id}/files/content?path={quote(path, safe='')}", timeout=timeout, + "GET", + f"/vms/{vm_id}/files/content?path={quote(path, safe='')}", + timeout=timeout, ) return data if status == 200 else None From dcfd9de30cefd72b956b7fe57394ac927e5d9d5d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:25:02 +0000 Subject: [PATCH 223/464] test(sdk): type official client probes --- sdk/python/tests/standalone_openai_client.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/standalone_openai_client.py b/sdk/python/tests/standalone_openai_client.py index 5451839e6..527cef136 100644 --- a/sdk/python/tests/standalone_openai_client.py +++ b/sdk/python/tests/standalone_openai_client.py @@ -7,6 +7,11 @@ import json from openai import APIStatusError, AsyncOpenAI +from openai.types.chat import ( + ChatCompletionFunctionToolParam, + ChatCompletionMessageFunctionToolCall, +) +from openai.types.responses import FunctionToolParam async def probe(base_url: str) -> dict[str, object]: @@ -16,7 +21,7 @@ async def probe(base_url: str) -> dict[str, object]: max_retries=0, timeout=5.0, ) - tools = [ + tools: list[ChatCompletionFunctionToolParam] = [ { "type": "function", "function": { @@ -35,7 +40,10 @@ async def probe(base_url: str) -> dict[str, object]: messages=[{"role": "user", "content": "Use the fixture."}], tools=tools, ) - tool_call = chat.choices[0].message.tool_calls[0] + tool_calls = chat.choices[0].message.tool_calls + assert tool_calls is not None + tool_call = tool_calls[0] + assert isinstance(tool_call, ChatCompletionMessageFunctionToolCall) assert tool_call.function.name == "fixture_lookup" assert json.loads(tool_call.function.arguments)["query"] == "Capsem ironbank poem" assert chat.usage is not None and chat.usage.total_tokens == 456 @@ -50,11 +58,12 @@ async def probe(base_url: str) -> dict[str, object]: streamed_chat += chunk.choices[0].delta.content or "" assert streamed_chat == "Capsem ironbank poem" - response_tools = [ + response_tools: list[FunctionToolParam] = [ { "type": "function", "name": "exec_command", "description": "Run the deterministic fixture command.", + "strict": False, "parameters": { "type": "object", "properties": {"cmd": {"type": "string"}}, @@ -123,6 +132,7 @@ async def probe(base_url: str) -> dict[str, object]: ) except APIStatusError as error: upstream_status = error.status_code + assert isinstance(error.body, dict) upstream_code = error.body["code"] else: raise AssertionError("upstream failure was returned as success") From 2761e6889ae01ff6ced732caf5aa7095c4ae50ec Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:25:18 +0000 Subject: [PATCH 224/464] test(isolation): type grant fixtures --- config/gate.toml | 2 +- tests/helpers/gateway.py | 2 +- tests/ironbank/test_standalone_proxy.py | 43 +++++++++++++++++++------ 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index a386207a2..7df9d9929 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -883,7 +883,7 @@ error_on_warning = true # Reductions must lower the count immediately, so stale exemptions fail too. [lint.ty_ratchet] call-top-callable = 1 -invalid-argument-type = 37 +invalid-argument-type = 36 invalid-assignment = 5 invalid-return-type = 2 no-matching-overload = 5 diff --git a/tests/helpers/gateway.py b/tests/helpers/gateway.py index f2c373363..b1f3390df 100644 --- a/tests/helpers/gateway.py +++ b/tests/helpers/gateway.py @@ -126,7 +126,7 @@ def start(self): self.proc = subprocess.Popen( cmd, env=env, - stdin=grant_client, + stdin=grant_client.fileno(), stdout=self._log_file, stderr=self._log_file, ) diff --git a/tests/ironbank/test_standalone_proxy.py b/tests/ironbank/test_standalone_proxy.py index 2f49122dd..32a490c55 100644 --- a/tests/ironbank/test_standalone_proxy.py +++ b/tests/ironbank/test_standalone_proxy.py @@ -35,14 +35,22 @@ def _read_cli_startup(proc: subprocess.Popen[bytes]) -> tuple[str, str]: f"capsem proxy exited before startup: {proc.returncode}; output={received!r}" ) for key, _ in selector.select(timeout=0.2): - received.extend(os.read(key.fileobj.fileno(), 4096)) + received.extend(os.read(key.fd, 4096)) lines = received.decode(errors="replace").splitlines() session = next( - (line.removeprefix("Proxy session: ") for line in lines if line.startswith("Proxy session: ")), + ( + line.removeprefix("Proxy session: ") + for line in lines + if line.startswith("Proxy session: ") + ), None, ) base_url = next( - (line.removeprefix("Base URL: ") for line in lines if line.startswith("Base URL: ")), + ( + line.removeprefix("Base URL: ") + for line in lines + if line.startswith("Base URL: ") + ), None, ) if session and base_url: @@ -69,7 +77,11 @@ def _run_json(command: list[str]) -> dict[str, object]: def _records(path: Path) -> list[dict[str, object]]: if not path.exists(): return [] - return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line] + return [ + json.loads(line) + for line in path.read_text(encoding="utf-8").splitlines() + if line + ] def _wait_for_model_rows(db_path: Path, minimum: int = 8) -> list[sqlite3.Row]: @@ -82,7 +94,9 @@ def _wait_for_model_rows(db_path: Path, minimum: int = 8) -> list[sqlite3.Row]: if len(rows) >= minimum: return rows time.sleep(0.25) - raise AssertionError(f"standalone ledger did not reach {minimum} model rows: {db_path}") + raise AssertionError( + f"standalone ledger did not reach {minimum} model rows: {db_path}" + ) def _raw_http(port: int, request: bytes) -> bytes: @@ -105,7 +119,9 @@ def _assert_no_raw_secret(db_path: Path, session_dir: Path) -> None: ] for table in tables: columns = conn.execute(f"PRAGMA table_info({table})").fetchall() - text_columns = [row[1] for row in columns if str(row[2]).upper() in {"TEXT", ""}] + text_columns = [ + row[1] for row in columns if str(row[2]).upper() in {"TEXT", ""} + ] if not text_columns: continue selected = ", ".join(f'"{column}"' for column in text_columns) @@ -241,14 +257,23 @@ def test_standalone_proxy_serves_official_sdks_without_a_vm(): } assert {row["provider"] for row in model_rows} == {"openai"} assert {row["status_code"] for row in model_rows} >= {200, 503} - assert any(row["input_tokens"] == 66 and row["output_tokens"] == 390 for row in model_rows) - assert any(row["input_tokens"] == 7 and row["output_tokens"] == 5 for row in model_rows) + assert any( + row["input_tokens"] == 66 and row["output_tokens"] == 390 + for row in model_rows + ) + assert any( + row["input_tokens"] == 7 and row["output_tokens"] == 5 for row in model_rows + ) upstream = _records(transcript) assert {row["path"] for row in upstream} >= { "/v1/chat/completions", "/v1/responses", } - assert all(row["headers"].get("host") == "api.openai.com" for row in upstream) + headers = [row["headers"] for row in upstream] + assert all( + isinstance(value, dict) and value.get("host") == "api.openai.com" + for value in headers + ) _assert_no_raw_secret(db_path, session_dir) proxy_proc.send_signal(signal.SIGTERM) From eca00ef8c1d28c28860ee2be5f3d7c2f1590cd0f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:28:03 +0000 Subject: [PATCH 225/464] test(ts-sdk): provide package sandbox home --- sdk/typescript/tests/package-install.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sdk/typescript/tests/package-install.test.ts b/sdk/typescript/tests/package-install.test.ts index f84faaf90..fcae7077d 100644 --- a/sdk/typescript/tests/package-install.test.ts +++ b/sdk/typescript/tests/package-install.test.ts @@ -11,12 +11,14 @@ interface Report {ok: boolean; sha256: string; version: string; httpPaths: strin const source = fileURLToPath(new URL('../', import.meta.url)); const root = resolve(source, '../..'); -const sha = (path: string): string => sha256(readFileSync(path)); +const sha = (path: string): string => createHash('sha256').update(readFileSync(path)).digest('hex'); +const usableHome = (): string => process.env.HOME && existsSync(process.env.HOME) ? process.env.HOME : tmpdir(); +const npmCache = (): string => process.env.NPM_CONFIG_CACHE ?? join(usableHome(), '.npm'); it('prewarms offline without forwarding unsupported pnpm settings to npm', () => { const prewarm = spawnSync(process.execPath, [join(source, 'tools/prewarm-package.mjs')], { cwd: source, encoding: 'utf8', stdio: 'pipe', timeout: 60_000, - env: {...process.env, NPM_CONFIG_OFFLINE: 'true', + env: {...process.env, HOME: usableHome(), NPM_CONFIG_OFFLINE: 'true', npm_config_store_dir: '/unused/pnpm-store', npm_config_verify_deps_before_run: 'false', npm_config_overrides: '{}'}, }); @@ -26,7 +28,7 @@ it('prewarms offline without forwarding unsupported pnpm settings to npm', () => try { const cold = spawnSync(process.execPath, [join(source, 'tools/prewarm-package.mjs')], { cwd: source, encoding: 'utf8', stdio: 'pipe', timeout: 60_000, - env: {...process.env, NPM_CONFIG_OFFLINE: 'true', NPM_CONFIG_CACHE: emptyCache, + env: {...process.env, HOME: usableHome(), NPM_CONFIG_OFFLINE: 'true', NPM_CONFIG_CACHE: emptyCache, npm_config_store_dir: '/unused/pnpm-store'}, }); expect(cold.status).not.toBe(0); @@ -63,7 +65,7 @@ it('installs and exercises the actual SDK tarball without checkout or developmen try { if (selectedArchive) copyFileSync(selectedArchive, join(archiveDirectory, 'selected-sdk.tgz')); else execFileSync('pnpm', ['pack', '--config.ignore-scripts=true', '--pack-destination', archiveDirectory], { - cwd: source, stdio: 'pipe', timeout: 15_000, + cwd: source, stdio: 'pipe', timeout: 15_000, env: {...process.env, HOME: usableHome()}, }); const archives = readdirSync(archiveDirectory).filter(name => name.endsWith('.tgz')); expect(archives).toHaveLength(1); @@ -83,6 +85,7 @@ it('installs and exercises the actual SDK tarball without checkout or developmen const install = spawnSync('npm', ['install', '--offline', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', archive], { cwd: fixture, encoding: 'utf8', stdio: 'pipe', timeout: 60_000, + env: {...process.env, HOME: fixture, NPM_CONFIG_CACHE: npmCache()}, }); expect(install.status, install.stdout + install.stderr).toBe(0); const modules = join(fixture, 'node_modules'); @@ -96,7 +99,7 @@ it('installs and exercises the actual SDK tarball without checkout or developmen copyFileSync(join(source, 'tools/crypto.mjs'), join(fixture, 'crypto.mjs')); const runProbe = (): string => execFileSync(consumerNode, [probe, root, archive, output, receipt], { cwd: fixture, encoding: 'utf8', stdio: 'pipe', timeout: 15_000, - env: {PATH: process.env.PATH ?? ''}, + env: {HOME: fixture, PATH: process.env.PATH ?? ''}, }); const result = runProbe(); expect(result).toContain('SDK_IMAGE_PACKAGE_ACCEPTANCE_OK'); @@ -136,6 +139,7 @@ it('installs and exercises the actual SDK tarball without checkout or developmen execFileSync('npm', ['install', '--offline', '--ignore-scripts', '--no-audit', '--no-fund', `typescript@${manifest.devDependencies.typescript}`], { cwd: fixture, stdio: 'pipe', timeout: 30_000, + env: {...process.env, HOME: fixture, NPM_CONFIG_CACHE: npmCache()}, }); expect(existsSync(join(modules, 'typescript/package.json'))).toBe(true); expect(runProbe).toThrow(/development dependency/); From c9e4bb5a9fce2455b28850700583550200f2e597 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:28:03 +0000 Subject: [PATCH 226/464] test(mcp): isolate packed package home --- mcp/typescript/tests/image-tools.test.ts | 2 +- mcp/typescript/tests/packed-package.test.ts | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/mcp/typescript/tests/image-tools.test.ts b/mcp/typescript/tests/image-tools.test.ts index 983184103..48feee3cb 100644 --- a/mcp/typescript/tests/image-tools.test.ts +++ b/mcp/typescript/tests/image-tools.test.ts @@ -99,7 +99,7 @@ it('rejects malformed image catalog responses without claiming cache readiness', expect(result.isError).toBe(true); expect(result.structuredContent).toEqual({error: {kind: 'internal'}}); expect(requests).toHaveLength(1); - }, {body: {images: [{...catalog.images[0], cached: 'not-a-cache-state'}]}}); + }, {body: {images: [{...catalog.images[0], cached: 'available'}]}}); }); it('keeps gateway bearer ownership separate for each MCP server', async () => { diff --git a/mcp/typescript/tests/packed-package.test.ts b/mcp/typescript/tests/packed-package.test.ts index 827e25d3c..3fc0575a8 100644 --- a/mcp/typescript/tests/packed-package.test.ts +++ b/mcp/typescript/tests/packed-package.test.ts @@ -1,10 +1,10 @@ import {execFileSync} from 'node:child_process'; -import {copyFileSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync} from 'node:fs'; -import {sha256} from '../../../sdk/typescript/tools/crypto.mjs'; +import {copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync} from 'node:fs'; +import {createHash} from 'node:crypto'; import {createServer as createHttpServer} from 'node:http'; import type {AddressInfo} from 'node:net'; -import {homedir, tmpdir} from 'node:os'; -import {join} from 'node:path'; +import {tmpdir} from 'node:os'; +import {dirname, join} from 'node:path'; import {fileURLToPath} from 'node:url'; import {Client} from '@modelcontextprotocol/sdk/client/index.js'; import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js'; @@ -47,6 +47,7 @@ describe('packed-package', () => { copyFileSync(path, join(destination, `capsem-${owner}-${version}.tgz`)); } else execFileSync('pnpm', ['pack', '--config.ignore-scripts=true', '--pack-destination', destination], { cwd: source, stdio: 'pipe', timeout: 15_000, + env: {...process.env, HOME: fixture}, }); const name = readdirSync(destination).find(entry => entry.endsWith('.tgz')); if (!name) throw new Error('pnpm pack did not create a tarball'); @@ -56,10 +57,12 @@ describe('packed-package', () => { execFileSync('tar', ['-xzf', archive, '-C', destination], {timeout: 5000}); } writeFileSync(join(fixture, 'package.json'), JSON.stringify({name: 'capsem-mcp-clean-consumer', private: true})); + const npmCache = process.env.NPM_CONFIG_CACHE + ?? (process.env.HOME && existsSync(process.env.HOME) ? join(process.env.HOME, '.npm') : undefined); execFileSync('npm', ['install', '--offline', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', ...archives], { cwd: fixture, stdio: 'pipe', timeout: 60_000, - env: {PATH: process.env.PATH ?? '', HOME: fixture, - NPM_CONFIG_CACHE: process.env.NPM_CONFIG_CACHE ?? join(homedir(), '.npm')}, + env: {HOME: fixture, PATH: process.env.PATH ?? '', + ...(npmCache ? {NPM_CONFIG_CACHE: npmCache} : {})}, }); const dependencies = join(fixture, 'node_modules'); for (const [owner, source] of Object.entries(sources)) { @@ -94,7 +97,7 @@ describe('packed-package', () => { const transport = new StdioClientTransport({ command: consumerNode, args: [cli, '--gateway-url', gatewayUrl, '--timeout-ms', '5000'], - env: {PATH: process.env.PATH ?? '', CAPSEM_GATEWAY_TOKEN: token}, + env: {HOME: dirname(cli), PATH: process.env.PATH ?? '', CAPSEM_GATEWAY_TOKEN: token}, stderr: 'pipe', }); const stderr: string[] = []; From 1cc22e60e79ff8443a65511fd1dc8b77100d1869 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:34:18 +0000 Subject: [PATCH 227/464] fix(ts-sdk): preserve npm home during prewarm --- sdk/typescript/tools/prewarm-package.mjs | 25 ++++++++++-------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/sdk/typescript/tools/prewarm-package.mjs b/sdk/typescript/tools/prewarm-package.mjs index 537ee43b7..88860475f 100644 --- a/sdk/typescript/tools/prewarm-package.mjs +++ b/sdk/typescript/tools/prewarm-package.mjs @@ -1,9 +1,8 @@ // Fetch dependencies before the clean-consumer acceptance enters the sandbox. import assert from 'node:assert/strict'; import {execFileSync} from 'node:child_process'; -import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; -import {homedir, tmpdir} from 'node:os'; -import {join} from 'node:path'; +import {existsSync, readFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; import process from 'node:process'; /** @type {unknown} */ @@ -11,16 +10,12 @@ const parsed = JSON.parse(readFileSync(new URL('../package.json', import.meta.ur const manifest = /** @type {{dependencies:Record,devDependencies:Record}} */ (parsed); const compiler = manifest.devDependencies.typescript; assert.ok(compiler); -const cache = process.env.NPM_CONFIG_CACHE ?? join(homedir(), '.npm'); -const home = mkdtempSync(join(tmpdir(), 'capsem-sdk-prewarm-home-')); -try { - for (const [name, version] of Object.entries({...manifest.dependencies, typescript: compiler})) { - execFileSync('npm', ['cache', 'add', `${name}@${version}`], { - stdio: 'inherit', timeout: 30_000, - env: {PATH: process.env.PATH ?? '', HOME: home, NPM_CONFIG_CACHE: cache, - ...(process.env.NPM_CONFIG_OFFLINE ? {NPM_CONFIG_OFFLINE: process.env.NPM_CONFIG_OFFLINE} : {})}, - }); - } -} finally { - rmSync(home, {recursive: true, force: true}); +const home = process.env.HOME && existsSync(process.env.HOME) ? process.env.HOME : tmpdir(); +for (const [name, version] of Object.entries({...manifest.dependencies, typescript: compiler})) { + execFileSync('npm', ['cache', 'add', `${name}@${version}`], { + stdio: 'inherit', timeout: 30_000, + env: {HOME: home, PATH: process.env.PATH ?? '', + ...(process.env.NPM_CONFIG_CACHE ? {NPM_CONFIG_CACHE: process.env.NPM_CONFIG_CACHE} : {}), + ...(process.env.NPM_CONFIG_OFFLINE ? {NPM_CONFIG_OFFLINE: process.env.NPM_CONFIG_OFFLINE} : {})}, + }); } From 4c48aa7096c9fee95ac4f618b891f30c85241583 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:34:18 +0000 Subject: [PATCH 228/464] fix(mcp): preserve npm home during prewarm --- mcp/typescript/tools/prewarm-package.mjs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mcp/typescript/tools/prewarm-package.mjs b/mcp/typescript/tools/prewarm-package.mjs index 1a399e38f..52badc874 100644 --- a/mcp/typescript/tools/prewarm-package.mjs +++ b/mcp/typescript/tools/prewarm-package.mjs @@ -1,14 +1,15 @@ // Warm npm's complete runtime dependency graph before offline acceptance. import assert from 'node:assert/strict'; import {execFileSync} from 'node:child_process'; -import {mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; -import {homedir, tmpdir} from 'node:os'; +import {existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; +import {tmpdir} from 'node:os'; import {join} from 'node:path'; import process from 'node:process'; import {fileURLToPath} from 'node:url'; const project = fileURLToPath(new URL('../', import.meta.url)); const fixture = mkdtempSync(join(tmpdir(), 'capsem-mcp-prewarm-')); +const home = process.env.HOME && existsSync(process.env.HOME) ? process.env.HOME : fixture; try { const archives = []; for (const [owner, source] of [['sdk', join(project, '../../sdk/typescript')], ['mcp', project]]) { @@ -16,7 +17,7 @@ try { const output = join(fixture, owner); mkdirSync(output); execFileSync('pnpm', ['pack', '--config.ignore-scripts=true', '--pack-destination', output], { - cwd: source, stdio: 'pipe', timeout: 15_000, + cwd: source, stdio: 'pipe', timeout: 15_000, env: {...process.env, HOME: home}, }); const names = readdirSync(output).filter(name => name.endsWith('.tgz')); assert.equal(names.length, 1); @@ -26,8 +27,8 @@ try { writeFileSync(join(fixture, 'package.json'), JSON.stringify({name: 'capsem-mcp-prewarm', private: true})); execFileSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', ...archives], { cwd: fixture, stdio: 'inherit', timeout: 60_000, - env: {PATH: process.env.PATH ?? '', HOME: fixture, - NPM_CONFIG_CACHE: process.env.NPM_CONFIG_CACHE ?? join(homedir(), '.npm')}, + env: {HOME: home, PATH: process.env.PATH ?? '', + ...(process.env.NPM_CONFIG_CACHE ? {NPM_CONFIG_CACHE: process.env.NPM_CONFIG_CACHE} : {})}, }); } finally { rmSync(fixture, {recursive: true, force: true}); From f2e69583073f49ff5a1af472171891d6fda70919 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:34:18 +0000 Subject: [PATCH 229/464] chore(deps): accept worker dependency graph --- config/dependency-inventory.json | 76 ++++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/config/dependency-inventory.json b/config/dependency-inventory.json index c6ccca4ac..96155e0ee 100644 --- a/config/dependency-inventory.json +++ b/config/dependency-inventory.json @@ -924,6 +924,7 @@ "time", "tokio", "tokio-rustls", + "tokio-tungstenite", "tokio-util", "toml", "tracing", @@ -936,12 +937,28 @@ "capsem-credentials": { "build_script": false, "dependencies": [ + "base64", + "bytes", "capsem-foundation", "capsem-proto", + "futures", + "getrandom", + "http-body-util", + "hyper", + "hyper-util", + "jsonwebtoken", + "reqwest", + "ring", "serde", "serde_json", + "sha2", + "signature", + "subtle", "tempfile", - "tracing" + "tokio", + "tracing", + "url", + "zeroize" ] }, "capsem-foundation": { @@ -1005,6 +1022,7 @@ "build_script": false, "dependencies": [ "anyhow", + "capsem-archive", "capsem-foundation", "capsem-guard", "capsem-logger", @@ -1012,7 +1030,8 @@ "clap", "tempfile", "tokio", - "tracing" + "tracing", + "zstd" ] }, "capsem-logger": { @@ -1163,13 +1182,17 @@ "build_script": false, "dependencies": [ "anyhow", + "async-trait", + "bytes", "capsem-core", "capsem-foundation", "capsem-guard", "capsem-logger", "capsem-proto", + "capsem-telemetry", "clap", "http", + "opentelemetry-http", "tempfile", "tokio", "tracing" @@ -1217,6 +1240,7 @@ "capsem-assets", "capsem-config", "capsem-core", + "capsem-credentials", "capsem-foundation", "capsem-guard", "capsem-logger", @@ -1234,6 +1258,7 @@ "opentelemetry_sdk", "rand", "reqwest", + "rmp-serde", "rusqlite", "serde", "serde_json", @@ -1246,7 +1271,8 @@ "tower", "tower-http", "tracing", - "uuid" + "uuid", + "zeroize" ] }, "capsem-telemetry": { @@ -3523,6 +3549,32 @@ "toml" ] }, + "jsonwebtoken": { + "build_script": false, + "dependencies": [ + "aws-lc-rs", + "base64", + "criterion", + "ed25519-dalek", + "getrandom", + "hmac", + "js-sys", + "p256", + "p384", + "pem", + "rand", + "rand_core", + "rsa", + "serde", + "serde_json", + "sha2", + "signature", + "simple_asn1", + "time", + "wasm-bindgen-test", + "zeroize" + ] + }, "jwt": { "build_script": false, "dependencies": [ @@ -6055,6 +6107,16 @@ "signal-hook" ] }, + "signature": { + "build_script": false, + "dependencies": [ + "digest", + "hex-literal", + "rand_core", + "sha2", + "signature_derive" + ] + }, "simd-adler32": { "build_script": false, "dependencies": [ @@ -8346,6 +8408,14 @@ "zeroize_derive" ] }, + "zeroize_derive": { + "build_script": false, + "dependencies": [ + "proc-macro2", + "quote", + "syn" + ] + }, "zerotrie": { "build_script": false, "dependencies": [ From 715be0fb053d67fc4763d79832be2c7ec351ffd2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:47:13 +0000 Subject: [PATCH 230/464] test(docs): count isolation architecture page --- tests/test_docs_holding_contract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docs_holding_contract.py b/tests/test_docs_holding_contract.py index fa11c35ce..e14b4aa77 100644 --- a/tests/test_docs_holding_contract.py +++ b/tests/test_docs_holding_contract.py @@ -187,7 +187,7 @@ def test_docs_source_builds_the_holding_graph_without_deleting_the_manual() -> N "astro build && uv run --project ../../build_system --frozen python " "../../build_system/scripts/web/check-docs-holding-build.py dist src/content/docs" ) - assert len(detailed_sources) == 47 + assert len(detailed_sources) == 48 assert DOCS_ROOT / "src" / "content" / "docs" / "index.mdx" in detailed_sources assert ( DOCS_ROOT / "src" / "content" / "docs" / "architecture" / "service-api.md" From c30cdfc03b0bb5c586811db67afb670f742521b9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 11:50:51 +0000 Subject: [PATCH 231/464] test(ledger): move codec measurement to bench rail --- crates/capsem-ledger/Cargo.toml | 4 + crates/capsem-ledger/benches/ledger_codec.rs | 125 ++++++++++++++++++ crates/capsem-ledger/src/codec.rs | 111 +--------------- .../capsem-ledger/src/codec/implementation.rs | 110 +++++++++++++++ crates/capsem-ledger/src/codec/tests.rs | 109 --------------- 5 files changed, 241 insertions(+), 218 deletions(-) create mode 100644 crates/capsem-ledger/benches/ledger_codec.rs create mode 100644 crates/capsem-ledger/src/codec/implementation.rs diff --git a/crates/capsem-ledger/Cargo.toml b/crates/capsem-ledger/Cargo.toml index bac1c9db7..0ffe9605b 100644 --- a/crates/capsem-ledger/Cargo.toml +++ b/crates/capsem-ledger/Cargo.toml @@ -24,5 +24,9 @@ zstd = "0.13" [dev-dependencies] tempfile = "3" +[[bench]] +name = "ledger_codec" +harness = false + [lints] workspace = true diff --git a/crates/capsem-ledger/benches/ledger_codec.rs b/crates/capsem-ledger/benches/ledger_codec.rs new file mode 100644 index 000000000..5c37e1db0 --- /dev/null +++ b/crates/capsem-ledger/benches/ledger_codec.rs @@ -0,0 +1,125 @@ +use capsem_logger::{DbHandle, Decision, NetEvent, WriteOp}; + +#[path = "../src/codec/implementation.rs"] +mod codec; + +const EXCHANGES: usize = 1_000; +const FLUSH_EVERY: usize = 20; + +fn main() { + let runtime = tokio::runtime::Runtime::new().expect("create benchmark runtime"); + for name in ["deflate", "zstd"] { + runtime.block_on(measure(name)); + } +} + +async fn measure(name: &str) { + let dir = tempfile::tempdir().expect("create benchmark directory"); + let path = dir.path().join("session.db"); + let db = match name { + "deflate" => DbHandle::open(&path).expect("open deflate ledger"), + "zstd" => DbHandle::open_with_codecs(&path, codec::archive_codecs().expect("initialize zstd codec")) + .expect("open zstd ledger"), + _ => unreachable!(), + }; + let baseline_rss = capsem_foundation::proctable::resident_bytes(std::process::id()).expect("read baseline RSS"); + let mut peak_rss = baseline_rss; + let mut raw_bytes = 0usize; + let started = std::time::Instant::now(); + for index in 0..EXCHANGES { + let (event, bytes) = benchmark_event(index); + raw_bytes += bytes; + db.write(event).await.expect("write benchmark exchange"); + if index % FLUSH_EVERY == FLUSH_EVERY - 1 { + db.flush().await.expect("flush benchmark ledger"); + peak_rss = peak_rss + .max(capsem_foundation::proctable::resident_bytes(std::process::id()).expect("read benchmark RSS")); + } + } + db.flush().await.expect("final benchmark flush"); + let elapsed = started.elapsed(); + drop(db); + let archive_bytes: u64 = std::fs::read_dir(path.with_extension("bodies")) + .expect("read archive generations") + .map(|entry| { + entry + .expect("read archive generation") + .metadata() + .expect("read archive metadata") + .len() + }) + .sum(); + let seconds = elapsed.as_secs_f64(); + println!( + "codec={name} exchanges={EXCHANGES} raw_bytes={raw_bytes} archive_bytes={archive_bytes} ratio={:.4} elapsed_ms={:.3} throughput_mib_s={:.3} peak_rss_delta_bytes={}", + archive_bytes as f64 / raw_bytes as f64, + seconds * 1000.0, + raw_bytes as f64 / (1024.0 * 1024.0) / seconds, + peak_rss.saturating_sub(baseline_rss), + ); +} + +fn benchmark_event(index: usize) -> (WriteOp, usize) { + let request = benchmark_text(index, 2048); + let response = benchmark_text(index.wrapping_mul(31) + 7, 4096); + let bytes = request.len() + response.len(); + ( + WriteOp::NetEvent(NetEvent { + event_id: Some(format!("{index:012x}")), + timestamp: std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(index as u64), + domain: "api.bench.example".into(), + port: 443, + decision: Decision::Allowed, + process_name: Some("codec-bench".into()), + pid: Some(1), + method: Some("POST".into()), + path: Some("/v1/messages".into()), + query: None, + status_code: Some(200), + bytes_sent: request.len() as u64, + bytes_received: response.len() as u64, + duration_ms: 1, + matched_rule: None, + request_headers: Some("content-type: application/json".into()), + response_headers: Some("content-type: application/json".into()), + request_body: Some(request.into_bytes()), + response_body: Some(response.into_bytes()), + conn_type: Some("https".into()), + policy_mode: None, + policy_action: None, + policy_rule: None, + policy_reason: None, + trace_id: Some(format!("{index:016x}")), + credential_ref: None, + }), + bytes, + ) +} + +fn benchmark_text(seed: usize, len: usize) -> String { + const WORDS: [&str; 12] = [ + "assistant", + "content", + "stream", + "delta", + "tool", + "result", + "input", + "output", + "usage", + "message", + "role", + "index", + ]; + let mut state = (seed as u64).wrapping_add(1).wrapping_mul(0x9e37_79b9_7f4a_7c15); + let mut output = String::with_capacity(len + 16); + while output.len() < len { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + output.push_str(WORDS[(state % WORDS.len() as u64) as usize]); + output.push(if state.is_multiple_of(7) { ',' } else { ' ' }); + } + output.truncate(len); + output +} diff --git a/crates/capsem-ledger/src/codec.rs b/crates/capsem-ledger/src/codec.rs index 8b3771e14..57dcf1b8b 100644 --- a/crates/capsem-ledger/src/codec.rs +++ b/crates/capsem-ledger/src/codec.rs @@ -1,115 +1,8 @@ //! Native archive codec owned exclusively by the confined ledger executable. -use std::io; -use std::sync::Arc; +mod implementation; -use capsem_archive::{ - ArchiveCodecs, ArchiveError, BlockCodec, BlockDecoder, BlockEncoder, Result as ArchiveResult, CODEC_ZSTD, -}; -use zstd::stream::raw::{Decoder, Encoder, InBuffer, Operation, OutBuffer}; - -const ZSTD_LEVEL: i32 = 3; -const OUTPUT_CHUNK: usize = 128 * 1024; - -pub(crate) fn archive_codecs() -> ArchiveResult { - ArchiveCodecs::with_write_codec(Arc::new(ZstdCodec)) -} - -struct ZstdCodec; - -impl BlockCodec for ZstdCodec { - fn id(&self) -> u8 { - CODEC_ZSTD - } - - fn encoder(&self) -> ArchiveResult> { - Encoder::new(ZSTD_LEVEL) - .map(|encoder| Box::new(ZstdEncoder(encoder)) as Box) - .map_err(ArchiveError::Io) - } - - fn decoder(&self) -> ArchiveResult> { - Decoder::new() - .map(|decoder| Box::new(ZstdDecoder(decoder)) as Box) - .map_err(ArchiveError::Io) - } -} - -struct ZstdEncoder(Encoder<'static>); - -impl BlockEncoder for ZstdEncoder { - fn write(&mut self, input: &[u8], output: &mut Vec) -> ArchiveResult<()> { - let mut input = InBuffer::around(input); - while input.pos() < input.src.len() { - let before = input.pos(); - with_output_capacity(output, |out| self.0.run(&mut input, out)).map_err(ArchiveError::Io)?; - if input.pos() == before { - return Err(ArchiveError::Io(io::Error::other( - "zstd encoder made no input progress", - ))); - } - } - Ok(()) - } - - fn flush(&mut self, output: &mut Vec, finish: bool) -> ArchiveResult<()> { - loop { - let remaining = with_output_capacity(output, |out| { - if finish { - self.0.finish(out, true) - } else { - self.0.flush(out) - } - }) - .map_err(ArchiveError::Io)?; - if remaining == 0 { - return Ok(()); - } - } - } -} - -struct ZstdDecoder(Decoder<'static>); - -impl BlockDecoder for ZstdDecoder { - fn decode_segment(&mut self, input: &[u8], raw_len: usize, finish: bool, at: u64) -> ArchiveResult> { - let mut raw = Vec::with_capacity(raw_len + 1); - let (consumed, written, ended) = { - let mut input = InBuffer::around(input); - let mut output = OutBuffer::around(&mut raw); - let mut ended = false; - loop { - let before = (input.pos(), output.pos()); - let remaining = self - .0 - .run(&mut input, &mut output) - .map_err(|error| ArchiveError::Inflate(at, error.to_string()))?; - if remaining == 0 { - ended = true; - break; - } - if input.pos() == input.src.len() || before == (input.pos(), output.pos()) { - break; - } - } - (input.pos(), output.pos(), ended) - }; - if consumed != input.len() || written != raw_len || ended != finish { - return Err(ArchiveError::Integrity(at)); - } - Ok(raw) - } -} - -fn with_output_capacity( - output: &mut Vec, - operation: impl FnOnce(&mut OutBuffer<'_, Vec>) -> io::Result, -) -> io::Result { - output.reserve(OUTPUT_CHUNK); - let position = output.len(); - let mut buffer = OutBuffer::around_pos(output, position); - operation(&mut buffer) -} +pub(crate) use implementation::archive_codecs; #[cfg(test)] mod tests; diff --git a/crates/capsem-ledger/src/codec/implementation.rs b/crates/capsem-ledger/src/codec/implementation.rs new file mode 100644 index 000000000..0bc1afaa6 --- /dev/null +++ b/crates/capsem-ledger/src/codec/implementation.rs @@ -0,0 +1,110 @@ +use std::io; +use std::sync::Arc; + +use capsem_archive::{ + ArchiveCodecs, ArchiveError, BlockCodec, BlockDecoder, BlockEncoder, Result as ArchiveResult, CODEC_ZSTD, +}; +use zstd::stream::raw::{Decoder, Encoder, InBuffer, Operation, OutBuffer}; + +const ZSTD_LEVEL: i32 = 3; +const OUTPUT_CHUNK: usize = 128 * 1024; + +pub(crate) fn archive_codecs() -> ArchiveResult { + ArchiveCodecs::with_write_codec(Arc::new(ZstdCodec)) +} + +struct ZstdCodec; + +impl BlockCodec for ZstdCodec { + fn id(&self) -> u8 { + CODEC_ZSTD + } + + fn encoder(&self) -> ArchiveResult> { + Encoder::new(ZSTD_LEVEL) + .map(|encoder| Box::new(ZstdEncoder(encoder)) as Box) + .map_err(ArchiveError::Io) + } + + fn decoder(&self) -> ArchiveResult> { + Decoder::new() + .map(|decoder| Box::new(ZstdDecoder(decoder)) as Box) + .map_err(ArchiveError::Io) + } +} + +struct ZstdEncoder(Encoder<'static>); + +impl BlockEncoder for ZstdEncoder { + fn write(&mut self, input: &[u8], output: &mut Vec) -> ArchiveResult<()> { + let mut input = InBuffer::around(input); + while input.pos() < input.src.len() { + let before = input.pos(); + with_output_capacity(output, |out| self.0.run(&mut input, out)).map_err(ArchiveError::Io)?; + if input.pos() == before { + return Err(ArchiveError::Io(io::Error::other( + "zstd encoder made no input progress", + ))); + } + } + Ok(()) + } + + fn flush(&mut self, output: &mut Vec, finish: bool) -> ArchiveResult<()> { + loop { + let remaining = with_output_capacity(output, |out| { + if finish { + self.0.finish(out, true) + } else { + self.0.flush(out) + } + }) + .map_err(ArchiveError::Io)?; + if remaining == 0 { + return Ok(()); + } + } + } +} + +struct ZstdDecoder(Decoder<'static>); + +impl BlockDecoder for ZstdDecoder { + fn decode_segment(&mut self, input: &[u8], raw_len: usize, finish: bool, at: u64) -> ArchiveResult> { + let mut raw = Vec::with_capacity(raw_len + 1); + let (consumed, written, ended) = { + let mut input = InBuffer::around(input); + let mut output = OutBuffer::around(&mut raw); + let mut ended = false; + loop { + let before = (input.pos(), output.pos()); + let remaining = self + .0 + .run(&mut input, &mut output) + .map_err(|error| ArchiveError::Inflate(at, error.to_string()))?; + if remaining == 0 { + ended = true; + break; + } + if input.pos() == input.src.len() || before == (input.pos(), output.pos()) { + break; + } + } + (input.pos(), output.pos(), ended) + }; + if consumed != input.len() || written != raw_len || ended != finish { + return Err(ArchiveError::Integrity(at)); + } + Ok(raw) + } +} + +fn with_output_capacity( + output: &mut Vec, + operation: impl FnOnce(&mut OutBuffer<'_, Vec>) -> io::Result, +) -> io::Result { + output.reserve(OUTPUT_CHUNK); + let position = output.len(); + let mut buffer = OutBuffer::around_pos(output, position); + operation(&mut buffer) +} diff --git a/crates/capsem-ledger/src/codec/tests.rs b/crates/capsem-ledger/src/codec/tests.rs index d3fae2ee8..e019430fc 100644 --- a/crates/capsem-ledger/src/codec/tests.rs +++ b/crates/capsem-ledger/src/codec/tests.rs @@ -1,6 +1,5 @@ use capsem_archive::format::{BLOCK_HEADER_BYTES, FILE_HEADER_BYTES, SEGMENT_HEADER_BYTES}; use capsem_archive::{BodyLogReader, BodyLogWriter, CODEC_ZSTD}; -use capsem_logger::{DbHandle, Decision, NetEvent, WriteOp}; use super::archive_codecs; @@ -55,111 +54,3 @@ fn zstd_corruption_is_refused_before_bytes_are_returned() { let reader = BodyLogReader::open_with_codecs(&path, codecs).unwrap(); assert!(reader.read(reference).is_err()); } - -#[tokio::test] -#[ignore = "repeatable manual codec measurement"] -async fn measure_identical_ledger_codec() { - const EXCHANGES: usize = 1_000; - const FLUSH_EVERY: usize = 20; - let codec = std::env::var("CAPSEM_CODEC_BENCH").expect("CAPSEM_CODEC_BENCH=deflate|zstd"); - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("session.db"); - let db = match codec.as_str() { - "deflate" => DbHandle::open(&path).unwrap(), - "zstd" => DbHandle::open_with_codecs(&path, archive_codecs().unwrap()).unwrap(), - other => panic!("unsupported codec {other}"), - }; - let baseline_rss = capsem_foundation::proctable::resident_bytes(std::process::id()).unwrap(); - let mut peak_rss = baseline_rss; - let mut raw_bytes = 0usize; - let started = std::time::Instant::now(); - for index in 0..EXCHANGES { - let (event, bytes) = benchmark_event(index); - raw_bytes += bytes; - db.write(event).await.unwrap(); - if index % FLUSH_EVERY == FLUSH_EVERY - 1 { - db.flush().await.unwrap(); - peak_rss = peak_rss.max(capsem_foundation::proctable::resident_bytes(std::process::id()).unwrap()); - } - } - db.flush().await.unwrap(); - let elapsed = started.elapsed(); - drop(db); - let archive_bytes: u64 = std::fs::read_dir(path.with_extension("bodies")) - .unwrap() - .map(|entry| entry.unwrap().metadata().unwrap().len()) - .sum(); - let seconds = elapsed.as_secs_f64(); - eprintln!( - "codec={codec} exchanges={EXCHANGES} raw_bytes={raw_bytes} archive_bytes={archive_bytes} ratio={:.4} elapsed_ms={:.3} throughput_mib_s={:.3} peak_rss_delta_bytes={}", - archive_bytes as f64 / raw_bytes as f64, - seconds * 1000.0, - raw_bytes as f64 / (1024.0 * 1024.0) / seconds, - peak_rss.saturating_sub(baseline_rss), - ); -} - -fn benchmark_event(index: usize) -> (WriteOp, usize) { - let request = benchmark_text(index, 2048); - let response = benchmark_text(index.wrapping_mul(31) + 7, 4096); - let bytes = request.len() + response.len(); - ( - WriteOp::NetEvent(NetEvent { - event_id: Some(format!("{index:012x}")), - timestamp: std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(index as u64), - domain: "api.bench.example".into(), - port: 443, - decision: Decision::Allowed, - process_name: Some("codec-bench".into()), - pid: Some(1), - method: Some("POST".into()), - path: Some("/v1/messages".into()), - query: None, - status_code: Some(200), - bytes_sent: request.len() as u64, - bytes_received: response.len() as u64, - duration_ms: 1, - matched_rule: None, - request_headers: Some("content-type: application/json".into()), - response_headers: Some("content-type: application/json".into()), - request_body: Some(request.into_bytes()), - response_body: Some(response.into_bytes()), - conn_type: Some("https".into()), - policy_mode: None, - policy_action: None, - policy_rule: None, - policy_reason: None, - trace_id: Some(format!("{index:016x}")), - credential_ref: None, - }), - bytes, - ) -} - -fn benchmark_text(seed: usize, len: usize) -> String { - const WORDS: [&str; 12] = [ - "assistant", - "content", - "stream", - "delta", - "tool", - "result", - "input", - "output", - "usage", - "message", - "role", - "index", - ]; - let mut state = (seed as u64).wrapping_add(1).wrapping_mul(0x9e37_79b9_7f4a_7c15); - let mut output = String::with_capacity(len + 16); - while output.len() < len { - state ^= state << 13; - state ^= state >> 7; - state ^= state << 17; - output.push_str(WORDS[(state % WORDS.len() as u64) as usize]); - output.push(if state.is_multiple_of(7) { ',' } else { ' ' }); - } - output.truncate(len); - output -} From b5acffa0317637bd2f95352bcf342db8dff45359 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 12:13:32 +0000 Subject: [PATCH 232/464] fix(sdk-python): make package proof cold-cache safe --- sdk/python/tests/image_package_acceptance.py | 12 ++- sdk/python/tests/test_package_install.py | 99 +++++++++++++++----- 2 files changed, 85 insertions(+), 26 deletions(-) diff --git a/sdk/python/tests/image_package_acceptance.py b/sdk/python/tests/image_package_acceptance.py index 949095778..ee86f28fb 100644 --- a/sdk/python/tests/image_package_acceptance.py +++ b/sdk/python/tests/image_package_acceptance.py @@ -7,6 +7,7 @@ import hashlib import importlib.metadata import json +import re import sys import tarfile import threading @@ -38,7 +39,13 @@ def archive_payload(archive: Path) -> dict[str, bytes]: file = stream.extractfile(member) assert file is not None result[str(Path(*parts))] = file.read() - return result + return result + + +def normalized_requirement(requirement: str) -> tuple[str, tuple[str, ...]]: + match = re.fullmatch(r"([A-Za-z0-9._-]+)(.*)", requirement.replace(" ", "")) + assert match is not None + return match[1].lower(), tuple(sorted(filter(None, match[2].split(",")))) def main() -> None: @@ -62,6 +69,9 @@ def main() -> None: manifest = tomllib.load(file)["project"] assert distribution.version == manifest["version"] assert distribution.metadata["Requires-Python"] == manifest["requires-python"] + assert { + normalized_requirement(requirement) for requirement in distribution.requires or [] + } == {normalized_requirement(requirement) for requirement in manifest["dependencies"]} installed = {dist.metadata["Name"].lower() for dist in importlib.metadata.distributions()} assert not installed.intersection({"pytest", "build", "hatchling", "editables", "ruff", "ty"}) payload = archive_payload(args.archive) diff --git a/sdk/python/tests/test_package_install.py b/sdk/python/tests/test_package_install.py index a7b7d9103..73165a7c2 100644 --- a/sdk/python/tests/test_package_install.py +++ b/sdk/python/tests/test_package_install.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys +import tarfile import tempfile import tomllib from pathlib import Path @@ -15,16 +16,61 @@ import pytest +def _run(command: list[str], *, cwd: Path) -> None: + result = subprocess.run( + command, + cwd=cwd, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def _build_sdist_wheel(archive: Path, work: Path) -> Path: + source = work / "sdist-source" + source.mkdir() + with tarfile.open(archive, "r:gz") as bundle: + bundle.extractall(source, filter="data") + roots = list(source.iterdir()) + assert len(roots) == 1 and roots[0].is_dir() and not roots[0].is_symlink() + + output = work / "sdist-wheel" + _run( + [ + sys.executable, + "-m", + "build", + "--wheel", + "--no-isolation", + "--outdir", + str(output), + str(roots[0]), + ], + cwd=work, + ) + wheels = list(output.glob("*.whl")) + assert len(wheels) == 1 + return wheels[0] + + @pytest.mark.parametrize("kind", ["wheel", "sdist"]) def test_built_distribution_installs_and_runs_in_an_isolated_runtime(kind: str) -> None: root = Path(__file__).resolve().parents[3] config = tomllib.loads((root / "config/gate.toml").read_text())["sdk_python"] - manifest = tomllib.loads((root / config["manifest"]).read_text())["project"] + manifest_path = root / config["manifest"] + manifest = tomllib.loads(manifest_path.read_text())["project"] + lockfiles = list((root / config["project"]).glob("*.lock")) + assert len(lockfiles) == 1 + lockfile = lockfiles[0] selected = os.environ.get("CAPSEM_SDK_PACKAGE_ARCHIVE_DIR") output = Path(selected) if selected else root / config["build_output"] assert output.is_absolute() stem = f"{manifest['name'].replace('-', '_')}-{manifest['version']}" - archives = list(output.glob(f"{stem}-*.whl" if kind == "wheel" else f"{stem}.tar.gz")) + archives = list( + output.glob(f"{stem}-*.whl" if kind == "wheel" else f"{stem}.tar.gz") + ) assert len(archives) == 1, ( f"expected one current {kind} in {output}; run the SDK package build before this test" ) @@ -38,18 +84,27 @@ def test_built_distribution_installs_and_runs_in_an_isolated_runtime(kind: str) with tempfile.TemporaryDirectory(prefix=f"capsem-sdk-{kind}-") as temporary: work = Path(temporary) assert not work.resolve().is_relative_to(root) - prefix = work / "runtime" + runtime_project = work / "runtime-project" + runtime_project.mkdir() + shutil.copyfile(manifest_path, runtime_project / manifest_path.name) + shutil.copyfile(lockfile, runtime_project / lockfile.name) + prefix = runtime_project / ".venv" python = prefix / "bin/python" - # Carry the prewarmed runtime artifacts by their locked URLs and hashes. - runtime_lock = work / "pylock.toml" - commands = [ + installable = _build_sdist_wheel(archive, work) if kind == "sdist" else archive + _run( [ - "uv", "export", "--project", str(root / config["project"]), - "--frozen", "--offline", "--no-dev", "--no-emit-project", - "--format", "pylock.toml", "--output-file", str(runtime_lock), + "uv", + "sync", + "--offline", + "--project", + str(runtime_project), + "--frozen", + "--no-dev", + "--no-install-project", ], - ["uv", "venv", "--offline", "--python", sys.executable, str(prefix)], - ["uv", "pip", "sync", "--offline", "--python", str(python), str(runtime_lock)], + cwd=work, + ) + _run( [ "uv", "pip", @@ -58,20 +113,11 @@ def test_built_distribution_installs_and_runs_in_an_isolated_runtime(kind: str) "--no-deps", "--python", str(python), - str(archive), + str(installable), ], - ["uv", "pip", "check", "--python", str(python)], - ] - for command in commands: - result = subprocess.run( - command, - cwd=work, - capture_output=True, - text=True, - timeout=60, - check=False, - ) - assert result.returncode == 0, result.stdout + result.stderr + cwd=work, + ) + _run(["uv", "pip", "check", "--python", str(python)], cwd=work) report_path = work / "acceptance.json" result = subprocess.run( @@ -98,7 +144,10 @@ def test_built_distribution_installs_and_runs_in_an_isolated_runtime(kind: str) assert report["ok"] and report["isolated"] assert report["sha256"] == digest assert report["version"] == manifest["version"] - assert report["credential_paths"] == ["/credentials/inject", "/credentials/inject"] + assert report["credential_paths"] == [ + "/credentials/inject", + "/credentials/inject", + ] assert report["private_input_redacted"] is True assert report["python_version"] == list(sys.version_info[:3]) assert Path(report["executable"]).resolve() == python.resolve() From 0b2ba62630c094d5f25b70850c3ff57996fbde47 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 12:24:30 +0000 Subject: [PATCH 233/464] security(foundation): support older Landlock ABIs --- CHANGELOG.md | 5 +++ .../src/unix/worker_sandbox/linux.rs | 41 ++++++++++------- .../src/unix/worker_sandbox/macos.rs | 19 ++++++-- .../src/unix/worker_sandbox/tests.rs | 44 ++++++++++++++++--- tests/citadel/unix_boundary_debt.toml | 4 +- 5 files changed, 85 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ba7f3ea8..2bac89306 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Keep host-worker confinement fail-closed across supported Linux Landlock + ABIs, including single-threaded startup on ABI 6 and 7, and canonicalize + macOS Seatbelt grants so system path aliases cannot silently deny or widen + the intended authority. + - Bind each standalone model-proxy lease to one service-selected provider and one revocable worker generation; client traffic receives only connected descriptors and cannot choose an upstream with `Host` or proxy semantics. diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs index b03031f3a..4c943644f 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs @@ -8,6 +8,7 @@ const CREATE_RULESET_VERSION: u32 = 1; const RESTRICT_SELF_TSYNC: u32 = 1 << 3; const RULE_PATH_BENEATH: u32 = 1; const MINIMUM_ABI: i32 = 6; +const TSYNC_ABI: i32 = 8; const FS_EXECUTE: u64 = 1 << 0; const FS_WRITE_FILE: u64 = 1 << 1; @@ -70,16 +71,16 @@ struct PathBeneathAttr { } pub(super) fn confine(policy: &Policy) -> io::Result<()> { - let ruleset = create_ruleset()?; + let (ruleset, abi) = create_ruleset()?; for rule in policy.paths() { add_path_rule(&ruleset, rule.path(), rule.access())?; } nix::sys::prctl::set_no_new_privs().map_err(super::super::errno::io)?; - restrict_all_threads(&ruleset)?; + restrict_all_threads(&ruleset, abi)?; install_seccomp(policy.role()) } -fn create_ruleset() -> io::Result { +fn create_ruleset() -> io::Result<(OwnedFd, i32)> { // nix has no Landlock API. The kernel reads no attribute when VERSION is // set, and returns the supported ABI directly. let abi = unsafe { @@ -118,7 +119,7 @@ fn create_ruleset() -> io::Result { return Err(io::Error::last_os_error()); } // SAFETY: a successful create_ruleset result is a new owned descriptor. - Ok(unsafe { OwnedFd::from_raw_fd(descriptor as i32) }) + Ok((unsafe { OwnedFd::from_raw_fd(descriptor as i32) }, abi as i32)) } fn add_path_rule(ruleset: &OwnedFd, path: &std::path::Path, access: Access) -> io::Result<()> { @@ -173,23 +174,33 @@ fn add_path_rule(ruleset: &OwnedFd, path: &std::path::Path, access: Access) -> i Ok(()) } -fn restrict_all_threads(ruleset: &OwnedFd) -> io::Result<()> { - // nix has no Landlock API. TSYNC is mandatory: workers create their Tokio - // runtime before confinement, and a single ambient sibling thread would - // invalidate the process boundary. - let result = unsafe { - libc::syscall( - libc::SYS_landlock_restrict_self, - ruleset.as_raw_fd(), - RESTRICT_SELF_TSYNC, - ) - }; +fn restrict_all_threads(ruleset: &OwnedFd, abi: i32) -> io::Result<()> { + let thread_count = + std::fs::read_dir("/proc/self/task")?.try_fold(0usize, |count, entry| entry.map(|_| count + 1))?; + let flags = restriction_flags_for(abi, thread_count)?; + // nix has no Landlock API. ABI 8 introduced TSYNC. Older supported + // kernels can safely confine only a process that has not created sibling + // threads yet; descendants inherit the caller's Landlock domain. + let result = unsafe { libc::syscall(libc::SYS_landlock_restrict_self, ruleset.as_raw_fd(), flags) }; if result < 0 { return Err(io::Error::last_os_error()); } Ok(()) } +pub(super) fn restriction_flags_for(abi: i32, thread_count: usize) -> io::Result { + if abi >= TSYNC_ABI { + return Ok(RESTRICT_SELF_TSYNC); + } + if thread_count != 1 { + return Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("Landlock ABI {abi} requires single-threaded worker startup; found {thread_count} threads"), + )); + } + Ok(0) +} + fn install_seccomp(role: Role) -> io::Result<()> { #[cfg(target_arch = "aarch64")] let architecture = 0xc00000b7; diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/macos.rs b/crates/capsem-foundation/src/unix/worker_sandbox/macos.rs index af2e13894..ec012cbcd 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/macos.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/macos.rs @@ -13,8 +13,8 @@ unsafe extern "C" { } pub(super) fn confine(policy: &Policy) -> io::Result<()> { - validate_paths(policy)?; - let compiled = super::seatbelt::compile(policy)?; + let policy = canonical_policy(policy)?; + let compiled = super::seatbelt::compile(&policy)?; let parameters = compiled.parameter_ptrs(); let mut error = std::ptr::null_mut(); // SAFETY: source and every alternating key/value parameter are NUL @@ -36,7 +36,8 @@ pub(super) fn confine(policy: &Policy) -> io::Result<()> { Err(io::Error::other(message)) } -fn validate_paths(policy: &Policy) -> io::Result<()> { +pub(super) fn canonical_policy(policy: &Policy) -> io::Result { + let mut canonical = Policy::new(policy.role()); for rule in policy.paths() { let metadata = std::fs::symlink_metadata(rule.path())?; if metadata.file_type().is_symlink() { @@ -51,6 +52,16 @@ fn validate_paths(policy: &Policy) -> io::Result<()> { format!("sandbox executable grant is not a file: {}", rule.path().display()), )); } + let path = std::fs::canonicalize(rule.path())?; + let canonical_metadata = std::fs::metadata(&path)?; + use std::os::unix::fs::MetadataExt; + if metadata.dev() != canonical_metadata.dev() || metadata.ino() != canonical_metadata.ino() { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("sandbox grant changed while it was prepared: {}", rule.path().display()), + )); + } + canonical = canonical.allow(path, rule.access()); } - Ok(()) + Ok(canonical) } diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index 430ceac0f..ba334e0a5 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -72,6 +72,30 @@ fn seatbelt_proxy_is_deny_by_default_and_descriptor_only() { assert!(compiled.parameters().is_empty()); } +#[cfg(target_os = "macos")] +#[test] +fn macos_policy_compiles_canonical_grant_paths() { + let directory = tempfile::tempdir().unwrap(); + let policy = Policy::new(Role::Ledger).allow(directory.path(), Access::ReadWrite); + let canonical = super::platform::canonical_policy(&policy).unwrap(); + assert_eq!(canonical.paths()[0].path(), directory.path().canonicalize().unwrap()); +} + +#[cfg(target_os = "linux")] +#[test] +fn old_landlock_abis_require_single_threaded_startup() { + assert_eq!(super::platform::restriction_flags_for(6, 1).unwrap(), 0); + let error = super::platform::restriction_flags_for(7, 2).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::Unsupported); + assert!(error.to_string().contains("single-threaded worker startup")); +} + +#[cfg(target_os = "linux")] +#[test] +fn landlock_abi_eight_synchronizes_existing_threads() { + assert_ne!(super::platform::restriction_flags_for(8, usize::MAX).unwrap(), 0); +} + #[cfg(target_os = "linux")] #[test] fn linux_policy_preserves_grants_and_denies_ambient_authority() { @@ -116,7 +140,7 @@ fn linux_policy_preserves_grants_and_denies_ambient_authority() { let status = child.wait().unwrap(); assert!(status.success(), "sandbox child failed: {status}"); read.unwrap(); - assert_eq!(&reply, b"pong"); + assert!(matches!(&reply, b"pong" | b"safe")); } #[cfg(target_os = "linux")] @@ -133,19 +157,24 @@ fn linux_sandbox_child() { let allowed = std::path::PathBuf::from(allowed); let stdin = std::io::stdin(); let mut inherited = TcpStream::from(super::super::fd::duplicate(stdin.as_fd()).unwrap()); - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) + // Tokio prepares its signal-driver socketpair while building a runtime. + // A current-thread runtime owns no sibling threads before confinement; + // its later blocking workers inherit the installed policy. + let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap(); - - confine( + if let Err(error) = confine( &Policy::new(Role::Ledger) .allow(&allowed, Access::ReadWrite) .allow(&readonly, Access::ReadOnly) .allow(&exact, Access::ReadOnly), - ) - .unwrap(); + ) { + assert_eq!(error.kind(), std::io::ErrorKind::Unsupported); + assert!(error.to_string().contains("single-threaded worker startup")); + inherited.write_all(b"safe").unwrap(); + std::process::exit(0); + } runtime.block_on(async { tokio::task::spawn_blocking(move || { @@ -204,6 +233,7 @@ fn macos_policy_preserves_grants_and_denies_ambient_authority() { .env("WORKER_SANDBOX_DENIED", &denied) .env("WORKER_SANDBOX_CONTROL", &control) .env("WORKER_SANDBOX_PORT", port.to_string()) + .env("LLVM_PROFILE_FILE", allowed.join("profile-%p-%m.profraw")) .stdin(Stdio::from(std::os::fd::OwnedFd::from(socket))) .spawn() .unwrap(); diff --git a/tests/citadel/unix_boundary_debt.toml b/tests/citadel/unix_boundary_debt.toml index cc6c699df..6aad41747 100644 --- a/tests/citadel/unix_boundary_debt.toml +++ b/tests/citadel/unix_boundary_debt.toml @@ -142,8 +142,8 @@ fingerprint = "d14267a19410254a6633ba264e8046ea3dbd9c9046b72006155d76ee76db008f" reason = "seccomp(2) filter install with TSYNC; nix has no seccomp" [foundation.raw_calls."crates/capsem-foundation/src/unix/worker_sandbox/linux.rs::syscall"] -fingerprint = "47970efa29a92b2183616ae27524b3453bc9c3abdd320c180f24a4423dd7488d" -reason = "Landlock rules and thread-synchronized seccomp installation have no nix API" +fingerprint = "7286c55845201710a915c8334492ab2a9cb35c11f6bcb1ebb4843554cf1f8986" +reason = "Landlock ABI-aware rules and thread-synchronized seccomp installation have no nix API" [foundation.raw_calls."crates/capsem-foundation/src/unix/worker_sandbox/macos.rs::sandbox_free_error"] fingerprint = "429f4efde365972763d15c7976b45af1d29819c8342fa1c4da5be3e123555fdb" From e09d00ecdb09af97277dba7d7ac63c98c704c9b9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 12:28:41 +0000 Subject: [PATCH 234/464] security(logger): prepare ledger before starting workers --- crates/capsem-logger/src/db.rs | 32 ++++- crates/capsem-logger/src/ledger_server.rs | 29 ++++- .../capsem-logger/src/ledger_server/tests.rs | 11 ++ crates/capsem-logger/src/writer.rs | 113 +++++++++++++----- 4 files changed, 145 insertions(+), 40 deletions(-) diff --git a/crates/capsem-logger/src/db.rs b/crates/capsem-logger/src/db.rs index f70c596b2..9a135b887 100644 --- a/crates/capsem-logger/src/db.rs +++ b/crates/capsem-logger/src/db.rs @@ -10,7 +10,7 @@ use capsem_telemetry::db::{ }; use crate::reader::DbReader; -use crate::writer::{DbWriter, WriteOp}; +use crate::writer::{DbWriter, PreparedDbWriter, WriteOp}; /// Public DB-boundary contract for Capsem session ledgers. /// @@ -235,6 +235,12 @@ struct DbHandleInner { query_many_pause: Mutex>, } +pub(crate) struct PreparedDbHandle { + path: PathBuf, + writer: PreparedDbWriter, + codecs: ArchiveCodecs, +} + /// Signals "the lookup is done" and waits for "go on". #[cfg(test)] type QueryManyPause = (tokio::sync::oneshot::Sender<()>, tokio::sync::oneshot::Receiver<()>); @@ -259,22 +265,29 @@ impl DbHandle { } pub fn open_with_codecs(path: &Path, codecs: ArchiveCodecs) -> rusqlite::Result { + Self::prepare_with_codecs(path, codecs).map(PreparedDbHandle::start) + } + + pub(crate) fn prepare_with_codecs(path: &Path, codecs: ArchiveCodecs) -> rusqlite::Result { let started = Instant::now(); - let writer = Arc::new(DbWriter::open_with_codecs(path, 1024, codecs.clone())?); + let writer = DbWriter::prepare_with_codecs(path, 1024, codecs.clone())?; // Reads go to the file, like every other handle's: the writer's // memory holds only rows it has not flushed, and SQLite's // `data_version` tells the reader when a flush landed. DbReader::open(path)?; - let handle = Self::open_with_writer(path.to_path_buf(), writer, codecs)?; tracing::debug!( db_path = %path.display(), - operation = "open", + operation = "prepare", duration_ms = elapsed_ms(started), - "session db handle opened" + "session db handle prepared" ); - Ok(handle) + Ok(PreparedDbHandle { + path: path.to_path_buf(), + writer, + codecs, + }) } /// Open a DB handle for a session DB written by another process. @@ -825,6 +838,13 @@ impl DbHandle { } } +impl PreparedDbHandle { + pub(crate) fn start(self) -> DbHandle { + DbHandle::open_with_writer(self.path, Arc::new(self.writer.start()), self.codecs) + .expect("prepared database reader must start") + } +} + impl SessionDb { /// Create a new SessionDb pointing at the given path. /// Does not open any connections; call `writer()` or `reader()` as needed. diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index a48a52b4d..a88e6422d 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -11,6 +11,7 @@ use capsem_proto::ledger::{ LedgerChannelGrant, LedgerFailure, LedgerFailureCode, LedgerRequest, LedgerResponse, LedgerWelcome, }; +use crate::db::PreparedDbHandle; use crate::ledger_protocol::{ LedgerBodyMetadata, LedgerClientMessage, LedgerCommand, LedgerExportSummary, LedgerReply, LedgerServerMessage, MAX_LEDGER_STREAM_CHUNK_BYTES, @@ -39,16 +40,28 @@ pub struct LedgerServer { admission: tokio::sync::Mutex, } +/// A ledger whose files, schema and exclusive writer authority are prepared, +/// but whose database workers have not started yet. Confining worker processes +/// between [`LedgerServer::prepare_with_codecs`] and [`Self::start`] ensures +/// every database thread inherits the process sandbox. +pub struct PreparedLedgerServer { + db: PreparedDbHandle, + session_dir: PathBuf, +} + impl LedgerServer { pub fn open(path: &Path) -> rusqlite::Result { Self::open_with_codecs(path, ArchiveCodecs::default()) } pub fn open_with_codecs(path: &Path, codecs: ArchiveCodecs) -> rusqlite::Result { - Ok(Self { - db: Arc::new(DbHandle::open_with_codecs(path, codecs)?), + Self::prepare_with_codecs(path, codecs).map(PreparedLedgerServer::start) + } + + pub fn prepare_with_codecs(path: &Path, codecs: ArchiveCodecs) -> rusqlite::Result { + Ok(PreparedLedgerServer { + db: DbHandle::prepare_with_codecs(path, codecs)?, session_dir: path.parent().unwrap_or_else(|| Path::new(".")).to_path_buf(), - admission: tokio::sync::Mutex::new(AdmissionState::default()), }) } @@ -275,6 +288,16 @@ impl LedgerServer { } } +impl PreparedLedgerServer { + pub fn start(self) -> LedgerServer { + LedgerServer { + db: Arc::new(self.db.start()), + session_dir: self.session_dir, + admission: tokio::sync::Mutex::new(AdmissionState::default()), + } + } +} + #[must_use] pub fn snapshot_directory_name(snapshot_id: [u8; 16]) -> String { format!(".ledger-snapshot-{}", uuid::Uuid::from_bytes(snapshot_id).simple()) diff --git a/crates/capsem-logger/src/ledger_server/tests.rs b/crates/capsem-logger/src/ledger_server/tests.rs index e6b313272..ea7ffb700 100644 --- a/crates/capsem-logger/src/ledger_server/tests.rs +++ b/crates/capsem-logger/src/ledger_server/tests.rs @@ -92,6 +92,17 @@ fn body_event(body: Vec) -> WriteOp { }) } +#[test] +fn prepared_server_holds_writer_authority_until_workers_start() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("session.db"); + let prepared = LedgerServer::prepare_with_codecs(&database, capsem_archive::ArchiveCodecs::default()).unwrap(); + assert!(LedgerServer::prepare_with_codecs(&database, capsem_archive::ArchiveCodecs::default()).is_err()); + let server = prepared.start(); + drop(server); + drop(LedgerServer::prepare_with_codecs(&database, capsem_archive::ArchiveCodecs::default()).unwrap()); +} + #[tokio::test] async fn accepted_durable_query_body_retention_and_warc_run_through_one_owner() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 7486e5fdf..fd4ba3fa9 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -303,8 +303,9 @@ impl WriteEnvelope { const _: () = assert!(std::mem::size_of::() <= 1024); type WriterSender = mpsc::SyncSender; +type WriterReceiver = mpsc::Receiver; -fn writer_channel(capacity: usize) -> (WriterSender, mpsc::Receiver) { +fn writer_channel(capacity: usize) -> (WriterSender, WriterReceiver) { mpsc::sync_channel(capacity.max(1)) } @@ -354,6 +355,18 @@ pub struct DbWriter { remote: Option, } +pub(crate) struct PreparedDbWriter { + tx: WriterSender, + rx: WriterReceiver, + held: capsem_foundation::unix::lock::FileLock, + conn: Connection, + db_path: PathBuf, + batch_capacity: usize, + pending_body_bytes: Arc, + bodies: BodyArchive, + tally: LedgerTally, +} + impl DbWriter { /// Spawn a dedicated writer thread that owns the DB connection. /// `capacity` controls the mpsc channel size (backpressure). @@ -362,7 +375,7 @@ impl DbWriter { } pub fn open_with_codecs(path: &Path, capacity: usize, codecs: ArchiveCodecs) -> rusqlite::Result { - Self::open_with_clock_and_codecs(path, capacity, SystemTime::now, codecs) + Self::prepare_with_clock_and_codecs(path, capacity, SystemTime::now, codecs).map(PreparedDbWriter::start) } /// `open`, with the archive index's timestamps read from `now`. @@ -370,23 +383,31 @@ impl DbWriter { /// Only a replay of an existing ledger has any business supplying one; see /// `LedgerClock`. pub fn open_with_clock(path: &Path, capacity: usize, now: LedgerClock) -> rusqlite::Result { - Self::open_with_clock_and_codecs(path, capacity, now, ArchiveCodecs::default()) + Self::prepare_with_clock_and_codecs(path, capacity, now, ArchiveCodecs::default()).map(PreparedDbWriter::start) + } + + pub(crate) fn prepare_with_codecs( + path: &Path, + capacity: usize, + codecs: ArchiveCodecs, + ) -> rusqlite::Result { + Self::prepare_with_clock_and_codecs(path, capacity, SystemTime::now, codecs) } - fn open_with_clock_and_codecs( + fn prepare_with_clock_and_codecs( path: &Path, capacity: usize, now: LedgerClock, codecs: ArchiveCodecs, - ) -> rusqlite::Result { + ) -> rusqlite::Result { if let Some(parent) = path.parent() { let _ = std::fs::create_dir_all(parent); } let mut last_busy = None; for _ in 0..50 { - match Self::open_once(path, capacity, now, codecs.clone()) { - Ok(writer) => return Ok(writer), + match Self::prepare_once(path, capacity, now, codecs.clone()) { + Ok(prepared) => return Ok(prepared), Err(error) if is_sqlite_busy(&error) => { last_busy = Some(error); std::thread::sleep(Duration::from_millis(20)); @@ -400,7 +421,12 @@ impl DbWriter { ))) } - fn open_once(path: &Path, capacity: usize, now: LedgerClock, codecs: ArchiveCodecs) -> rusqlite::Result { + fn prepare_once( + path: &Path, + capacity: usize, + now: LedgerClock, + codecs: ArchiveCodecs, + ) -> rusqlite::Result { let held = writer_lock::acquire(path)?; legacy::set_aside_pre_archive_ledger(path)?; let flags = OpenFlags::SQLITE_OPEN_READ_WRITE @@ -442,32 +468,17 @@ impl DbWriter { let (tx, rx) = writer_channel(schedule.capacity); let db_path = path.to_path_buf(); - let writer_loop_db_path = Some(db_path.clone()); let pending_body_bytes = Arc::new(AtomicU64::new(0)); - let loop_pending_body_bytes = Arc::clone(&pending_body_bytes); - - let join_handle = std::thread::Builder::new() - .name("capsem-db-writer".into()) - .spawn(move || { - let _held = held; - writer_loop( - conn, - rx, - writer_loop_db_path, - schedule, - &loop_pending_body_bytes, - bodies, - tally, - ) - }) - .expect("failed to spawn db writer thread"); - - Ok(Self { - tx: std::sync::Mutex::new(Some(tx)), - join_handle: std::sync::Mutex::new(Some(join_handle)), + Ok(PreparedDbWriter { + tx, + rx, + held, + conn, db_path, + batch_capacity, pending_body_bytes, - remote: None, + bodies, + tally, }) } @@ -643,6 +654,46 @@ impl DbWriter { } } +impl PreparedDbWriter { + pub(crate) fn start(self) -> DbWriter { + let Self { + tx, + rx, + held, + conn, + db_path, + batch_capacity, + pending_body_bytes, + bodies, + tally, + } = self; + let writer_loop_db_path = Some(db_path.clone()); + let loop_pending_body_bytes = Arc::clone(&pending_body_bytes); + let join_handle = std::thread::Builder::new() + .name("capsem-db-writer".into()) + .spawn(move || { + let _held = held; + writer_loop( + conn, + rx, + writer_loop_db_path, + batch_capacity, + &loop_pending_body_bytes, + bodies, + tally, + ) + }) + .expect("failed to spawn db writer thread"); + DbWriter { + tx: std::sync::Mutex::new(Some(tx)), + join_handle: std::sync::Mutex::new(Some(join_handle)), + db_path, + pending_body_bytes, + remote: None, + } + } +} + impl Drop for DbWriter { fn drop(&mut self) { self.shutdown_blocking(); From b357a6d1236c4652e5e1718277378129f08c6685 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 12:30:04 +0000 Subject: [PATCH 235/464] security(ledger): start workers after confinement --- crates/capsem-ledger/src/main.rs | 26 ++++++++++++++++++------ crates/capsem-ledger/tests/subprocess.rs | 2 +- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/crates/capsem-ledger/src/main.rs b/crates/capsem-ledger/src/main.rs index 96952201b..1a608f694 100644 --- a/crates/capsem-ledger/src/main.rs +++ b/crates/capsem-ledger/src/main.rs @@ -53,19 +53,22 @@ fn main() -> Result<()> { default_filter: "capsem_ledger=info,capsem_logger=info,capsem_foundation=warn", })?; let args = Args::parse(); - capsem_guard::watch_parent_or_exit(Some(args.parent_pid))?; let stdin = io::stdin(); let control = UnixStream::from(fd::duplicate(stdin.as_fd())?); let session_dir = session_directory(&args.database)?; + validate_session_directory(&session_dir)?; let denied_file = std::env::current_exe().context("resolve ledger executable before confinement")?; let codecs = codec::archive_codecs().context("initialize confined archive codecs")?; - let server = Arc::new(LedgerServer::open_with_codecs(&args.database, codecs).context("open session ledger")?); - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build()?; + let prepared = LedgerServer::prepare_with_codecs(&args.database, codecs).context("prepare session ledger")?; + // The current-thread runtime prepares Tokio's internal descriptors without + // creating sibling threads. This keeps Landlock ABI 6 and 7 safe while + // allowing tasks and blocking workers created after confinement to inherit + // the worker's authority. + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Ledger).allow(&session_dir, Access::ReadWrite)) .context("confine ledger worker before readiness")?; + capsem_guard::watch_parent_or_exit(Some(args.parent_pid))?; + let server = Arc::new(prepared.start()); attest_confinement(&session_dir, &denied_file, args.parent_pid, args.generation)?; runtime.block_on(run_control(control, server, args.generation, CLIENT_LIMIT)) } @@ -81,6 +84,17 @@ fn session_directory(database: &Path) -> Result { Ok(directory.to_path_buf()) } +fn validate_session_directory(directory: &Path) -> Result<()> { + let metadata = std::fs::symlink_metadata(directory).context("inspect ledger session directory")?; + if metadata.file_type().is_symlink() { + bail!("sandbox grant is a symlink: {}", directory.display()); + } + if !metadata.is_dir() { + bail!("ledger session path is not a directory: {}", directory.display()); + } + Ok(()) +} + fn attest_confinement( session_dir: &Path, denied_file: &Path, diff --git a/crates/capsem-ledger/tests/subprocess.rs b/crates/capsem-ledger/tests/subprocess.rs index 8c84a564b..64e9e88ba 100644 --- a/crates/capsem-ledger/tests/subprocess.rs +++ b/crates/capsem-ledger/tests/subprocess.rs @@ -247,7 +247,7 @@ async fn executable_refuses_a_symlinked_session_before_readiness() { assert!(!output.status.success()); let stderr = String::from_utf8_lossy(&output.stderr); assert!( - stderr.contains("open session ledger") || stderr.contains("confine ledger worker before readiness"), + stderr.contains("sandbox grant is a symlink"), "unexpected refusal: {}", stderr ); From 5f85cc30714247541ff7edd7fe82008420d94274 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 12:32:00 +0000 Subject: [PATCH 236/464] security(proxy): confine before starting worker threads --- crates/capsem-proxy/src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 310e3aed5..624fb1f73 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -67,16 +67,16 @@ fn main() -> Result<()> { default_filter: "capsem_proxy=info,capsem_core=info,capsem_foundation=warn", })?; let args = Args::parse(); - capsem_guard::watch_parent_or_exit(Some(args.parent_pid))?; let stdin = io::stdin(); let control = UnixStream::from(fd::duplicate(stdin.as_fd())?); let denied_file = std::env::current_exe().context("resolve proxy executable before confinement")?; - let runtime = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build()?; + // Prepare Tokio's internal descriptors without creating sibling threads. + // Tasks and blocking workers born after confinement inherit the policy on + // Linux kernels whose Landlock ABI predates thread synchronization. + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Proxy)) .context("confine proxy worker before readiness")?; + capsem_guard::watch_parent_or_exit(Some(args.parent_pid))?; attest_confinement(&denied_file, args.parent_pid)?; runtime.block_on(run_control(control, args.generation, args.standalone_provider)) } From d9c94fe81436aeaadd496de5774cc7de40df9d98 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 12:32:00 +0000 Subject: [PATCH 237/464] test(proxy): authenticate ledger commitment channel --- crates/capsem-proxy/tests/subprocess.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs index 07ca4ee29..2978ba839 100644 --- a/crates/capsem-proxy/tests/subprocess.rs +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -4,8 +4,10 @@ use std::os::unix::net::UnixStream; use std::process::{Child, Command, Stdio}; use std::time::Duration; +use capsem_foundation::ipc_channel; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; -use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration, LedgerWelcome}; +use capsem_proto::ledger_commitment::{CommitmentClientMessage, CommitmentServerMessage}; use capsem_proto::proxy_control::{ decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyChannelGrant, ProxyControlEvent, ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, @@ -232,16 +234,34 @@ async fn ledger_grant_is_authenticated_before_adoption() { } }); let (server_stream, granted) = UnixStream::pair().unwrap(); + let (commitment_server, commitment_granted) = UnixStream::pair().unwrap(); let ledger_grant = LedgerChannelGrant::new(LedgerGeneration::new([4; 16]), 17, LedgerClientRole::Proxy).unwrap(); + let commitment_serving = tokio::spawn(async move { + let (sender, receiver) = + ipc_channel::channel_from_std::(commitment_server) + .unwrap(); + let CommitmentClientMessage::Hello { hello } = receiver.recv().await.unwrap() else { + panic!("commitment client omitted hello") + }; + ledger_grant.validate_hello(&hello).unwrap(); + sender + .send(CommitmentServerMessage::Welcome { + welcome: LedgerWelcome::for_grant(&ledger_grant), + }) + .await + .unwrap(); + while receiver.recv().await.is_ok() {} + }); let grant = ProxyChannelGrant::with_ledger(GENERATION, 1, ledger_grant).unwrap(); requests .send( &encode_proxy_control_request(ProxyControlRequest::Attach(grant)), - &[granted.as_raw_fd()], + &[granted.as_raw_fd(), commitment_granted.as_raw_fd()], ) .await .unwrap(); drop(granted); + drop(commitment_granted); assert!(tokio::time::timeout(Duration::from_millis(100), event_rx.recv()) .await @@ -289,6 +309,7 @@ async fn ledger_grant_is_authenticated_before_adoption() { ); event_pump.await.unwrap(); assert!(serving.await.unwrap().is_ok()); + commitment_serving.await.unwrap(); let child = child.take(); let output = tokio::task::spawn_blocking(move || child.wait_with_output()) .await From 315447fb37af114a9d751f55aea65df24a2e0103 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 12:34:22 +0000 Subject: [PATCH 238/464] security(foundation): prepare logs before confinement --- crates/capsem-foundation/src/telemetry.rs | 17 +++++++++++++++++ crates/capsem-foundation/src/telemetry/tests.rs | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/crates/capsem-foundation/src/telemetry.rs b/crates/capsem-foundation/src/telemetry.rs index bb03611e2..09e7587cd 100644 --- a/crates/capsem-foundation/src/telemetry.rs +++ b/crates/capsem-foundation/src/telemetry.rs @@ -41,6 +41,18 @@ pub enum LogSink { /// File (json) + stderr (pretty). Used by capsem-app so the file /// feeds the support bundle and stderr feeds `pnpm tauri dev` output. FileAndPretty { path: PathBuf }, + /// Write JSON-per-line to an already-open private file. A worker that must + /// confine before creating the non-blocking writer thread prepares this + /// descriptor while it still has startup filesystem authority. + PreparedFile { file: std::fs::File }, +} + +/// Open a private append-only log before a worker installs filesystem +/// confinement. Passing the returned sink to [`init`] starts the writer thread +/// later, so that thread inherits the worker sandbox. +pub fn prepare_file_sink(path: &std::path::Path) -> std::io::Result { + let file = crate::unix::fs::open_private_append_no_follow(path)?; + Ok(LogSink::PreparedFile { file }) } /// Static per-binary telemetry config. `service` is the binary name (also @@ -425,6 +437,11 @@ pub fn init(cfg: TelemetryConfig) -> std::io::Result { .with(stderr_pretty_layer()) .init(); } + LogSink::PreparedFile { file } => { + let (nb, guard) = tracing_appender::non_blocking(file); + file_guard = Some(guard); + registry.with(fmt::layer().json().with_writer(nb).boxed()).init(); + } } // Once the subscriber is wired, emit a "service started" line that diff --git a/crates/capsem-foundation/src/telemetry/tests.rs b/crates/capsem-foundation/src/telemetry/tests.rs index f574bfabc..eebbd0c30 100644 --- a/crates/capsem-foundation/src/telemetry/tests.rs +++ b/crates/capsem-foundation/src/telemetry/tests.rs @@ -106,6 +106,20 @@ fn rolling_parts_survives_a_path_with_no_extension_or_parent() { assert_eq!(suffix, "log"); } +#[test] +fn prepared_file_sink_opens_one_private_unrotated_log() { + use std::os::unix::fs::PermissionsExt; + + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("worker.log"); + let LogSink::PreparedFile { file } = prepare_file_sink(&path).unwrap() else { + panic!("prepared sink returned the wrong variant") + }; + assert!(file.metadata().unwrap().is_file()); + assert_eq!(file.metadata().unwrap().permissions().mode() & 0o777, 0o600); + assert!(path.is_file()); +} + #[test] fn rolling_appender_writes_a_dated_file_beside_the_requested_path() { use std::io::Write; From 77e8c1ce3eb86110bb230fd7f1df6855e8335cd4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 12:36:28 +0000 Subject: [PATCH 239/464] security(gateway): confine before starting runtime threads --- crates/capsem-gateway/src/main.rs | 99 +++++++++++++++--------------- crates/capsem-gateway/src/tests.rs | 16 ++++- 2 files changed, 66 insertions(+), 49 deletions(-) diff --git a/crates/capsem-gateway/src/main.rs b/crates/capsem-gateway/src/main.rs index 5424eb7b3..387f2c972 100644 --- a/crates/capsem-gateway/src/main.rs +++ b/crates/capsem-gateway/src/main.rs @@ -81,7 +81,7 @@ fn main() -> Result<()> { // SAFETY: argument parsing has opened no descriptors or threads; process // entry still precedes telemetry, descriptor owners and runtime threads. unsafe { capsem_foundation::unix::fd::close_inherited_descriptors()? }; - let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; runtime.block_on(run(args)) } @@ -92,54 +92,25 @@ async fn run(args: Args) -> Result<()> { ); let run_dir = gateway_run_dir(&args); let _ = std::fs::create_dir_all(&run_dir); - let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { - service: "capsem-gateway", - sink: capsem_foundation::telemetry::LogSink::File { - path: run_dir.join("gateway.log"), - }, - // tower_http + hyper at debug so request-level and connection-level - // failures (parse errors, early RST, malformed headers) land in the - // gateway log; without these, auth-path flakes surface as curl "000" - // with nothing on the gateway side to explain it. - default_filter: "capsem_gateway=info,tower_http=debug,hyper=info", - })?; - - // Surface any gateway panic in the log instead of letting it vanish into - // the void -- under test load a panicked task would otherwise just drop - // the connection, leaving the client with no response and no trace. - capsem_foundation::telemetry::install_panic_logger("capsem-gateway"); + let log_path = run_dir.join("gateway.log"); + let log_sink = capsem_foundation::telemetry::prepare_file_sink(&log_path)?; // Companion guards: refuse to run without a live parent service, and // refuse if another gateway already holds the singleton lock for this // run_dir. Both conditions are expected (stale launch, double-spawn race) // and resolved by exiting 0 -- standalone launches become no-ops. let lock_path = args.lock_path.clone().unwrap_or_else(|| run_dir.join("gateway.lock")); - match capsem_guard::install(args.parent_pid, &lock_path) { - Ok(Some(guards)) => { - // Keep the guards alive for the process's lifetime. - Box::leak(Box::new(guards)); - } - Ok(None) => { - tracing::info!( - lock = %lock_path.display(), - "another capsem-gateway is already running; exiting 0" - ); - return Ok(()); - } - Err(e) => { - tracing::info!( - error = %e, - "gateway refusing to run without a live capsem-service; exiting 0" - ); - return Ok(()); - } - } + let Some(parent_pid) = args + .parent_pid + .filter(|parent_pid| capsem_guard::parent_is_expected(*parent_pid)) + else { + return Ok(()); + }; + let Some(_singleton) = capsem_guard::Singleton::try_acquire(&lock_path)? else { + return Ok(()); + }; let uds_path = args.uds_path.unwrap_or_else(|| run_dir.join("service.sock")); - - // Check if service socket exists (warning only -- service may start later) - if !uds_path.exists() { - tracing::warn!(path = %uds_path.display(), "service socket not found -- requests will return 502 until service starts"); - } + let service_socket_missing = !uds_path.exists(); // Bind TCP listener first so the runtime file records the real bound port // (args.port may be 0 to request an OS-assigned port). @@ -163,6 +134,24 @@ async fn run(args: Args) -> Result<()> { // SAFETY: the service gives this child sole ownership of stdin as its // descriptor grant socket. No standard-input owner has been created. let grant_socket = unsafe { std::os::unix::net::UnixStream::from_raw_fd(0) }; + + confine_gateway(&auth_state, &uds_path, listener.local_addr()?, &log_path)?; + + let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { + service: "capsem-gateway", + sink: log_sink, + // tower_http + hyper at debug so request-level and connection-level + // failures (parse errors, early RST, malformed headers) land in the + // gateway log; without these, auth-path flakes surface as curl "000" + // with nothing on the gateway side to explain it. + default_filter: "capsem_gateway=info,tower_http=debug,hyper=info", + })?; + capsem_foundation::telemetry::install_panic_logger("capsem-gateway"); + capsem_guard::watch_parent_or_exit(Some(parent_pid))?; + + if service_socket_missing { + tracing::warn!(path = %uds_path.display(), "service socket not found -- requests will return 502 until service starts"); + } let service_client = ServiceClient::granted(service_grant::GatewayGrantClient::start(grant_socket)?); let state = Arc::new(AppState { token, @@ -174,8 +163,6 @@ async fn run(args: Args) -> Result<()> { previews: preview::PreviewState::new(preview_port), }); - confine_gateway(&auth_state, &uds_path, listener.local_addr()?)?; - let preview_state = state.clone(); tokio::spawn(async move { preview::serve(preview_listener, preview_state).await; @@ -229,7 +216,12 @@ async fn run(args: Args) -> Result<()> { } #[cfg(target_os = "linux")] -fn confine_gateway(_auth_state: &AuthState, uds_path: &std::path::Path, listener: SocketAddr) -> Result<()> { +fn confine_gateway( + _auth_state: &AuthState, + uds_path: &std::path::Path, + listener: SocketAddr, + _log_path: &std::path::Path, +) -> Result<()> { use capsem_foundation::unix::worker_sandbox::{Policy, Role}; capsem_foundation::unix::worker_sandbox::confine(&Policy::new(Role::Gateway)) .context("confine gateway before readiness")?; @@ -237,13 +229,19 @@ fn confine_gateway(_auth_state: &AuthState, uds_path: &std::path::Path, listener } #[cfg(target_os = "macos")] -fn confine_gateway(auth_state: &AuthState, uds_path: &std::path::Path, listener: SocketAddr) -> Result<()> { +fn confine_gateway( + auth_state: &AuthState, + uds_path: &std::path::Path, + listener: SocketAddr, + log_path: &std::path::Path, +) -> Result<()> { use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; let policy = Policy::new(Role::Gateway) .allow(&auth_state.token_path, Access::ReadWrite) .allow(&auth_state.port_path, Access::ReadWrite) .allow(&auth_state.pid_path, Access::ReadWrite) - .allow(&auth_state.preview_port_path, Access::ReadWrite); + .allow(&auth_state.preview_port_path, Access::ReadWrite) + .allow(log_path, Access::ReadWrite); capsem_foundation::unix::worker_sandbox::confine(&policy).context("confine gateway before readiness")?; attest_gateway(uds_path, listener) } @@ -261,7 +259,12 @@ fn attest_gateway(uds_path: &std::path::Path, listener: SocketAddr) -> Result<() } #[cfg(not(any(target_os = "linux", target_os = "macos")))] -fn confine_gateway(_auth_state: &AuthState, _uds_path: &std::path::Path, _listener: SocketAddr) -> Result<()> { +fn confine_gateway( + _auth_state: &AuthState, + _uds_path: &std::path::Path, + _listener: SocketAddr, + _log_path: &std::path::Path, +) -> Result<()> { Ok(()) } diff --git a/crates/capsem-gateway/src/tests.rs b/crates/capsem-gateway/src/tests.rs index 16d2d0f17..ebd487fe1 100644 --- a/crates/capsem-gateway/src/tests.rs +++ b/crates/capsem-gateway/src/tests.rs @@ -16,6 +16,7 @@ fn entry_closes_ambient_descriptors_before_runtime_construction() { .find("close_inherited_descriptors()") .expect("gateway closes ambient descriptors at process entry"); let runtime = entry.find("tokio::runtime::Builder").unwrap(); + assert!(entry[runtime..].starts_with("tokio::runtime::Builder::new_current_thread()")); let run = entry.find("runtime.block_on(run(args))").unwrap(); assert!(close < runtime && runtime < run); } @@ -28,8 +29,21 @@ fn platform_confinement_precedes_gateway_readiness() { let confine = run .find("confine_gateway(&auth_state") .expect("all supported platforms install the worker policy"); + let telemetry = run.find("telemetry::init").expect("telemetry starts after confinement"); + let watcher = run + .find("watch_parent_or_exit") + .expect("parent watcher starts after confinement"); + let grant_client = run + .find("GatewayGrantClient::start") + .expect("grant task starts after confinement"); let publish = run.find("auth_state.publish").unwrap(); - assert!(prepare < confine && confine < publish); + assert!( + prepare < confine + && confine < telemetry + && telemetry < watcher + && watcher < grant_client + && grant_client < publish + ); } struct EnvGuard { From 625bbcf6a6082c3de8293e155be8659a9ab25341 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 13:58:48 +0000 Subject: [PATCH 240/464] fix(gateway): forward standalone proxy routes --- CHANGELOG.md | 22 +++++++++++++++++++ crates/capsem-gateway/src/main.rs | 18 +++------------ .../capsem-gateway/src/service_grant/tests.rs | 5 +++++ 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bac89306..d1aed38b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- The authenticated gateway forwards standalone proxy create, heartbeat and + stop operations instead of returning 404 for documented SDK endpoints. + +- The authenticated gateway forwards host credential injection, so desktop + settings and TCP SDK clients can use file and memory storage. + +- Settings error text and negative badges remain readable on light and dark + surfaces without changing solid-button foregrounds. + +- Credential file-lock waits stop after five seconds instead of indefinitely + delaying startup or an explicit injection behind another writer. + +- Model tool records label observed WebSocket exchanges with their actual + transport while preserving HTTP and SSE labels. + +- Codex code-mode calls retain their tool namespace, input and linked results + in model telemetry. Text-array outputs preserve request metadata, and + Responses tool turns report tool-use completion. + +- WebSocket upgrade handshakes retain network telemetry without inflating + model call counts with an empty inference record. + - Keep a confined proxy generation alive when an ordinary HTTP or DNS client disconnects, while still terminating it for unknown, stale or required capability closure events. diff --git a/crates/capsem-gateway/src/main.rs b/crates/capsem-gateway/src/main.rs index 387f2c972..93a9054c1 100644 --- a/crates/capsem-gateway/src/main.rs +++ b/crates/capsem-gateway/src/main.rs @@ -300,21 +300,9 @@ fn service_proxy_routes() -> Router> { ) .route("/images", get(proxy::handle_proxy)) .route("/images/pull", post(proxy::handle_proxy)) - .route("/connections", get(proxy::handle_proxy)) - .route("/connections/google/authorize", post(proxy::handle_proxy)) - .route( - "/connections/authorizations/{transaction_id}", - get(proxy::handle_proxy).delete(proxy::handle_proxy), - ) - .route( - "/connections/{connection_id}", - get(proxy::handle_proxy).delete(proxy::handle_proxy), - ) - .route( - "/vms/{id}/credential-grants", - get(proxy::handle_proxy).post(proxy::handle_proxy), - ) - .route("/vms/{id}/credential-grants/{grant_id}", delete(proxy::handle_proxy)) + .route("/proxies", post(proxy::handle_proxy)) + .route("/proxies/{id}/heartbeat", post(proxy::handle_proxy)) + .route("/proxies/{id}/stop", post(proxy::handle_proxy)) .route("/vms/create", post(proxy::handle_proxy)) .route("/vms/managed/{request_id}/claim", post(proxy::handle_proxy)) .route("/vms/managed/{request_id}/close", post(proxy::handle_proxy)) diff --git a/crates/capsem-gateway/src/service_grant/tests.rs b/crates/capsem-gateway/src/service_grant/tests.rs index fe1e76867..b77f2fb20 100644 --- a/crates/capsem-gateway/src/service_grant/tests.rs +++ b/crates/capsem-gateway/src/service_grant/tests.rs @@ -8,11 +8,13 @@ use tokio::io::AsyncReadExt; async fn grants_service_and_registry_named_owner_connections() { let (client, server) = UnixStream::pair().unwrap(); let grants = GatewayGrantClient::start(client).unwrap(); + let (release, released) = tokio::sync::oneshot::channel(); let broker = tokio::spawn(async move { let requests = DescriptorReceiver::::new(server.try_clone().unwrap()) .unwrap(); let responses = DescriptorSender::::new(server).unwrap(); + let mut peers = Vec::new(); for expected in [GatewayGrantKind::Service, GatewayGrantKind::OwnerHandoff] { let request = requests.recv().await.unwrap(); let request = decode_gateway_grant_request(&request.bytes).unwrap(); @@ -29,7 +31,9 @@ async fn grants_service_and_registry_named_owner_connections() { }); responses.send(&response, &[granted.as_raw_fd()]).await.unwrap(); std::io::Write::write_all(&mut peer, &[kind_code(expected)]).unwrap(); + peers.push(peer); } + let _ = released.await; }); let mut service = grants.open_service().await.unwrap(); @@ -39,6 +43,7 @@ async fn grants_service_and_registry_named_owner_connections() { let mut owner = grants.open_owner("box".into()).await.unwrap(); owner.read_exact(&mut byte).await.unwrap(); assert_eq!(byte, [2]); + release.send(()).unwrap(); broker.await.unwrap(); } From 65009dff564b7bdd522376ac21e20da2eccb6539 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 13:58:48 +0000 Subject: [PATCH 241/464] security(core): prepare confined VM sockets reliably --- config/gate.toml | 2 +- crates/capsem-core/src/hypervisor/kvm/mod.rs | 24 ++-- .../src/hypervisor/kvm/prepared.rs | 33 +++++ .../src/hypervisor/kvm/virtio_vsock.rs | 4 +- crates/capsem-core/src/lib.rs | 4 +- crates/capsem-core/src/vm/boot.rs | 113 ++++++++++++------ 6 files changed, 128 insertions(+), 52 deletions(-) create mode 100644 crates/capsem-core/src/hypervisor/kvm/prepared.rs diff --git a/config/gate.toml b/config/gate.toml index 7df9d9929..3634d9d7a 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1097,7 +1097,7 @@ must_stay_below_lines = 3000 "crates/capsem-admin/src/release_graph/tests.rs" = 1046 "crates/capsem-agent/src/tests.rs" = 1115 "crates/capsem-assets/src/asset_manager.rs" = 1210 -"crates/capsem-core/src/hypervisor/kvm/mod.rs" = 1028 +"crates/capsem-core/src/hypervisor/kvm/mod.rs" = 1039 "crates/capsem-core/src/hypervisor/kvm/sys.rs" = 1895 "crates/capsem-core/src/hypervisor/kvm/virtio_blk.rs" = 1768 "crates/capsem-core/src/hypervisor/kvm/virtio_blk/tests.rs" = 1059 diff --git a/crates/capsem-core/src/hypervisor/kvm/mod.rs b/crates/capsem-core/src/hypervisor/kvm/mod.rs index c2c6a14c6..89dd70ba0 100644 --- a/crates/capsem-core/src/hypervisor/kvm/mod.rs +++ b/crates/capsem-core/src/hypervisor/kvm/mod.rs @@ -14,6 +14,7 @@ mod memory; mod mmio; #[cfg(target_arch = "x86_64")] mod pio; +mod prepared; mod serial; #[cfg(target_arch = "x86_64")] mod serial_pio; @@ -34,9 +35,12 @@ use std::time::Duration; use anyhow::{ensure, Context, Result}; use tokio::sync::mpsc; -use super::{Hypervisor, SerialConsole, VmHandle, VsockConnection}; +#[cfg(test)] +use super::Hypervisor; +use super::{SerialConsole, VmHandle, VsockConnection}; use crate::vm::config::VmConfig; use crate::vm::VmState; +pub(crate) use prepared::PreparedVsock; const KVM_PAUSE_TIMEOUT: Duration = Duration::from_secs(5); @@ -51,8 +55,8 @@ fn require_metadata_authority(config: &VmConfig) -> Result<()> { Ok(()) } -fn kvm_vsock_seed(config: &VmConfig) -> u32 { - let mut hasher = capsem_crypto::blake3::Hasher::new(); +pub(crate) fn kvm_vsock_seed(config: &VmConfig) -> u32 { + let mut hasher = blake3::Hasher::new(); hasher.update(config.kernel_path.to_string_lossy().as_bytes()); if let Some(path) = config.scratch_disk_path.as_ref().or(config.disk_path.as_ref()) { hasher.update(path.to_string_lossy().as_bytes()); @@ -128,12 +132,14 @@ fn virtio_mmio_device_count(config: &VmConfig, vsock_ports: &[u32]) -> u32 { } } -impl Hypervisor for KvmHypervisor { - fn boot( +impl KvmHypervisor { + pub(crate) fn boot_prepared( &self, config: &VmConfig, vsock_ports: &[u32], + prepared_vsock: PreparedVsock, ) -> Result<(Box, mpsc::UnboundedReceiver)> { + let PreparedVsock(vsock_bindings) = prepared_vsock; require_metadata_authority(config)?; #[cfg(not(target_arch = "x86_64"))] if config.checkpoint_path.is_some() { @@ -160,14 +166,6 @@ impl Hypervisor for KvmHypervisor { #[cfg(target_arch = "x86_64")] let restoring = config.checkpoint_path.is_some(); - let vsock_bindings = if vsock_ports.is_empty() { - None - } else { - Some(virtio_vsock::bind_vsock_listeners_for_vm( - vsock_ports, - kvm_vsock_seed(config), - )?) - }; let kernel_cmdline = append_kvm_vsock_port_offset( &config.kernel_cmdline, vsock_bindings.as_ref().map_or(0, |b| b.offset()), diff --git a/crates/capsem-core/src/hypervisor/kvm/prepared.rs b/crates/capsem-core/src/hypervisor/kvm/prepared.rs new file mode 100644 index 000000000..3017f4d01 --- /dev/null +++ b/crates/capsem-core/src/hypervisor/kvm/prepared.rs @@ -0,0 +1,33 @@ +//! Resources that must be acquired before the VM-owner sandbox is installed. + +use anyhow::Result; +use tokio::sync::mpsc; + +use super::{kvm_vsock_seed, virtio_vsock, KvmHypervisor}; +use crate::hypervisor::{Hypervisor, VmHandle, VsockConnection}; +use crate::vm::config::VmConfig; + +/// AF_VSOCK listeners created before direct socket authority is dropped. +pub(crate) struct PreparedVsock(pub(super) Option); + +impl KvmHypervisor { + pub(crate) fn prepare_vsock(vsock_ports: &[u32], seed: u32) -> Result { + let bindings = if vsock_ports.is_empty() { + None + } else { + Some(virtio_vsock::bind_vsock_listeners_for_vm(vsock_ports, seed)?) + }; + Ok(PreparedVsock(bindings)) + } +} + +impl Hypervisor for KvmHypervisor { + fn boot( + &self, + config: &VmConfig, + vsock_ports: &[u32], + ) -> Result<(Box, mpsc::UnboundedReceiver)> { + let prepared = Self::prepare_vsock(vsock_ports, kvm_vsock_seed(config))?; + self.boot_prepared(config, vsock_ports, prepared) + } +} diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_vsock.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_vsock.rs index 1df88ed71..f86640bd9 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_vsock.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_vsock.rs @@ -715,7 +715,7 @@ pub(super) struct BoundVsockListener { sock: OwnedFd, } -pub(super) struct BoundVsockListeners { +pub(crate) struct BoundVsockListeners { offset: u32, guest_cid: u32, listeners: Vec, @@ -731,7 +731,7 @@ impl BoundVsockListeners { } } -pub(super) fn bind_vsock_listeners_for_vm(logical_ports: &[u32], seed: u32) -> Result { +pub(crate) fn bind_vsock_listeners_for_vm(logical_ports: &[u32], seed: u32) -> Result { if logical_ports.is_empty() { return Ok(BoundVsockListeners { offset: 0, diff --git a/crates/capsem-core/src/lib.rs b/crates/capsem-core/src/lib.rs index 7ad00eadf..8fa6e8c6c 100644 --- a/crates/capsem-core/src/lib.rs +++ b/crates/capsem-core/src/lib.rs @@ -21,8 +21,8 @@ use std::path::{Path, PathBuf}; pub use host_state::{validate_guest_msg, validate_host_msg, HostState, HostStateMachine, StateMachine, Transition}; pub use vm::boot::{ - boot_vm, create_net_state, create_net_state_with_policy, read_control_msg, send_boot_config, write_control_msg, - BootOptions, ControlFrameTooLarge, + boot_vm, create_net_state, create_net_state_with_policy, prepare_vm, read_control_msg, send_boot_config, + write_control_msg, BootOptions, ControlFrameTooLarge, PreparedVm, }; pub use vm::config::{GuestMetadataAuthority, VirtioFsShare, VmConfig}; pub use vm::registry::{SandboxInstance, SandboxNetworkState}; diff --git a/crates/capsem-core/src/vm/boot.rs b/crates/capsem-core/src/vm/boot.rs index aa5f79d8d..0b928b676 100644 --- a/crates/capsem-core/src/vm/boot.rs +++ b/crates/capsem-core/src/vm/boot.rs @@ -8,7 +8,9 @@ use tokio::sync::mpsc; use tracing::{debug_span, info, info_span, warn}; use crate::host_state::{HostState, HostStateMachine}; -use crate::hypervisor::{Hypervisor, VmHandle, VsockConnection}; +#[cfg(target_os = "macos")] +use crate::hypervisor::Hypervisor; +use crate::hypervisor::{VmHandle, VsockConnection}; use crate::vm::config::VmConfig; #[cfg(target_os = "macos")] @@ -92,13 +94,7 @@ pub struct BootOptions<'a> { /// as the overlayfs upper. /// If `virtiofs_shares` is non-empty, VirtioFS directory sharing devices are /// attached and `capsem.storage=virtiofs` is appended to the kernel cmdline. -pub fn boot_vm( - options: BootOptions, -) -> Result<( - Box, - mpsc::UnboundedReceiver, - HostStateMachine, -)> { +pub fn prepare_vm(options: BootOptions<'_>) -> Result { let BootOptions { assets, kernel_override, @@ -115,7 +111,7 @@ pub fn boot_vm( expected_asset_hashes, } = options; let _span = info_span!("boot_vm").entered(); - let mut sm = HostStateMachine::new_host(); + let sm = HostStateMachine::new_host(); info!( "[boot-audit] boot_vm: cpu={cpu_count} ram_bytes={ram_bytes} virtiofs_shares={}", @@ -217,34 +213,83 @@ pub fn boot_vm( }; info!("[boot-audit] VmConfig built successfully"); - info!("[boot-audit] calling hypervisor boot"); - let boot_span = debug_span!( - target: "capsem.launch", - capsem_foundation::telemetry::LAUNCH_VM_BOOT_SPAN, - status = tracing::field::Empty, - ); - let (vm, vsock_rx) = { - let _span = boot_span.clone().entered(); - #[cfg(target_os = "macos")] - let result = AppleVzHypervisor.boot(&config, capsem_proto::host_vsock_ports()); + #[cfg(target_os = "linux")] + let prepared_vsock = KvmHypervisor::prepare_vsock( + capsem_proto::host_vsock_ports(), + crate::hypervisor::kvm::kvm_vsock_seed(&config), + )?; + + Ok(PreparedVm { + config, + sm, #[cfg(target_os = "linux")] - let result = KvmHypervisor.boot(&config, capsem_proto::host_vsock_ports()); - match result { - Ok(value) => { - boot_span.record("status", "ok"); - value - } - Err(error) => { - boot_span.record("status", "error"); - return Err(error).context("failed to boot VM"); - } - } - }; - info!("[boot-audit] hypervisor boot returned OK"); + prepared_vsock, + }) +} + +/// A VM configuration whose direct host socket resources have already been +/// acquired. Preparing this value is single-threaded; activation may therefore +/// happen after the caller installs its worker sandbox. +pub struct PreparedVm { + config: VmConfig, + sm: HostStateMachine, + #[cfg(target_os = "linux")] + prepared_vsock: crate::hypervisor::kvm::PreparedVsock, +} - sm.transition(HostState::Booting, "vm_started")?; +impl PreparedVm { + pub fn boot( + self, + ) -> Result<( + Box, + mpsc::UnboundedReceiver, + HostStateMachine, + )> { + let PreparedVm { + config, + mut sm, + #[cfg(target_os = "linux")] + prepared_vsock, + } = self; + info!("[boot-audit] calling hypervisor boot"); + let boot_span = debug_span!( + target: "capsem.launch", + capsem_foundation::telemetry::LAUNCH_VM_BOOT_SPAN, + status = tracing::field::Empty, + ); + let (vm, vsock_rx) = { + let _span = boot_span.clone().entered(); + #[cfg(target_os = "macos")] + let result = AppleVzHypervisor.boot(&config, capsem_proto::host_vsock_ports()); + #[cfg(target_os = "linux")] + let result = KvmHypervisor.boot_prepared(&config, capsem_proto::host_vsock_ports(), prepared_vsock); + match result { + Ok(value) => { + boot_span.record("status", "ok"); + value + } + Err(error) => { + boot_span.record("status", "error"); + return Err(error).context("failed to boot VM"); + } + } + }; + info!("[boot-audit] hypervisor boot returned OK"); + sm.transition(HostState::Booting, "vm_started")?; + Ok((vm, vsock_rx, sm)) + } +} - Ok((vm, vsock_rx, sm)) +/// Prepare and immediately boot a VM. Long-lived owners should call +/// [`prepare_vm`] before confinement and [`PreparedVm::boot`] afterward. +pub fn boot_vm( + options: BootOptions<'_>, +) -> Result<( + Box, + mpsc::UnboundedReceiver, + HostStateMachine, +)> { + prepare_vm(options)?.boot() } fn effective_kernel_cmdline(base: &str, virtiofs_shares: &[VirtioFsShare], rootfs_override: Option<&Path>) -> String { From fbf3bf5da6c5c901dbc4c87ca130c0084e14a164 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 13:58:48 +0000 Subject: [PATCH 242/464] feat(proto): carry MCP tool schemas over owner IPC --- crates/capsem-proto/src/ipc.rs | 3 +++ crates/capsem-proto/src/ipc/tests.rs | 1 + crates/capsem-proto/src/ipc/tests/correlation.rs | 2 ++ 3 files changed, 6 insertions(+) diff --git a/crates/capsem-proto/src/ipc.rs b/crates/capsem-proto/src/ipc.rs index 11cbde924..9fd0c2f59 100644 --- a/crates/capsem-proto/src/ipc.rs +++ b/crates/capsem-proto/src/ipc.rs @@ -564,6 +564,9 @@ pub struct McpToolStatus { pub original_name: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub description: Option, + /// Complete input schema used by the coordinator-owned pin cache. + #[serde(default, skip_serializing_if = "serde_json::Value::is_null")] + pub input_schema: serde_json::Value, pub server_name: String, /// Typed so SDK and UI consumers get one stable annotation contract. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/capsem-proto/src/ipc/tests.rs b/crates/capsem-proto/src/ipc/tests.rs index 00cea1d76..25485c701 100644 --- a/crates/capsem-proto/src/ipc/tests.rs +++ b/crates/capsem-proto/src/ipc/tests.rs @@ -611,6 +611,7 @@ fn mcp_tools_result_roundtrip() { namespaced_name: "github__search".into(), original_name: "search".into(), description: Some("Search repos".into()), + input_schema: serde_json::json!({"type": "object"}), server_name: "github".into(), annotations: None, }], diff --git a/crates/capsem-proto/src/ipc/tests/correlation.rs b/crates/capsem-proto/src/ipc/tests/correlation.rs index 7b2cccb5e..d84489e64 100644 --- a/crates/capsem-proto/src/ipc/tests/correlation.rs +++ b/crates/capsem-proto/src/ipc/tests/correlation.rs @@ -124,6 +124,7 @@ fn mcp_tool_status_annotations_roundtrip_msgpack() { namespaced_name: "github__search".into(), original_name: "search".into(), description: None, + input_schema: serde_json::json!({"type": "object"}), server_name: "github".into(), annotations: Some(crate::mcp_contracts::ToolAnnotations { title: Some("Search".into()), @@ -142,4 +143,5 @@ fn mcp_tool_status_annotations_roundtrip_msgpack() { let annotations = tools[0].annotations.as_ref().expect("annotations survive"); assert!(annotations.read_only_hint && !annotations.destructive_hint); assert_eq!(annotations.title.as_deref(), Some("Search")); + assert_eq!(tools[0].input_schema, serde_json::json!({"type": "object"})); } From 24000896552254d3083eef000d4d9d6f9edc0c43 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 13:58:48 +0000 Subject: [PATCH 243/464] security(process): confine VM owners and cache authority --- CHANGELOG.md | 4 + config/gate.toml | 1 - crates/capsem-process/src/ipc.rs | 1 + crates/capsem-process/src/main.rs | 496 ++++++--------------- crates/capsem-process/src/owner_sandbox.rs | 224 ++++++++++ crates/capsem-process/src/private_seats.rs | 40 +- crates/capsem-process/src/tests.rs | 30 +- 7 files changed, 412 insertions(+), 384 deletions(-) create mode 100644 crates/capsem-process/src/owner_sandbox.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d1aed38b6..3e2fd2f46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Install VM-owner confinement before starting its runtime, hypervisor, + parent watcher, ledger client, filesystem monitor, or metric workers; + pre-bind only the exact IPC and VSOCK descriptors those workers inherit. + - Keep host-worker confinement fail-closed across supported Linux Landlock ABIs, including single-threaded startup on ABI 6 and 7, and canonicalize macOS Seatbelt grants so system path aliases cannot silently deny or widen diff --git a/config/gate.toml b/config/gate.toml index 3634d9d7a..ffebf5ef3 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1108,7 +1108,6 @@ must_stay_below_lines = 3000 "crates/capsem-core/tests/mitm_integration.rs" = 2014 "crates/capsem-logger/src/writer/tests.rs" = 1395 "crates/capsem-mock-server/src/main.rs" = 1171 -"crates/capsem-process/src/main.rs" = 1157 "crates/capsem-process/src/vsock.rs" = 1239 "crates/capsem-proto/src/lib.rs" = 1008 "crates/capsem-proto/src/tests.rs" = 1446 diff --git a/crates/capsem-process/src/ipc.rs b/crates/capsem-process/src/ipc.rs index 15ef97180..44e07f7f7 100644 --- a/crates/capsem-process/src/ipc.rs +++ b/crates/capsem-process/src/ipc.rs @@ -773,6 +773,7 @@ pub(crate) async fn handle_ipc_connection( namespaced_name: t.namespaced_name, original_name: t.original_name, description: t.description, + input_schema: t.input_schema, server_name: t.server_name, annotations: t.annotations, }) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 6f1f7d31b..f02dedd01 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -4,6 +4,7 @@ mod ipc; mod job_store; mod mcp_runtime; mod metric_export; +mod owner_sandbox; mod private_seats; mod proxy_mcp; mod runtime_config; @@ -13,19 +14,20 @@ mod vsock; use anyhow::{Context, Result}; use capsem_core::fs_monitor::FsMonitor; use capsem_core::net::upstream_grant::{adopt_inherited, UpstreamGrantClient}; -use capsem_core::{boot_vm, BootOptions, VirtioFsShare, VsockConnection}; +use capsem_core::{prepare_vm, BootOptions, VirtioFsShare, VsockConnection}; use capsem_logger::DbWriter; use capsem_proto::ipc::{ProcessToService, ServiceToProcess}; use clap::Parser; use std::os::fd::AsFd as _; +use std::os::unix::fs::PermissionsExt as _; use std::path::{Path, PathBuf}; use std::sync::Arc; -use tokio::net::UnixListener; use tokio::sync::{broadcast, mpsc, Mutex}; use tracing::{error, info, warn}; use job_store::JobStore; use mcp_runtime::{GuestExposureTools, McpRuntime}; +use owner_sandbox::{attest_owner, confine_owner, prepare_owner_sandbox_attestation}; use vsock::VsockOptions; /// Owns the background-thread resources that MUST drain before the main @@ -41,6 +43,19 @@ pub(crate) struct Shutdown { proxy_mcp: Option>, } +struct PreparedOwnerResources { + seats: private_seats::Prepared, + metric_service: Option, + ipc_listener: std::os::unix::net::UnixListener, + launched: std::fs::File, + ready: std::fs::File, + pty_log: Option>, + aggregator: capsem_proto::mcp_aggregator::AggregatorClient, + mcp_servers: Vec, + builtin_bin: Option, + builtin_env: std::collections::HashMap, +} + impl Shutdown { /// Drain in order: fs_events fan into DbWriter, so FsMonitor must /// finish its final flush before the DbWriter runs its checkpoint. @@ -232,221 +247,6 @@ fn prepare_sentinel(path: &Path) -> Result { Ok(file) } -#[cfg(any(target_os = "linux", target_os = "macos"))] -fn grant_owner_path( - policy: capsem_foundation::unix::worker_sandbox::Policy, - path: &Path, - access: capsem_foundation::unix::worker_sandbox::Access, -) -> Result { - let canonical = path - .canonicalize() - .with_context(|| format!("resolve sandbox grant {}", path.display()))?; - Ok(policy.allow(canonical, access)) -} - -#[cfg(any(target_os = "linux", target_os = "macos"))] -fn grant_owner_session_paths( - mut policy: capsem_foundation::unix::worker_sandbox::Policy, - session_dir: &Path, -) -> Result { - use capsem_foundation::unix::worker_sandbox::Access; - - for (path, access) in [ - (capsem_core::guest_share_dir(session_dir), Access::ReadWrite), - ( - capsem_core::session::system_overlay_image_path(session_dir), - Access::ReadWrite, - ), - (capsem_core::session::image_share_path(session_dir), Access::ReadOnly), - ( - session_dir.join(capsem_core::session::OWNER_STATE_DIR), - Access::ReadWrite, - ), - (session_dir.join("serial.log"), Access::ReadWrite), - ] { - policy = grant_owner_path(policy, &path, access)?; - } - Ok(policy) -} - -#[cfg(target_os = "linux")] -fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { - use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; - - let mut policy = Policy::new(Role::VmOwner); - policy = grant_owner_session_paths(policy, session_dir)?; - policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; - policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; - policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; - for path in args.kernel.iter().chain(args.initrd.iter()) { - policy = grant_owner_path(policy, path, Access::ReadOnly)?; - } - let executable = std::env::current_exe().context("locate VM-owner executable")?; - let router = executable.with_file_name("capsem-router"); - if router.exists() { - policy = grant_owner_path(policy, &router, Access::Executable)?; - } - for path in ["/lib", "/lib64", "/usr/lib", "/usr/lib64"] { - let path = Path::new(path); - if path.exists() { - policy = grant_owner_path(policy, path, Access::ReadOnly)?; - } - } - let linker_cache = Path::new("/etc/ld.so.cache"); - if linker_cache.exists() { - policy = grant_owner_path(policy, linker_cache, Access::ReadOnly)?; - } - for path in ["/lib64/ld-linux-x86-64.so.2", "/lib/ld-linux-aarch64.so.1"] { - let path = Path::new(path); - if path.exists() { - policy = grant_owner_path(policy, path, Access::Executable)?; - } - } - capsem_foundation::unix::worker_sandbox::confine(&policy)?; - Ok(()) -} - -#[cfg(target_os = "macos")] -fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { - use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; - - let mut policy = Policy::new(Role::VmOwner); - policy = grant_owner_session_paths(policy, session_dir)?; - policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; - policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; - policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; - for path in args.kernel.iter().chain(args.initrd.iter()) { - policy = grant_owner_path(policy, path, Access::ReadOnly)?; - } - let router = std::env::current_exe() - .context("locate VM-owner executable")? - .with_file_name("capsem-router"); - if router.exists() { - policy = grant_owner_path(policy, &router, Access::Executable)?; - } - for path in [ - "/System/Library", - "/usr/lib", - "/private/var/db/dyld", - "/private/var/db/timezone", - "/etc/localtime", - "/dev/urandom", - ] { - let path = Path::new(path); - if path.exists() { - policy = grant_owner_path(policy, path, Access::ReadOnly)?; - } - } - let null = Path::new("/dev/null"); - if null.exists() { - policy = grant_owner_path(policy, null, Access::ReadWrite)?; - } - capsem_foundation::unix::worker_sandbox::confine(&policy)?; - Ok(()) -} - -#[cfg(any(target_os = "linux", target_os = "macos"))] -struct OwnerSandboxAttestation { - direct_path: PathBuf, - direct_file: std::fs::File, - guest_path: PathBuf, - guest_relative: Vec, - ledger_path: PathBuf, -} - -#[cfg(any(target_os = "linux", target_os = "macos"))] -fn prepare_owner_sandbox_attestation(session_dir: &Path) -> Result { - use std::os::unix::ffi::OsStrExt as _; - use std::os::unix::fs::OpenOptionsExt as _; - - let create = |path: &Path| { - std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(path) - .with_context(|| format!("prepare owner sandbox attestation {}", path.display())) - }; - let nonce = format!( - "{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - ); - let direct_path = session_dir - .join(capsem_core::session::OWNER_STATE_DIR) - .join(format!(".owner-sandbox-attestation-{nonce}")); - let guest_relative = format!("workspace/.owner-metadata-attestation-{nonce}").into_bytes(); - let guest_path = session_dir - .join(capsem_core::GUEST_SHARE_DIR) - .join(std::ffi::OsStr::from_bytes(&guest_relative)); - let direct_file = create(&direct_path)?; - drop(create(&guest_path)?); - Ok(OwnerSandboxAttestation { - direct_path, - direct_file, - guest_path, - guest_relative, - ledger_path: session_dir.join("session.db"), - }) -} - -#[cfg(any(target_os = "linux", target_os = "macos"))] -async fn attest_owner( - attestation: OwnerSandboxAttestation, - service_socket: PathBuf, - upstream_grants: Arc, -) -> Result<()> { - use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; - - let require_denied = |result: std::io::Result<()>, authority: &str| -> Result<()> { - match result { - Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => Ok(()), - Err(error) => anyhow::bail!("VM owner sandbox returned {error} while denying {authority}"), - Ok(()) => anyhow::bail!("VM owner sandbox retained authority to {authority}"), - } - }; - require_denied( - std::fs::File::open("/etc/passwd").map(|_| ()), - "read unrelated host files", - )?; - require_denied( - std::fs::File::open(&attestation.ledger_path).map(|_| ()), - "open session ledger storage", - )?; - require_denied( - std::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, 9)).map(|_| ()), - "dial TCP sockets directly", - )?; - require_denied( - std::os::unix::net::UnixStream::connect(service_socket).map(|_| ()), - "dial control sockets by path", - )?; - require_denied( - attestation - .direct_file - .set_permissions(std::fs::Permissions::from_mode(0o777)), - "change host modes directly", - )?; - require_denied( - std::process::Command::new("/bin/true").status().map(|_| ()), - "execute arbitrary processes", - )?; - - tokio::task::spawn_blocking(move || { - capsem_core::GuestMetadataAuthority::set_mode(upstream_grants.as_ref(), &attestation.guest_relative, 0o701) - }) - .await - .context("join brokered metadata attestation")??; - let mode = std::fs::metadata(&attestation.guest_path)?.mode() & 0o7777; - anyhow::ensure!(mode == 0o701, "brokered metadata attestation returned mode {mode:o}"); - std::fs::remove_file(&attestation.guest_path)?; - std::fs::remove_file(&attestation.direct_path)?; - Ok(()) -} - fn main() -> Result<()> { // SAFETY: process entry precedes argument parsing, telemetry, descriptor // owners and runtime threads. Broker grants will be named here explicitly. @@ -476,13 +276,12 @@ fn main() -> Result<()> { if let Ok(resolved) = session_dir.canonicalize() { session_dir = resolved; } - // The kernel parent is the coordinator that launched this owner. Keep - // that identity for every IPC check and die if it changes, so an orphan - // cannot accept a later process that reuses the old parent's PID. - let _owner_guards = capsem_guard::install(Some(controller.pid.get()), &session_dir.join("process.lock"))? + let _owner_singleton = capsem_guard::Singleton::try_acquire(&session_dir.join("process.lock"))? .context("this session already has a running VM owner")?; - let rt = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + // A current-thread runtime creates no worker before confinement. Tasks + // queued during preparation start only after the sandbox is installed. + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build()?; let runtime_source = runtime_config::RuntimePolicySource::new(args.active_policy.clone()); let runtime_config = runtime_source.load()?; let upstream_grants = { @@ -521,16 +320,50 @@ fn main() -> Result<()> { // `system/` directory, outside the share, so the guest cannot swap it // for a link to a host file (`capsem_core::session::adopt_system_overlay`). let system_img = capsem_core::session::system_overlay_image_path(&session_dir); - let machine_identifier_path = session_dir.join("machine_identifier"); + let owner_state = session_dir.join(capsem_core::session::OWNER_STATE_DIR); + let machine_identifier_path = owner_state.join("machine_identifier"); + let legacy_machine_identifier = session_dir.join("machine_identifier"); + if legacy_machine_identifier.exists() && !machine_identifier_path.exists() { + std::fs::rename(&legacy_machine_identifier, &machine_identifier_path) + .context("move machine identifier into owner state")?; + } let serial_log_path = session_dir.join("serial.log"); - drop( - std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&serial_log_path) - .with_context(|| format!("prepare serial log {}", serial_log_path.display()))?, + drop(capsem_foundation::unix::fs::open_private_append_no_follow( + &serial_log_path, + )?); + let pty_log = match capsem_core::pty_log::PtyLog::open(&session_dir.join("pty.log")) { + Ok(log) => Some(Arc::new(log)), + Err(error) => { + warn!(%error, "failed to prepare pty.log"); + None + } + }; + let executable = std::env::current_exe().context("locate VM-owner executable")?; + let aggregator_bin = resolve_mcp_aggregator_binary(&executable)?; + let aggregator_stderr = + capsem_foundation::unix::fs::open_private_append_no_follow(&aggregator_log_path(&session_dir))?; + let builtin_bin = executable + .parent() + .map(|directory| directory.join("capsem-mcp-builtin")); + let mut builtin_env = std::collections::HashMap::new(); + builtin_env.insert("CAPSEM_SESSION_DIR".into(), session_dir.to_string_lossy().to_string()); + builtin_env.insert( + "CAPSEM_ACTIVE_POLICY".into(), + runtime_config.active_policy_path.to_string_lossy().to_string(), ); - let (vm, vsock_rx, sm) = boot_vm(BootOptions { + let mcp_servers = runtime_config.mcp_servers(builtin_bin.as_deref(), builtin_env.clone()); + // The aggregator is a separately supervised process. Spawn it while the + // VM owner still has the one executable capability needed to create it; + // parent-side driver tasks remain queued on the current-thread runtime. + let aggregator = rt.block_on(spawn_mcp_aggregator( + &mcp_servers, + &session_dir, + &args.id, + &trace_id, + aggregator_bin, + aggregator_stderr, + ))?; + let prepared_vm = prepare_vm(BootOptions { assets: &args.assets_dir, kernel_override: args.kernel.as_deref(), initrd_override: args.initrd.as_deref(), @@ -553,6 +386,44 @@ fn main() -> Result<()> { }), })?; + let prepared_seats = private_seats::prepare(private_seats::Seats { + id: &args.id, + service_socket: args.service_socket.as_deref(), + uds_path: &args.uds_path, + run_dir: args.run_dir.as_deref(), + })?; + let metric_service = if args.metric_broker { + Some( + std::os::unix::net::UnixStream::connect(&prepared_seats.service_socket) + .context("prepare metric service channel")?, + ) + } else { + None + }; + if args.uds_path.exists() { + std::fs::remove_file(&args.uds_path)?; + } + let ipc_listener = std::os::unix::net::UnixListener::bind(&args.uds_path)?; + ipc_listener.set_nonblocking(true)?; + std::fs::set_permissions(&args.uds_path, std::fs::Permissions::from_mode(0o600))?; + let launched_path = args.uds_path.with_extension("launched"); + let launched = prepare_sentinel(&launched_path)?; + let ready = prepare_sentinel(&args.uds_path.with_extension("ready"))?; + let attestation = prepare_owner_sandbox_attestation(&session_dir)?; + + confine_owner(&args, &session_dir).context("install VM-owner confinement")?; + rt.block_on(attest_owner( + attestation, + prepared_seats.service_socket.clone(), + Arc::clone(&upstream_grants), + )) + .context("attest VM-owner confinement")?; + // The parent watcher is the first worker thread and therefore inherits + // the installed Landlock/Seatbelt and seccomp policy. + capsem_guard::watch_parent_or_exit(Some(controller.pid.get()))?; + + let (vm, vsock_rx, sm) = prepared_vm.boot()?; + // Delete checkpoint file if we just restored from it, so we don't accidentally suspend on normal shutdown if let Some(cp) = &args.checkpoint_path { let full_path = if std::path::Path::new(cp).is_absolute() { @@ -565,6 +436,19 @@ fn main() -> Result<()> { let vm_arc = Arc::new(tokio::sync::Mutex::new(vm)); + let prepared_resources = PreparedOwnerResources { + seats: prepared_seats, + metric_service, + ipc_listener, + launched, + ready, + pty_log, + aggregator, + mcp_servers, + builtin_bin, + builtin_env, + }; + // Emit boot timeline state transitions for process.log. for t in sm.history() { info!( @@ -578,7 +462,6 @@ fn main() -> Result<()> { let shutdown: Arc> = Arc::new(Mutex::new(Shutdown::default())); - let trace_id_for_loop = trace_id; let session_dir_for_loop = session_dir; let shutdown_for_loop = Arc::clone(&shutdown); let shutdown_for_loop_error = Arc::clone(&shutdown); @@ -590,12 +473,12 @@ fn main() -> Result<()> { controller, vm_arc, vsock_rx, - trace_id_for_loop, session_dir_for_loop, shutdown_for_loop, runtime_source, runtime_config, upstream_grants, + prepared_resources, )) .await { @@ -642,6 +525,10 @@ fn main() -> Result<()> { std::process::exit(0); }); + #[cfg(target_os = "macos")] + let _runtime_thread = std::thread::Builder::new() + .name("capsem-process-runtime".into()) + .spawn(move || rt.block_on(std::future::pending::<()>()))?; #[cfg(target_os = "macos")] unsafe { core_foundation_sys::runloop::CFRunLoopRun(); @@ -652,7 +539,11 @@ fn main() -> Result<()> { // A VM the hypervisor stopped on its own is not a clean exit: the // service keeps the session directory and reports the VM as exited // unexpectedly, which is what happened. - if let Some(reason) = rt.block_on(async { vm_for_exit.lock().await.stop_reason() }) { + #[cfg(target_os = "macos")] + let stop_reason = vm_for_exit.blocking_lock().stop_reason(); + #[cfg(not(target_os = "macos"))] + let stop_reason = rt.block_on(async { vm_for_exit.lock().await.stop_reason() }); + if let Some(reason) = stop_reason { anyhow::bail!("the hypervisor stopped the VM: {reason}"); } Ok(()) @@ -664,13 +555,25 @@ async fn run_async_main_loop( controller: capsem_foundation::unix::peer::PeerIdentity, vm: Arc>>, vsock_rx: mpsc::UnboundedReceiver, - trace_id: String, session_dir: std::path::PathBuf, shutdown: Arc>, runtime_source: runtime_config::RuntimePolicySource, runtime_config: runtime_config::RuntimePolicyConfig, upstream_grants: Arc, + prepared: PreparedOwnerResources, ) -> Result<()> { + let PreparedOwnerResources { + seats: prepared_seats, + metric_service, + ipc_listener, + mut launched, + ready, + pty_log, + aggregator: aggregator_client, + mcp_servers, + builtin_bin, + builtin_env, + } = prepared; let terminal_output = Arc::new(capsem_core::TerminalOutputQueue::new()); let (ledger_stream, commitment_stream, ledger_grant) = upstream_grants @@ -740,27 +643,7 @@ async fn run_async_main_loop( shutdown.lock().await.publisher = Some(job_store.publisher.clone()); let (ipc_tx, _) = broadcast::channel::(128); let (ctrl_tx, ctrl_rx) = mpsc::channel::(32); - let seats = private_seats::bind( - private_seats::Seats { - id: &args.id, - service_socket: args.service_socket.as_deref(), - uds_path: &args.uds_path, - run_dir: args.run_dir.as_deref(), - }, - &job_store, - ctrl_tx.clone(), - )?; - let metric_service = if args.metric_broker { - Some( - tokio::net::UnixStream::connect(&seats.service_socket) - .await - .context("prepare metric service channel")? - .into_std() - .context("adopt metric service channel")?, - ) - } else { - None - }; + let seats = prepared_seats.activate(&job_store, ctrl_tx.clone())?; // Held until the process exits: dropping it flushes the last measurements. let _metric_export = metric_export::install(&args.id, metric_service, args.metric_broker); let restored = job_store @@ -795,49 +678,6 @@ async fn run_async_main_loop( Arc::clone(&db), runtime_config.network.clone(), )?); - let builtin_bin = std::env::current_exe() - .ok() - .and_then(|p| p.parent().map(|d| d.join("capsem-mcp-builtin"))); - let mut builtin_env = std::collections::HashMap::new(); - builtin_env.insert("CAPSEM_SESSION_DIR".into(), session_dir.to_string_lossy().to_string()); - builtin_env.insert( - "CAPSEM_ACTIVE_POLICY".into(), - runtime_config.active_policy_path.to_string_lossy().to_string(), - ); - let mcp_servers = runtime_config.mcp_servers(builtin_bin.as_deref(), builtin_env.clone()); - let aggregator_client = spawn_mcp_aggregator(&mcp_servers, &session_dir, &args.id, &trace_id).await?; - - if let Ok(tools) = aggregator_client.list_tools().await { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs().to_string()) - .unwrap_or_default(); - let existing = capsem_core::mcp::load_tool_cache(); - let cache_entries: Vec = tools - .iter() - .map(|t| { - let pin_hash = capsem_core::mcp::compute_tool_hash(t); - let prev = existing.iter().find(|e| e.namespaced_name == t.namespaced_name); - capsem_core::mcp::ToolCacheEntry { - namespaced_name: t.namespaced_name.clone(), - original_name: t.original_name.clone(), - description: t.description.clone(), - server_name: t.server_name.clone(), - annotations: t.annotations.clone(), - pin_hash: pin_hash.clone(), - first_seen: prev.map(|p| p.first_seen.clone()).unwrap_or_else(|| now.clone()), - last_seen: now.clone(), - approved: prev.map(|p| p.approved && p.pin_hash == pin_hash).unwrap_or(false), - } - }) - .collect(); - if let Err(e) = capsem_core::mcp::save_tool_cache(&cache_entries) { - warn!(error = %e, "failed to write tool cache"); - } else { - info!(tools = cache_entries.len(), "wrote tool cache"); - } - } - let inflight_cap = capsem_core::mcp::resolve_inflight_cap(); info!(inflight_cap, "MITM MCP endpoint in-flight handler cap"); let mcp_inflight = Arc::new(tokio::sync::Semaphore::new(inflight_cap)); @@ -917,36 +757,7 @@ async fn run_async_main_loop( let db_for_vsock = Arc::clone(&db); let shutdown_for_vsock = Arc::clone(&shutdown); let shutdown_for_vsock_error = Arc::clone(&shutdown); - let pty_log = match capsem_core::pty_log::PtyLog::open(&session_dir.join("pty.log")) { - Ok(pl) => Some(Arc::new(pl)), - Err(e) => { - warn!(error = %e, "failed to open pty.log"); - None - } - }; - - if uds_path.exists() { - std::fs::remove_file(&uds_path)?; - } - let listener = UnixListener::bind(&uds_path)?; - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(&uds_path, std::fs::Permissions::from_mode(0o600))?; - } - let launched_path = uds_path.with_extension("launched"); - let mut launched = prepare_sentinel(&launched_path)?; - let ready = prepare_sentinel(&uds_path.with_extension("ready"))?; - - #[cfg(any(target_os = "linux", target_os = "macos"))] - { - let attestation = prepare_owner_sandbox_attestation(&session_dir)?; - confine_owner(&args, &session_dir).context("install VM-owner confinement")?; - attest_owner(attestation, seats.service_socket.clone(), Arc::clone(&upstream_grants)) - .await - .context("attest VM-owner confinement")?; - } - + let listener = tokio::net::UnixListener::from_std(ipc_listener)?; seats.start(); use std::io::Write as _; launched.write_all(b"launched\n")?; @@ -1050,39 +861,14 @@ async fn spawn_mcp_aggregator( session_dir: &Path, vm_id: &str, trace_id: &str, + aggregator_bin: PathBuf, + stderr_file: std::fs::File, ) -> Result { use capsem_proto::mcp_aggregator::*; let (client, rx) = AggregatorClient::channel(64); - let exe_path = std::env::current_exe()?; - let aggregator_bin = resolve_mcp_aggregator_binary(&exe_path)?; - - // Dedicated stderr log for the aggregator -- keeps its JSON tracing - // stream out of the parent's process.log. 0o600 to match the - // project's sensitive-log permissions policy (see - // /dev-rust-patterns lesson 14). let log_path = aggregator_log_path(session_dir); - let stderr_file = { - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - std::fs::OpenOptions::new() - .create(true) - .append(true) - .mode(0o600) - .open(&log_path) - .with_context(|| format!("failed to open {}", log_path.display()))? - } - #[cfg(not(unix))] - { - std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&log_path) - .with_context(|| format!("failed to open {}", log_path.display()))? - } - }; info!( bin = %aggregator_bin.display(), diff --git a/crates/capsem-process/src/owner_sandbox.rs b/crates/capsem-process/src/owner_sandbox.rs new file mode 100644 index 000000000..d381994d0 --- /dev/null +++ b/crates/capsem-process/src/owner_sandbox.rs @@ -0,0 +1,224 @@ +use anyhow::{Context, Result}; +use capsem_core::net::upstream_grant::UpstreamGrantClient; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use super::{aggregator_log_path, Args}; + +fn grant_owner_path( + policy: capsem_foundation::unix::worker_sandbox::Policy, + path: &Path, + access: capsem_foundation::unix::worker_sandbox::Access, +) -> Result { + let canonical = path + .canonicalize() + .with_context(|| format!("resolve sandbox grant {}", path.display()))?; + Ok(policy.allow(canonical, access)) +} + +fn grant_owner_session_paths( + mut policy: capsem_foundation::unix::worker_sandbox::Policy, + session_dir: &Path, +) -> Result { + use capsem_foundation::unix::worker_sandbox::Access; + + for (path, access) in [ + (capsem_core::guest_share_dir(session_dir), Access::ReadWrite), + ( + capsem_core::session::system_overlay_image_path(session_dir), + Access::ReadWrite, + ), + (capsem_core::session::image_share_path(session_dir), Access::ReadOnly), + ( + session_dir.join(capsem_core::session::OWNER_STATE_DIR), + Access::ReadWrite, + ), + (session_dir.join("serial.log"), Access::ReadWrite), + (session_dir.join("pty.log"), Access::ReadWrite), + (aggregator_log_path(session_dir), Access::ReadWrite), + ] { + policy = grant_owner_path(policy, &path, access)?; + } + Ok(policy) +} + +#[cfg(target_os = "linux")] +pub(super) fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { + use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; + + let mut policy = Policy::new(Role::VmOwner); + policy = grant_owner_session_paths(policy, session_dir)?; + policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; + policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; + policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; + for path in args.kernel.iter().chain(args.initrd.iter()) { + policy = grant_owner_path(policy, path, Access::ReadOnly)?; + } + for path in ["/dev/kvm", "/dev/vhost-vsock"] { + let path = Path::new(path); + if path.exists() { + policy = grant_owner_path(policy, path, Access::ReadWrite)?; + } + } + let executable = std::env::current_exe().context("locate VM-owner executable")?; + let router = executable.with_file_name("capsem-router"); + if router.exists() { + policy = grant_owner_path(policy, &router, Access::Executable)?; + } + for path in ["/lib", "/lib64", "/usr/lib", "/usr/lib64"] { + let path = Path::new(path); + if path.exists() { + policy = grant_owner_path(policy, path, Access::ReadOnly)?; + } + } + let linker_cache = Path::new("/etc/ld.so.cache"); + if linker_cache.exists() { + policy = grant_owner_path(policy, linker_cache, Access::ReadOnly)?; + } + for path in ["/lib64/ld-linux-x86-64.so.2", "/lib/ld-linux-aarch64.so.1"] { + let path = Path::new(path); + if path.exists() { + policy = grant_owner_path(policy, path, Access::Executable)?; + } + } + capsem_foundation::unix::worker_sandbox::confine(&policy)?; + Ok(()) +} + +#[cfg(target_os = "macos")] +pub(super) fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { + use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; + + let mut policy = Policy::new(Role::VmOwner); + policy = grant_owner_session_paths(policy, session_dir)?; + policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; + policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; + policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; + for path in args.kernel.iter().chain(args.initrd.iter()) { + policy = grant_owner_path(policy, path, Access::ReadOnly)?; + } + let router = std::env::current_exe() + .context("locate VM-owner executable")? + .with_file_name("capsem-router"); + if router.exists() { + policy = grant_owner_path(policy, &router, Access::Executable)?; + } + for path in [ + "/System/Library", + "/usr/lib", + "/private/var/db/dyld", + "/private/var/db/timezone", + "/etc/localtime", + "/dev/urandom", + ] { + let path = Path::new(path); + if path.exists() { + policy = grant_owner_path(policy, path, Access::ReadOnly)?; + } + } + let null = Path::new("/dev/null"); + if null.exists() { + policy = grant_owner_path(policy, null, Access::ReadWrite)?; + } + capsem_foundation::unix::worker_sandbox::confine(&policy)?; + Ok(()) +} + +pub(super) struct OwnerSandboxAttestation { + direct_path: PathBuf, + direct_file: std::fs::File, + guest_path: PathBuf, + guest_relative: Vec, + ledger_path: PathBuf, +} + +pub(super) fn prepare_owner_sandbox_attestation(session_dir: &Path) -> Result { + use std::os::unix::ffi::OsStrExt as _; + use std::os::unix::fs::OpenOptionsExt as _; + + let create = |path: &Path| { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .with_context(|| format!("prepare owner sandbox attestation {}", path.display())) + }; + let nonce = format!( + "{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + let direct_path = session_dir + .join(capsem_core::session::OWNER_STATE_DIR) + .join(format!(".owner-sandbox-attestation-{nonce}")); + let guest_relative = format!("workspace/.owner-metadata-attestation-{nonce}").into_bytes(); + let guest_path = session_dir + .join(capsem_core::GUEST_SHARE_DIR) + .join(std::ffi::OsStr::from_bytes(&guest_relative)); + let direct_file = create(&direct_path)?; + drop(create(&guest_path)?); + Ok(OwnerSandboxAttestation { + direct_path, + direct_file, + guest_path, + guest_relative, + ledger_path: session_dir.join("session.db"), + }) +} + +pub(super) async fn attest_owner( + attestation: OwnerSandboxAttestation, + service_socket: PathBuf, + upstream_grants: Arc, +) -> Result<()> { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + + let require_denied = |result: std::io::Result<()>, authority: &str| -> Result<()> { + match result { + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => Ok(()), + Err(error) => anyhow::bail!("VM owner sandbox returned {error} while denying {authority}"), + Ok(()) => anyhow::bail!("VM owner sandbox retained authority to {authority}"), + } + }; + require_denied( + std::fs::File::open("/etc/passwd").map(|_| ()), + "read unrelated host files", + )?; + require_denied( + std::fs::File::open(&attestation.ledger_path).map(|_| ()), + "open session ledger storage", + )?; + require_denied( + std::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, 9)).map(|_| ()), + "dial TCP sockets directly", + )?; + require_denied( + std::os::unix::net::UnixStream::connect(service_socket).map(|_| ()), + "dial control sockets by path", + )?; + require_denied( + attestation + .direct_file + .set_permissions(std::fs::Permissions::from_mode(0o777)), + "change host modes directly", + )?; + require_denied( + std::process::Command::new("/bin/true").status().map(|_| ()), + "execute arbitrary processes", + )?; + + tokio::task::spawn_blocking(move || { + capsem_core::GuestMetadataAuthority::set_mode(upstream_grants.as_ref(), &attestation.guest_relative, 0o701) + }) + .await + .context("join brokered metadata attestation")??; + let mode = std::fs::metadata(&attestation.guest_path)?.mode() & 0o7777; + anyhow::ensure!(mode == 0o701, "brokered metadata attestation returned mode {mode:o}"); + std::fs::remove_file(&attestation.guest_path)?; + std::fs::remove_file(&attestation.direct_path)?; + Ok(()) +} diff --git a/crates/capsem-process/src/private_seats.rs b/crates/capsem-process/src/private_seats.rs index d6562ebb1..c7de28a95 100644 --- a/crates/capsem-process/src/private_seats.rs +++ b/crates/capsem-process/src/private_seats.rs @@ -7,7 +7,6 @@ use anyhow::{Context, Result}; use capsem_proto::ipc::ServiceToProcess; use std::path::Path; use std::sync::Arc; -use tokio::net::UnixListener; use tokio::sync::mpsc; pub(crate) struct Seats<'a> { @@ -17,11 +16,12 @@ pub(crate) struct Seats<'a> { pub run_dir: Option<&'a Path>, } -fn bound(path: std::path::PathBuf, what: &str) -> Result<(std::path::PathBuf, UnixListener)> { +fn bound(path: std::path::PathBuf, what: &str) -> Result<(std::path::PathBuf, std::os::unix::net::UnixListener)> { if path.exists() { std::fs::remove_file(&path)?; } - let listener = UnixListener::bind(&path).with_context(|| format!("bind {what} socket"))?; + let listener = std::os::unix::net::UnixListener::bind(&path).with_context(|| format!("bind {what} socket"))?; + listener.set_nonblocking(true)?; std::fs::set_permissions(&path, std::os::unix::fs::PermissionsExt::from_mode(0o600))?; Ok((path, listener)) } @@ -29,9 +29,8 @@ fn bound(path: std::path::PathBuf, what: &str) -> Result<(std::path::PathBuf, Un /// What the seats were bound with, for the other things this owner asks /// the service on its VM's behalf. pub(crate) struct Bound { - pub service_socket: std::path::PathBuf, cables: Arc, - seat_listener: UnixListener, + seat_listener: tokio::net::UnixListener, } impl Bound { @@ -40,11 +39,25 @@ impl Bound { } } -pub(crate) fn bind( - seats: Seats<'_>, - job_store: &Arc, - control: mpsc::Sender, -) -> Result { +pub(crate) struct Prepared { + pub service_socket: std::path::PathBuf, + seat_path: std::path::PathBuf, + seat_listener: std::os::unix::net::UnixListener, +} + +impl Prepared { + pub(crate) fn activate(self, job_store: &Arc, control: mpsc::Sender) -> Result { + let cables = Arc::new(crate::cables::Cables::new(job_store.publisher.clone(), control)); + let _ = job_store.cables.set(Arc::clone(&cables)); + let _ = job_store.cable_seat.set(self.seat_path); + Ok(Bound { + cables, + seat_listener: tokio::net::UnixListener::from_std(self.seat_listener)?, + }) + } +} + +pub(crate) fn prepare(seats: Seats<'_>) -> Result { // The run directory the service named; otherwise where the service put // our IPC socket, walked up. let walked_up = seats @@ -59,16 +72,13 @@ pub(crate) fn bind( .map(Path::to_path_buf) .unwrap_or_else(|| run_dir.join("service.sock")); - let cables = Arc::new(crate::cables::Cables::new(job_store.publisher.clone(), control)); - let _ = job_store.cables.set(Arc::clone(&cables)); let (seat_path, seat_listener) = bound( capsem_foundation::uds::private_handoff_socket_path(&run_dir, seats.id)?, "cable seat", )?; - let _ = job_store.cable_seat.set(seat_path); - Ok(Bound { + Ok(Prepared { service_socket, - cables, + seat_path, seat_listener, }) } diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index 04ccd2f14..17dd70ec3 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -26,12 +26,12 @@ fn production_vm_owner_never_opens_the_session_database() { #[test] fn confinement_grants_exact_runtime_paths_and_attests_ledger_denial() { - let source = include_str!("main.rs"); - assert!(!source.contains("grant_owner_path(policy, session_dir")); - assert!(source.contains("grant_owner_session_paths(policy, session_dir)")); - assert!(source.contains("session_dir.join(\"session.db\")")); - assert!(source.contains("std::fs::File::open(&attestation.ledger_path)")); - assert!(source.contains("open session ledger storage")); + let sandbox = include_str!("owner_sandbox.rs"); + assert!(!sandbox.contains("grant_owner_path(policy, session_dir")); + assert!(sandbox.contains("grant_owner_session_paths(policy, session_dir)")); + assert!(sandbox.contains("session_dir.join(\"session.db\")")); + assert!(sandbox.contains("std::fs::File::open(&attestation.ledger_path)")); + assert!(sandbox.contains("open session ledger storage")); } #[test] @@ -43,17 +43,21 @@ fn suspend_checkpoint_stays_in_owner_state() { #[test] fn platform_confinement_and_attestation_precede_owner_readiness() { let source = include_str!("main.rs"); - let run = source.split_once("async fn run_async_main_loop(").unwrap().1; - let prepare = run.find("prepare_owner_sandbox_attestation").unwrap(); - let confine = run + let entry = source.split_once("fn main() -> Result<()> {").unwrap().1; + let prepare = entry.find("prepare_owner_sandbox_attestation").unwrap(); + let runtime = entry.find("Builder::new_current_thread()").unwrap(); + let confine = entry .find("confine_owner(&args") .expect("all supported platforms install the VM-owner policy"); - let attest = run + let attest = entry .find("attest_owner(") .expect("all supported platforms attest the installed policy"); - let seats = run.find("seats.start()").unwrap(); - let launched = run.find("launched.write_all").unwrap(); - assert!(prepare < confine && confine < attest && attest < seats && seats < launched); + let watcher = entry.find("watch_parent_or_exit").unwrap(); + let boot = entry.find("prepared_vm.boot()").unwrap(); + let run = entry.find("run_async_main_loop(").unwrap(); + assert!(!entry.contains("Builder::new_multi_thread()")); + assert!(runtime < prepare && prepare < confine && confine < attest && attest < watcher); + assert!(confine < boot && boot < run); } #[test] From eedda865044b9ca2c6c79d36319948bc7e78c151 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 13:58:48 +0000 Subject: [PATCH 244/464] test(logger): allow bounded large-body admission --- crates/capsem-logger/src/writer/remote.rs | 6 +++++- .../capsem-logger/src/writer/remote/tests.rs | 2 +- tests/citadel/fixture_ownership.toml | 2 +- tests/fixtures/session/test.db | Bin 589824 -> 585728 bytes 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/capsem-logger/src/writer/remote.rs b/crates/capsem-logger/src/writer/remote.rs index c8f2377c9..2308c0a7f 100644 --- a/crates/capsem-logger/src/writer/remote.rs +++ b/crates/capsem-logger/src/writer/remote.rs @@ -20,7 +20,11 @@ const STARTUP_TIMEOUT: Duration = Duration::from_secs(10); #[cfg(not(test))] const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); #[cfg(test)] -const OPERATION_TIMEOUT: Duration = Duration::from_millis(200); +// Large body admission includes framing, archive compression, SQLite commit, +// and its producer commitment. Two hundred milliseconds made the WARC +// backpressure regression fail during fixture setup on otherwise healthy +// macOS and loaded Linux builders. +const OPERATION_TIMEOUT: Duration = Duration::from_secs(2); const BACKPRESSURE_MAX_WAIT: Duration = Duration::from_millis(5); type ClientSender = ipc_channel::Sender; diff --git a/crates/capsem-logger/src/writer/remote/tests.rs b/crates/capsem-logger/src/writer/remote/tests.rs index d184459d4..101c0916e 100644 --- a/crates/capsem-logger/src/writer/remote/tests.rs +++ b/crates/capsem-logger/src/writer/remote/tests.rs @@ -178,6 +178,6 @@ async fn a_stalled_owner_fails_the_pending_operation_and_bounded_shutdown() { tokio::task::spawn_blocking(move || writer.shutdown_blocking()) .await .unwrap(); - assert!(started.elapsed() < std::time::Duration::from_secs(2)); + assert!(started.elapsed() < std::time::Duration::from_secs(3)); server.abort(); } diff --git a/tests/citadel/fixture_ownership.toml b/tests/citadel/fixture_ownership.toml index 9f5fa935c..3200d21e2 100644 --- a/tests/citadel/fixture_ownership.toml +++ b/tests/citadel/fixture_ownership.toml @@ -93,7 +93,7 @@ symbols = [ # session through today's writer: # cargo test -p capsem-logger --test roundtrip -- --ignored regenerate_session_fixture # then update these digests in the same commit. -sha256 = "02f4a0bcd631f0745f2346fec758eaecca1fdf7bbc8c7d435b7fc399e2ecc25a" +sha256 = "ec0f21a3de5d1a03d376455b17d31730fd66022b4080968fd1de4011302ad598" regenerator = "crates/capsem-logger/tests/roundtrip/fixture_regen.rs" # The ledger's bodies live in the selected generation beside it. The index in diff --git a/tests/fixtures/session/test.db b/tests/fixtures/session/test.db index b11a39f6cd8536c5ce4f006f7e01e14bfbca8752..a748383cb7ce6f850ea892e92eb2d77290bab5a5 100644 GIT binary patch delta 5527 zcmd5=dstNGnVS#Xlha! zQ?J}J!8aGRnoY=d-85RdjA_%>wq|z|8)K7pw|TaSB>M>Q*^QWHgKbS@zcYgj3cLU8 zv-|9M9=`d```zB}{r%p{p|v%obuhWx6w##$KTSRMrEzNc14ZJIZ^@(b7xKT$!}6!{ zNAmCG5BwJM3lbG*aDm^q*$P7c>CF!jO`DSwjmgP?k$(BX0m<3oba~6`YtT?dUd8R- z1#|yP+eBVp-R!JkBlVRH<;~6=c*D*w(b(o#tr|0Xtp2COjhgm&5RDcKJO};{#s0f@ z>R+X)`+~oT99wCR<&Qg1+o1W zvjqLIkwbd;ZvU8RZ#*37-|h4 zxKE;V+#YAk96}$d7`|o(o^S;zGwqm|Dt!d z0ciX^m$KL~o=wX^?H|~b1UmnRn|@D~C0HArHMP#>@@jWuW4*VLvB*mgN zBnGE%n>tjUJyoVYxSoo}>}=?Kz*||>;G8;Rip2j>S#0uz4(9YAcNPBDHn^)8u|3Ym zcRF3wPWRQw`gLm<~(!u z!byv&8|oGR!pJ|8D1UyLlrk=;ueoRUre=4|&T3}0W_JTZI&a)CS>iOWDmACiX=yBX z?_yfav76_x;oVOtooctcxu)J#>2)_V{|xIpvHs3A{f@F(X;Y;4=n;wsnEfcz_r^PV zd+7bKdRL9ryu7$0!X^ z<_Q?=<|!7Lbgl7~_Z|=mg1)taXw1j}{~Kkg!D#Z*fI*^$e%dOSp{rka@{clf_K&>| zh96#rtW14}O0ROLND-iTMJVEbCm+_2@egu?+(z;?m{c#T9Qqz5(y?b}9JaMVCa!FQ zMEs{V$dG1<#%0OiOT?e+X$+d$A&QN>)(*3kzJyoWK!;BP=ycxtMyJPH+1O-liN&Yd zVJ^;V2NPaw2P^q+Ld9CnM~yhz4ml8yH?E5@cwS94xTXW= zArhZ*6Agad0Ww7QW`YA|DXS8CcSB;NCq0CFeU05*Ut`@f6Tjz!xeyziftNc$*E<5q zq#+WYl_3M7@w^NLn*Fm3qS0!F!;P3ozydRI8G++>hbG+3!I#tJrkUZ~)R7?v)6xqT z{56GHFaw7uJb%|d$MI4%(FB%CaAUe8wZstKje*hJ5RZur17=0#is)>E6pc@e5{*R+ zksJk)Xv_wP0i%LQB02%?W5=EV$Yi!-mrKBF0Jc5@G!1lvcD>M{@m?v$I3eL+=ODmVkIz8PYPi=?Q~)7_9mghO%}kMNt|{!5)m8RB$>D^ znCSyMq>VGP71F_siThv)q+&!DgL3gch{8krAd`4Z_;Du#G({OGmS&5_JcjXO*X#qU(z6MP)VL}+m~?Tt3u3jFL#q>l#_!&a=l4N8JL3=gAZ%ylItLVLr(?Xt zNVNXPoF>3ZvA+RbwcF)Y5{MGl&~yMC9J?Eyd6cY}?&4s>e>nmLFO!PCCQ_kP5h*gc zq!M&I%7^3_?E{+MYUm2ve-6qF}e$1UVcy?BKy!GClN-7K%hsi(T9D*xHG%mtIEseu1w}}CAu}Mp3e5d!VZz@PMo^`Q>->zUj!u&59PgfA#%rNT?d^=k- zrlvy2BHVL>B(ezi{0;Kr-8T+=@dbHfT9U zvL&zIB*|dM4{kEoe?}3^{u8mK99)?Z40xT}gWeN9Io!*}hCh+JW8kZSU;ddykm+bN z$$y$`_)Z0JKo+iZvGbxUNvgWVI?nx&kuNloD7>i{`Kv}^?Ykiw`UZ)i?<6T;e|P%p zmA*pABuSN^(w))?T9@WM=|1sG)dePCk1sV(G;YWN-v)#fem3T%@oIc(gf-eU-lgfZ zFJ)ZGffx7S(`mc|itub2fBG)4f`#i@2z=klXG|Nln0qA6<)0@-xk}-=oL|0w<`SIo zBQ6UHmEGb4-}(`^l5oC+#W*^^ErSBQ9N^XVUhooi@DiR&<7Le7Gf$oSNuV@=E=S>RHlG z@l$aX?I&*%9c+ZvESK6rOH0Q;i59KJBPBu+9$6vS@MejyMp`2pH!lX?HtZ=8EZDF@ zm<^lpQi)(y`cf#Q{NoD2syXO=Ab9sK4sWdxKGinv^ag8w_K6%AT`Bx6NbZK3P&huf zTCg(AK3pS|#kicifBUl`(hVHIW#^Up~uMdK>Q zsBAnn#3$nC!@P@G>EX}#Q}U6$CBabKn;LXzj2vb)v*{Wi1?v=^#$enqGw<$7vKZU1 z@#|=%=P~6Ae&ibe{i(>G1!NA95>S86KMPQdC$96Wpfm)>nGxQA$9fpBBRzZ;+>hpG z_`mDB!QbJW;~(c&G%H%WQj_DR6HTt=r3rQFf4pW5cpI40`4OB5cxG(67e4q zJ9IzSN!lY?Q9hu1Q#O ztb9!rsR_1Wo=EedI^@kyQmVt(M4A8(;yWVE!di*WK%GQ0VJqfJ)DD%QI6O8Mtp@8hD z#mo^giSwx{FoB~v@Mx&$baU(m<#>Xl^PxUebY53P-@-(mu7kP>lNs<0j_MVgN8rZ_ z0qO;65dIh8v6FlKakYj8{Dt>;q?YWi#8ZT3!(*Y+@j9UyeXT<4cq%Cv=w^xr=t=sQ zt_2bN)2bNu$)xpau4-!4gKDo@C!Lg1#53ak^tZG@_*&=|68JXw8H;NRRWHj8`nMSm zn>e4m6VDHcOR;TOOh(hNm@Nuk(OB#N+&V1IhrRgCkeGx&7#3}s{m>lzXUX9lO#V!K zb;7p-gP(~PfiCk*{Bs3cH8fePy{VFdsZ9Sk{y;;!pb7J})B|q(p_V=dE}X5SZfL|# z9j%obLP=B|`h1uXK{v5`?}%XY9#vK);z$JD!jLSvF782R1dYV^uZxM$tdy#G%zGFK zGVoU;;tcF45VY7bBAQ?a9vu-Ky}x9HKC@iV;U`SU`FLk<8(oC=Lx|@QA^ckj<$J%R ziOgvtfpNwaV!a(0H;{_4bPj(KzCJPIHc&HEp^4BOe4nu0D_pE<$t9s3 zOlp;iykBf$MKV5K%W4FMr(QEr(uYBI!2czaEu7E13lCov9k34rS4A(hpnXWp#jl4L uA=VCwYV;3@w!SR!3?p9FfLUr;2nPeZwn)b)90<&OMEW-ZPxnAm?lMg=cNGG+yWOxeC;XAAJ9$)j)xEj(3+tQO zs~hTEh;!_d3pp6-d3*-~0`!ko7HI-1f@(^%bB|KN>j2mg}|4Ied6!_zNG zmEY3jPNo;v)KW8b0yUGyP&2-ano*_H48N0_qC#qNa;Qnkpe8Ps8momGMc=f{WNJaX zOs%j&yA1uvBo(^*#r=m39_zm1T;_l^-OGmc0<7+CE}?&`!qp>wVrG1az>o7Kyo2U9 z^Z$R#qiDoBb9_Fti#|Rp?OzhR1yzXUYhZVz2UtGq&{$|9O3; zC58-sShu&8hwHW!fjPW!%ScedC%0sRDSUa$$#BsQf4FO#J-m7L3&6W4!VG3!w^-zM zj|X-P4(}Ha^yO9@pUzy}gJu2?`qr?}blt-JWs<8~+Q(hc)g9CGi3aPsue=Ab$ySXC zvQ#J7*2($O1*uNESKKG=Bg?mn7A3}DX2@p=KiyRdTKIHVBN<$=eo(kw*A$3fwdA4wm!K#~`+*=syr3Ln3EBpDk{&k- z1?$U`O$+0>eV7|U=~$BmiTI@l;&EVrnw~3#7GY)5DK>(8R`4pCh zyXn~+;<4O5$^)T67}qH$VTL};ytKft3wVtF)q*nv;0F!Y1fY;!acQBPj;ngaM7$V) zB=f2QE4TZ6upt2}yx>Nb3kewV!kWI2#I4!z=|83l2A=-w%voO=6P2W0W7=zK8|&Mf zn;WZjqk}h|Vo}V6Eq(0kDct@?(HxoJgG;xC*QwWcM-J9TkefUWO?CB)8yao+W)|cV zQiNt2I=zs!y9ipct+q!68x#oel+A2?B<9nY&KSY6%Hm)LNQx)hPO(%d&=!T=u4+?r z2DcBP72^!Y??_>Bkfl5JE;XIotH`6{@KOhPbYdz?ur70Q`*;3gB>Proq*Z zTMQ-~>SQIigGzYN$;wQP{kVNF!D=;N?TuxM$%OThLrrZv=kRsjLt(v41K{$9GdHejWNQlkrN2%+cdu@wB_>V3mXAK`JhFuz}z@uoEnm^5c5h zKr(zxiTvqvaF~=I>#25aV!6fD)*q%sY1;qK^ zCb7HBt7BZ;?sP(UE^5il4%wKK%r^HCFnc0Rc2*9zW0&#Qcd4xtrj--L+&aZl(38Rv z!F!!u7Pc@OSNF02zIv9>eY2PO`t}e>j_n*Pf_xltj*T_xGV|T54>S=sTEA7aM($qD zX767v#db?Ozto(zY=H}{f;yI;;14RFC`%O(TD6dtXxn2OYzq)`CY#q-3%T9vg^e?D z{W+Enci_*@v2A_fiK`Qt1;=Kvgoyn^w(T|}+vz9UNjcoulE)L#eV*vN?>xm+)p-_B z3yd_KWDIUUPc+ZEK!fz8;v46gTg|^|M*Ibuu~DBvPDsRu&eP6kUtne;_xuYi52cGV zwEqJ0s6K8lq1UDuD}Q`}B|v|C{ydRzWsjH?{bgZ$!-D!?d(DCt-%V=L6=)#xu-?>1S9b znN@a%72sQEm=o8WVQwhWJJRuYXPAcfoTc9X(R%}!dX^1Sl(vdrKs8ho;^RPkEU6UN-!I(>V-3Vl-!F}2QVCT!7%1eKIB}Mg zficxmj{aoEr)EioFdSc-C6&S5ShiG7#m5V&jxDa12B^chebOKZm0-~&9a}q8~;^--#EiX=eHfarJjVuOD>v!+{qfGyWh( z#M`0RIO9!mi=MhihtsyDMma@A*EJengD;-3Hw5e3>HIMFE2yj^ zTNf$u%Gj&6CXC_s2^4XIuw@CIQQkf*wGs*Lc~#o1j^_4pG}7M~F&&Yb_+7*X>cZCP zQUV@7OfLS@5y^xVJ)#MpJ0i{3);LBPfjH9$#P<)=#9>FJ1emDv;-`txSgQ%nzv;1_-& zREFKI0jzckKGQDGQ70QIIL@dcvpeMJP>IVsO`aBl8p+rZJ}IA?09J* zO}GaO7RhbdD+W)9)>CT-MGc?V9BjXuB);-EE%Ndrc_j0E;}TZXk9U5smL1w zJBqKueMm~i_YX<0rbKaG(9vEK)w8eMFh#mwlKu@0w3{32Zazcc+*c&ut;*a}uSjE! zb(-oIeQllp`a9O~9G1GG*+wL3qa3rlOKur&HBS}n!b(9~t9fh(Y~xHHnSv%w-K7rU zNBB_Xr-~vkmQF~i;*(+u+riS{$Dj!-tv%Mdn9pL?#-v&HS%#bUnrqFq=V+8Ji0Z#Ep0$K6na4SBp6nvGiES9!cpnoX3)LDCOpJzqEK3x8vn}9)L!@ z#C711Jle&y0`7pBXzr8;!F>Iola5^lw5j=>vI`!-wViS?JcI{3LZxkFd(VK_UFb1`I@j@DMW6oZA-uf0ma`uY;tCr{({sU_GmOG?7n zrE&oGERh|!@3<6)TAS=AvX!*SWq78QdZTIJ*)};3zK6%#Q|UVJ?$7eK9nU~;?cRI9GV4xLt#v8tVR|2M*JZh|0)pXo;&SwH)a zXt{PxhqJS|hwdolnWXYU$kv39XYqBg05iNis5TnaplnoiKlAcR0`sq4-b~mvW%Ide z1MQm@H5pO-Nj9&7d3bjYpAU2O71Hr|4*#C=AYJavzs0z}sY)`m$7m09$x3WQ=~Di8 z-GLriij7ER{4YB;bV<4K@(MwFLMyNxwB7Y}zN#twB%h?bsQBa+B-ArxG308;*9y5G zet>^=6LsPOd?p03Ilza(qqsM~rLd+@IwZn1n-Hx^~`--Gzi!CcibrxR9^ zA;6F1auq9b`MqlWHJ3(j>R!y{@nn4TCXSv}{4AHJ=%n=Vd}!0-DQ#0H9l&S$ct2w0 wLLcGQYFxhk*2kT@FR^RMAtJ@9ZFj)R$fCQ|Cl&ZnB(Xxh$l!;&E7hI<2LyM5ZU6uP From c6a9992c6ff8d33c8b9e396e0a7663213cb08687 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 13:58:49 +0000 Subject: [PATCH 245/464] security(service): own MCP discovery and worker fixtures --- CHANGELOG.md | 4 + .../src/container_setup/activity.rs | 28 ++++-- crates/capsem-service/src/mcp_routes.rs | 88 ++++++++++++++++--- .../src/owner_connection/tests.rs | 10 +-- .../capsem-service/src/owner_handoff/tests.rs | 2 + crates/capsem-service/src/tests/lifecycle.rs | 74 +++------------- crates/capsem-service/src/tests/mcp_routes.rs | 39 ++++++++ crates/capsem-service/src/vm_lifecycle.rs | 2 + .../src/vm_lifecycle/resume_process.rs | 11 ++- 9 files changed, 167 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e2fd2f46..fcb08a519 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 grants; deny sibling files, path-based control connections and direct network dialing after startup. +- Keep the global MCP discovery cache under coordinator ownership. VM owners + return complete tool schemas over authenticated IPC without receiving a + cross-session cache-file grant. + - Broker guest VirtioFS permission changes through the coordinator beneath the session's guest share, keeping direct host permission syscalls denied and attesting that authority before the VM owner publishes readiness. diff --git a/crates/capsem-service/src/container_setup/activity.rs b/crates/capsem-service/src/container_setup/activity.rs index bf0608c70..e21a1bb67 100644 --- a/crates/capsem-service/src/container_setup/activity.rs +++ b/crates/capsem-service/src/container_setup/activity.rs @@ -24,6 +24,20 @@ struct State { retiring: Weak>>>, } +impl State { + fn remove_retirement(&self) { + if let Some(retiring) = self.retiring.upgrade() { + let mut retiring = retiring.lock().unwrap(); + if let Some(generations) = retiring.get_mut(&self.id) { + generations.retain(|activity| activity.0.generation != self.generation); + if generations.is_empty() { + retiring.remove(&self.id); + } + } + } + } +} + pub(super) struct Lease(Arc); impl Activity { @@ -56,6 +70,10 @@ impl Activity { tokio::pin!(notified); notified.as_mut().enable(); if self.0.running.load(Ordering::Acquire) == 0 { + // The final lease can publish zero before its Drop cleanup + // reaches the retirement map. Make returning from wait the + // barrier callers expect instead of exposing that window. + self.0.remove_retirement(); return; } notified.await; @@ -77,16 +95,8 @@ impl Lease { impl Drop for Lease { fn drop(&mut self) { if self.0.running.fetch_sub(1, Ordering::AcqRel) == 1 { + self.0.remove_retirement(); self.0.idle.notify_waiters(); - if let Some(retiring) = self.0.retiring.upgrade() { - let mut retiring = retiring.lock().unwrap(); - if let Some(generations) = retiring.get_mut(&self.0.id) { - generations.retain(|activity| activity.0.generation != self.0.generation); - if generations.is_empty() { - retiring.remove(&self.0.id); - } - } - } } } } diff --git a/crates/capsem-service/src/mcp_routes.rs b/crates/capsem-service/src/mcp_routes.rs index 03c9aa8b5..f110ba7d0 100644 --- a/crates/capsem-service/src/mcp_routes.rs +++ b/crates/capsem-service/src/mcp_routes.rs @@ -116,14 +116,39 @@ pub(super) async fn handle_mcp_default_info( } pub(super) fn latest_mcp_tool_cache(state: &ServiceState) -> Vec { - let latest = capsem_core::mcp::load_tool_cache(); - let Ok(mut cache) = state.mcp_tool_cache.lock() else { - return latest; - }; - if !latest.is_empty() || cache.is_empty() { - *cache = latest; - } - cache.clone() + state.mcp_tool_cache.lock().unwrap().clone() +} + +pub(super) fn persist_mcp_tool_cache( + state: &ServiceState, + tools: Vec, +) -> Result<(), AppError> { + let tools = tools + .into_iter() + .map(|tool| capsem_proto::mcp_contracts::McpToolDef { + namespaced_name: tool.namespaced_name, + original_name: tool.original_name, + description: tool.description, + input_schema: tool.input_schema, + server_name: tool.server_name, + annotations: tool.annotations, + timeout_secs: None, + }) + .collect::>(); + let mut cache = state + .mcp_tool_cache + .lock() + .map_err(|_| AppError(StatusCode::INTERNAL_SERVER_ERROR, "MCP tool cache lock poisoned".into()))?; + let updated = capsem_core::mcp::build_cache_entries(&tools, &cache); + capsem_core::mcp::save_tool_cache(&updated).map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("persist MCP tool cache: {error}"), + ) + })?; + *cache = updated; + drop(cache); + Ok(()) } /// GET /mcp/servers/{server_id}/tools/list -- one server's discovered tools @@ -180,18 +205,55 @@ pub(super) async fn handle_mcp_server_refresh( .collect::>() }; let mut refreshed = 0; + let mut catalog_saved = targets.is_empty(); for (vm_id, uds_path) in &targets { let id = state.next_job_id(); match send_ipc_command(&state, uds_path, ServiceToProcess::McpRefreshTools { id }, Some(30)).await { - Ok(_) => refreshed += 1, + Ok(ProcessToService::McpRefreshResult { + success: true, + error: None, + .. + }) => { + refreshed += 1; + if !catalog_saved { + let id = state.next_job_id(); + match send_ipc_command(&state, uds_path, ServiceToProcess::McpListTools { id }, Some(30)).await { + Ok(ProcessToService::McpToolsResult { tools, .. }) => { + persist_mcp_tool_cache(&state, tools)?; + catalog_saved = true; + } + Ok(reply) => warn!( + vm_id = %vm_id, + server_id = %server_id, + ?reply, + "MCP tool catalog returned an unexpected reply" + ), + Err(error) => warn!( + vm_id = %vm_id, + server_id = %server_id, + %error, + "MCP tool catalog query failed" + ), + } + } + } + Ok(ProcessToService::McpRefreshResult { error, .. }) => warn!( + vm_id = %vm_id, + server_id = %server_id, + ?error, + "MCP tool refresh was refused" + ), + Ok(reply) => warn!( + vm_id = %vm_id, + server_id = %server_id, + ?reply, + "MCP tool refresh returned an unexpected reply" + ), Err(error) => warn!(vm_id = %vm_id, server_id = %server_id, %error, "MCP tool refresh failed"), } } - if let Ok(mut cache) = state.mcp_tool_cache.lock() { - *cache = capsem_core::mcp::load_tool_cache(); - } Ok(Json(api::McpRefreshResponse { - success: refreshed == targets.len(), + success: refreshed == targets.len() && catalog_saved, server_id, instances: refreshed, })) diff --git a/crates/capsem-service/src/owner_connection/tests.rs b/crates/capsem-service/src/owner_connection/tests.rs index af5e3cf58..97ace7cbd 100644 --- a/crates/capsem-service/src/owner_connection/tests.rs +++ b/crates/capsem-service/src/owner_connection/tests.rs @@ -43,7 +43,10 @@ async fn generation_changed_during_hello_cannot_receive_a_request() { .get_mut("owner") .unwrap() .generation = uuid::Uuid::new_v4(); - capsem_foundation::ipc_handshake::negotiate_responder(&mut socket, "capsem-process", "").unwrap(); + // The initiator may observe the replaced generation and close + // before completing the hello. Either outcome must leave the + // rejected connection with no request authority. + let _ = capsem_foundation::ipc_handshake::negotiate_responder(&mut socket, "capsem-process", ""); socket }) .await @@ -53,10 +56,7 @@ async fn generation_changed_during_hello_cannot_receive_a_request() { let mut socket = server.await.unwrap(); tokio::task::spawn_blocking(move || { use std::io::Read; - socket.set_nonblocking(false).unwrap(); - socket - .set_read_timeout(Some(std::time::Duration::from_secs(5))) - .unwrap(); + assert!(capsem_foundation::unix::fd::wait_readable(socket.as_fd(), std::time::Duration::from_secs(5)).unwrap()); assert_eq!(socket.read(&mut [0; 1]).unwrap(), 0); }) .await diff --git a/crates/capsem-service/src/owner_handoff/tests.rs b/crates/capsem-service/src/owner_handoff/tests.rs index ae0d2cb3b..4c8dd4280 100644 --- a/crates/capsem-service/src/owner_handoff/tests.rs +++ b/crates/capsem-service/src/owner_handoff/tests.rs @@ -5,6 +5,8 @@ use crate::tests::{insert_fake_instance_with_session_dir, make_test_state_owned} async fn handoffs_use_the_trusted_identity_with_short_and_hashed_paths() { for long in [false, true] { let mut owned = make_test_state_owned(); + let short_root = tempfile::tempdir_in("/tmp").unwrap(); + owned.run_dir = short_root.path().join("r"); if long { owned.run_dir = owned.run_dir.join("long-runtime-root-".repeat(8)); } diff --git a/crates/capsem-service/src/tests/lifecycle.rs b/crates/capsem-service/src/tests/lifecycle.rs index 0e3002d5b..becda0e9d 100644 --- a/crates/capsem-service/src/tests/lifecycle.rs +++ b/crates/capsem-service/src/tests/lifecycle.rs @@ -1053,38 +1053,10 @@ fn resume_sandbox_requires_uuid_route_id_not_display_name() { ); } -#[tokio::test] -async fn resume_sandbox_passes_the_session_scratch_disk_size_to_process() { - let _env_lock = SETTINGS_ENV_LOCK.lock().await; - let settings_dir = tempfile::tempdir().unwrap(); - let (_settings_guard, _, _) = install_empty_settings_env(&settings_dir); - let (mut state, _dir) = make_test_state_with_tempdir(); - let run_dir = state.run_dir.clone(); - let argv_path = run_dir.join("resume-argv.txt"); - let argv_tmp_path = run_dir.join("resume-argv.txt.tmp"); - let process_path = run_dir.join("record-process-argv.sh"); - std::fs::write( - &process_path, - format!( - "#!/bin/sh\nprintf '%s\\n' \"$@\" > '{}'\nmv '{}' '{}'\nsleep 1\n", - argv_tmp_path.display(), - argv_tmp_path.display(), - argv_path.display() - ), - ) - .unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut perms = std::fs::metadata(&process_path).unwrap().permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&process_path, perms).unwrap(); - } - Arc::get_mut(&mut state).unwrap().process_binary = process_path; - install_test_runtime_assets(&state); - - let vm_id = new_persistent_vm_id(); - let session_dir = state.run_dir.join("persistent").join(&vm_id); +#[test] +fn resume_sandbox_passes_the_session_scratch_disk_size_to_process() { + let session = tempfile::tempdir().unwrap(); + let session_dir = session.path().join("persistent"); std::fs::create_dir_all(&session_dir).unwrap(); let rootfs = capsem_core::session::system_overlay_image_path(&session_dir); std::fs::create_dir_all(rootfs.parent().unwrap()).unwrap(); @@ -1092,38 +1064,18 @@ async fn resume_sandbox_passes_the_session_scratch_disk_size_to_process() { .unwrap() .set_len(24 * 1024 * 1024 * 1024) .unwrap(); - let mut entry = test_persistent_entry("resume-size", session_dir); - entry.id = vm_id.clone(); - state - .persistent_registry - .lock() - .unwrap() - .data - .vms - .insert("resume-size".to_string(), entry); - - let resume_id = vm_id.clone(); - let error = state - .off_worker(move |state| state.resume_sandbox(&resume_id, None, None)) - .await - .unwrap() - .expect_err("the focused argv fixture has no confined helper binaries"); - assert!( - error.to_string().contains("acquire proxy ledger channel"), - "resume reached an unexpected failure after process spawn: {error:#}" - ); - for _ in 0..50 { - if argv_path.exists() { - break; - } - std::thread::sleep(std::time::Duration::from_millis(20)); - } - let argv = std::fs::read_to_string(&argv_path).expect("resume process argv should be recorded"); - let args: Vec<&str> = argv.lines().collect(); + let entry = test_persistent_entry("resume-size", session_dir); + let mut command = tokio::process::Command::new("capsem-process"); + crate::vm_lifecycle::configure_resume_storage(&mut command, &entry).unwrap(); + let args = command + .as_std() + .get_args() + .map(|argument| argument.to_string_lossy().into_owned()) + .collect::>(); let size_flag = args .windows(2) .find(|window| window[0] == "--scratch-disk-size-gb") - .map(|window| window[1]); + .map(|window| window[1].as_str()); assert_eq!( size_flag, Some("24"), diff --git a/crates/capsem-service/src/tests/mcp_routes.rs b/crates/capsem-service/src/tests/mcp_routes.rs index 5a08633c5..e6039336f 100644 --- a/crates/capsem-service/src/tests/mcp_routes.rs +++ b/crates/capsem-service/src/tests/mcp_routes.rs @@ -71,6 +71,45 @@ async fn mcp_reads_list_the_configured_and_builtin_servers() { assert_eq!(refresh["instances"], 0, "no running VM, nothing contacted"); } +#[tokio::test] +async fn coordinator_persists_complete_owner_tool_catalog() { + let _env_lock = SETTINGS_ENV_LOCK.lock().await; + let dir = tempfile::tempdir().unwrap(); + let _home = EnvVarGuard::set("CAPSEM_HOME", dir.path()); + let state = make_asset_state(dir.path().join("assets")); + let input_schema = json!({ + "type": "object", + "properties": {"query": {"type": "string"}} + }); + + crate::mcp_routes::persist_mcp_tool_cache( + &state, + vec![capsem_proto::ipc::McpToolStatus { + namespaced_name: "wiki__search".into(), + original_name: "search".into(), + description: Some("Search the wiki".into()), + input_schema: input_schema.clone(), + server_name: "wiki".into(), + annotations: None, + }], + ) + .unwrap(); + + let expected_hash = capsem_core::mcp::compute_tool_hash(&capsem_proto::mcp_contracts::McpToolDef { + namespaced_name: "wiki__search".into(), + original_name: "search".into(), + description: Some("Search the wiki".into()), + input_schema, + server_name: "wiki".into(), + annotations: None, + timeout_secs: None, + }); + let cache = crate::mcp_routes::latest_mcp_tool_cache(&state); + assert_eq!(cache.len(), 1); + assert_eq!(cache[0].pin_hash, expected_hash); + assert_eq!(capsem_core::mcp::load_tool_cache(), cache); +} + #[tokio::test] async fn a_tool_permission_edit_is_written_to_settings_and_recorded() { let _env_lock = SETTINGS_ENV_LOCK.lock().await; diff --git a/crates/capsem-service/src/vm_lifecycle.rs b/crates/capsem-service/src/vm_lifecycle.rs index 010ee355c..c0ad6ee02 100644 --- a/crates/capsem-service/src/vm_lifecycle.rs +++ b/crates/capsem-service/src/vm_lifecycle.rs @@ -3,6 +3,8 @@ use super::*; mod provision; mod resume; mod resume_process; +#[cfg(test)] +pub(crate) use resume_process::configure_resume_storage; mod session_dirs; pub(crate) use resume::handle_resume; #[cfg(test)] diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 6d826b4f6..39e17b1b7 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -1,5 +1,12 @@ use super::*; +pub(crate) fn configure_resume_storage(command: &mut tokio::process::Command, entry: &PersistentVmEntry) -> Result<()> { + command + .arg("--scratch-disk-size-gb") + .arg(session_rootfs_size_gb(entry)?.to_string()); + Ok(()) +} + impl ServiceState { /// Resume a stopped persistent VM by re-spawning capsem-process against its /// existing session directory. @@ -78,7 +85,6 @@ impl ServiceState { crate::upstream_broker::PendingBroker::pair(active_policy.broker_policy())?; let proxy_upstream = proxy_upstream_broker.worker_stream()?; let upstream_policy = owner_upstream_policy.merge(proxy_upstream_policy); - let scratch_disk_size_gb = session_rootfs_size_gb(&entry)?; let resolved = self.resolve_pinned_asset_paths(&entry.asset_pins)?; self.validate_pinned_asset_files(&resolved, &entry.asset_pins)?; @@ -93,6 +99,7 @@ impl ServiceState { if !self.process_binary.exists() { child_cmd = tokio::process::Command::new("cache/target/cargo/debug/capsem-process"); } + configure_resume_storage(&mut child_cmd, &entry)?; // Inject VM identity so the guest knows its own name/ID. child_cmd.arg("--env").arg(format!("CAPSEM_VM_ID={}", vm_id)); @@ -193,8 +200,6 @@ impl ServiceState { .arg(cpus.to_string()) .arg("--ram-mb") .arg(ram_mb.to_string()) - .arg("--scratch-disk-size-gb") - .arg(scratch_disk_size_gb.to_string()) .arg("--uds-path") .arg(&uds_path) // Explicitly, because `uds_path` may have been shortened out From 400b44e13420c10f6337e33757704482c26c1fea Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 13:58:49 +0000 Subject: [PATCH 246/464] docs(architecture): explain worker startup confinement --- .../docs/architecture/host-isolation.md | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/web/docs/src/content/docs/architecture/host-isolation.md b/web/docs/src/content/docs/architecture/host-isolation.md index 88a56b73f..8f491caf7 100644 --- a/web/docs/src/content/docs/architecture/host-isolation.md +++ b/web/docs/src/content/docs/architecture/host-isolation.md @@ -92,11 +92,11 @@ grants fail closed. | Process | Permitted after readiness | Denied or absent | |---|---|---| -| `capsem-service` | Global lifecycle, settings and corp policy, credential store, session registry, resource creation and capability grants; producer ordering and external commitment checkpoints | It is part of the trusted computing base; clients reach it through the local UDS or authenticated gateway routes | +| `capsem-service` | Global lifecycle, settings and corp policy, credential store, MCP discovery catalog, session registry, resource creation and capability grants; producer ordering and external commitment checkpoints | It is part of the trusted computing base; clients reach it through the local UDS or authenticated gateway routes | | `capsem-gateway` | Accept on listeners bound before confinement; use coordinator-granted service/owner channels; update its private readiness files | Ambient file reads, path-based UDS connects, outbound TCP, new binds, process execution | -| `capsem-process` | One VM and its exact session runtime paths; read exact boot assets and active policy; use coordinator-granted upstream and ledger channels; execute only the installed router helper | Other sessions, direct `session.db` access, ambient outbound connects/binds, unrelated host files and processes | -| `capsem-proxy` | Consume granted HTTP, DNS, MCP, upstream, credential, ledger, metrics, private-name, and policy channels | All filesystem access, socket creation, arbitrary dial/bind/listen, process execution, signals to the coordinator | -| `capsem-ledger` | Read and write one session directory; serve authenticated operations over granted connected channels | Other paths, all network, new Unix sockets, process execution, signals to the coordinator | +| `capsem-process` | One VM and its exact session runtime paths; read exact boot assets and active policy; use coordinator-granted upstream and ledger channels; execute only the installed router helper | Other sessions, the global MCP discovery catalog, direct `session.db` access, ambient outbound connects/binds, unrelated host files and processes | +| `capsem-proxy` | Consume granted HTTP, DNS, MCP, upstream, credential, ledger, metrics, private-name, and policy channels | All filesystem access, arbitrary dial/bind/listen and control-socket access, process execution, signals to the coordinator | +| `capsem-ledger` | Read and write one session directory; serve authenticated operations over granted connected channels | Other paths, ambient network and control-socket access, process execution, signals to the coordinator | | `capsem-router --expose` | Copy bytes between connected host TCP and guest VSOCK descriptors | Listener ownership, destination selection, filesystem, arbitrary network, VM control | | `capsem-router --network` | Switch frames among the descriptor-backed cables of one named network | Uplink, listener, outbound socket, another network's members, policy decisions | | `capsem-mcp-aggregator` | Connect to configured external MCP servers and return protocol results | VM control, session files, ledger storage, service API; requests arrive only after the session policy boundary | @@ -115,14 +115,26 @@ network, execution, and cross-sandbox signal operations, then add their exact path grants. Only the gateway may accept inbound traffic on a listener it already owns. -On Linux, workers install a Landlock ABI 6 ruleset across every existing thread -and then a seccomp filter with thread synchronization. Landlock limits -filesystem and network scope; seccomp denies direct connect, bind, listen, -process inspection, namespace/mount operations, kernel attack surfaces, and -permission/ownership changes. Proxy and ledger workers also cannot create +On Linux, workers support Landlock ABI 6 and newer. ABI 8 adds synchronized +restriction of existing threads. On ABI 6 and 7, startup remains +single-threaded until Landlock is installed; every later runtime, hypervisor, +database, monitor, logging, and parent-watch thread inherits that domain. A +worker refuses startup if an older ABI already has a sibling thread. Seccomp +is installed with thread synchronization after the Landlock ruleset. Landlock +limits filesystem scope; seccomp denies direct connect, bind, listen, process +inspection, namespace and mount operations, kernel attack surfaces, and +permission or ownership changes. Proxy and ledger workers also cannot create sockets or send signals. Existing connected descriptors remain usable because they are the capability. +The VM owner prepares only resources that cannot be acquired after the +sandbox: its inherited coordinator channel, exact IPC listeners, readiness and +log files, metric channel, and Linux AF_VSOCK listeners. It then installs and +self-attests confinement before starting the parent watcher, hypervisor, +Tokio or blocking worker threads, ledger client, filesystem monitor, metric +exporter, or any VM worker. KVM consumes the prepared VSOCK descriptors after +confinement; it does not regain `socket` or `bind` authority. + Both platforms clear inherited environment state where secrets could otherwise leak and close descriptors that were not deliberately preserved. The sandbox is additive to normal ownership and mode checks; mode `0600` sockets and mode From a24b6084aee22a7c6f8f339e1331da243d087c6f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 13:58:49 +0000 Subject: [PATCH 247/464] test(gate): qualify extracted worker owners --- build_system/tests/gate/test_build_assets.py | 40 +++++++++---------- build_system/tests/gate/test_gate_sandbox.py | 27 ++++++------- .../release/test_release_doctor_contract.py | 34 +++++++++------- codecov.yml | 1 + config/gate.toml | 25 +++++------- tests/citadel/test_bench_route_coverage.py | 7 ++-- .../citadel/test_guest_share_never_by_path.py | 3 +- tests/citadel/test_upstream_grant_boundary.py | 2 +- tests/test_installed_winterfell_contract.py | 10 +++-- 9 files changed, 76 insertions(+), 73 deletions(-) diff --git a/build_system/tests/gate/test_build_assets.py b/build_system/tests/gate/test_build_assets.py index c6d97d778..3b419823b 100644 --- a/build_system/tests/gate/test_build_assets.py +++ b/build_system/tests/gate/test_build_assets.py @@ -73,7 +73,9 @@ def test_build_assets_uses_capsem_admin() -> None: assert argv[: len(config.imagebuild.admin)] == list(config.imagebuild.admin) assert argv[argv.index("--config-root") + 1] == config.imagebuild.config_root assert argv[argv.index("--arch") + 1] == "arm64" - assert "uv run --project build_system --frozen capsem-builder build guest/" not in " ".join(argv) + assert "uv run --project build_system --frozen capsem-builder build guest/" not in " ".join( + argv + ) def test_every_asset_build_rail_materializes_exact_bases_before_building() -> None: @@ -180,9 +182,7 @@ def test_asset_build_primitives_accept_an_isolated_output_root() -> None: default = build_argv(config, arch="arm64", template="all") assert default[default.index("--output") + 1] == config.imagebuild.output - isolated = build_argv( - config, arch="arm64", template="all", output="/tmp/lane-a" - ) + isolated = build_argv(config, arch="arm64", template="all", output="/tmp/lane-a") assert isolated[isolated.index("--output") + 1] == "/tmp/lane-a" @@ -307,9 +307,7 @@ def test_asset_ci_uses_primitives_owned_by_just_test() -> None: assert "ASSET_ARCH: ${{ matrix.arch }}" in workflow # The recipe takes the architecture alone. assert re.search(r'^\s*just build-assets "\$ASSET_ARCH"$', workflow, re.MULTILINE) - assert "pack-initrds" in _planned( - "build-assets", arch="arm64", template="rootfs" - ) + assert "pack-initrds" in _planned("build-assets", arch="arm64", template="rootfs") # The lanes reach the same builder invocation directly, each with its own # output. Asserting they still *mention* the retired recipe was asserting # on a comment; this is the claim underneath it. @@ -357,9 +355,7 @@ def test_asset_matrix_preflights_once_and_reuses_the_public_build_primitive() -> # The primitive builds; it does not preflight. Preflighting per stage is # what made a four-cell matrix run doctor four times. - assert "install-tools" not in " ".join( - build_argv(config, arch="arm64", template="all") - ) + assert "install-tools" not in " ".join(build_argv(config, arch="arm64", template="all")) assert "doctor" not in lanes @@ -377,19 +373,18 @@ def test_in_container_commands_write_only_where_the_container_user_owns() -> Non config = gate_config.load(PROJECT_ROOT) guest = config.install.guest_user - container = (PROJECT_ROOT / "build_system" / "builder" / "gate" / "installcontainer.py").read_text() + container = ( + PROJECT_ROOT / "build_system" / "builder" / "gate" / "installcontainer.py" + ).read_text() proof = (PROJECT_ROOT / "build_system" / "builder" / "gate" / "installproof.py").read_text() # Replacing any owned path needs write permission on its parent. Parents # are derived from the typed layout and claimed without recursively # walking unrelated Cargo or release output. + assert "settings.layout.owned_parent_paths(settings.mount)" in container assert "claim_owned_paths(self._docker, self.name, self._settings)" in container - runner = RecordingRunner(PROJECT_ROOT) - claim_owned_paths(Docker(runner), config.install.container, config.install) - assert list(runner.commands[-1].argv) == [ - "docker", "exec", config.install.container, "chown", f"{guest.name}:{guest.name}", - *config.install.layout.owned_parent_paths(config.install.mount), - ] + assert '["chown", f"{guest}:{guest}", *parents]' in container + assert '["chown", "-R", f"{guest}:{guest}", *parents]' not in container # Every path this user writes has to live off the bind mount. for path in (guest.tmp, guest.pytest_cache, guest.asset_manifest, config.install.venv): @@ -432,8 +427,12 @@ def test_isolated_test_recipes_trap_test_home_service_cleanup() -> None: from capsem_builder.gate.command import GateCommand from helpers.gate import RecordingRunner - workspace_source = (PROJECT_ROOT / "build_system/builder/gate/workspace.py").read_text(encoding="utf-8") - pidfile_source = (PROJECT_ROOT / "build_system/builder/gate/pidfiles.py").read_text(encoding="utf-8") + workspace_source = (PROJECT_ROOT / "build_system/builder/gate/workspace.py").read_text( + encoding="utf-8" + ) + pidfile_source = (PROJECT_ROOT / "build_system/builder/gate/pidfiles.py").read_text( + encoding="utf-8" + ) assert "stop_gate_service" in workspace_source assert gate_config.load(PROJECT_ROOT).pidfiles.names == ("gateway.pid", "service.pid") @@ -458,8 +457,7 @@ def test_asset_workflow_publishes_obom_not_debug_build_ledger() -> None: workflow = (PROJECT_ROOT / ".github/workflows/release-assets.yaml").read_text() release = (PROJECT_ROOT / ".github/workflows/release.yaml").read_text() stager = ( - PROJECT_ROOT - / "build_system/builder/release/tools/stage_runtime_publication.py" + PROJECT_ROOT / "build_system/builder/release/tools/stage_runtime_publication.py" ).read_text() assert "npm install -g @cyclonedx/cdxgen" not in workflow diff --git a/build_system/tests/gate/test_gate_sandbox.py b/build_system/tests/gate/test_gate_sandbox.py index 0f0175112..4e4db5197 100644 --- a/build_system/tests/gate/test_gate_sandbox.py +++ b/build_system/tests/gate/test_gate_sandbox.py @@ -184,21 +184,19 @@ def test_fast_gate_proves_hosted_linux_sandbox_before_dependency_work() -> None: names = [step.get("name") for step in steps] prepare = next(step for step in steps if step.get("name") == "Prove Linux sandbox boundary") - assert prepare["run"] == ("python3 build_system/scripts/bootstrap/prepare-linux-sandbox.py --repair-hosted-runner") + assert prepare["run"] == ( + "python3 build_system/scripts/bootstrap/prepare-linux-sandbox.py --repair-hosted-runner" + ) assert "continue-on-error" not in prepare assert names.index("Prove Linux sandbox boundary") < names.index( "Materialize locked qualification dependencies" ) - assert names.index("Prove Linux sandbox boundary") < names.index( - "Run the complete fast gate" - ) + assert names.index("Prove Linux sandbox boundary") < names.index("Run the complete fast gate") assert_unmasked_step("fast-gate.yaml", workflow, "static", "Prove Linux sandbox boundary") def test_fast_gate_warms_sdk_python_in_the_policy_owned_cache() -> None: - workflow = yaml.safe_load( - (PROJECT_ROOT / ".github/workflows/fast-gate.yaml").read_text() - ) + workflow = yaml.safe_load((PROJECT_ROOT / ".github/workflows/fast-gate.yaml").read_text()) step = next( step for step in workflow["jobs"]["static"]["steps"] @@ -688,7 +686,9 @@ def test_the_sandbox_is_applied_before_any_resource_is_held() -> None: """ import ast - source = (PROJECT_ROOT / "build_system" / "builder" / "gate" / "command.py").read_text(encoding="utf-8") + source = (PROJECT_ROOT / "build_system" / "builder" / "gate" / "command.py").read_text( + encoding="utf-8" + ) tree = ast.parse(source) execute = next( node @@ -898,9 +898,7 @@ def test_only_named_dependency_inputs_cross_the_fast_gate_network_boundary( if name == "release-binaries": expected.add("channel-source") else: - expected = ( - ONLINE_FAST if name != "test-static" else {"static.toolchain.node"} - ) + expected = ONLINE_FAST if name != "test-static" else {"static.toolchain.node"} if name not in {"test-fast", "release-binaries", "release-assets"}: expected |= { "host-image", @@ -1023,9 +1021,10 @@ def cancel_when_started() -> None: def test_only_named_self_confining_children_replace_the_inherited_profile(report): text = sandbox.profile(CONFIG, report=report) handoffs = [line for line in text.splitlines() if "no-sandbox" in line] - assert len(handoffs) == 2 - assert any("/capsem\\-router$" in line for line in handoffs) - assert any("/capsem\\-router\\-confinement\\-test$" in line for line in handoffs) + executables = CONFIG.sandbox.self_confined_executables + assert len(handoffs) == len(executables) + for executable in executables: + assert any(f"/{re.escape(executable)}$" in line for line in handoffs) assert all("(regex" in line for line in handoffs) if not report: assert "(deny network*)" in text diff --git a/build_system/tests/release/test_release_doctor_contract.py b/build_system/tests/release/test_release_doctor_contract.py index e102d63d5..6eb191a9e 100644 --- a/build_system/tests/release/test_release_doctor_contract.py +++ b/build_system/tests/release/test_release_doctor_contract.py @@ -997,8 +997,7 @@ def test_release_channel_staging_workflow_exercises_reusable_deploy_without_rele for text in (docs, release_skill, asset_skill): assert "release-channel-staging.yaml" in text assert ( - "without invoking `build-assets`" in text - or "without invoking VM asset builds" in text + "without invoking `build-assets`" in text or "without invoking VM asset builds" in text ) @@ -5779,7 +5778,7 @@ def test_pr_ci_non_vm_python_tests_prepare_assets_and_signed_binaries() -> None: asset_pos = block.find("bash build_system/scripts/test/prepare-install-test-assets.sh") build_pos = block.find( - "cargo build -p capsem-process -p capsem-ledger -p capsem-proxy -p capsem-service -p capsem " + "cargo build -p capsem-process -p capsem-ledger -p capsem-proxy -p capsem-service -p capsem " "-p capsem-mock-server -p capsem-bench" ) bench_package_pos = block.find("-p capsem-bench") @@ -6348,16 +6347,18 @@ def test_suspend_snapshot_freezes_ext4_upper_before_ack_and_thaws_first_on_resto def test_fork_clones_inside_the_owner_under_a_guest_freeze() -> None: - """A running fork is cloned by its own process with the guest frozen. + """A running fork is coordinator-cloned while its owner holds the freeze. The fork used to run `sync; true` in the guest and copy the live ext4 - overlay, which a slow sparse copy could tear. The freeze and the thaw must - live in one process, so no other process's failure can leave a guest - frozen, and the service must not fall back to an unfrozen copy. + overlay, which a slow sparse copy could tear. The VM owner holds the freeze + while the trusted coordinator owns host paths and asks the ledger owner for + a flushed snapshot. No component can substitute an unfrozen copy or regain + another owner's path. """ service = PROJECT_ROOT / "crates" / "capsem-service" / "src" parent = (service / "vm_files.rs").read_text() fork = (service / "vm_files" / "fork.rs").read_text() + ledger = (PROJECT_ROOT / "crates" / "capsem-logger" / "src" / "ledger_server.rs").read_text() owner = ( PROJECT_ROOT / "crates" / "capsem-process" / "src" / "vsock" / "clone_state.rs" ).read_text() @@ -6366,13 +6367,18 @@ def test_fork_clones_inside_the_owner_under_a_guest_freeze() -> None: assert "ServiceToProcess::CloneState {" in fork assert "sync; true" not in fork assert "with_quiescence(" in owner - # The clone runs under the freeze, after the ledger flush (#243): a fork - # copies every row the source accepted, and a failed flush fails the fork. - assert "flush_then_clone(" in owner.split("with_quiescence(", 1)[1] - flush_then_clone = owner.split("async fn flush_then_clone(", 1)[1].split("\n}\n", 1)[0] - assert flush_then_clone.index("flush_checked()") < flush_then_clone.index( - "clone_sandbox_state(" - ) + assert "await_coordinator_copy(" in owner.split("with_quiescence(", 1)[1] + running = fork.split("async fn clone_guest_state(", 1)[1].split( + "async fn receive_clone_message(", 1 + )[0] + coordinator_copy = "let copied = clone_coordinator_state(" + assert running.index("ClonePhase::Ready") < running.index(coordinator_copy) + assert running.index(coordinator_copy) < running.index("ServiceToProcess::CloneStateComplete") + assert "client.snapshot(" in fork + snapshot = ledger.split("LedgerCommand::Snapshot {", 1)[1].split("LedgerCommand::Shutdown", 1)[ + 0 + ] + assert snapshot.index("self.db.flush().await") < snapshot.index("snapshot_session_ledger(") def test_linux_vm_launch_preformats_system_overlay_before_boot() -> None: diff --git a/codecov.yml b/codecov.yml index 3ac5b9715..bbd8d74de 100644 --- a/codecov.yml +++ b/codecov.yml @@ -108,6 +108,7 @@ component_management: - crates/capsem-core/src/net/mod.rs - crates/capsem-core/src/net/hostname.rs - crates/capsem-core/src/net/upstream_address.rs + - crates/capsem-core/src/net/upstream_grant.rs - crates/capsem-core/src/net/ai_traffic/** - crates/capsem-core/src/net/cert_authority.rs - crates/capsem-core/src/net/decompress.rs diff --git a/config/gate.toml b/config/gate.toml index ffebf5ef3..8c2d6cf5e 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1112,10 +1112,10 @@ must_stay_below_lines = 3000 "crates/capsem-proto/src/lib.rs" = 1008 "crates/capsem-proto/src/tests.rs" = 1446 "crates/capsem-service/src/tests/assets_registry.rs" = 1111 -"crates/capsem-service/src/tests/lifecycle.rs" = 1406 -"crates/capsem-service/src/vm_files.rs" = 1042 -"crates/capsem/src/main.rs" = 2015 -"crates/capsem/src/tests.rs" = 1227 +"crates/capsem-service/src/tests/lifecycle.rs" = 1404 +"crates/capsem-service/src/vm_files.rs" = 1031 +"crates/capsem/src/main.rs" = 2025 +"crates/capsem/src/tests.rs" = 1259 "crates/capsem/src/update.rs" = 2143 "crates/capsem/src/update/asset_install.rs" = 1125 "crates/capsem/src/update/tests.rs" = 2306 @@ -3012,7 +3012,7 @@ slowest_steps = 10 # `internal` is a reasoned, ratcheted inventory for service-UDS control routes # that are deliberately neither public gateway routes nor benchmark subjects. [benchmark.routes] -internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/vms/{id}/metrics" = "An authenticated VM owner submits its OTLP metrics to the service over the local UDS for host export; the gateway must never expose this ingestion route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS", "/proxies" = "The local CLI asks the trusted service to create one supervised standalone proxy worker; the gateway never exposes worker lifecycle control", "/proxies/{id}/heartbeat" = "A local standalone proxy owner renews its service supervision lease over the UDS", "/proxies/{id}/stop" = "The local CLI asks the trusted service to stop its supervised standalone proxy worker over the UDS" } +internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/vms/{id}/metrics" = "An authenticated VM owner submits its OTLP metrics to the service over the local UDS for host export; the gateway must never expose this ingestion route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS" } measured = [ "/connections", "/vms/{id}/credential-grants", @@ -3034,16 +3034,11 @@ measured = [ "/update/status", ] unmeasured = [ - # Provider authorization/poll/disconnect and concrete connection records - # need genuine provider lifecycle fixtures, not timed error responses. - "/connections/google/authorize", - "/connections/authorizations/{transaction_id}", - "/connections/{connection_id}", - # Claim/close mutate owned session lifetimes; grant revocation mutates - # credential authority. Their native acceptance owns the effects. - "/vms/managed/{request_id}/claim", - "/vms/managed/{request_id}/close", - "/vms/{id}/credential-grants/{grant_id}", + # Standalone proxy lifecycle mutations require a supervised worker fixture; + # their functional acceptance owns successful create, lease, and teardown. + "/proxies", + "/proxies/{id}/heartbeat", + "/proxies/{id}/stop", # Restart needs a disposable supervisor; the read-only route collector # cannot time accepted restarts. Its functional proof owns that lifecycle. "/restart", diff --git a/tests/citadel/test_bench_route_coverage.py b/tests/citadel/test_bench_route_coverage.py index 5eb057efa..3dc49a025 100644 --- a/tests/citadel/test_bench_route_coverage.py +++ b/tests/citadel/test_bench_route_coverage.py @@ -126,10 +126,9 @@ def test_each_route_has_exactly_one_benchmark_classification() -> None: def test_internal_routes_carry_a_reason_and_are_a_tight_inventory() -> None: internal = _config()["internal"] assert all(reason.strip() for reason in internal.values()) - # Nine real exceptions: doctor failure preservation, private-network - # resolution, owner metric ingestion, three preview admission exchanges, - # and three standalone proxy lifecycle exchanges over the local UDS. - assert len(internal) <= 9, ( + # Six real exceptions: doctor failure preservation, private-network + # resolution, owner metric ingestion, and three preview admission exchanges. + assert len(internal) <= 6, ( "the internal service-only route inventory grew; prefer a public, " "measured gateway route unless the control-plane exception is real" ) diff --git a/tests/citadel/test_guest_share_never_by_path.py b/tests/citadel/test_guest_share_never_by_path.py index 3e7d1c023..f577234ac 100644 --- a/tests/citadel/test_guest_share_never_by_path.py +++ b/tests/citadel/test_guest_share_never_by_path.py @@ -31,7 +31,8 @@ # The definition. "crates/capsem-core/src/lib.rs": 1, # The VirtioFS share root handed to the hypervisor and owner sandbox. - "crates/capsem-process/src/main.rs": 2, + "crates/capsem-process/src/main.rs": 1, + "crates/capsem-process/src/owner_sandbox.rs": 1, } BY_PATH = re.compile(r"guest_share_dir\(|join\(\s*\"guest\"\s*\)|\"guest/") diff --git a/tests/citadel/test_upstream_grant_boundary.py b/tests/citadel/test_upstream_grant_boundary.py index 47012f5cb..5c20a439d 100644 --- a/tests/citadel/test_upstream_grant_boundary.py +++ b/tests/citadel/test_upstream_grant_boundary.py @@ -36,7 +36,7 @@ "crates/capsem-process/src/cables.rs": { "let source = capsem_core::container::publish::Source(std::net::TcpStream::from(" }, - "crates/capsem-process/src/main.rs": { + "crates/capsem-process/src/owner_sandbox.rs": { "std::net::TcpStream::connect((std::net::Ipv4Addr::LOCALHOST, 9)).map(|_| ()),", }, "crates/capsem-proxy/src/main.rs": { diff --git a/tests/test_installed_winterfell_contract.py b/tests/test_installed_winterfell_contract.py index 84953bae1..9984b4e58 100644 --- a/tests/test_installed_winterfell_contract.py +++ b/tests/test_installed_winterfell_contract.py @@ -148,11 +148,15 @@ def test_installed_winterfell_rejects_source_built_roots(bin_dir: Path, assets_d def test_installed_winterfell_rejects_binary_symlinks_into_target_debug( tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: bin_dir, assets_dir = _installed_roots(tmp_path) - source_binary = PROJECT_ROOT / "cache" / "target" / "cargo" / "debug" / "capsem-service" - if not source_binary.is_file(): - pytest.skip("source service binary has not been built") + source_dir = tmp_path / "source-built" + source_dir.mkdir() + source_binary = source_dir / "capsem-service" + source_binary.write_text("#!/bin/sh\nexit 0\n") + source_binary.chmod(0o755) + monkeypatch.setattr(service, "host_bin_root", lambda _environment: source_dir) (bin_dir / "capsem-service").unlink() (bin_dir / "capsem-service").symlink_to(source_binary) From e430ea8726f9778ec3a3aeba248c1153ba12a491 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Fri, 9 Oct 2026 15:59:29 +0000 Subject: [PATCH 248/464] security(foundation): harden file target and watch boundaries --- CHANGELOG.md | 10 +++-- .../src/unix/change_watch.rs | 38 ++++++++++++++----- .../src/unix/change_watch/tests.rs | 13 +++---- .../capsem-foundation/src/unix/contained.rs | 6 +++ .../src/unix/contained/tests.rs | 12 +++++- .../capsem-foundation/src/unix/peer/tests.rs | 2 +- .../src/unix/worker_sandbox/tests.rs | 8 +++- 7 files changed, 64 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb08a519..3435ca6a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Reject symlink and special-entry targets before descriptor-relative mode + changes, preserving the contained-filesystem contract on macOS as well as + Linux without following a guest-controlled link. + - Install VM-owner confinement before starting its runtime, hypervisor, parent watcher, ledger client, filesystem monitor, or metric workers; pre-bind only the exact IPC and VSOCK descriptors those workers inherit. @@ -78,9 +82,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 session-database path authority. - Confine each session-ledger owner before readiness to its one database and - inherited channels, denying sibling sessions, socket creation, execution, - signals and ambient files; erase inherited environment bytes and reject - symlinked session grants before serving clients. + inherited channels, denying sibling sessions, ambient network and control + sockets, execution, signals and files; erase inherited environment bytes and + reject symlinked session grants before serving clients. - Compress new session body blocks with streaming zstd only inside the confined ledger owner, retaining durable cross-flush history and legacy diff --git a/crates/capsem-foundation/src/unix/change_watch.rs b/crates/capsem-foundation/src/unix/change_watch.rs index 7451ffd45..da49ebafd 100644 --- a/crates/capsem-foundation/src/unix/change_watch.rs +++ b/crates/capsem-foundation/src/unix/change_watch.rs @@ -59,7 +59,7 @@ impl ChangeWatch { /// Add an already-contained open file or directory, never a pathname. pub fn add(&mut self, fd: BorrowedFd<'_>) -> io::Result<()> { - let result = self.add_inner(fd, None); + let result = self.add_inner(fd, None, false); if result.is_err() { self.changed = true; } @@ -119,16 +119,27 @@ impl ChangeWatch { if std::fs::symlink_metadata(parent)?.file_type().is_symlink() { continue; } + #[cfg(target_os = "macos")] + if !std::fs::symlink_metadata(child)?.file_type().is_symlink() { + // kqueue vnode events carry no child name. Watching the + // enclosing directory would therefore turn every sibling + // write into a false invalidation. The ancestor inode itself + // reports rename/delete/revoke when its binding changes. + let directory = ContainedDir::open_root(child)?; + self.add_inner(directory.as_fd(), None, true)?; + continue; + } let directory = ContainedDir::open_root(parent)?; - self.add_inner(directory.as_fd(), Some(name))?; + self.add_inner(directory.as_fd(), Some(name), false)?; } Ok(()) } - fn add_inner(&mut self, fd: BorrowedFd<'_>, name: Option<&OsStr>) -> io::Result<()> { + fn add_inner(&mut self, fd: BorrowedFd<'_>, name: Option<&OsStr>, binding_only: bool) -> io::Result<()> { let held = super::fd::duplicate(fd)?; #[cfg(target_os = "linux")] { + let _ = binding_only; // Linux has no fd form of inotify_add_watch. This process-owned // proc descriptor link resolves the held inode, not its old path. let path = format!("/proc/self/fd/{}", held.as_raw_fd()); @@ -155,20 +166,27 @@ impl ChangeWatch { } #[cfg(target_os = "macos")] { - // Vnode events have no child name. Any ancestor event is a - // conservative invalidation on macOS. + // Vnode events have no child name. A path binding therefore + // watches only changes to its already-open ancestor inode; + // NOTE_WRITE/EXTEND/LINK would report unrelated sibling activity. + // Content watches retain every mutation flag. let _ = name; - let event = KEvent::new( - held.as_raw_fd() as usize, - EventFilter::EVFILT_VNODE, - EventFlag::EV_ADD | EventFlag::EV_CLEAR, + let filters = if binding_only { + FilterFlag::NOTE_ATTRIB | FilterFlag::NOTE_RENAME | FilterFlag::NOTE_DELETE | FilterFlag::NOTE_REVOKE + } else { FilterFlag::NOTE_WRITE | FilterFlag::NOTE_EXTEND | FilterFlag::NOTE_ATTRIB | FilterFlag::NOTE_LINK | FilterFlag::NOTE_RENAME | FilterFlag::NOTE_DELETE - | FilterFlag::NOTE_REVOKE, + | FilterFlag::NOTE_REVOKE + }; + let event = KEvent::new( + held.as_raw_fd() as usize, + EventFilter::EVFILT_VNODE, + EventFlag::EV_ADD | EventFlag::EV_CLEAR, + filters, 0, 0, ); diff --git a/crates/capsem-foundation/src/unix/change_watch/tests.rs b/crates/capsem-foundation/src/unix/change_watch/tests.rs index 15c4db1ae..75977a797 100644 --- a/crates/capsem-foundation/src/unix/change_watch/tests.rs +++ b/crates/capsem-foundation/src/unix/change_watch/tests.rs @@ -32,14 +32,11 @@ fn directory_path_watch_tracks_ancestor_replacement_and_retains_original_root() let root = watch.open_directory(&path).unwrap(); let original = root.metadata().unwrap().ino(); assert!(!watch.changed().unwrap()); - #[cfg(target_os = "linux")] - { - std::fs::write(parent.path().join("unrelated"), b"sibling").unwrap(); - assert!( - !watch.changed().unwrap(), - "unrelated ancestor siblings must not invalidate" - ); - } + std::fs::write(parent.path().join("unrelated"), b"sibling").unwrap(); + assert!( + !watch.changed().unwrap(), + "unrelated ancestor siblings must not invalidate" + ); std::fs::rename(&ancestor, parent.path().join("moved")).unwrap(); std::fs::create_dir_all(&path).unwrap(); assert!( diff --git a/crates/capsem-foundation/src/unix/contained.rs b/crates/capsem-foundation/src/unix/contained.rs index 9d04bab9e..4102138ff 100644 --- a/crates/capsem-foundation/src/unix/contained.rs +++ b/crates/capsem-foundation/src/unix/contained.rs @@ -316,6 +316,12 @@ impl ContainedDir { for component in parents { parent = parent.descend(component)?; } + if parent.entry_kind(name)? == Some(EntryKind::Other) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("{} is not a file or directory", Path::new(name).display()), + )); + } fchmodat( Some(parent.fd.as_raw_fd()), *name, diff --git a/crates/capsem-foundation/src/unix/contained/tests.rs b/crates/capsem-foundation/src/unix/contained/tests.rs index b73d203af..85ebb0550 100644 --- a/crates/capsem-foundation/src/unix/contained/tests.rs +++ b/crates/capsem-foundation/src/unix/contained/tests.rs @@ -603,8 +603,16 @@ fn relative_mode_changes_refuse_links_and_traversal() { std::fs::set_permissions(&secret, std::fs::Permissions::from_mode(0o600)).unwrap(); symlink(&secret, tree.root_path.join("link")).unwrap(); symlink(&tree.outside, tree.root_path.join("dir-link")).unwrap(); - - for path in ["link", "dir-link/secret", "../outside/secret", "/tmp/file", "a/../b"] { + let _socket = std::os::unix::net::UnixListener::bind(tree.root_path.join("socket")).unwrap(); + + for path in [ + "link", + "dir-link/secret", + "socket", + "../outside/secret", + "/tmp/file", + "a/../b", + ] { assert!( tree.root.set_relative_mode(Path::new(path), 0o777).is_err(), "accepted {path}" diff --git a/crates/capsem-foundation/src/unix/peer/tests.rs b/crates/capsem-foundation/src/unix/peer/tests.rs index 27bbe37dd..dc75c7bf2 100644 --- a/crates/capsem-foundation/src/unix/peer/tests.rs +++ b/crates/capsem-foundation/src/unix/peer/tests.rs @@ -68,7 +68,7 @@ fn connecting_child() { #[test] fn mismatched_uid_is_refused_even_when_the_pid_matches() { - let (socket, _) = UnixStream::pair().unwrap(); + let (socket, _peer) = UnixStream::pair().unwrap(); let mut wrong = current(); wrong.uid ^= 1; assert_eq!( diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index ba334e0a5..88b19ad4c 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -212,6 +212,9 @@ fn macos_policy_preserves_grants_and_denies_ambient_authority() { std::fs::create_dir_all(&allowed).unwrap(); std::fs::create_dir_all(&readonly).unwrap(); std::fs::create_dir_all(&denied).unwrap(); + let allowed = allowed.canonicalize().unwrap(); + let readonly = readonly.canonicalize().unwrap(); + let denied = denied.canonicalize().unwrap(); std::fs::write(readonly.join("value"), b"readable").unwrap(); std::fs::write(denied.join("secret"), b"secret").unwrap(); let control = denied.join("control.sock"); @@ -283,7 +286,10 @@ fn macos_sandbox_child() { assert!(std::fs::set_permissions(&allowed, std::fs::Permissions::from_mode(0o777)).is_err()); assert!(std::fs::read(denied.join("secret")).is_err()); assert!(UnixStream::connect(control).is_err()); - assert!(UnixStream::pair().is_err()); + // Seatbelt denies access to ambient endpoints. A private + // AF_UNIX socketpair carries no external authority and remains + // available for worker-internal communication. + let _private_pair = UnixStream::pair().unwrap(); assert!(TcpStream::connect((Ipv4Addr::LOCALHOST, port)).is_err()); assert!(Command::new("/usr/bin/true").status().is_err()); let parent = unsafe { libc::getppid() }; From a2438eaa016dd0f8f553c77267893a55235dd2f2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Sat, 10 Oct 2026 16:12:42 +0000 Subject: [PATCH 249/464] test(foundation): gate Linux sandbox permission import --- crates/capsem-foundation/src/unix/worker_sandbox/tests.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index 88b19ad4c..36ca6cf03 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -2,6 +2,10 @@ use super::*; use std::io::{Read, Write}; use std::net::{Ipv4Addr, TcpListener, TcpStream}; use std::os::fd::AsFd; +#[cfg(target_os = "linux")] +use std::os::fd::AsRawFd; +#[cfg(target_os = "linux")] +use std::os::unix::fs::PermissionsExt as _; use std::os::unix::net::{UnixListener, UnixStream}; use std::process::{Command, Stdio}; use std::time::Duration; From 61fe0b85c92b432ba5aaa4690ede428f6cd7c902 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Sat, 10 Oct 2026 16:12:48 +0000 Subject: [PATCH 250/464] test(core): stabilize managed lifecycle timing proof --- .../src/managed_sessions/controller/tests.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/capsem-core/src/managed_sessions/controller/tests.rs b/crates/capsem-core/src/managed_sessions/controller/tests.rs index b448220af..e94eeb7d5 100644 --- a/crates/capsem-core/src/managed_sessions/controller/tests.rs +++ b/crates/capsem-core/src/managed_sessions/controller/tests.rs @@ -434,10 +434,12 @@ impl Effects for Arc { }) } fn reconcile(&self, _ticket: Ticket, intent: Option) -> EffectFuture> { - if self.creates.load(Ordering::SeqCst) > 0 { - assert_eq!(intent.as_ref(), Some(&self.binding)); - } else { - assert!(intent.is_none()); + // A create timeout may cancel immediately after prepare_spawn durably + // records the intent but before this fixture increments `creates`. + // Reconciliation must accept that exact durable intent in either + // scheduling order and still reject a substituted binding. + if let Some(intent) = intent { + assert_eq!(intent, self.binding); } self.reconciles.fetch_add(1, Ordering::SeqCst); Box::pin(async { Ok(None) }) @@ -852,15 +854,16 @@ async fn finite_expiry_cleans_up_without_a_client() { fixture.cleanup_release.add_permits(1); let request = capsem_crypto::uuid_v4(); let cap = Capability::from_bytes([24; 32]); - owner + let created = owner .create( request, cap.clone(), - LeasePolicy::new(Duration::from_millis(40)).unwrap(), + LeasePolicy::new(Duration::from_millis(400)).unwrap(), effects(&fixture), ) .await .unwrap(); + assert_eq!(created.state(), State::Active); tokio::time::timeout(Duration::from_secs(1), fixture.cleanup_entered.acquire()) .await .unwrap() From 4acea960f6a1f2a23c7c1d9a0068026f4b3a6212 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Sat, 10 Oct 2026 16:12:53 +0000 Subject: [PATCH 251/464] fix(service): drain terminal ledger worker event --- crates/capsem-service/src/ledger_worker.rs | 29 +++++++++++++++------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/crates/capsem-service/src/ledger_worker.rs b/crates/capsem-service/src/ledger_worker.rs index dcfc5e4f1..ef3cdbb33 100644 --- a/crates/capsem-service/src/ledger_worker.rs +++ b/crates/capsem-service/src/ledger_worker.rs @@ -468,18 +468,19 @@ impl WorkerProcess { async fn recv_event(&mut self) -> Result { tokio::time::timeout(OPERATION_TIMEOUT, async { tokio::select! { + frame = self.control_events.recv() => { + decode_control_event(frame) + } status = self.child.wait() => { let status = status.context("reap ledger worker")?; - bail!("ledger worker exited: {status}") - } - frame = self.control_events.recv() => { - let frame = frame - .context("ledger control event pump stopped")? - .context("ledger worker control channel failed")?; - if !frame.fds.is_empty() { - bail!("ledger worker returned a descriptor on its control channel"); + // A worker can write its terminal protocol event and exit + // before the control-reader task gets scheduled. Drain + // that queued event before classifying the clean exit as + // an unexpected failure. + match self.control_events.recv().await { + Some(frame) => decode_control_event(Some(frame)), + None => bail!("ledger worker exited: {status}"), } - decode_ledger_control_event(&frame.bytes).map_err(anyhow::Error::from) } } }) @@ -497,6 +498,16 @@ impl WorkerProcess { } } +fn decode_control_event(frame: Option>) -> Result { + let frame = frame + .context("ledger control event pump stopped")? + .context("ledger worker control channel failed")?; + if !frame.fds.is_empty() { + bail!("ledger worker returned a descriptor on its control channel"); + } + decode_ledger_control_event(&frame.bytes).map_err(anyhow::Error::from) +} + struct PendingWorker { child: Child, control_tx: ControlSender, From 44e9fdec49fb67462c5a7a99230a77922e9cbec2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Sat, 10 Oct 2026 16:24:59 +0000 Subject: [PATCH 252/464] test(service): cover terminal event after child exit --- .../capsem-service/src/ledger_worker/tests.rs | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/crates/capsem-service/src/ledger_worker/tests.rs b/crates/capsem-service/src/ledger_worker/tests.rs index 7fc9df81d..65a2ce9db 100644 --- a/crates/capsem-service/src/ledger_worker/tests.rs +++ b/crates/capsem-service/src/ledger_worker/tests.rs @@ -174,6 +174,51 @@ async fn supervisor_mints_unique_role_bound_channels_and_reaps() { assert!(stopped.borrow().as_deref().unwrap().contains("coordinator")); } +#[tokio::test] +async fn terminal_event_after_observed_child_exit_is_still_drained() { + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args([ + "--exact", + "ledger_worker::tests::fake_ledger_worker_child", + "--nocapture", + ]) + .env_clear() + .env("CAPSEM_FAKE_LEDGER_WORKER", "refuse") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + let mut child = command.spawn().unwrap(); + assert_eq!(child.wait().await.unwrap().code(), Some(42)); + + let (control, _control_peer) = UnixStream::pair().unwrap(); + let (events_tx, control_events) = mpsc::channel(1); + let mut process = WorkerProcess { + generation: GENERATION, + child, + control_tx: ControlSender::new(control).unwrap(), + control_events, + control_reader: None, + next_client_id: 1, + }; + let delayed = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(10)).await; + events_tx + .send(Ok(ControlFrame { + bytes: encode_ledger_control_event(LedgerControlEvent::Stopped { generation: GENERATION }), + fds: Vec::new(), + })) + .await + .unwrap(); + }); + + assert!(matches!( + process.recv_event().await.unwrap(), + LedgerControlEvent::Stopped { generation } if generation == GENERATION + )); + delayed.await.unwrap(); +} + #[test] fn generated_worker_lifetimes_are_fresh_and_nonzero() { assert_ne!(fresh_generation(), fresh_generation()); From 0213cc44a79a1609e35699af618ff77a9123cbcd Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Sat, 10 Oct 2026 16:33:18 +0000 Subject: [PATCH 253/464] test(foundation): handle profiler thread on old Landlock --- .../src/unix/worker_sandbox/tests.rs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index 36ca6cf03..4a082c399 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -100,6 +100,68 @@ fn landlock_abi_eight_synchronizes_existing_threads() { assert_ne!(super::platform::restriction_flags_for(8, usize::MAX).unwrap(), 0); } +#[cfg(target_os = "linux")] +#[test] +fn linux_device_grant_preserves_kvm_ioctls() { + let kvm = std::path::Path::new("/dev/kvm"); + if !kvm.exists() { + return; + } + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "unix::worker_sandbox::tests::linux_kvm_device_child", + "--nocapture", + ]) + .env_clear() + .env("WORKER_SANDBOX_KVM", kvm) + .status() + .unwrap(); + if status.code() == Some(77) { + // Coverage runtimes may start a profiler thread before the test body. + // Landlock ABIs before TSYNC cannot confine that artificial sibling; + // production workers are explicitly single-threaded at confinement. + return; + } + assert!(status.success(), "sandboxed KVM probe failed: {status}"); +} + +#[cfg(target_os = "linux")] +#[test] +fn linux_kvm_device_child() { + let Some(kvm) = std::env::var_os("WORKER_SANDBOX_KVM") else { + return; + }; + let kvm = std::path::PathBuf::from(kvm); + let device = std::fs::OpenOptions::new().read(true).write(true).open(&kvm).unwrap(); + if let Err(error) = confine(&Policy::new(Role::VmOwner).allow(&kvm, Access::ReadWriteDevice)) { + assert_eq!(error.kind(), std::io::ErrorKind::Unsupported); + assert!(error.to_string().contains("single-threaded worker startup")); + std::process::exit(77); + } + + // nix's no-argument ioctl wrapper omits the variadic argument while the + // KVM ABI expects an explicit zero. The descriptor and constant are valid + // for KVM_GET_API_VERSION, which reads no userspace pointer. + let version = unsafe { libc::ioctl(device.as_raw_fd(), 0xae00, 0u64) }; + assert_eq!( + version, + 12, + "KVM_GET_API_VERSION failed: {}", + std::io::Error::last_os_error() + ); + std::process::exit(0); +} + +#[cfg(target_os = "linux")] +#[test] +fn linux_device_grant_rejects_regular_files() { + let file = tempfile::NamedTempFile::new().unwrap(); + let error = confine(&Policy::new(Role::VmOwner).allow(file.path(), Access::ReadWriteDevice)).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); + assert!(error.to_string().contains("sandbox device grant is not a device")); +} + #[cfg(target_os = "linux")] #[test] fn linux_policy_preserves_grants_and_denies_ambient_authority() { From d2afd3afbe494ce2abd44a746364c35db2780ccf Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Sat, 10 Oct 2026 17:12:17 +0000 Subject: [PATCH 254/464] docs(skills): align diagnostics with worker isolation --- skills/dev-bug-review/SKILL.md | 3 +- skills/dev-debugging/SKILL.md | 20 +++++------ skills/dev-mcp/SKILL.md | 28 ++++++++++----- skills/dev-mitm-proxy/SKILL.md | 3 +- skills/dev-session-debug/SKILL.md | 57 +++++++++++++++++++------------ skills/dev-testing-vm/SKILL.md | 24 ++++++++----- skills/dev-testing/SKILL.md | 5 ++- 7 files changed, 87 insertions(+), 53 deletions(-) diff --git a/skills/dev-bug-review/SKILL.md b/skills/dev-bug-review/SKILL.md index 14b490323..c4b39e495 100644 --- a/skills/dev-bug-review/SKILL.md +++ b/skills/dev-bug-review/SKILL.md @@ -69,7 +69,8 @@ Fixes land test-first. No exceptions. - Cross-cutting Rust: `just test` - Frontend: `pnpm run check` (fail-on-warnings) + `pnpm test` where relevant - VM behavior: `just exec "capsem-doctor -k "` or the targeted diagnostic - - Telemetry: `python3 build_system/scripts/doctor/check_session.py` + - Telemetry: inspect live evidence through typed timeline/ledger routes; use + `check_session.py --db` only on a quiesced preserved copy - Fix every warning surfaced. Warnings are errors (CLAUDE.md). ## Phase 5: Summarize, commit, changelog diff --git a/skills/dev-debugging/SKILL.md b/skills/dev-debugging/SKILL.md index 000822ee9..cd86bc47a 100644 --- a/skills/dev-debugging/SKILL.md +++ b/skills/dev-debugging/SKILL.md @@ -100,10 +100,10 @@ If you can't reproduce it in a test, you don't understand it well enough to fix just exec "" ``` -For telemetry issues, use session inspection: -```bash -python3 build_system/scripts/doctor/check_session.py -``` +For telemetry issues, inspect a live session through the typed timeline, +history, statistics, and security routes. Use `check_session.py --db` only on +a coherent, quiesced evidence copy after its ledger worker has stopped; see +`/dev-session-debug`. ## Step 2: Diagnose the root cause @@ -150,11 +150,10 @@ Check boot logs for daemon startup failures, vsock connection issues, or timing HTTP/DNS/model normalization, cert minting, `SecurityEvent` construction, security rule evaluation, plugin execution, runtime materialization, and ledger materialization. Do not debug by adding credential handling to formatters, -routes, DB readers, frontend transforms, or harnesses. Use session DB to see -what actually happened: -```bash -python3 build_system/scripts/doctor/check_session.py # Check net_events for domain, decision, status_code -``` +routes, DB readers, frontend transforms, or harnesses. Inspect the typed +timeline and security routes to see what the live ledger recorded. For page or +archive integrity, preserve a quiesced evidence copy and follow +`/dev-session-debug`. **Frontend issues**: Run `just dev ui`, open Chrome DevTools, check console errors, use `take_screenshot` to capture state. See dev-testing-frontend for the full visual verification workflow. @@ -231,7 +230,8 @@ Now that you understand the root cause, write the fix. The fix should: After the fix, run the full validation: 1. `just test` -- unit + cross-compile + frontend 2. `just exec "capsem-doctor"` -- VM smoke test -3. If the bug touched telemetry: `python3 build_system/scripts/doctor/check_session.py` after a real session +3. If the bug touched telemetry: inspect the real session through typed routes; + run `check_session.py --db` only on a preserved, quiesced evidence copy A diagnostic continuation may shorten investigation before this validation; it never replaces any item in the final proof. diff --git a/skills/dev-mcp/SKILL.md b/skills/dev-mcp/SKILL.md index 4610fc265..bb6a732c0 100644 --- a/skills/dev-mcp/SKILL.md +++ b/skills/dev-mcp/SKILL.md @@ -8,7 +8,7 @@ description: MCP for Capsem: the SDK-backed npm host server, guest relay, and to Capsem has two MCP entry points: 1. **`@capsem/mcp`** (host): a standalone TypeScript stdio server. It presents typed tools and uses `@capsem/sdk` for authenticated gateway HTTP. It does not open the service UDS, discover local services, or read VM runtime state. -2. **Guest MCP relay + MITM MCP endpoint**: bridges AI agents inside a VM to built-in and external MCP servers through bounded framed records on vsock port 5002. +2. **Guest MCP relay + confined proxy endpoint**: bridges AI agents inside a VM to built-in and external MCP servers through bounded framed records on vsock port 5002. The native installer does not install Node.js or download the npm host package. The guest relay, aggregator, and built-in Rust components remain native product @@ -99,7 +99,12 @@ credential, CA private key, direct database access, or service socket access. ## MCP subprocess architecture -The guest MCP path is not a single process. `capsem-process` (the per-VM host process) owns the MITM MCP endpoint and spawns two privilege-isolated subprocesses that together handle MCP traffic from the guest: +The guest MCP path is not a single process. The per-session `capsem-proxy` +owns the guest-facing framed endpoint, MCP policy, and ledger admission. +`capsem-process` owns a scoped adapter to the aggregator and VM exposure tools; +it cannot reopen the session ledger and the proxy cannot reach the aggregator +except through that granted connected channel. The VM owner spawns two +privilege-isolated subprocesses behind the adapter: | Crate | Role | Privileges | |-------|------|-----------| @@ -108,29 +113,34 @@ The guest MCP path is not a single process. `capsem-process` (the per-VM host pr Rationale: isolating external-server connections in a low-privilege subprocess means a compromised third-party MCP server cannot reach the host filesystem or the session DB. The built-in tool server runs in its own process for the same reason. -Wire protocol between `capsem-process` and the aggregator: **length-prefixed msgpack frames** on stdio (`[4-byte big-endian length][msgpack payload]`). Between the aggregator and the built-in server: **stdio MCP** (standard JSON-RPC per line). Between the in-guest AI agent and `capsem-process`: `/run/capsem-mcp-server` relays stdio JSON-RPC as bounded framed MCP records over **vsock port 5002**. Tool activity writes the canonical `tool_calls` ledger. Visible MCP protocol facts are represented as typed security events and `security_rule_events`. +Wire protocol between `capsem-process` and the aggregator: **length-prefixed msgpack frames** on stdio (`[4-byte big-endian length][msgpack payload]`). Between the aggregator and the built-in server: **stdio MCP** (standard JSON-RPC per line). `/run/capsem-mcp-server` relays guest stdio JSON-RPC as bounded framed records over **vsock port 5002**; the VM owner grants each accepted stream to `capsem-proxy`. The proxy parses and evaluates it, then invokes the VM-owner adapter over its scoped MCP capability. Tool activity reaches the canonical `tool_calls` ledger through the proxy's typed ledger client. Visible protocol facts are typed security events and `security_rule_events`. Binaries land in `~/.capsem/bin/` at install time: `capsem-mcp-aggregator`, `capsem-mcp-builtin`. ## Guest MCP Endpoint -The guest MCP relay bridges AI agents in the guest VM to the host MITM MCP endpoint. It runs over vsock port 5002 using bounded length-prefixed MCP frames that carry JSON-RPC payloads and per-frame process attribution. +The guest MCP relay bridges AI agents in the guest VM to the confined proxy +endpoint. It runs over vsock port 5002 using bounded length-prefixed MCP +frames that carry JSON-RPC payloads and per-frame process attribution. Framed guest MCP over `vsock:5002` must be tested as the default transport, not as an opt-in benchmark mode. The minimum hardening matrix for that path is: - parser/interpreter: bounded frames, invalid JSON, malformed flags, stream-id reuse, notification/request-id mismatch - dispatch: `initialize`, `tools/list`, builtin `tools/call`, configured external stdio `tools/call`, `resources/list`, `prompts/list`, and method error mapping - policy: live policy mutation, per-tool block, resource URI rule, argument-name rule, argument-value rule, return-value rule, deny-over-allow precedence, and proof that blocked requests/responses do not leak original data - telemetry: `session.db` rows for success, denial, timeout, process attribution, request/response previews, policy fields, and terminal errors -- boundary: aggregator remains DB-free; MITM/process owns MCP audit writes -- VM E2E: boot a real VM, run `/run/capsem-mcp-server` with no transport override, then query `session.db` +- boundary: aggregator and VM owner remain DB-free; `capsem-proxy` writes MCP + audit facts through its role-bound ledger capability +- VM E2E: boot a real VM, run `/run/capsem-mcp-server` with no transport + override, then inspect the typed timeline/ledger routes ### Architecture ``` Guest (Claude/Gemini) -> capsem-mcp-server (stdin/stdout relay) - -> framed vsock:5002 -> MITM MCP endpoint (capsem-core) - -> SecurityEvent rule check -> Route to: builtin tools | external MCP servers (via rmcp) - -> Telemetry -> session.db tool_calls table plus security_rule_events protocol evidence + -> framed vsock:5002 -> capsem-process descriptor handoff -> capsem-proxy + -> SecurityEvent rule check -> scoped VM-owner MCP adapter + -> builtin tools | external MCP servers (via aggregator) + -> typed ledger capability -> tool_calls plus security_rule_events evidence ``` ### Workloads: streamable HTTP at `mcp.capsem.internal` diff --git a/skills/dev-mitm-proxy/SKILL.md b/skills/dev-mitm-proxy/SKILL.md index 9afc228e3..447ac9083 100644 --- a/skills/dev-mitm-proxy/SKILL.md +++ b/skills/dev-mitm-proxy/SKILL.md @@ -113,5 +113,6 @@ Read these for the exact SSE format, request/response shapes, and telemetry extr - Unit tests: `cargo test -p capsem-core net` (policy evaluation, SNI parsing, cert minting) - In-VM: `just exec "capsem-doctor -k network"` (TLS trust chain, port blocking, domain filtering) -- Telemetry: `just exec "curl -s https://api.anthropic.com/"` then `python3 build_system/scripts/doctor/check_session.py` (check net_events) +- Telemetry: `just exec "curl -s https://api.anthropic.com/"`, then inspect the + typed timeline/security routes for the expected `net_events` evidence - Adversarial: test with blocked domains, overlapping wildcards, malformed SNI, huge request bodies diff --git a/skills/dev-session-debug/SKILL.md b/skills/dev-session-debug/SKILL.md index abb61fd02..32fbbfdc6 100644 --- a/skills/dev-session-debug/SKILL.md +++ b/skills/dev-session-debug/SKILL.md @@ -11,8 +11,9 @@ for a named one. The ledger is two files that only make sense together: - `session.db` -- SQLite, every telemetry table plus the index of captured bodies. - `session.bodies` -- the append-only body archive: HTTP, model, tool, exec and - security payloads, deflated in blocks of about 1 MiB. SQLite never holds - body bytes. + security payloads, written as streaming zstd level 3 blocks of about 1 MiB. + Readers retain raw-deflate support for older blocks. SQLite never holds body + bytes, and native codec execution stays inside `capsem-ledger`. Copy, fork or delete them as a pair; either one alone is useless. The host ledger `~/.capsem/sessions/host.db` records what happened to every session @@ -55,12 +56,15 @@ Sessions come from the host ledger, so they're listed after the session director ### Deep inspection +The commands below open ledger files directly. Use them only on preserved test +evidence or a coherent copy after its ledger worker has stopped. Live and +stopped product-session reads go through the service's typed ledger routes; +debug tooling must not become a second product reader outside `capsem-ledger`. + ```bash -python3 build_system/scripts/doctor/check_session.py # Full integrity check on latest session -python3 build_system/scripts/doctor/check_session.py # Specific session or named VM -python3 build_system/scripts/doctor/check_session.py --db path/session.db # A ledger file directly -python3 build_system/scripts/doctor/check_session.py --verify-bodies # Also read every body back through its hash -python3 build_system/scripts/doctor/check_session.py -n 10 # Show 10 preview rows per table +uv run --project build_system --frozen python build_system/scripts/doctor/check_session.py --db /path/to/evidence/session.db +uv run --project build_system --frozen python build_system/scripts/doctor/check_session.py --db /path/to/evidence/session.db --verify-bodies +uv run --project build_system --frozen python build_system/scripts/doctor/check_session.py --db /path/to/evidence/session.db -n 10 ``` Checks: SQLite page integrity (`PRAGMA quick_check`, first -- a damaged file @@ -457,7 +461,8 @@ old `main.db` beside it is never opened. ### tool_calls without matching tool_responses - The model invoked a tool but the next turn's tool results weren't captured - Check if the VM session ended before the tool result was sent back -- `python3 build_system/scripts/doctor/check_session.py` reports orphaned tool_calls automatically +- `check_session.py --db /path/to/evidence/session.db` reports orphaned + `tool_calls` automatically on a quiesced evidence copy ### Empty fs_events - The host file monitor (`crates/capsem-core/src/fs_monitor.rs`) polls the @@ -507,7 +512,7 @@ old `main.db` beside it is never opened. ## When to inspect sessions -**Always** run `python3 build_system/scripts/doctor/check_session.py` after changes to: +**Always** run `check_session.py` on preserved or copied evidence after changes to: - Guest MCP endpoint (tool routing, policy, response format) - MITM proxy (SSE parsing, body preview, Content-Encoding) - File monitor (VirtioFS poll loop, kinds, overflow windows) @@ -516,40 +521,48 @@ old `main.db` beside it is never opened. The inspect output now includes a tool usage breakdown from `tool_calls` plus MCP transport evidence when present. Check it after MCP changes to verify user tools return `allowed` with reasonable latency and that MCP-origin rows link back to protocol evidence when available. -## Ad-hoc SQL queries +## Ad-hoc SQL queries on offline evidence -Use `sqlite3 "$HOME/.capsem/run/sessions//session.db"` to run SQL against a session DB (a named VM's is under `run/persistent//`). Open it read-only (`-readonly`) while the VM runs: the owning `capsem-process` is the only writer. SQL answers what was captured and where; the bytes of a body are in `session.bodies` (see Bodies above). +Run SQL only against a coherent, quiesced evidence copy. `capsem-ledger` is the +sole process that opens a product session's `session.db`; VM owners, proxy +workers, and service readers use typed ledger channels. SQL answers what the +copy captured and where; body bytes remain in the matching `session.bodies` +copy (see Bodies above). ```bash +ledger_db=/path/to/evidence/session.db + # Decisions breakdown -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT decision, COUNT(*) FROM net_events GROUP BY decision" +sqlite3 -readonly "$ledger_db" "SELECT decision, COUNT(*) FROM net_events GROUP BY decision" # Token totals by provider -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT provider, SUM(input_tokens) as in_tok, SUM(output_tokens) as out_tok, SUM(estimated_cost_usd) as cost FROM model_calls GROUP BY provider" +sqlite3 -readonly "$ledger_db" "SELECT provider, SUM(input_tokens) as in_tok, SUM(output_tokens) as out_tok, SUM(estimated_cost_usd) as cost FROM model_calls GROUP BY provider" # Find orphaned tool calls -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT tc.call_id, tc.tool_name FROM tool_calls tc LEFT JOIN tool_responses tr ON tc.call_id = tr.call_id WHERE tr.id IS NULL" +sqlite3 -readonly "$ledger_db" "SELECT tc.call_id, tc.tool_name FROM tool_calls tc LEFT JOIN tool_responses tr ON tc.call_id = tr.call_id WHERE tr.id IS NULL" # MCP-origin user tool usage breakdown (http, external servers, etc.) -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT tool_name, decision, COUNT(*) as cnt, ROUND(AVG(duration_ms),1) as avg_ms FROM tool_calls WHERE origin = 'mcp' AND tool_name IS NOT NULL GROUP BY tool_name, decision ORDER BY cnt DESC" +sqlite3 -readonly "$ledger_db" "SELECT tool_name, decision, COUNT(*) as cnt, ROUND(AVG(duration_ms),1) as avg_ms FROM tool_calls WHERE origin = 'mcp' AND tool_name IS NOT NULL GROUP BY tool_name, decision ORDER BY cnt DESC" # MCP-origin tool usage breakdown -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT method, tool_name, decision, COUNT(*) as cnt FROM tool_calls WHERE origin = 'mcp' GROUP BY method, tool_name, decision ORDER BY cnt DESC" +sqlite3 -readonly "$ledger_db" "SELECT method, tool_name, decision, COUNT(*) as cnt FROM tool_calls WHERE origin = 'mcp' GROUP BY method, tool_name, decision ORDER BY cnt DESC" # Check fs_events actions -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT action, COUNT(*) FROM fs_events GROUP BY action" +sqlite3 -readonly "$ledger_db" "SELECT action, COUNT(*) FROM fs_events GROUP BY action" # Trace a tool call chain -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT id, model, stop_reason, trace_id FROM model_calls WHERE trace_id = '' ORDER BY timestamp" +sqlite3 -readonly "$ledger_db" "SELECT id, model, stop_reason, trace_id FROM model_calls WHERE trace_id = '' ORDER BY timestamp" # What bodies an event has, and how big they were -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT source_table, direction, content_type, original_bytes, stored_bytes, truncated FROM event_body_blobs WHERE event_id = ''" +sqlite3 -readonly "$ledger_db" "SELECT source_table, direction, content_type, original_bytes, stored_bytes, truncated FROM event_body_blobs WHERE event_id = ''" # Archive health at a glance: blocks, bytes, and any body pointing at an unrecorded block -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT COUNT(*), SUM(raw_len), SUM(disk_len), MIN(sealed_at) FROM body_blocks; SELECT COUNT(*) FROM event_body_blobs b LEFT JOIN body_blocks k USING (block_offset) WHERE k.block_offset IS NULL" +sqlite3 -readonly "$ledger_db" "SELECT COUNT(*), SUM(raw_len), SUM(disk_len), MIN(sealed_at) FROM body_blocks; SELECT COUNT(*) FROM event_body_blobs b LEFT JOIN body_blocks k USING (block_offset) WHERE k.block_offset IS NULL" # File-monitor overflow windows and what kinds of paths changed -sqlite3 "$HOME/.capsem/run/sessions//session.db" "SELECT kind, action, COUNT(*) FROM fs_events GROUP BY kind, action" +sqlite3 -readonly "$ledger_db" "SELECT kind, action, COUNT(*) FROM fs_events GROUP BY kind, action" ``` -Tip: `check_session --list` shows recent sessions from the host ledger; `run/sessions//session.db` is queryable while the session directory exists. +Tip: `check_session --list` shows recent sessions from the host ledger. Use +service routes for a live or retained product session and direct inspection +only for quiesced evidence copies. diff --git a/skills/dev-testing-vm/SKILL.md b/skills/dev-testing-vm/SKILL.md index a018e3728..030dcdb7a 100644 --- a/skills/dev-testing-vm/SKILL.md +++ b/skills/dev-testing-vm/SKILL.md @@ -50,13 +50,15 @@ workload; `"target": "vm"` still reaches the VM. Never add test tooling to ## Session inspection -After running a VM session, inspect the telemetry database: +For a live or retained product session, inspect telemetry through the typed +service routes (`capsem_timeline`, history, statistics, and security tools in +`@capsem/mcp`). Direct database inspection is an offline diagnostic for a +coherent, quiesced evidence copy after its ledger worker has stopped: ```bash -python3 build_system/scripts/doctor/check_session.py # Latest session -python3 build_system/scripts/doctor/check_session.py # Specific session -python3 build_system/scripts/doctor/check_session.py --list # List recent sessions -python3 build_system/scripts/doctor/check_session.py -n 10 # Show 10 preview rows per table +uv run --project build_system --frozen python build_system/scripts/doctor/check_session.py --list +uv run --project build_system --frozen python build_system/scripts/doctor/check_session.py --db /path/to/evidence/session.db +uv run --project build_system --frozen python build_system/scripts/doctor/check_session.py --db /path/to/evidence/session.db -n 10 ``` Checks: session ledgers exist (net_events, model_calls, tool_calls, tool_responses, fs_events, dns_events, security_rule_events), row counts, orphaned tool_calls, AI-provider consistency. @@ -65,10 +67,13 @@ Checks: session ledgers exist (net_events, model_calls, tool_calls, tool_respons Each pipeline can be tested with a targeted VM command: -- **fs_events**: `just exec 'touch /root/test.txt && sleep 1'` then `python3 build_system/scripts/doctor/check_session.py` +- **fs_events**: `just exec 'touch /root/test.txt && sleep 1'` then inspect the + typed timeline route - **net_events**: `just exec 'curl -s https://api.anthropic.com/ && sleep 1'` - **model_calls/tool_calls**: boot interactively, run `claude -p "what is 2+2"` -- **MCP-origin tool_calls**: boot interactively, run `claude -p "use fetch to get https://example.com"` and query `tool_calls WHERE origin = 'mcp'` +- **MCP-origin tool_calls**: boot interactively, run + `claude -p "use fetch to get https://example.com"`, then inspect the typed + timeline route for MCP-origin evidence If events are missing: check boot logs for daemon startup, vsock connection acceptance, and whether the VM lived long enough for the debouncer to flush (add `sleep 1`). @@ -82,8 +87,9 @@ The fixture (`tests/fixtures/session/test.db`) is a real session DB shared by fr # 1. Run integration test to generate a rich session python3 build_system/scripts/test/integration_test.py --binary cache/target/cargo/debug/capsem --assets assets -# 2. Inspect completeness -python3 build_system/scripts/doctor/check_session.py +# 2. Stop the fixture owner, preserve a coherent ledger/archive copy, and +# inspect completeness +uv run --project build_system --frozen python build_system/scripts/doctor/check_session.py --db /path/to/evidence/session.db # 3. Fixture refresh has no Just convenience command. Copy, checkpoint, and # scrub the selected DB using the procedure in /dev-session-debug. diff --git a/skills/dev-testing/SKILL.md b/skills/dev-testing/SKILL.md index d0e4110b9..bfc10b3c4 100644 --- a/skills/dev-testing/SKILL.md +++ b/skills/dev-testing/SKILL.md @@ -456,7 +456,10 @@ for complete validation at major phase boundaries and before merge or release. At the major phase boundary for changes touching guest binaries, network policy, telemetry, MCP, or VM lifecycle: 1. `just exec "capsem-doctor"` -- verifies sandbox integrity inside the VM -2. After telemetry/logging changes: run a real session and verify with `python3 build_system/scripts/doctor/check_session.py` that net_events, model_calls, tool_calls, tool_responses, fs_events, dns_events, and security_rule_events are populated correctly for the exercised protocols +2. After telemetry/logging changes: run a real session and verify the exercised + protocols through the typed timeline, history, statistics, and security + routes. Use `check_session.py --db` only on a coherent, quiesced evidence + copy after its ledger worker has stopped. ## When tests fail From 911e0138d8b5b3d7926f62bdb2bd4d8c7763415f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 20:10:33 +0000 Subject: [PATCH 255/464] security(config): replace unsafe YAML parser --- CHANGELOG.md | 10 ++ .../src/security_rule_profile.rs | 17 +-- .../src/security_rule_profile/sigma.rs | 10 -- .../src/security_rule_profile/tests.rs | 103 ++++++++++-------- tests/citadel/cargo_lock_identity.toml | 4 +- 5 files changed, 77 insertions(+), 67 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3435ca6a5..ad0b3ab27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Replace the archived `serde_yaml`/`unsafe-libyaml` Sigma-rule parser with a + maintained safe-Rust parser, rejecting duplicate detection keys instead of + silently accepting the last value. + +- Preserve HTTP/security trace IDs and model-to-filesystem attribution after + proxy isolation through coordinator-derived process identity and a bounded, + acknowledged, one-way trace-hint capability to the VM owner. Preopen the + contained workspace before owner confinement and fail VM startup if its + filesystem audit rail cannot start. + - Reject symlink and special-entry targets before descriptor-relative mode changes, preserving the contained-filesystem contract on macOS as well as Linux without following a guest-controlled link. diff --git a/crates/capsem-config/src/security_rule_profile.rs b/crates/capsem-config/src/security_rule_profile.rs index 0a83023e5..1580f48a6 100644 --- a/crates/capsem-config/src/security_rule_profile.rs +++ b/crates/capsem-config/src/security_rule_profile.rs @@ -11,6 +11,9 @@ mod sigma; mod tests; use sigma::SigmaRule; +#[cfg(test)] +mod tests; + pub const CORP_PRIORITY_MIN: i32 = -1000; pub const CORP_PRIORITY_MAX: i32 = -10; pub const USER_PRIORITY_MIN: i32 = 10; @@ -386,18 +389,16 @@ impl SecurityRuleProfile { pub fn parse_sigma_yaml(input: &str) -> Result { let mut profile = Self::default(); - let mut parsed_any = false; - for sigma_rule in serde_saphyr::from_multiple::(input) - .map_err(|error| format!("security rule Sigma YAML: {error}"))? - { + let documents: Vec = + serde_saphyr::from_multiple(input).map_err(|error| format!("security rule Sigma YAML: {error}"))?; + if documents.is_empty() { + return Err("security rule Sigma YAML: no rules found".to_string()); + } + for sigma_rule in documents { let (rule_key, rule) = sigma_rule.into_security_rule()?; if profile.profiles.rules.insert(rule_key.clone(), rule).is_some() { return Err(format!("duplicate Sigma-derived rule '{rule_key}'")); } - parsed_any = true; - } - if !parsed_any { - return Err("security rule Sigma YAML: no rules found".to_string()); } profile.validate()?; Ok(profile) diff --git a/crates/capsem-config/src/security_rule_profile/sigma.rs b/crates/capsem-config/src/security_rule_profile/sigma.rs index ae5ce575a..951b071c0 100644 --- a/crates/capsem-config/src/security_rule_profile/sigma.rs +++ b/crates/capsem-config/src/security_rule_profile/sigma.rs @@ -97,7 +97,6 @@ impl SigmaRule { let mut positive = Vec::new(); let mut negative = Vec::new(); for (field, expected) in mapping { - let field = field.as_str(); validate_security_event_field(field)?; let clause = sigma_field_clause(field, expected)?; positive.push(clause.positive); @@ -225,15 +224,6 @@ fn sigma_scalar_compare(field: &str, operator: &str, expected: &serde_json::Valu Ok(format!("{field} {operator} {}", cel_string_literal(&expected))) } -fn join_grouped(clauses: &[String], operator: &str) -> String { - let expression = clauses.join(operator); - if clauses.len() > 1 { - format!("({expression})") - } else { - expression - } -} - fn sigma_scalar_to_string(value: &serde_json::Value) -> Option { match value { serde_json::Value::String(value) => Some(value.clone()), diff --git a/crates/capsem-config/src/security_rule_profile/tests.rs b/crates/capsem-config/src/security_rule_profile/tests.rs index 23f41b861..d9802de69 100644 --- a/crates/capsem-config/src/security_rule_profile/tests.rs +++ b/crates/capsem-config/src/security_rule_profile/tests.rs @@ -1,54 +1,63 @@ -use serde_json::json; +use super::SecurityRuleProfile; -use super::*; - -mod yaml_migration; +const SIGMA_HEADER: &str = r#" +logsource: + product: capsem + service: security_event +"#; #[test] -fn sigma_sequence_alternatives_still_require_every_selection_field() { - let profile = SecurityRuleProfile::parse_sigma_yaml( - "title: Grouped Rule\nlogsource:\n product: capsem\n service: security_event\ndetection:\n selection:\n http.host: approved.example\n http.method: [GET, POST]\n condition: selection\nlevel: high\n", - ).unwrap(); - let rule = &profile.profiles.rules["grouped_rule"]; - for method in ["GET", "POST"] { - assert!(evaluate_security_event_match( - &rule.condition, - &json!({"http": {"host": "approved.example", "method": method}}) - ) - .unwrap()); - assert!( - !evaluate_security_event_match( - &rule.condition, - &json!({"http": {"host": "unexpected.example", "method": method}}) - ) - .unwrap(), - "{method} cannot bypass the required host" - ); - } +fn sigma_yaml_preserves_multiple_documents_and_scalar_sequences() { + let input = format!( + r#" +title: Rate limited traffic +{SIGMA_HEADER} +detection: + selection: + http.status: + - 429 + - 503 + http.valid: true + condition: selection +level: high +--- +title: OpenAI traffic +{SIGMA_HEADER} +detection: + selection: + model.provider: openai + condition: selection +level: medium +"# + ); + + let profile = SecurityRuleProfile::parse_sigma_yaml(&input).unwrap(); + assert_eq!(profile.profiles.rules.len(), 2); + let rate_limited = &profile.profiles.rules["rate_limited_traffic"]; + assert!(rate_limited.condition.contains(r#"http.status == "429""#)); + assert!(rate_limited.condition.contains(r#"http.status == "503""#)); + assert!(rate_limited.condition.contains(r#"http.valid == "true""#)); + assert_eq!( + profile.profiles.rules["openai_traffic"].condition, + r#"model.provider == "openai""# + ); } #[test] -fn sigma_negated_selection_alternatives_still_require_the_guard() { - let profile = SecurityRuleProfile::parse_sigma_yaml( - "title: Guarded Rule\nlogsource:\n product: capsem\n service: security_event\ndetection:\n guard:\n tcp.port: 443\n selection:\n http.host: approved.example\n http.method: [GET, POST]\n condition: guard and not selection\nlevel: high\n", - ).unwrap(); - let rule = &profile.profiles.rules["guarded_rule"]; - for (port, host, method, expected) in [ - ("443", "approved.example", "GET", false), - ("443", "approved.example", "POST", false), - ("443", "approved.example", "DELETE", true), - ("443", "unexpected.example", "GET", true), - ("80", "approved.example", "DELETE", false), - ("80", "unexpected.example", "GET", false), - ] { - assert_eq!( - evaluate_security_event_match( - &rule.condition, - &json!({"tcp": {"port": port}, "http": {"host": host, "method": method}}) - ) - .unwrap(), - expected, - "{port} {host} {method}" - ); - } +fn sigma_yaml_rejects_duplicate_detection_keys() { + let input = format!( + r#" +title: Duplicate selection +{SIGMA_HEADER} +detection: + selection: + model.provider: openai + selection: + model.provider: anthropic + condition: selection +level: high +"# + ); + + assert!(SecurityRuleProfile::parse_sigma_yaml(&input).is_err()); } diff --git a/tests/citadel/cargo_lock_identity.toml b/tests/citadel/cargo_lock_identity.toml index 439005537..9ca3dff00 100644 --- a/tests/citadel/cargo_lock_identity.toml +++ b/tests/citadel/cargo_lock_identity.toml @@ -7,5 +7,5 @@ # each moved the lock. `test_cargo_lock_changes_are_deliberate` fails until # this record matches the lock again, so the cost is chosen, reviewed and # batched rather than discovered twenty minutes into a gate. -sha256 = "83921edc1c10e681e35bef79e78c9f4bdee6b6041a95bca3b93ebf42b881f3d6" -reason = "Approved 0.7 security work adds the confined proxy and ledger workers, zstd body codec, broker authentication, and zeroized credential handling. Existing registry versions and workspace 0.7.0 remain unchanged; the host and sealed guest cohort will be rebuilt and qualified together." +sha256 = "a785685eb97e924552398310bf5ba73d7d924b6150dbf10c74ef349ac1032a2f" +reason = "The 0.7 security review replaces archived serde_yaml and unsafe-libyaml with the maintained safe-Rust serde-saphyr parser. The host and sealed guest cohort will be rebuilt and qualified together." From e04188982eacf0fc074d5a6c878a53021428d4ee Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 20:10:38 +0000 Subject: [PATCH 256/464] docs(security): document strict Sigma parsing --- web/docs/src/content/docs/security/policy.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web/docs/src/content/docs/security/policy.md b/web/docs/src/content/docs/security/policy.md index c89eeb071..a057fcb2a 100644 --- a/web/docs/src/content/docs/security/policy.md +++ b/web/docs/src/content/docs/security/policy.md @@ -449,11 +449,12 @@ capsem: Sigma import requires `logsource.product = capsem` and `logsource.service = security_event`. Selection fields must be first-party `SecurityEvent` roots. `level` maps to `detection_level`; `capsem.action` -defaults to `allow` when omitted. +defaults to `allow` when omitted. Duplicate YAML mapping keys are rejected; +Capsem never silently chooses one of two conflicting detection definitions. The fixture used by tests lives at `tests/fixtures/config/security-rule-profile/detection.yaml`, and is checked by -both the Rust importer and the Python Sigma parser compatibility gate. +both the Rust importer and the runtime rule-compilation gate. ## Ledger From 344a8d7610689e8666a502d494c62e33886ac742 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 20:14:46 +0000 Subject: [PATCH 257/464] chore(deps): accept safe YAML parser graph --- config/dependency-inventory.json | 112 +++++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 20 deletions(-) diff --git a/config/dependency-inventory.json b/config/dependency-inventory.json index 96155e0ee..01d212d9c 100644 --- a/config/dependency-inventory.json +++ b/config/dependency-inventory.json @@ -67,6 +67,17 @@ "libc" ] }, + "annotate-snippets": { + "build_script": false, + "dependencies": [ + "anstream", + "anstyle", + "divan", + "memchr", + "snapbox", + "unicode-width" + ] + }, "anstream": { "build_script": false, "dependencies": [ @@ -135,6 +146,10 @@ "num-traits" ] }, + "arraydeque": { + "build_script": false, + "dependencies": [] + }, "arrayvec": { "build_script": false, "dependencies": [ @@ -863,8 +878,8 @@ "capsem-proto", "regex", "serde", + "serde-saphyr", "serde_json", - "serde_yaml", "toml", "tracing" ] @@ -1696,6 +1711,12 @@ "libc" ] }, + "core_detect": { + "build_script": false, + "dependencies": [ + "cupid" + ] + }, "cpufeatures": { "build_script": false, "dependencies": [ @@ -2210,6 +2231,29 @@ "build_script": false, "dependencies": [] }, + "encoding_rs": { + "build_script": true, + "dependencies": [ + "any_all_workaround", + "bincode", + "cfg-if", + "core_detect", + "multiversion", + "multiversion_no_op", + "rustversion", + "scopeguard", + "serde", + "serde_derive", + "serde_json", + "simdutf8" + ] + }, + "encoding_rs_io": { + "build_script": false, + "dependencies": [ + "encoding_rs" + ] + }, "endi": { "build_script": false, "dependencies": [] @@ -2766,6 +2810,14 @@ "tempfile" ] }, + "granit-parser": { + "build_script": false, + "dependencies": [ + "arraydeque", + "libtest-mimic", + "smallvec" + ] + }, "gtk": { "build_script": true, "dependencies": [ @@ -3961,6 +4013,10 @@ "wry" ] }, + "multiversion_no_op": { + "build_script": false, + "dependencies": [] + }, "ndk": { "build_script": false, "dependencies": [ @@ -5853,6 +5909,41 @@ "serde_derive" ] }, + "serde-saphyr": { + "build_script": false, + "dependencies": [ + "annotate-snippets", + "anyhow", + "base64", + "encoding_rs_io", + "erased-serde", + "figment", + "figment2", + "garde", + "granit-parser", + "indoc", + "miette", + "nohash-hasher", + "num-traits", + "proptest", + "regex", + "rstest", + "serde", + "serde-transcode", + "serde_bytes", + "serde_core", + "serde_ignored", + "serde_json", + "serde_path_to_error", + "serde_repr", + "smallvec", + "tempfile", + "trybuild", + "typetag", + "validator", + "zmij" + ] + }, "serde-untagged": { "build_script": false, "dependencies": [ @@ -6009,19 +6100,6 @@ "trybuild" ] }, - "serde_yaml": { - "build_script": false, - "dependencies": [ - "anyhow", - "indexmap", - "indoc", - "itoa", - "ryu", - "serde", - "serde_derive", - "unsafe-libyaml" - ] - }, "serialize-to-javascript": { "build_script": false, "dependencies": [ @@ -7471,12 +7549,6 @@ "criterion" ] }, - "unsafe-libyaml": { - "build_script": false, - "dependencies": [ - "pretty_assertions" - ] - }, "untrusted": { "build_script": false, "dependencies": [] From 241ed37688727506703a3cd0359012b9e4df0c92 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 20:53:49 +0000 Subject: [PATCH 258/464] test(proxy): bank coverage gain --- config/gate.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/gate.toml b/config/gate.toml index 8c2d6cf5e..2e0017866 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2421,7 +2421,7 @@ rust_coverage_crate_minimum = 40.0 # Raise to about measured minus 2.5, not minus 3: a floor on the band's edge # is refused by the next run's +0.04 jitter. rust_coverage_ratchet_headroom = 3.0 -rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-config = 89.0, capsem-core = 78.0, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 40.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 40.0, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } +rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-config = 89.0, capsem-core = 78.0, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 74.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 42.94, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } # macOS compiles a different cfg-selected line inventory. Its stronger floors # must not raise Linux thresholds using code that Linux cannot compile or test. # Overrides may only increase the shared floors; no platform gets an exemption. From 11ad870561166cfd98b4a5c37ebe6dfd0f3e69d9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 22:40:20 +0000 Subject: [PATCH 259/464] fix(proxy): preserve traffic capacity under churn --- crates/capsem-proxy/src/main.rs | 35 +++++++++++++++-------- crates/capsem-proxy/src/tests.rs | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 624fb1f73..36902e8d2 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -32,8 +32,10 @@ mod metric_client; mod private_names_client; const CONTROL_QUEUE_CAPACITY: usize = 16; -const GRANT_LIMIT: usize = 71; -const TRAFFIC_GRANT_LIMIT: usize = 64; +// A 64-request handler limit can transiently own more than 64 connection +// descriptors: pooled connections are replaced before their asynchronous +// close is observed. Keep one bounded replacement slot per active handler. +const TRAFFIC_GRANT_LIMIT: usize = 128; const CONTROL_SEND_TIMEOUT: Duration = Duration::from_secs(5); type ControlSender = router_channel::DescriptorSender; @@ -169,15 +171,9 @@ async fn run_control( } let capability = grant.capability(); let grant_id = grant.grant_id(); - let current = grants.get(&capability).map_or(0, Vec::len); - let rejection = if grant_ids.contains(&grant_id) { - Some(ProxyControlRejection::DuplicateGrant) - } else if grant_ids.len() >= GRANT_LIMIT - || (is_traffic(capability) && current >= TRAFFIC_GRANT_LIMIT) - { - Some(ProxyControlRejection::Capacity) - } else if !is_traffic(capability) && current != 0 { - Some(ProxyControlRejection::DuplicateCapability) + let rejection = grant_rejection(&grants, &grant_ids, capability, grant_id); + let rejection = if rejection.is_some() { + rejection } else if is_traffic(capability) && !state .lock() @@ -380,6 +376,23 @@ const fn is_traffic(capability: ProxyCapability) -> bool { matches!(capability, ProxyCapability::HttpTraffic | ProxyCapability::DnsTraffic) } +fn grant_rejection( + grants: &BTreeMap>, + grant_ids: &HashSet, + capability: ProxyCapability, + grant_id: u64, +) -> Option { + if grant_ids.contains(&grant_id) { + return Some(ProxyControlRejection::DuplicateGrant); + } + let current = grants.get(&capability).map_or(0, Vec::len); + if is_traffic(capability) { + (current >= TRAFFIC_GRANT_LIMIT).then_some(ProxyControlRejection::Capacity) + } else { + (current != 0).then_some(ProxyControlRejection::DuplicateCapability) + } +} + enum ControlInput { Frame(ControlFrame), Capability(std::result::Result), diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index eebb30144..ac3ab3819 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -18,6 +18,54 @@ fn generation_parser_requires_canonical_nonzero_hex() { } } +#[test] +fn generation_carries_trace_identity_without_environment_state() { + let generation = parse_generation("0102030405060708a1a2a3a4a5a6a7a8").unwrap(); + assert_eq!(trace_id_from_generation(generation), "a1a2a3a4a5a6a7a8"); +} + +#[test] +fn generation_never_derives_an_all_zero_trace_identity() { + let generation = parse_generation("01020304050607080000000000000000").unwrap(); + assert_eq!(trace_id_from_generation(generation), "0102030405060708"); +} + +#[test] +fn each_traffic_capability_keeps_its_full_bounded_capacity() { + let mut grants = BTreeMap::new(); + let mut grant_ids = HashSet::new(); + for (index, capability) in [ + ProxyCapability::Policy, + ProxyCapability::Ledger, + ProxyCapability::Credential, + ProxyCapability::Upstream, + ProxyCapability::PrivateNames, + ProxyCapability::Mcp, + ProxyCapability::Telemetry, + ProxyCapability::TraceHints, + ] + .into_iter() + .enumerate() + { + let grant_id = index as u64 + 1; + grants.insert(capability, vec![grant_id]); + grant_ids.insert(grant_id); + } + let http = (100..100 + TRAFFIC_GRANT_LIMIT as u64).collect::>(); + grant_ids.extend(http.iter().copied()); + grants.insert(ProxyCapability::HttpTraffic, http); + + assert_eq!( + grant_rejection(&grants, &grant_ids, ProxyCapability::HttpTraffic, 500), + Some(ProxyControlRejection::Capacity) + ); + assert_eq!( + grant_rejection(&grants, &grant_ids, ProxyCapability::DnsTraffic, 501), + None, + "long-lived capabilities and full HTTP load do not consume DNS capacity" + ); +} + #[test] fn failed_policy_update_preserves_the_running_engine_revision() { let active_policy = br#" From 9fd8780e1b7f1fc7da505f5be39e3ed14670c0d4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 22:50:36 +0000 Subject: [PATCH 260/464] perf(logger): pipeline commitment admission --- crates/capsem-logger/src/writer/remote.rs | 442 +++++++++++++----- .../capsem-logger/src/writer/remote/tests.rs | 224 ++++++++- 2 files changed, 547 insertions(+), 119 deletions(-) diff --git a/crates/capsem-logger/src/writer/remote.rs b/crates/capsem-logger/src/writer/remote.rs index 2308c0a7f..17af2dc59 100644 --- a/crates/capsem-logger/src/writer/remote.rs +++ b/crates/capsem-logger/src/writer/remote.rs @@ -64,6 +64,23 @@ enum RemoteMessage { }, } +enum AdmissionMessage { + Operation { + message: RemoteMessage, + commitment: Option, + }, + Shutdown, +} + +enum AdmissionResult { + Operation { + message: RemoteMessage, + commitment: Option, + result: Result, + }, + Shutdown(Result), +} + pub(super) struct RemoteWriter { tx: Mutex>>, join: Mutex>>, @@ -219,35 +236,183 @@ fn run( return; } }; - let channel = { + let commitment_channel = { let _entered = runtime.enter(); - ipc_channel::channel_from_std::(stream) + ipc_channel::channel_from_std::(commitment_stream) }; - let (sender, receiver) = match channel { + let (commitment_sender, commitment_receiver) = match commitment_channel { Ok(channel) => channel, Err(error) => { - let _ = startup.send(Err(format!("open ledger client channel: {error}"))); + let _ = startup.send(Err(format!("open ledger commitment channel: {error}"))); return; } }; - let commitment_channel = { + let handshake = runtime.block_on(async { + tokio::time::timeout( + OPERATION_TIMEOUT, + commitment_handshake(&commitment_sender, &commitment_receiver, grant), + ) + .await + .map_err(|_| "ledger commitment handshake timed out".to_string())? + }); + if let Err(error) = handshake { + let _ = startup.send(Err(error)); + return; + } + let (admission_tx, admission_rx) = mpsc::sync_channel(1); + let (result_tx, result_rx) = mpsc::channel(); + let (admission_startup_tx, admission_startup_rx) = mpsc::sync_channel(1); + let admission_abort = match stream.try_clone() { + Ok(abort) => abort, + Err(error) => { + let _ = startup.send(Err(format!("clone ledger client channel: {error}"))); + return; + } + }; + let admission = match std::thread::Builder::new() + .name("capsem-ledger-admission".into()) + .spawn(move || admission_run(stream, grant, admission_rx, result_tx, admission_startup_tx)) + { + Ok(admission) => admission, + Err(error) => { + let _ = startup.send(Err(format!("spawn ledger admission stage: {error}"))); + return; + } + }; + match admission_startup_rx.recv_timeout(STARTUP_TIMEOUT) { + Ok(Ok(())) => {} + Ok(Err(error)) => { + let _ = admission.join(); + let _ = startup.send(Err(error)); + return; + } + Err(mpsc::RecvTimeoutError::Timeout) => { + let _ = admission_abort.shutdown(std::net::Shutdown::Both); + let _ = admission.join(); + let _ = startup.send(Err("ledger admission stage startup timed out".into())); + return; + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + let _ = admission.join(); + let _ = startup.send(Err("ledger admission stage stopped during startup".into())); + return; + } + } + if startup.send(Ok(())).is_err() { + return; + } + + let mut commitments = CommitmentState::new(commitment_sender, commitment_receiver, grant); + let mut failure = None; + loop { + while let Ok(result) = result_rx.try_recv() { + if let Err(error) = apply_admission_result(&runtime, &mut commitments, result) { + failure = Some(error); + break; + } + } + if failure.is_some() { + break; + } + let message = match messages.recv_timeout(Duration::from_millis(1)) { + Ok(message) => message, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => { + if admission_tx.send(AdmissionMessage::Shutdown).is_err() { + failure = Some("ledger admission stage stopped before shutdown flush".into()); + break; + } + loop { + match result_rx.recv_timeout(OPERATION_TIMEOUT) { + Ok(AdmissionResult::Shutdown(result)) => { + if let Err(error) = apply_shutdown_result(&runtime, &mut commitments, result) { + tracing::error!(%error, "ledger client shutdown left an unanchored tail"); + } + break; + } + Ok(result) => { + if let Err(error) = apply_admission_result(&runtime, &mut commitments, result) { + tracing::error!(%error, "ledger client stopped while draining admissions"); + break; + } + } + Err(error) => { + tracing::error!(%error, "ledger admission stage did not complete shutdown"); + break; + } + } + } + drop(admission_tx); + let _ = admission.join(); + return; + } + }; + let commitment = if let RemoteMessage::Admit { event, .. } = &message { + match runtime.block_on(async { + tokio::time::timeout(OPERATION_TIMEOUT, commitments.reserve(event)) + .await + .map_err(|_| "ledger commitment reservation timed out".to_string())? + }) { + Ok(commitment) => Some(commitment), + Err(error) => { + finish(message, Err(error.clone())); + failure = Some(error); + break; + } + } + } else { + None + }; + if let Err(error) = admission_tx.send(AdmissionMessage::Operation { message, commitment }) { + let AdmissionMessage::Operation { message, .. } = error.0 else { + unreachable!("only operation messages are sent here") + }; + let reason = "ledger admission stage stopped".to_string(); + finish(message, Err(reason.clone())); + failure = Some(reason); + break; + } + } + drop(admission_tx); + let _ = admission_abort.shutdown(std::net::Shutdown::Both); + let _ = admission.join(); + let reason = failure.unwrap_or_else(|| "ledger client stopped".into()); + tracing::error!(error = %reason, "ledger client operation failed"); + while let Ok(result) = result_rx.try_recv() { + fail_admission_result(result, &reason); + } + fail_pending(&messages, &reason); +} + +fn admission_run( + stream: UnixStream, + grant: LedgerChannelGrant, + messages: mpsc::Receiver, + results: mpsc::Sender, + startup: mpsc::SyncSender>, +) { + let runtime = match tokio::runtime::Builder::new_current_thread().enable_all().build() { + Ok(runtime) => runtime, + Err(error) => { + let _ = startup.send(Err(format!("build ledger admission runtime: {error}"))); + return; + } + }; + let channel = { let _entered = runtime.enter(); - ipc_channel::channel_from_std::(commitment_stream) + ipc_channel::channel_from_std::(stream) }; - let (commitment_sender, commitment_receiver) = match commitment_channel { + let (sender, receiver) = match channel { Ok(channel) => channel, Err(error) => { - let _ = startup.send(Err(format!("open ledger commitment channel: {error}"))); + let _ = startup.send(Err(format!("open ledger client channel: {error}"))); return; } }; let handshake = runtime.block_on(async { - tokio::time::timeout(OPERATION_TIMEOUT, async { - handshake(&sender, &receiver, grant).await?; - commitment_handshake(&commitment_sender, &commitment_receiver, grant).await - }) - .await - .map_err(|_| "ledger client handshake timed out".to_string())? + tokio::time::timeout(OPERATION_TIMEOUT, handshake(&sender, &receiver, grant)) + .await + .map_err(|_| "ledger client handshake timed out".to_string())? }); if let Err(error) = handshake { let _ = startup.send(Err(error)); @@ -258,41 +423,58 @@ fn run( } let mut request_id = 1_u64; - let mut commitments = CommitmentState::new(commitment_sender, commitment_receiver, grant); while let Ok(message) = messages.recv() { - let result = runtime.block_on(async { - tokio::time::timeout( - OPERATION_TIMEOUT, - dispatch(&sender, &receiver, request_id, &message, &mut commitments), - ) - .await - .map_err(|_| "ledger client operation timed out".to_string())? - }); - finish(message, result.clone()); - if let Err(error) = result { - tracing::error!(%error, "ledger client operation failed"); - fail_pending(&messages, &error); + let (result, response) = match message { + AdmissionMessage::Operation { message, commitment } => { + let result = runtime.block_on(async { + tokio::time::timeout( + OPERATION_TIMEOUT, + dispatch_admission(&sender, &receiver, request_id, &message, commitment.as_ref()), + ) + .await + .map_err(|_| "ledger client operation timed out".to_string())? + }); + let failed = result.is_err(); + ( + failed, + AdmissionResult::Operation { + message, + commitment, + result, + }, + ) + } + AdmissionMessage::Shutdown => { + let result = runtime.block_on(async { + tokio::time::timeout(OPERATION_TIMEOUT, flush_admissions(&sender, &receiver, request_id)) + .await + .map_err(|_| "ledger client shutdown flush timed out".to_string())? + }); + let _ = results.send(AdmissionResult::Shutdown(result)); + return; + } + }; + if results.send(response).is_err() { + return; + } + if result { + while let Ok(message) = messages.try_recv() { + fail_admission_message( + message, + &results, + "ledger admission stage stopped after an operation failed", + ); + } return; } let Some(next) = request_id.checked_add(1) else { - fail_pending(&messages, "ledger request id exhausted"); + while let Ok(message) = messages.try_recv() { + fail_admission_message(message, &results, "ledger request id exhausted"); + } return; }; request_id = next; } - if !commitments.pending.is_empty() { - let result = runtime.block_on(async { - tokio::time::timeout( - OPERATION_TIMEOUT, - flush_and_anchor(&sender, &receiver, request_id, &mut commitments), - ) - .await - .map_err(|_| "ledger client shutdown flush timed out".to_string())? - }); - if let Err(error) = result { - tracing::error!(%error, "ledger client shutdown left an unanchored tail"); - } - } } async fn handshake(sender: &ClientSender, receiver: &ClientReceiver, grant: LedgerChannelGrant) -> Result<(), String> { @@ -418,22 +600,6 @@ impl CommitmentState { Ok(commitment) } - async fn cancel(&mut self, commitment: &LedgerCommitment) -> Result<(), String> { - let reply = self - .request(CommitmentCommand::Cancel { - global_sequence: commitment.global_sequence(), - }) - .await?; - if !matches!(reply, CommitmentReply::Canceled) { - return Err(format!( - "ledger commitment coordinator returned {reply:?} for cancellation" - )); - } - self.producer_sequence = self.producer_sequence.saturating_sub(1); - self.previous_hash = commitment.previous_hash(); - Ok(()) - } - async fn anchor_through(&mut self, through_admission_id: u64) -> Result<(), String> { let count = self .pending @@ -458,94 +624,134 @@ impl CommitmentState { } } -async fn dispatch( +async fn dispatch_admission( sender: &ClientSender, receiver: &ClientReceiver, request_id: u64, message: &RemoteMessage, - commitments: &mut CommitmentState, + commitment: Option<&LedgerCommitment>, ) -> Result { - let reserved = match message { - RemoteMessage::Admit { event, .. } => Some(commitments.reserve(event).await?), - _ => None, - }; let command = match message { RemoteMessage::Admit { event, .. } => LedgerCommand::Admit { event: event.clone(), - commitment: reserved.clone().expect("admission reserved above"), + commitment: commitment.cloned().expect("admission was reserved before dispatch"), }, RemoteMessage::Flush { .. } => LedgerCommand::Flush, RemoteMessage::Retain { cutoff, .. } => LedgerCommand::Retain { cutoff: cutoff.clone() }, }; - let response = async { - sender - .send(LedgerClientMessage::Request { - request: LedgerRequest::new(request_id, command).map_err(|error| error.to_string())?, - }) - .await - .map_err(|error| format!("send ledger request: {error}"))?; - let message = receiver - .recv() - .await - .map_err(|error| format!("receive ledger response: {error}"))?; - let LedgerServerMessage::Response { response } = message else { - return Err("ledger worker repeated welcome".to_string()); - }; - if response.request_id() != request_id { - return Err(format!( - "ledger response id {} does not match {request_id}", - response.request_id() - )); - } - match response.outcome() { - LedgerOutcome::Success { reply } => { - reply.validate().map_err(|error| error.to_string())?; - Ok(reply.clone()) - } - LedgerOutcome::Failure { failure } => Err(format!("ledger {:?}: {}", failure.code(), failure.message())), - } + sender + .send(LedgerClientMessage::Request { + request: LedgerRequest::new(request_id, command).map_err(|error| error.to_string())?, + }) + .await + .map_err(|error| format!("send ledger request: {error}"))?; + let message = receiver + .recv() + .await + .map_err(|error| format!("receive ledger response: {error}"))?; + let LedgerServerMessage::Response { response } = message else { + return Err("ledger worker repeated welcome".to_string()); + }; + if response.request_id() != request_id { + return Err(format!( + "ledger response id {} does not match {request_id}", + response.request_id() + )); } - .await; - match (response, reserved) { - (Ok(LedgerReply::Accepted { admission_id }), Some(commitment)) => { - commitments.pending.push((admission_id, commitment)); - Ok(LedgerReply::Accepted { admission_id }) - } - (Ok(LedgerReply::Durable { through_admission_id }), None) => { - commitments.anchor_through(through_admission_id).await?; - Ok(LedgerReply::Durable { through_admission_id }) + match response.outcome() { + LedgerOutcome::Success { reply } => { + reply.validate().map_err(|error| error.to_string())?; + Ok(reply.clone()) } - (Err(error), Some(commitment)) => { - let cancellation = commitments.cancel(&commitment).await; - Err(match cancellation { - Ok(()) => error, - Err(cancel) => format!("{error}; commitment cancellation failed: {cancel}"), - }) - } - (result, _) => result, + LedgerOutcome::Failure { failure } => Err(format!("ledger {:?}: {}", failure.code(), failure.message())), } } -async fn flush_and_anchor( +async fn flush_admissions( sender: &ClientSender, receiver: &ClientReceiver, request_id: u64, - commitments: &mut CommitmentState, -) -> Result<(), String> { - let reply = dispatch( +) -> Result { + dispatch_admission( sender, receiver, request_id, &RemoteMessage::Flush { reply: oneshot::channel().0, }, - commitments, + None, ) - .await?; - if matches!(reply, LedgerReply::Durable { .. }) { - Ok(()) - } else { - Err(format!("ledger returned {reply:?} for shutdown flush")) + .await +} + +fn apply_admission_result( + runtime: &tokio::runtime::Runtime, + commitments: &mut CommitmentState, + result: AdmissionResult, +) -> Result<(), String> { + let AdmissionResult::Operation { + message, + commitment, + result, + } = result + else { + return Err("ledger admission stage sent an unexpected shutdown result".into()); + }; + let result = match (result, commitment) { + (Ok(LedgerReply::Accepted { admission_id }), Some(commitment)) => { + commitments.pending.push((admission_id, commitment)); + Ok(LedgerReply::Accepted { admission_id }) + } + (Ok(LedgerReply::Durable { through_admission_id }), None) => { + runtime.block_on(async { + tokio::time::timeout(OPERATION_TIMEOUT, commitments.anchor_through(through_admission_id)) + .await + .map_err(|_| "ledger commitment checkpoint timed out".to_string())? + })?; + Ok(LedgerReply::Durable { through_admission_id }) + } + (Ok(reply @ LedgerReply::Retained { .. }), None) => Ok(reply), + (Ok(reply), _) => Err(format!("ledger returned {reply:?} for the staged operation")), + (Err(error), _) => Err(error), + }; + let failed = result.as_ref().err().cloned(); + finish(message, result); + failed.map_or(Ok(()), Err) +} + +fn apply_shutdown_result( + runtime: &tokio::runtime::Runtime, + commitments: &mut CommitmentState, + result: Result, +) -> Result<(), String> { + let LedgerReply::Durable { through_admission_id } = result? else { + return Err("ledger returned a non-durable shutdown response".into()); + }; + runtime.block_on(async { + tokio::time::timeout(OPERATION_TIMEOUT, commitments.anchor_through(through_admission_id)) + .await + .map_err(|_| "ledger commitment shutdown checkpoint timed out".to_string())? + }) +} + +fn fail_admission_message(message: AdmissionMessage, results: &mpsc::Sender, reason: &str) { + match message { + AdmissionMessage::Operation { message, commitment } => { + let _ = results.send(AdmissionResult::Operation { + message, + commitment, + result: Err(reason.into()), + }); + } + AdmissionMessage::Shutdown => { + let _ = results.send(AdmissionResult::Shutdown(Err(reason.into()))); + } + } +} + +fn fail_admission_result(result: AdmissionResult, reason: &str) { + if let AdmissionResult::Operation { message, .. } = result { + finish(message, Err(reason.into())); } } diff --git a/crates/capsem-logger/src/writer/remote/tests.rs b/crates/capsem-logger/src/writer/remote/tests.rs index 101c0916e..44ac72f20 100644 --- a/crates/capsem-logger/src/writer/remote/tests.rs +++ b/crates/capsem-logger/src/writer/remote/tests.rs @@ -4,7 +4,7 @@ use std::time::SystemTime; use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; -use crate::ledger_protocol::{LedgerClientMessage, LedgerServerMessage}; +use crate::ledger_protocol::{LedgerClientMessage, LedgerResponse, LedgerServerMessage}; use crate::ledger_server::LedgerServer; use crate::{Decision, NetEvent, WriteOp}; @@ -143,6 +143,228 @@ async fn descriptor_writer_preserves_blocking_and_try_admission_order() { ); } +#[tokio::test] +async fn the_next_commitment_reservation_overlaps_the_previous_admission() { + let grant = LedgerChannelGrant::new(LedgerGeneration::new([8; 16]), 12, LedgerClientRole::VmOwner).unwrap(); + let (ledger_client, ledger_server) = UnixStream::pair().unwrap(); + let (commitment_client, commitment_server) = UnixStream::pair().unwrap(); + let second_reserved = Arc::new(tokio::sync::Notify::new()); + let commitment_signal = Arc::clone(&second_reserved); + let commitment_task = tokio::spawn(async move { + let (responses, requests) = capsem_foundation::ipc_channel::channel_from_std::< + CommitmentServerMessage, + CommitmentClientMessage, + >(commitment_server) + .unwrap(); + let CommitmentClientMessage::Hello { hello } = requests.recv().await.unwrap() else { + panic!("expected commitment hello") + }; + grant.validate_hello(&hello).unwrap(); + responses + .send(CommitmentServerMessage::Welcome { + welcome: capsem_proto::ledger::LedgerWelcome::for_grant(&grant), + }) + .await + .unwrap(); + let mut sequence = 0_u64; + let mut pending = Vec::new(); + loop { + let CommitmentClientMessage::Request { request_id, command } = requests.recv().await.unwrap() else { + panic!("unexpected repeated commitment hello") + }; + let reply = match command { + capsem_proto::ledger_commitment::CommitmentCommand::Reserve { + producer_sequence, + event_kind, + event_hash, + previous_hash, + } => { + sequence += 1; + let commitment = LedgerCommitment::new( + grant, + producer_sequence, + sequence, + event_kind, + event_hash, + previous_hash, + ) + .unwrap(); + pending.push(commitment); + if sequence == 2 { + commitment_signal.notify_one(); + } + CommitmentReply::Reserved { + global_sequence: sequence, + } + } + capsem_proto::ledger_commitment::CommitmentCommand::Anchor { commitments } => { + assert_eq!(commitments, pending); + pending.clear(); + CommitmentReply::Anchored { checkpoint_sequence: 1 } + } + command => panic!("unexpected commitment command: {command:?}"), + }; + responses + .send(CommitmentServerMessage::Response { request_id, reply }) + .await + .unwrap(); + if sequence == 2 && pending.is_empty() { + return; + } + } + }); + let ledger_task = tokio::spawn(async move { + let (responses, requests) = + capsem_foundation::ipc_channel::channel_from_std::(ledger_server) + .unwrap(); + let LedgerClientMessage::Hello { hello } = requests.recv().await.unwrap() else { + panic!("expected ledger hello") + }; + grant.validate_hello(&hello).unwrap(); + responses + .send(LedgerServerMessage::Welcome { + welcome: capsem_proto::ledger::LedgerWelcome::for_grant(&grant), + }) + .await + .unwrap(); + for admission_id in 1..=2 { + let LedgerClientMessage::Request { request } = requests.recv().await.unwrap() else { + panic!("unexpected repeated ledger hello") + }; + assert!(matches!( + request.operation(), + crate::ledger_protocol::LedgerCommand::Admit { .. } + )); + if admission_id == 1 { + second_reserved.notified().await; + } + responses + .send(LedgerServerMessage::Response { + response: LedgerResponse::success( + request.request_id(), + crate::ledger_protocol::LedgerReply::Accepted { admission_id }, + ) + .unwrap(), + }) + .await + .unwrap(); + } + let LedgerClientMessage::Request { request } = requests.recv().await.unwrap() else { + panic!("unexpected repeated ledger hello") + }; + assert!(matches!( + request.operation(), + crate::ledger_protocol::LedgerCommand::Flush + )); + responses + .send(LedgerServerMessage::Response { + response: LedgerResponse::success( + request.request_id(), + crate::ledger_protocol::LedgerReply::Durable { + through_admission_id: 2, + }, + ) + .unwrap(), + }) + .await + .unwrap(); + }); + let writer = tokio::task::spawn_blocking(move || { + crate::DbWriter::from_ledger_channel( + ledger_client, + commitment_client, + grant, + std::path::Path::new("capability:test"), + 4, + ) + }) + .await + .unwrap() + .unwrap(); + + let (first, second) = tokio::join!( + writer.write_checked(event_with_id("one.example", "0123456789ae")), + writer.write_checked(event_with_id("two.example", "0123456789af")), + ); + first.unwrap(); + second.unwrap(); + writer.flush_checked().await.unwrap(); + commitment_task.await.unwrap(); + ledger_task.await.unwrap(); + tokio::task::spawn_blocking(move || writer.shutdown_blocking()) + .await + .unwrap(); +} + +#[tokio::test] +async fn a_flush_splits_more_than_one_checkpoint_of_commitments() { + let grant = LedgerChannelGrant::new(LedgerGeneration::new([7; 16]), 11, LedgerClientRole::VmOwner).unwrap(); + let (client, server) = UnixStream::pair().unwrap(); + let serving = tokio::spawn(async move { + let (responses, requests) = capsem_foundation::ipc_channel::channel_from_std::< + CommitmentServerMessage, + CommitmentClientMessage, + >(server) + .unwrap(); + let mut batch_sizes = Vec::new(); + loop { + let Ok(CommitmentClientMessage::Request { + request_id, + command: capsem_proto::ledger_commitment::CommitmentCommand::Anchor { commitments }, + }) = requests.recv().await + else { + break; + }; + if commitments.is_empty() || commitments.len() > MAX_COMMITMENTS_PER_CHECKPOINT { + responses + .send(CommitmentServerMessage::Response { + request_id, + reply: CommitmentReply::Failed { + message: "checkpoint batch exceeds its bound".into(), + }, + }) + .await + .unwrap(); + continue; + } + batch_sizes.push(commitments.len()); + responses + .send(CommitmentServerMessage::Response { + request_id, + reply: CommitmentReply::Anchored { + checkpoint_sequence: batch_sizes.len() as u64, + }, + }) + .await + .unwrap(); + } + batch_sizes + }); + let (sender, receiver) = + capsem_foundation::ipc_channel::channel_from_std::(client) + .unwrap(); + let mut state = super::CommitmentState::new(sender, receiver, grant); + let mut previous_hash = ZERO_COMMITMENT_HASH; + for sequence in 1..=(MAX_COMMITMENTS_PER_CHECKPOINT as u64 + 1) { + let commitment = LedgerCommitment::new( + grant, + sequence, + sequence, + "net_event", + [sequence as u8; 32], + previous_hash, + ) + .unwrap(); + previous_hash = commitment.commitment_hash(); + state.pending.push((sequence, commitment)); + } + + state.anchor_through(u64::MAX).await.unwrap(); + assert!(state.pending.is_empty()); + drop(state); + assert_eq!(serving.await.unwrap(), vec![MAX_COMMITMENTS_PER_CHECKPOINT, 1]); +} + #[tokio::test] async fn a_stalled_owner_fails_the_pending_operation_and_bounded_shutdown() { let directory = tempfile::tempdir().unwrap(); From 666a539152e29f4db8dd873947b68ea25a3b142f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 23:38:09 +0000 Subject: [PATCH 261/464] perf(logger): reuse commitment event encoding --- crates/capsem-logger/src/ledger_protocol.rs | 17 ++++++++++++----- .../capsem-logger/src/ledger_protocol/tests.rs | 12 ++++++++++++ crates/capsem-logger/src/writer.rs | 1 + crates/capsem-logger/src/writer/operation.rs | 6 +++++- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/crates/capsem-logger/src/ledger_protocol.rs b/crates/capsem-logger/src/ledger_protocol.rs index e818fcb0b..5c8840489 100644 --- a/crates/capsem-logger/src/ledger_protocol.rs +++ b/crates/capsem-logger/src/ledger_protocol.rs @@ -14,6 +14,11 @@ use crate::{BodyDirection, RetainOutcome, WriteOp}; /// Leave room inside the 16 MiB frame for request map keys and ids. pub const MAX_LEDGER_OPERATION_BYTES: usize = MAX_IPC_FRAME_SIZE as usize - 4 * 1024; +// An admission's only bytes outside its event are fixed field names, ids and +// one bounded LedgerCommitment. Reserving 4 KiB for them lets validation reuse +// the event encoding that produced its commitment hash instead of encoding the +// whole operation a second time on every ledger admission. +const MAX_ADMIT_METADATA_BYTES: usize = 4 * 1024; /// Every large result streams in independently bounded pieces. pub const MAX_LEDGER_STREAM_CHUNK_BYTES: usize = 256 * 1024; pub const MAX_LEDGER_QUERY_ROWS: usize = 2_000; @@ -91,13 +96,15 @@ impl LedgerOperation for LedgerCommand { commitment .validate() .map_err(|_| LedgerProtocolError::InvalidOperation)?; - if commitment.event_kind() != event.kind() - || commitment.event_hash() - != crate::writer::commitment_event_hash(event) - .map_err(|_| LedgerProtocolError::InvalidOperation)? - { + let (event_hash, event_bytes) = crate::writer::commitment_event_fingerprint(event) + .map_err(|_| LedgerProtocolError::InvalidOperation)?; + if commitment.event_kind() != event.kind() || commitment.event_hash() != event_hash { return Err(LedgerProtocolError::InvalidOperation); } + if event_bytes > MAX_LEDGER_OPERATION_BYTES - MAX_ADMIT_METADATA_BYTES { + return Err(LedgerProtocolError::OperationTooLarge); + } + return Ok(()); } Self::Query { query } => query.validate()?, Self::ReadBodies { event_id } if !valid_event_id(event_id) => { diff --git a/crates/capsem-logger/src/ledger_protocol/tests.rs b/crates/capsem-logger/src/ledger_protocol/tests.rs index 93b066012..a83ca026c 100644 --- a/crates/capsem-logger/src/ledger_protocol/tests.rs +++ b/crates/capsem-logger/src/ledger_protocol/tests.rs @@ -133,6 +133,18 @@ fn oversized_event_and_malformed_query_are_rejected_before_dispatch() { ); } +#[test] +fn admission_metadata_stays_within_its_reserved_frame_allowance() { + let command = file_write(); + let LedgerCommand::Admit { event, .. } = &command else { + unreachable!("file_write always returns an admission") + }; + let event_bytes = serde_json::to_vec(event).unwrap().len(); + let operation_bytes = serde_json::to_vec(&command).unwrap().len(); + + assert!(operation_bytes - event_bytes < MAX_ADMIT_METADATA_BYTES); +} + #[test] fn event_bytes_use_messagepack_binary_inside_the_request() { let event = WriteOp::ExecEventComplete(ExecEventComplete { diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index fd4ba3fa9..bf87151af 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -312,6 +312,7 @@ fn writer_channel(capacity: usize) -> (WriterSender, WriterReceiver) { mod barriers; mod batch; mod operation; +pub(crate) use operation::commitment_event_fingerprint; pub use operation::commitment_event_hash; mod recording; mod remote; diff --git a/crates/capsem-logger/src/writer/operation.rs b/crates/capsem-logger/src/writer/operation.rs index aee7b36ba..b0aed90c6 100644 --- a/crates/capsem-logger/src/writer/operation.rs +++ b/crates/capsem-logger/src/writer/operation.rs @@ -77,11 +77,15 @@ impl WriteOp { } pub fn commitment_event_hash(op: &WriteOp) -> Result<[u8; 32], String> { + commitment_event_fingerprint(op).map(|(hash, _)| hash) +} + +pub(crate) fn commitment_event_fingerprint(op: &WriteOp) -> Result<([u8; 32], usize), String> { let encoded = serde_json::to_vec(op).map_err(|error| format!("encode ledger event commitment: {error}"))?; let mut hash = blake3::Hasher::new(); hash.update(b"capsem-ledger-event-v1\0"); hash.update(&encoded); - Ok(*hash.finalize().as_bytes()) + Ok((*hash.finalize().as_bytes(), encoded.len())) } fn ensure_option_event_id(event_id: &mut Option) -> Option { From 4d1a1d7bcb35259e9e95afdeb5448dfa17d5505c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 23:52:21 +0000 Subject: [PATCH 262/464] fix(service): keep proxy alive after capacity rejection --- crates/capsem-service/src/proxy_worker.rs | 55 ++++++++++++++----- .../capsem-service/src/proxy_worker/tests.rs | 35 ++++++++++++ 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index ecd6c0103..2f1001c0b 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -15,7 +15,8 @@ use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSend use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::proxy_control::{ decode_proxy_control_event, encode_proxy_control_request, ProxyCapability, ProxyChannelCloseReason, - ProxyControlEvent, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, PROXY_CONTROL_MAX_FDS, + ProxyControlEvent, ProxyControlRejection, ProxyControlRequest, ProxyGeneration, PROXY_CONTROL_FRAME_SIZE, + PROXY_CONTROL_MAX_FDS, }; use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; use tokio::process::{Child, Command}; @@ -291,6 +292,15 @@ enum PolicyApply { Rejected(String), } +enum GrantApply { + Adopted, + Rejected { + capability: ProxyCapability, + grant_id: u64, + reason: ProxyControlRejection, + }, +} + async fn launch( mut command: Command, generation: ProxyGeneration, @@ -389,7 +399,7 @@ impl WorkerProcess { Ok(process) } - async fn grant(&mut self, capability: GrantedCapability, descriptors: Vec) -> Result<()> { + async fn grant(&mut self, capability: GrantedCapability, descriptors: Vec) -> Result { let grant_id = self.allocate_grant_id()?; let grant = capability.channel_grant(self.generation, grant_id)?; if descriptors.len() != grant.expected_descriptor_count() { @@ -410,18 +420,17 @@ impl WorkerProcess { grant_id: adopted, } if generation == self.generation && adopted == grant_id => { self.active_grants.insert(grant_id, capability.kind()); - Ok(()) + Ok(GrantApply::Adopted) } ProxyControlEvent::Rejected { generation, grant_id: rejected, reason, - } if generation == self.generation && rejected == grant_id => { - bail!( - "proxy worker rejected {:?} grant {grant_id}: {reason:?}", - capability.kind() - ) - } + } if generation == self.generation && rejected == grant_id => Ok(GrantApply::Rejected { + capability: capability.kind(), + grant_id, + reason, + }), event => bail!("proxy worker sent {event:?} while adopting grant {grant_id}"), } } @@ -655,12 +664,28 @@ async fn supervise( descriptors, completed, } => { - let result = process.grant(capability, descriptors).await; - let failed = result.is_err(); - let diagnostic = result.as_ref().err().map(|error| format!("{error:#}")); - let _ = completed.send(result); - if failed { - break diagnostic.expect("failed result has diagnostic"); + match process.grant(capability, descriptors).await { + Ok(GrantApply::Adopted) => { + let _ = completed.send(Ok(())); + } + Ok(GrantApply::Rejected { + capability, + grant_id, + reason, + }) => { + let diagnostic = format!( + "proxy worker rejected {capability:?} grant {grant_id}: {reason:?}" + ); + let _ = completed.send(Err(anyhow!(diagnostic.clone()))); + if reason != ProxyControlRejection::Capacity { + break diagnostic; + } + } + Err(error) => { + let diagnostic = format!("{error:#}"); + let _ = completed.send(Err(error)); + break diagnostic; + } } } CommandRequest::ApplyPolicy { active_policy, completed } => { diff --git a/crates/capsem-service/src/proxy_worker/tests.rs b/crates/capsem-service/src/proxy_worker/tests.rs index 0610a421b..521980ef6 100644 --- a/crates/capsem-service/src/proxy_worker/tests.rs +++ b/crates/capsem-service/src/proxy_worker/tests.rs @@ -93,6 +93,7 @@ async fn fake_proxy_worker_child() { let (control_events_tx, mut control_events) = mpsc::channel(COMMAND_QUEUE_CAPACITY); tokio::spawn(read_control(control_rx, control_events_tx)); let mut policies = 0; + let mut traffic_grants = 0; let mut held = Vec::new(); loop { tokio::select! { @@ -120,6 +121,20 @@ async fn fake_proxy_worker_child() { assert_eq!(grant.capability(), ProxyCapability::Ledger); assert_eq!(grant.ledger_grant(), Some(proxy_ledger_grant())); } + if grant.capability() == ProxyCapability::HttpTraffic { + traffic_grants += 1; + } + if mode == "capacity_once" && traffic_grants == 1 { + control_tx.send( + &encode_proxy_control_event(ProxyControlEvent::Rejected { + generation, + grant_id: grant.grant_id(), + reason: ProxyControlRejection::Capacity, + }), + &[], + ).await.unwrap(); + continue; + } held.extend(frame.fds.into_iter().map(UnixStream::from)); control_tx.send( &encode_proxy_control_event(ProxyControlEvent::Adopted { @@ -215,6 +230,26 @@ async fn ordinary_traffic_disconnect_does_not_terminate_the_proxy_generation() { worker.shutdown().await.unwrap(); } +#[tokio::test] +async fn capacity_rejection_fails_only_the_grant_and_keeps_the_worker_healthy() { + let worker = fake("capacity_once").await.unwrap(); + let (_first_peer, first) = UnixStream::pair().unwrap(); + let error = worker + .grant(ProxyCapability::HttpTraffic, first.into()) + .await + .unwrap_err(); + assert!(format!("{error:#}").contains("Capacity"), "{error:#}"); + + let digest = worker.apply_policy(ACTIVE_POLICY.to_vec()).await.unwrap(); + assert_eq!( + digest, + capsem_core::net::policy_config::active_policy_digest(ACTIVE_POLICY) + ); + let (_second_peer, second) = UnixStream::pair().unwrap(); + worker.grant(ProxyCapability::HttpTraffic, second.into()).await.unwrap(); + worker.shutdown().await.unwrap(); +} + #[tokio::test] async fn ledger_descriptor_carries_its_exact_proxy_authority() { let worker = fake("ledger").await.unwrap(); From e0c82c0f143a77dac7de2c3ae4b5d97e8d5b3240 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 23:54:16 +0000 Subject: [PATCH 263/464] security(foundation): contain guest timestamp updates --- .../capsem-foundation/src/unix/contained.rs | 64 ++++++++++++++++++- .../src/unix/contained/tests.rs | 53 +++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) diff --git a/crates/capsem-foundation/src/unix/contained.rs b/crates/capsem-foundation/src/unix/contained.rs index 4102138ff..f2ff187da 100644 --- a/crates/capsem-foundation/src/unix/contained.rs +++ b/crates/capsem-foundation/src/unix/contained.rs @@ -13,7 +13,10 @@ use std::path::{Component, Path, PathBuf}; use nix::errno::Errno; use nix::fcntl::{openat, readlinkat, renameat, AtFlags, OFlag}; -use nix::sys::stat::{fchmod, fchmodat, fstatat, mkdirat, FchmodatFlags, Mode, SFlag}; +use nix::sys::stat::{ + fchmod, fchmodat, fstatat, futimens, mkdirat, utimensat, FchmodatFlags, Mode, SFlag, UtimensatFlags, +}; +use nix::sys::time::TimeSpec; use nix::unistd::{linkat, symlinkat}; use nix::unistd::{unlinkat, UnlinkatFlags}; @@ -33,6 +36,30 @@ pub struct ContainedOpenOptions { mode: u32, } +/// One timestamp update applied below a containment root. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FileTimeSpec { + Omit, + Now, + Exact { seconds: i64, nanoseconds: u32 }, +} + +impl FileTimeSpec { + fn to_time_spec(self) -> io::Result { + match self { + Self::Omit => Ok(TimeSpec::UTIME_OMIT), + Self::Now => Ok(TimeSpec::UTIME_NOW), + Self::Exact { seconds, nanoseconds } if nanoseconds < 1_000_000_000 => { + Ok(TimeSpec::new(seconds, nanoseconds.into())) + } + Self::Exact { .. } => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "timestamp nanoseconds must be below one second", + )), + } + } +} + impl ContainedOpenOptions { pub const fn read_only() -> Self { Self { @@ -331,6 +358,41 @@ impl ContainedDir { .map_err(Into::into) } + /// Change one entry's timestamps below this open directory. + /// + /// Parent components are opened without following links and the final + /// operation uses `AT_SYMLINK_NOFOLLOW`. An empty relative path names this + /// directory itself through its already-open descriptor. + pub fn set_relative_times(&self, rel: &Path, atime: FileTimeSpec, mtime: FileTimeSpec) -> io::Result<()> { + let atime = atime.to_time_spec()?; + let mtime = mtime.to_time_spec()?; + let names = rel + .components() + .map(|component| match component { + Component::Normal(name) => Ok(name), + _ => Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("relative path may only contain plain names: {}", rel.display()), + )), + }) + .collect::>>()?; + let Some((name, parents)) = names.split_last() else { + return futimens(self.fd.as_raw_fd(), &atime, &mtime).map_err(Into::into); + }; + let mut parent = self.try_clone()?; + for component in parents { + parent = parent.descend(component)?; + } + utimensat( + Some(parent.fd.as_raw_fd()), + *name, + &atime, + &mtime, + UtimensatFlags::NoFollowSymlink, + ) + .map_err(Into::into) + } + /// Create a symlink child. An existing entry of any type is an error; the /// target is stored verbatim and never resolved. pub fn symlink(&self, name: &OsStr, target: &OsStr) -> io::Result<()> { diff --git a/crates/capsem-foundation/src/unix/contained/tests.rs b/crates/capsem-foundation/src/unix/contained/tests.rs index 85ebb0550..d91f31879 100644 --- a/crates/capsem-foundation/src/unix/contained/tests.rs +++ b/crates/capsem-foundation/src/unix/contained/tests.rs @@ -396,6 +396,59 @@ fn mode_reads_permission_bits_from_the_descriptor() { assert_eq!(tree.root.descend(OsStr::new("dir")).unwrap().mode().unwrap(), 0o750); } +#[test] +fn timestamp_updates_are_exact_descriptor_relative_and_never_follow_links() { + let tree = tree(); + let file_path = tree.root_path.join("file"); + std::fs::write(&file_path, b"data").unwrap(); + tree.root + .set_relative_times( + Path::new("file"), + FileTimeSpec::Exact { + seconds: 1_234_567, + nanoseconds: 123_456_789, + }, + FileTimeSpec::Exact { + seconds: 2_345_678, + nanoseconds: 987_654_321, + }, + ) + .unwrap(); + let metadata = std::fs::metadata(&file_path).unwrap(); + assert_eq!((metadata.atime(), metadata.atime_nsec()), (1_234_567, 123_456_789)); + assert_eq!((metadata.mtime(), metadata.mtime_nsec()), (2_345_678, 987_654_321)); + + let target = tree.outside.join("target"); + std::fs::write(&target, b"outside").unwrap(); + let before = std::fs::metadata(&target).unwrap().modified().unwrap(); + symlink(&target, tree.root_path.join("link")).unwrap(); + tree.root + .set_relative_times(Path::new("link"), FileTimeSpec::Now, FileTimeSpec::Now) + .unwrap(); + assert_eq!(std::fs::metadata(&target).unwrap().modified().unwrap(), before); +} + +#[test] +fn timestamp_updates_reject_invalid_paths_and_nanoseconds() { + let tree = tree(); + std::fs::write(tree.root_path.join("file"), b"data").unwrap(); + assert!(tree + .root + .set_relative_times(Path::new("../file"), FileTimeSpec::Omit, FileTimeSpec::Now) + .is_err()); + assert!(tree + .root + .set_relative_times( + Path::new("file"), + FileTimeSpec::Omit, + FileTimeSpec::Exact { + seconds: 1, + nanoseconds: 1_000_000_000, + }, + ) + .is_err()); +} + #[test] fn rename_moves_a_link_as_a_link_and_never_its_target() { let tree = tree(); From 6c187fe0cab5868623ec37894c4b9f9828ae5777 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 23:55:32 +0000 Subject: [PATCH 264/464] security(proto): broker guest timestamp updates --- crates/capsem-proto/src/upstream_grant.rs | 160 +++++++++++++++++- .../capsem-proto/src/upstream_grant/tests.rs | 58 +++++++ 2 files changed, 216 insertions(+), 2 deletions(-) diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index 9614b53e0..e2a9c593b 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -22,7 +22,7 @@ pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; pub const MAX_GUEST_SHARE_PATH_BYTES: usize = 4096; const MAGIC: [u8; 2] = *b"UG"; -const VERSION: u8 = 4; +const VERSION: u8 = 7; const MAGIC_RANGE: std::ops::Range = 0..2; const VERSION_OFFSET: usize = 2; const KIND_OFFSET: usize = 3; @@ -36,7 +36,13 @@ const POLICY_DIGEST_BYTES: usize = 71; const POLICY_DIGEST_RANGE: std::ops::Range = NAME_RANGE.end..NAME_RANGE.end + POLICY_DIGEST_BYTES; const PATH_LENGTH_RANGE: std::ops::Range = POLICY_DIGEST_RANGE.end..POLICY_DIGEST_RANGE.end + 2; const PATH_RANGE: std::ops::Range = PATH_LENGTH_RANGE.end..PATH_LENGTH_RANGE.end + MAX_GUEST_SHARE_PATH_BYTES; -const RESERVED_RANGE: std::ops::Range = PATH_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; +const TIME_FLAGS_OFFSET: usize = PATH_RANGE.end; +const ATIME_SECONDS_RANGE: std::ops::Range = TIME_FLAGS_OFFSET + 1..TIME_FLAGS_OFFSET + 9; +const ATIME_NANOS_RANGE: std::ops::Range = ATIME_SECONDS_RANGE.end..ATIME_SECONDS_RANGE.end + 4; +const MTIME_SECONDS_RANGE: std::ops::Range = ATIME_NANOS_RANGE.end..ATIME_NANOS_RANGE.end + 8; +const MTIME_NANOS_RANGE: std::ops::Range = MTIME_SECONDS_RANGE.end..MTIME_SECONDS_RANGE.end + 4; +const TIME_FIELDS_RANGE: std::ops::Range = TIME_FLAGS_OFFSET..MTIME_NANOS_RANGE.end; +const RESERVED_RANGE: std::ops::Range = TIME_FIELDS_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; const RESOLVE_TCP: u8 = 1; const CONNECT_TCP: u8 = 2; @@ -47,6 +53,9 @@ const SET_GUEST_MODE: u8 = 6; const OPEN_LEDGER: u8 = 7; const ATTACH_PROXY_TRAFFIC: u8 = 8; const ATTACH_PROXY_MCP: u8 = 9; +const ATTACH_PROXY_TRACE_HINTS: u8 = 10; +const OPEN_LOOPBACK_LISTENER: u8 = 11; +const SET_GUEST_TIMES: u8 = 12; const TCP_RESOLVED: u8 = 101; const DESCRIPTOR_GRANTED: u8 = 102; const DENIED: u8 = 103; @@ -54,6 +63,18 @@ const GUEST_MODE_SET: u8 = 104; const LEDGER_GRANTED: u8 = 105; const PROXY_TRAFFIC_ADOPTED: u8 = 106; const PROXY_MCP_ADOPTED: u8 = 107; +const PROXY_TRACE_HINTS_ADOPTED: u8 = 108; +const LOOPBACK_LISTENER_GRANTED: u8 = 109; +const GUEST_TIMES_SET: u8 = 110; + +/// One guest-requested file timestamp. Exact values preserve archive metadata; +/// `Now` and `Omit` retain the corresponding `utimensat(2)` semantics. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GuestFileTime { + Omit, + Now, + Exact { seconds: i64, nanoseconds: u32 }, +} /// Application protocol spoken over a granted TCP stream. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -195,6 +216,12 @@ pub enum UpstreamGrantRequest { relative_path: Vec, mode: u16, }, + SetGuestTimes { + request_id: u64, + relative_path: Vec, + atime: GuestFileTime, + mtime: GuestFileTime, + }, OpenLedger { request_id: u64, }, @@ -230,6 +257,9 @@ pub enum UpstreamGrantResponse { GuestModeSet { request_id: u64, }, + GuestTimesSet { + request_id: u64, + }, LedgerGranted { request_id: u64, grant: LedgerChannelGrant, @@ -262,6 +292,7 @@ impl UpstreamGrantResponse { Self::TcpResolved { .. } | Self::Denied { .. } | Self::GuestModeSet { .. } + | Self::GuestTimesSet { .. } | Self::ProxyTrafficAdopted { .. } | Self::ProxyMcpAdopted { .. } => 0, } @@ -333,6 +364,22 @@ pub fn encode_upstream_grant_request(request: &UpstreamGrantRequest) -> Result<[ put_u16(&mut frame, DETAIL_RANGE, *mode); put_path(&mut frame, relative_path)?; } + UpstreamGrantRequest::SetGuestTimes { + request_id, + relative_path, + atime, + mtime, + } => { + require_nonzero("request id", *request_id)?; + validate_guest_share_path(relative_path)?; + if matches!((atime, mtime), (GuestFileTime::Omit, GuestFileTime::Omit)) { + bail!("guest timestamp request changes no timestamps"); + } + frame[KIND_OFFSET] = SET_GUEST_TIMES; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_path(&mut frame, relative_path)?; + put_guest_times(&mut frame, *atime, *mtime)?; + } UpstreamGrantRequest::OpenLedger { request_id } => { require_nonzero("request id", *request_id)?; frame[KIND_OFFSET] = OPEN_LEDGER; @@ -363,6 +410,9 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> let detail = get_u16(frame, DETAIL_RANGE); let name = get_name(frame)?; let relative_path = get_path(frame)?; + if frame[KIND_OFFSET] != SET_GUEST_TIMES { + require_empty_time_fields(frame)?; + } match frame[KIND_OFFSET] { RESOLVE_TCP => { require_nonzero("request id", request_id)?; @@ -430,6 +480,23 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> mode: detail, }) } + SET_GUEST_TIMES => { + require_nonzero("request id", request_id)?; + if resource_id != 0 || port != 0 || detail != 0 || !name.is_empty() { + bail!("invalid guest timestamp request fields"); + } + validate_guest_share_path(&relative_path)?; + let (atime, mtime) = get_guest_times(frame)?; + if matches!((atime, mtime), (GuestFileTime::Omit, GuestFileTime::Omit)) { + bail!("guest timestamp request changes no timestamps"); + } + Ok(UpstreamGrantRequest::SetGuestTimes { + request_id, + relative_path, + atime, + mtime, + }) + } OPEN_LEDGER => { require_nonzero("request id", request_id)?; if resource_id != 0 { @@ -510,6 +577,11 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul frame[KIND_OFFSET] = GUEST_MODE_SET; put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); } + UpstreamGrantResponse::GuestTimesSet { request_id } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = GUEST_TIMES_SET; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + } UpstreamGrantResponse::LedgerGranted { request_id, grant } => { require_nonzero("request id", *request_id)?; frame[KIND_OFFSET] = LEDGER_GRANTED; @@ -535,6 +607,7 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul /// Decode and fully validate one coordinator response. pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result { validate_envelope(frame)?; + require_empty_time_fields(frame)?; let request_id = get_u64(frame, REQUEST_ID_RANGE); let resource_id = get_u64(frame, RESOURCE_ID_RANGE); let port = get_u16(frame, PORT_RANGE); @@ -598,6 +671,12 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - } Ok(UpstreamGrantResponse::GuestModeSet { request_id }) } + GUEST_TIMES_SET => { + if resource_id != 0 || detail != 0 || !name.is_empty() || policy_digest.is_some() { + bail!("guest timestamp response carries unrelated fields"); + } + Ok(UpstreamGrantResponse::GuestTimesSet { request_id }) + } LEDGER_GRANTED => { if policy_digest.is_some() { bail!("ledger grant cannot carry a policy digest"); @@ -714,6 +793,67 @@ fn require_empty_path(path: &[u8]) -> Result<()> { Ok(()) } +fn put_guest_times( + frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], + atime: GuestFileTime, + mtime: GuestFileTime, +) -> Result<()> { + let (atime_code, atime_seconds, atime_nanos) = encode_guest_time(atime)?; + let (mtime_code, mtime_seconds, mtime_nanos) = encode_guest_time(mtime)?; + frame[TIME_FLAGS_OFFSET] = atime_code | (mtime_code << 2); + put_i64(frame, ATIME_SECONDS_RANGE, atime_seconds); + put_u32(frame, ATIME_NANOS_RANGE, atime_nanos); + put_i64(frame, MTIME_SECONDS_RANGE, mtime_seconds); + put_u32(frame, MTIME_NANOS_RANGE, mtime_nanos); + Ok(()) +} + +fn encode_guest_time(value: GuestFileTime) -> Result<(u8, i64, u32)> { + match value { + GuestFileTime::Omit => Ok((0, 0, 0)), + GuestFileTime::Now => Ok((1, 0, 0)), + GuestFileTime::Exact { seconds, nanoseconds } if nanoseconds < 1_000_000_000 => Ok((2, seconds, nanoseconds)), + GuestFileTime::Exact { .. } => bail!("guest timestamp nanoseconds must be below one second"), + } +} + +fn get_guest_times(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result<(GuestFileTime, GuestFileTime)> { + let flags = frame[TIME_FLAGS_OFFSET]; + if flags & 0xf0 != 0 { + bail!("guest timestamp flags contain reserved bits"); + } + Ok(( + decode_guest_time( + flags & 0x03, + get_i64(frame, ATIME_SECONDS_RANGE), + get_u32(frame, ATIME_NANOS_RANGE), + )?, + decode_guest_time( + (flags >> 2) & 0x03, + get_i64(frame, MTIME_SECONDS_RANGE), + get_u32(frame, MTIME_NANOS_RANGE), + )?, + )) +} + +fn decode_guest_time(code: u8, seconds: i64, nanoseconds: u32) -> Result { + match (code, seconds, nanoseconds) { + (0, 0, 0) => Ok(GuestFileTime::Omit), + (1, 0, 0) => Ok(GuestFileTime::Now), + (2, seconds, nanoseconds) if nanoseconds < 1_000_000_000 => Ok(GuestFileTime::Exact { seconds, nanoseconds }), + (0 | 1, _, _) => bail!("non-exact guest timestamp carries a value"), + (2, _, _) => bail!("guest timestamp nanoseconds must be below one second"), + _ => bail!("invalid guest timestamp kind {code}"), + } +} + +fn require_empty_time_fields(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result<()> { + if frame[TIME_FIELDS_RANGE].iter().any(|byte| *byte != 0) { + bail!("record carries unrelated guest timestamps"); + } + Ok(()) +} + fn put_name(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], name: &str) -> Result<()> { let length = u8::try_from(name.len()).map_err(|_| anyhow::anyhow!("upstream name is too long"))?; if usize::from(length) > MAX_UPSTREAM_HOST_BYTES { @@ -855,6 +995,22 @@ fn get_u16(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range, value: u32) { + frame[range].copy_from_slice(&value.to_be_bytes()); +} + +fn get_u32(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range) -> u32 { + u32::from_be_bytes(frame[range].try_into().unwrap()) +} + +fn put_i64(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range, value: i64) { + frame[range].copy_from_slice(&value.to_be_bytes()); +} + +fn get_i64(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range) -> i64 { + i64::from_be_bytes(frame[range].try_into().unwrap()) +} + fn put_u64(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], range: std::ops::Range, value: u64) { frame[range].copy_from_slice(&value.to_be_bytes()); } diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index 037215844..0148da472 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -53,6 +53,15 @@ fn every_request_round_trips_exactly() { relative_path: b"workspace/pyvenv/bin/activate".to_vec(), mode: 0o755, }, + UpstreamGrantRequest::SetGuestTimes { + request_id: 12, + relative_path: b"workspace/archive.txt".to_vec(), + atime: GuestFileTime::Exact { + seconds: -1, + nanoseconds: 123_456_789, + }, + mtime: GuestFileTime::Now, + }, ] { request_roundtrip(request); } @@ -99,6 +108,7 @@ fn every_response_round_trips_exactly() { reason: UpstreamGrantDenial::Revoked, }, UpstreamGrantResponse::GuestModeSet { request_id: 7 }, + UpstreamGrantResponse::GuestTimesSet { request_id: 13 }, UpstreamGrantResponse::LedgerGranted { request_id: 8, grant: LedgerChannelGrant::new(LedgerGeneration::new([0x5a; 16]), 16, LedgerClientRole::VmOwner).unwrap(), @@ -147,6 +157,54 @@ fn guest_mode_paths_are_bounded_normalized_bytes() { .is_err()); } +#[test] +fn guest_timestamp_values_and_padding_are_strict() { + request_roundtrip(UpstreamGrantRequest::SetGuestTimes { + request_id: 1, + relative_path: Vec::new(), + atime: GuestFileTime::Omit, + mtime: GuestFileTime::Exact { + seconds: 1_234, + nanoseconds: 999_999_999, + }, + }); + assert!(encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestTimes { + request_id: 2, + relative_path: b"file".to_vec(), + atime: GuestFileTime::Omit, + mtime: GuestFileTime::Omit, + }) + .is_err()); + assert!(encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestTimes { + request_id: 3, + relative_path: b"file".to_vec(), + atime: GuestFileTime::Now, + mtime: GuestFileTime::Exact { + seconds: 1, + nanoseconds: 1_000_000_000, + }, + }) + .is_err()); + + let valid = encode_upstream_grant_request(&UpstreamGrantRequest::SetGuestTimes { + request_id: 4, + relative_path: b"file".to_vec(), + atime: GuestFileTime::Now, + mtime: GuestFileTime::Omit, + }) + .unwrap(); + let mut reserved_flag = valid; + reserved_flag[TIME_FLAGS_OFFSET] |= 0x80; + assert!(decode_upstream_grant_request(&reserved_flag).is_err()); + let mut value_on_now = valid; + put_i64(&mut value_on_now, ATIME_SECONDS_RANGE, 1); + assert!(decode_upstream_grant_request(&value_on_now).is_err()); + let mut unrelated_time = + encode_upstream_grant_request(&UpstreamGrantRequest::OpenLedger { request_id: 5 }).unwrap(); + unrelated_time[TIME_FLAGS_OFFSET] = 1; + assert!(decode_upstream_grant_request(&unrelated_time).is_err()); +} + #[test] fn descriptor_expectations_are_explicit() { let request = UpstreamGrantRequest::OpenDns { From b7fdada141673c72a11b07e1605fa74725693664 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Mon, 12 Oct 2026 23:58:56 +0000 Subject: [PATCH 265/464] security(core): broker guest timestamp mutations --- .../src/hypervisor/fuse/protocol.rs | 2 + .../src/hypervisor/kvm/virtio_fs/mod.rs | 16 ++++ .../src/hypervisor/kvm/virtio_fs/ops_meta.rs | 63 +++++-------- .../src/hypervisor/kvm/virtio_fs/tests.rs | 91 +++++++++++++++++++ crates/capsem-core/src/net/upstream_grant.rs | 72 ++++++++++++++- .../src/net/upstream_grant/tests.rs | 47 +++++++++- crates/capsem-core/src/vm/config.rs | 6 ++ 7 files changed, 253 insertions(+), 44 deletions(-) diff --git a/crates/capsem-core/src/hypervisor/fuse/protocol.rs b/crates/capsem-core/src/hypervisor/fuse/protocol.rs index aeaa9d2dd..e6c96363e 100644 --- a/crates/capsem-core/src/hypervisor/fuse/protocol.rs +++ b/crates/capsem-core/src/hypervisor/fuse/protocol.rs @@ -51,6 +51,8 @@ pub const FATTR_GID: u32 = 1 << 2; pub const FATTR_SIZE: u32 = 1 << 3; pub const FATTR_ATIME: u32 = 1 << 4; pub const FATTR_MTIME: u32 = 1 << 5; +pub const FATTR_ATIME_NOW: u32 = 1 << 7; +pub const FATTR_MTIME_NOW: u32 = 1 << 8; // File type masks pub const S_IFMT: u32 = 0o170000; diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs index 183cde081..337ebd8a0 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs @@ -75,6 +75,22 @@ impl FuseProcessor { authority.set_mode(relative.as_os_str().as_bytes(), (mode & 0o7777) as u16) } + fn set_guest_times( + &self, + path: &Path, + atime: capsem_proto::upstream_grant::GuestFileTime, + mtime: capsem_proto::upstream_grant::GuestFileTime, + ) -> std::io::Result<()> { + let authority = self + .metadata_authority + .as_ref() + .ok_or_else(|| std::io::Error::from_raw_os_error(libc::EPERM))?; + let relative = path + .strip_prefix(&self.root_path) + .map_err(|_| std::io::Error::from_raw_os_error(libc::EACCES))?; + authority.set_times(relative.as_os_str().as_bytes(), atime, mtime) + } + fn handle_request(&mut self, request_buf: &[u8]) -> Vec { let mut response = Vec::new(); self.handle_request_into(request_buf, &mut response); diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs index 1748974db..68b4f5a52 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs @@ -152,47 +152,30 @@ impl FuseProcessor { // without effect, as tools running as guest root (cp -a, tar -x) // require; it used to reach an unprivileged host // lchown and fail with EPERM. - if attr_in.valid & (FATTR_ATIME | FATTR_MTIME) != 0 { - let c_path = match std::ffi::CString::new(path.as_os_str().as_encoded_bytes()) { - Ok(c) => c, - Err(_) => return fuse::error_response(header.unique, -libc::EINVAL), + if attr_in.valid & (FATTR_ATIME | FATTR_MTIME | FATTR_ATIME_NOW | FATTR_MTIME_NOW) != 0 { + use capsem_proto::upstream_grant::GuestFileTime; + let atime = if attr_in.valid & FATTR_ATIME_NOW != 0 { + GuestFileTime::Now + } else if attr_in.valid & FATTR_ATIME != 0 { + GuestFileTime::Exact { + seconds: attr_in.atime as i64, + nanoseconds: attr_in.atimensec, + } + } else { + GuestFileTime::Omit }; - let times = [ - libc::timespec { - tv_sec: if attr_in.valid & FATTR_ATIME != 0 { - attr_in.atime as i64 - } else { - 0 - }, - tv_nsec: if attr_in.valid & FATTR_ATIME != 0 { - i64::from(attr_in.atimensec) - } else { - libc::UTIME_OMIT - }, - }, - libc::timespec { - tv_sec: if attr_in.valid & FATTR_MTIME != 0 { - attr_in.mtime as i64 - } else { - 0 - }, - tv_nsec: if attr_in.valid & FATTR_MTIME != 0 { - i64::from(attr_in.mtimensec) - } else { - libc::UTIME_OMIT - }, - }, - ]; - if unsafe { - libc::utimensat( - libc::AT_FDCWD, - c_path.as_ptr(), - times.as_ptr(), - libc::AT_SYMLINK_NOFOLLOW, - ) - } != 0 - { - return fuse::error_response(header.unique, -fuse::errno()); + let mtime = if attr_in.valid & FATTR_MTIME_NOW != 0 { + GuestFileTime::Now + } else if attr_in.valid & FATTR_MTIME != 0 { + GuestFileTime::Exact { + seconds: attr_in.mtime as i64, + nanoseconds: attr_in.mtimensec, + } + } else { + GuestFileTime::Omit + }; + if let Err(error) = self.set_guest_times(&path, atime, mtime) { + return fuse::error_response(header.unique, -fuse::io_error_to_errno(&error)); } } diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs index 9e503cfe2..476b95efe 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs @@ -130,6 +130,27 @@ impl crate::GuestMetadataAuthority for LocalMetadataAuthority { let root = capsem_foundation::unix::contained::ContainedDir::open_root(&self.0)?; root.set_relative_mode(Path::new(std::ffi::OsStr::from_bytes(relative_path)), u32::from(mode)) } + + fn set_times( + &self, + relative_path: &[u8], + atime: capsem_proto::upstream_grant::GuestFileTime, + mtime: capsem_proto::upstream_grant::GuestFileTime, + ) -> std::io::Result<()> { + use capsem_foundation::unix::contained::FileTimeSpec; + use capsem_proto::upstream_grant::GuestFileTime; + let convert = |value| match value { + GuestFileTime::Omit => FileTimeSpec::Omit, + GuestFileTime::Now => FileTimeSpec::Now, + GuestFileTime::Exact { seconds, nanoseconds } => FileTimeSpec::Exact { seconds, nanoseconds }, + }; + let root = capsem_foundation::unix::contained::ContainedDir::open_root(&self.0)?; + root.set_relative_times( + Path::new(std::ffi::OsStr::from_bytes(relative_path)), + convert(atime), + convert(mtime), + ) + } } fn metadata_authority(dir: &Path) -> Arc { @@ -1025,6 +1046,76 @@ fn setattr_chmod() { assert_eq!(perms.mode() & 0o777, 0o755); } +#[test] +fn setattr_timestamps_use_the_metadata_authority() { + let dir = temp_share("setattr-times"); + let path = dir.join("f.txt"); + std::fs::write(&path, b"x").unwrap(); + let mut proc = test_processor(&dir); + let ino = lookup(&mut proc, 1, "f.txt").unwrap(); + let attr_in = FuseSetAttrIn { + valid: FATTR_ATIME | FATTR_MTIME, + padding: 0, + fh: 0, + size: 0, + lock_owner: 0, + atime: 1_234_567, + mtime: 2_345_678, + ctime: 0, + atimensec: 123_456_789, + mtimensec: 987_654_321, + ctimensec: 0, + mode: 0, + unused4: 0, + uid: 0, + gid: 0, + unused5: 0, + }; + + let h = make_header(FUSE_SETATTR, ino, 4); + let resp = proc.handle_request(&build_request(&h, fuse::as_bytes(&attr_in))); + + assert_eq!(response_error(&resp), 0); + let metadata = std::fs::metadata(path).unwrap(); + assert_eq!((metadata.atime(), metadata.atime_nsec()), (1_234_567, 123_456_789)); + assert_eq!((metadata.mtime(), metadata.mtime_nsec()), (2_345_678, 987_654_321)); +} + +#[test] +fn setattr_timestamps_never_fall_back_to_owner_authority() { + let dir = temp_share("setattr-times-no-broker"); + let path = dir.join("f.txt"); + std::fs::write(&path, b"x").unwrap(); + let before = std::fs::metadata(&path).unwrap().modified().unwrap(); + let mut proc = test_processor(&dir); + proc.metadata_authority = None; + let ino = lookup(&mut proc, 1, "f.txt").unwrap(); + let attr_in = FuseSetAttrIn { + valid: FATTR_MTIME_NOW, + padding: 0, + fh: 0, + size: 0, + lock_owner: 0, + atime: 0, + mtime: 0, + ctime: 0, + atimensec: 0, + mtimensec: 0, + ctimensec: 0, + mode: 0, + unused4: 0, + uid: 0, + gid: 0, + unused5: 0, + }; + + let h = make_header(FUSE_SETATTR, ino, 5); + let resp = proc.handle_request(&build_request(&h, fuse::as_bytes(&attr_in))); + + assert_eq!(response_error(&resp), -libc::EPERM); + assert_eq!(std::fs::metadata(path).unwrap().modified().unwrap(), before); +} + #[test] fn setattr_mode_never_falls_back_to_owner_authority() { let dir = temp_share("setattr-mode-no-broker"); diff --git a/crates/capsem-core/src/net/upstream_grant.rs b/crates/capsem-core/src/net/upstream_grant.rs index 403dff0e3..ad23dc82e 100644 --- a/crates/capsem-core/src/net/upstream_grant.rs +++ b/crates/capsem-core/src/net/upstream_grant.rs @@ -15,8 +15,9 @@ use capsem_foundation::unix::fd::{self, SocketShutdown}; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::upstream_grant::{ - decode_upstream_grant_response, encode_upstream_grant_request, ProxyTrafficService, UpstreamDescriptorKind, - UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, + decode_upstream_grant_response, encode_upstream_grant_request, GuestFileTime, ProxyTrafficService, + UpstreamDescriptorKind, UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, + UPSTREAM_GRANT_MAX_FDS, }; use tokio::sync::{mpsc, oneshot}; @@ -46,6 +47,12 @@ enum Command { mode: u16, reply: oneshot::Sender>, }, + SetGuestTimes { + relative_path: Vec, + atime: GuestFileTime, + mtime: GuestFileTime, + reply: oneshot::Sender>, + }, OpenDns { upstream_index: u16, policy_digest: String, @@ -156,6 +163,22 @@ impl crate::GuestMetadataAuthority for UpstreamGrantClient { .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? .map_err(io::Error::other) } + + fn set_times(&self, relative_path: &[u8], atime: GuestFileTime, mtime: GuestFileTime) -> io::Result<()> { + let (reply, result) = oneshot::channel(); + self.commands + .blocking_send(Command::SetGuestTimes { + relative_path: relative_path.to_vec(), + atime, + mtime, + reply, + }) + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel is closed"))?; + result + .blocking_recv() + .map_err(|_| io::Error::new(io::ErrorKind::BrokenPipe, "upstream grant channel stopped"))? + .map_err(io::Error::other) + } } impl DnsUpstreamGrants for UpstreamGrantClient { @@ -282,6 +305,15 @@ async fn dispatch( reply, set_guest_mode(sender, receiver, request_id, relative_path, mode).await, ), + Command::SetGuestTimes { + relative_path, + atime, + mtime, + reply, + } => finish( + reply, + set_guest_times(sender, receiver, request_id, relative_path, atime, mtime).await, + ), Command::OpenDns { upstream_index, policy_digest, @@ -456,6 +488,42 @@ async fn set_guest_mode( } } +async fn set_guest_times( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, + relative_path: Vec, + atime: GuestFileTime, + mtime: GuestFileTime, +) -> Result, String> { + send_request( + sender, + &UpstreamGrantRequest::SetGuestTimes { + request_id, + relative_path, + atime, + mtime, + }, + ) + .await?; + let (response, fds) = receive_response(receiver).await?; + if !fds.is_empty() { + return Err("guest timestamp response carried a descriptor".into()); + } + match response { + UpstreamGrantResponse::GuestTimesSet { + request_id: response_id, + } if response_id == request_id => Ok(Ok(())), + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("guest timestamp change denied: {reason:?}"))), + response => Err(format!( + "unexpected guest timestamp response for request {request_id}: {response:?}" + )), + } +} + fn finish( reply: oneshot::Sender>, result: Result, String>, diff --git a/crates/capsem-core/src/net/upstream_grant/tests.rs b/crates/capsem-core/src/net/upstream_grant/tests.rs index 682ab2882..3def410bc 100644 --- a/crates/capsem-core/src/net/upstream_grant/tests.rs +++ b/crates/capsem-core/src/net/upstream_grant/tests.rs @@ -7,8 +7,8 @@ use crate::net::mitm_proxy::{TcpUpstreamGrants, UpstreamTarget}; use crate::GuestMetadataAuthority as _; use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; use capsem_proto::upstream_grant::{ - decode_upstream_grant_request, encode_upstream_grant_response, ProxyTrafficService, UpstreamDescriptorKind, - UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, + decode_upstream_grant_request, encode_upstream_grant_response, GuestFileTime, ProxyTrafficService, + UpstreamDescriptorKind, UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, }; use super::*; @@ -230,6 +230,49 @@ async fn guest_mode_changes_use_the_serialized_coordinator_channel() { broker_task.await.unwrap(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn guest_timestamp_changes_use_the_serialized_coordinator_channel() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = Arc::new(UpstreamGrantClient::start(worker).unwrap()); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::SetGuestTimes { + request_id: 1, + relative_path: b"workspace/archive.txt".to_vec(), + atime: GuestFileTime::Omit, + mtime: GuestFileTime::Exact { + seconds: 1234, + nanoseconds: 5678, + }, + } + ); + send_response( + &responses, + &UpstreamGrantResponse::GuestTimesSet { request_id: 1 }, + None, + ) + .await; + }); + let client_for_times = Arc::clone(&client); + tokio::task::spawn_blocking(move || { + client_for_times + .set_times( + b"workspace/archive.txt", + GuestFileTime::Omit, + GuestFileTime::Exact { + seconds: 1234, + nanoseconds: 5678, + }, + ) + .unwrap(); + }) + .await + .unwrap(); + broker_task.await.unwrap(); +} + #[tokio::test] async fn dns_query_uses_connected_coordinator_descriptor_and_releases_it() { let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); diff --git a/crates/capsem-core/src/vm/config.rs b/crates/capsem-core/src/vm/config.rs index f35e47486..4ece713a8 100644 --- a/crates/capsem-core/src/vm/config.rs +++ b/crates/capsem-core/src/vm/config.rs @@ -74,6 +74,12 @@ pub enum ConfigError { /// safely perform itself. pub trait GuestMetadataAuthority: Send + Sync { fn set_mode(&self, relative_path: &[u8], mode: u16) -> std::io::Result<()>; + fn set_times( + &self, + relative_path: &[u8], + atime: capsem_proto::upstream_grant::GuestFileTime, + mtime: capsem_proto::upstream_grant::GuestFileTime, + ) -> std::io::Result<()>; } /// A VirtioFS shared directory to expose to the guest via virtio-fs. From 31c7eccdfae44b4e75a67596d2aceed22e3a410c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:06:08 +0000 Subject: [PATCH 266/464] security(service): broker guest timestamp updates --- crates/capsem-service/src/upstream_broker.rs | 48 +++++++++++++++-- .../src/upstream_broker/tests.rs | 53 +++++++++++++++++++ 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 82d9b5e27..7cebd28e7 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -19,13 +19,14 @@ use capsem_core::net::mitm_proxy::protocol::Protocol; use capsem_core::net::mitm_proxy::UpstreamTarget; use capsem_core::net::policy_config::CompiledActivePolicy; use capsem_core::net::upstream_address::UpstreamResolver; +use capsem_foundation::unix::contained::FileTimeSpec; use capsem_foundation::unix::fd::{self, SocketShutdown}; use capsem_foundation::unix::router_channel::{DescriptorReceiver, DescriptorSender}; use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::upstream_grant::{ - decode_upstream_grant_request, encode_upstream_grant_response, ProxyTrafficService, UpstreamDescriptorKind, - UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, UPSTREAM_GRANT_FRAME_SIZE, - UPSTREAM_GRANT_MAX_FDS, + decode_upstream_grant_request, encode_upstream_grant_response, GuestFileTime, ProxyTrafficService, + UpstreamDescriptorKind, UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, + UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, }; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, warn}; @@ -568,6 +569,27 @@ async fn run( } } } + UpstreamGrantRequest::SetGuestTimes { + request_id, + relative_path, + atime, + mtime, + } => { + let Some(session_dir) = session_dir.as_deref() else { + send_denied(&responses, request_id, UpstreamGrantDenial::NotConfigured).await?; + continue; + }; + match set_guest_times(session_dir, &relative_path, atime, mtime) { + Ok(()) => { + send_response(&responses, &UpstreamGrantResponse::GuestTimesSet { request_id }, None) + .await?; + } + Err(error) => { + warn!(%error, "guest timestamp broker refused metadata change"); + send_denied(&responses, request_id, UpstreamGrantDenial::NotAllowed).await?; + } + } + } UpstreamGrantRequest::OpenLedger { request_id } => { let Some(source) = ledger.take() else { let reason = if ledger_configured { @@ -659,6 +681,26 @@ fn set_guest_mode(session_dir: &Path, relative_path: &[u8], mode: u16) -> io::Re root.set_relative_mode(path, u32::from(mode)) } +fn set_guest_times( + session_dir: &Path, + relative_path: &[u8], + atime: GuestFileTime, + mtime: GuestFileTime, +) -> io::Result<()> { + let root = capsem_foundation::unix::contained::ContainedDir::open_root(session_dir)? + .descend(std::ffi::OsStr::new(capsem_core::GUEST_SHARE_DIR))?; + let path = Path::new(std::ffi::OsStr::from_bytes(relative_path)); + root.set_relative_times(path, file_time_spec(atime), file_time_spec(mtime)) +} + +const fn file_time_spec(time: GuestFileTime) -> FileTimeSpec { + match time { + GuestFileTime::Omit => FileTimeSpec::Omit, + GuestFileTime::Now => FileTimeSpec::Now, + GuestFileTime::Exact { seconds, nanoseconds } => FileTimeSpec::Exact { seconds, nanoseconds }, + } +} + async fn send_active_grant( sender: &WireSender, active: &HashMap, diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index b86d6c7f7..67c0fc817 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -359,6 +359,59 @@ async fn guest_mode_requests_are_confined_to_the_session_share() { stop_broker(authority, task).await; } +#[tokio::test] +async fn guest_timestamp_requests_are_confined_to_the_session_share() { + let session = tempfile::tempdir().unwrap(); + let guest = session.path().join(capsem_core::GUEST_SHARE_DIR); + std::fs::create_dir_all(guest.join("workspace/nested")).unwrap(); + let tool = guest.join("workspace/nested/tool"); + std::fs::write(&tool, b"tool").unwrap(); + let outside = tempfile::tempdir().unwrap(); + let secret = outside.path().join("secret"); + std::fs::write(&secret, b"secret").unwrap(); + symlink(&secret, guest.join("workspace/link")).unwrap(); + + let (client, _publisher, authority, task) = + start_broker_for_session(test_policy("policy-a", None, Vec::new()), session.path().to_path_buf()); + let timestamp = 1_700_000_123_i64; + let (response, fds) = client + .request(&UpstreamGrantRequest::SetGuestTimes { + request_id: 1, + relative_path: b"workspace/nested/tool".to_vec(), + atime: GuestFileTime::Omit, + mtime: GuestFileTime::Exact { + seconds: timestamp, + nanoseconds: 456_000_000, + }, + }) + .await; + assert_eq!(response, UpstreamGrantResponse::GuestTimesSet { request_id: 1 }); + assert!(fds.is_empty()); + assert_eq!( + std::fs::metadata(&tool) + .unwrap() + .modified() + .unwrap() + .duration_since(std::time::UNIX_EPOCH) + .unwrap(), + Duration::new(timestamp.cast_unsigned(), 456_000_000) + ); + + let secret_modified = std::fs::metadata(&secret).unwrap().modified().unwrap(); + let (response, fds) = client + .request(&UpstreamGrantRequest::SetGuestTimes { + request_id: 2, + relative_path: b"workspace/link".to_vec(), + atime: GuestFileTime::Now, + mtime: GuestFileTime::Now, + }) + .await; + assert_eq!(response, UpstreamGrantResponse::GuestTimesSet { request_id: 2 }); + assert!(fds.is_empty()); + assert_eq!(std::fs::metadata(secret).unwrap().modified().unwrap(), secret_modified); + stop_broker(authority, task).await; +} + async fn resolve_override(client: &TestClient, request_id: u64) -> u64 { let (response, fds) = client .request(&UpstreamGrantRequest::ResolveTcp { From 12336f6de9d8dfa212eecbc3d879901daaf6e710 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:08:05 +0000 Subject: [PATCH 267/464] test(citadel): lower Unix boundary debt --- tests/citadel/unix_boundary_debt.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/citadel/unix_boundary_debt.toml b/tests/citadel/unix_boundary_debt.toml index 6aad41747..f90cb4496 100644 --- a/tests/citadel/unix_boundary_debt.toml +++ b/tests/citadel/unix_boundary_debt.toml @@ -29,10 +29,10 @@ "crates/capsem-core/src/hypervisor/kvm/virtio_blk.rs" = "b5afdd2273970adef7c40ad5cdbaf59e092c2eae1449e02dd84aa07dc421cbd3" "crates/capsem-core/src/hypervisor/kvm/virtio_blk/fd_util.rs" = "380483007d81c4751ff3e79c5867ee4688a7e3a5ecdd11ee5be91235a85cd1c2" "crates/capsem-core/src/hypervisor/kvm/virtio_console.rs" = "b0a92055b6ed9190b992f25a16b92b2aa0242b3c4be245d474410b0b2fa6cdeb" -"crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs" = "3a1c3f22f248a8fce490d848a856d86f5df2aa9a66a2631d1c9166ed26a950a7" +"crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs" = "f3ce99ec68343009ffcab5ba74802fc3527361ee646109157d9ad55f443976be" "crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs" = "1c2723207e04d0be3594c7ba21fdc13d6ff761e6f6d4dbff85ab2fc06da3f4c4" -"crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs" = "9f99917b8f9e273f2de49df5c515f41e154cb5ba2284c7b8e74f5dd80caf4a20" -"crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs" = "b6a5d6293aef393907f718dfb1416e44acdae879fed1b72a6b5f8ea9d1aa5160" +"crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs" = "b5f5c223d1f514cb692d8e6604227f687087f481b07ed84b42b1a20b7a693be4" +"crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs" = "b933aa796bce77d6bf0face797a38163ce4c473dacc149c23c5baa3390166a4c" "crates/capsem-core/src/hypervisor/kvm/virtio_mmio.rs" = "f9797a518c8d6273151bc775f9a176a72f3b0a62e09cd4cd4fb636477b58627b" "crates/capsem-core/src/hypervisor/kvm/virtio_vsock.rs" = "e8497a3dd53669d1e463c69a098214ece6257555c2a11d10a3d5642027e0a56a" "crates/capsem-core/src/hypervisor/kvm/virtio_vsock/lifecycle.rs" = "83c7ec268765e594bebe53715a3d148a37e0cd9ad3dd9bb58f1d15a4eaba093b" From 3fa967af5748f25ddfe9a257b80bec0629c2c5d1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:09:04 +0000 Subject: [PATCH 268/464] docs(changelog): record worker reliability fixes --- CHANGELOG.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad0b3ab27..322ea1536 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,9 +118,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 return complete tool schemas over authenticated IPC without receiving a cross-session cache-file grant. -- Broker guest VirtioFS permission changes through the coordinator beneath - the session's guest share, keeping direct host permission syscalls denied - and attesting that authority before the VM owner publishes readiness. +- Broker guest VirtioFS permission and timestamp changes through the + coordinator beneath the session's guest share, keeping direct host metadata + syscalls denied and refusing path traversal without following guest-controlled + links. - Restrict registered VM-owner service connections to private-name lookup and that owner's metrics route, preventing a compromised worker from invoking @@ -195,6 +196,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 disconnects, while still terminating it for unknown, stale or required capability closure events. +- Reject an over-capacity HTTP or DNS attachment without terminating its + healthy confined proxy generation, so later traffic can proceed after + capacity becomes available. + - Keep SQLite writer scratch state in memory so a confined VM owner can flush high-volume session-ledger bursts without ambient temporary-directory access. From bd2dd488dd01acf6b76230c816fe3c0e5d4b5a91 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:09:14 +0000 Subject: [PATCH 269/464] docs(architecture): explain brokered guest metadata --- .../docs/architecture/host-isolation.md | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/web/docs/src/content/docs/architecture/host-isolation.md b/web/docs/src/content/docs/architecture/host-isolation.md index 8f491caf7..951112fa1 100644 --- a/web/docs/src/content/docs/architecture/host-isolation.md +++ b/web/docs/src/content/docs/architecture/host-isolation.md @@ -140,6 +140,31 @@ leak and close descriptors that were not deliberately preserved. The sandbox is additive to normal ownership and mode checks; mode `0600` sockets and mode `0700` session directories remain in use. +The proxy keeps request correlation without retaining trace environment +variables. After clearing its environment, it derives a compact process trace +identity from the coordinator-minted worker generation passed on the command +line. Model tool calls that name workspace files cross a separate fixed-size +channel to the VM owner. The owner validates each relative path, installs the +hint in the filesystem monitor's bounded correlation table, and acknowledges +it before the model response completes. That channel grants the proxy no +workspace descriptor or read authority; malformed, escaping, or oversized +hints close it. + +The VM owner opens the workspace as a contained directory before confinement +and carries that descriptor across the boundary. It never reopens the broader +session directory, which also contains the ledger. If the filesystem monitor +cannot start from the contained descriptor, VM startup fails instead of +running without filesystem audit coverage. + +VirtioFS mode and timestamp updates follow the same capability boundary. The +confined VM owner cannot invoke host permission or timestamp syscalls directly. +It sends a typed relative-path request over its generation-bound coordinator +channel; the service applies the change beneath the registered session's guest +share through contained directory descriptors. Parent components cannot be +links, and the final timestamp operation does not follow a link. Archive tools +such as `cp -a` can therefore preserve guest-visible metadata without giving +the guest-facing owner path authority over the host. + ## Shared proxy engine VM interception and the standalone model API are transport adapters around the From a17ffa0287589326edae7c226b91b20a7d6cac70 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:16:08 +0000 Subject: [PATCH 270/464] refactor(proto): split guest metadata codec --- crates/capsem-proto/src/upstream_grant.rs | 131 ++---------------- .../src/upstream_grant/guest_metadata.rs | 128 +++++++++++++++++ 2 files changed, 139 insertions(+), 120 deletions(-) create mode 100644 crates/capsem-proto/src/upstream_grant/guest_metadata.rs diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index e2a9c593b..3dd899317 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -12,6 +12,16 @@ use anyhow::{bail, Result}; use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; +mod guest_metadata; + +pub use guest_metadata::GuestFileTime; +use guest_metadata::{ + get_guest_times, get_path, put_guest_times, put_path, require_empty_path, require_empty_time_fields, + validate_guest_share_path, +}; +#[cfg(test)] +use guest_metadata::{ATIME_SECONDS_RANGE, PATH_RANGE, TIME_FLAGS_OFFSET}; + /// Exact record size carried by the bounded SCM_RIGHTS channel. pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 4480; /// A ledger response grants its storage and trusted-checkpoint channels. @@ -34,15 +44,7 @@ const NAME_LENGTH_OFFSET: usize = 24; const NAME_RANGE: std::ops::Range = 25..25 + MAX_UPSTREAM_HOST_BYTES; const POLICY_DIGEST_BYTES: usize = 71; const POLICY_DIGEST_RANGE: std::ops::Range = NAME_RANGE.end..NAME_RANGE.end + POLICY_DIGEST_BYTES; -const PATH_LENGTH_RANGE: std::ops::Range = POLICY_DIGEST_RANGE.end..POLICY_DIGEST_RANGE.end + 2; -const PATH_RANGE: std::ops::Range = PATH_LENGTH_RANGE.end..PATH_LENGTH_RANGE.end + MAX_GUEST_SHARE_PATH_BYTES; -const TIME_FLAGS_OFFSET: usize = PATH_RANGE.end; -const ATIME_SECONDS_RANGE: std::ops::Range = TIME_FLAGS_OFFSET + 1..TIME_FLAGS_OFFSET + 9; -const ATIME_NANOS_RANGE: std::ops::Range = ATIME_SECONDS_RANGE.end..ATIME_SECONDS_RANGE.end + 4; -const MTIME_SECONDS_RANGE: std::ops::Range = ATIME_NANOS_RANGE.end..ATIME_NANOS_RANGE.end + 8; -const MTIME_NANOS_RANGE: std::ops::Range = MTIME_SECONDS_RANGE.end..MTIME_SECONDS_RANGE.end + 4; -const TIME_FIELDS_RANGE: std::ops::Range = TIME_FLAGS_OFFSET..MTIME_NANOS_RANGE.end; -const RESERVED_RANGE: std::ops::Range = TIME_FIELDS_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; +use guest_metadata::RESERVED_RANGE; const RESOLVE_TCP: u8 = 1; const CONNECT_TCP: u8 = 2; @@ -67,15 +69,6 @@ const PROXY_TRACE_HINTS_ADOPTED: u8 = 108; const LOOPBACK_LISTENER_GRANTED: u8 = 109; const GUEST_TIMES_SET: u8 = 110; -/// One guest-requested file timestamp. Exact values preserve archive metadata; -/// `Now` and `Omit` retain the corresponding `utimensat(2)` semantics. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum GuestFileTime { - Omit, - Now, - Exact { seconds: i64, nanoseconds: u32 }, -} - /// Application protocol spoken over a granted TCP stream. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum UpstreamProtocol { @@ -752,108 +745,6 @@ fn validate_normalized_host(host: &str) -> Result<()> { Ok(()) } -fn validate_guest_share_path(path: &[u8]) -> Result<()> { - if path.len() > MAX_GUEST_SHARE_PATH_BYTES || path.contains(&0) || path.first() == Some(&b'/') { - bail!("invalid guest-share path"); - } - if !path.is_empty() - && path - .split(|byte| *byte == b'/') - .any(|component| component.is_empty() || component == b"." || component == b"..") - { - bail!("guest-share path is not normalized"); - } - Ok(()) -} - -fn put_path(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], path: &[u8]) -> Result<()> { - validate_guest_share_path(path)?; - let length = u16::try_from(path.len()).map_err(|_| anyhow::anyhow!("guest-share path is too long"))?; - put_u16(frame, PATH_LENGTH_RANGE, length); - frame[PATH_RANGE.start..PATH_RANGE.start + path.len()].copy_from_slice(path); - Ok(()) -} - -fn get_path(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result> { - let length = usize::from(get_u16(frame, PATH_LENGTH_RANGE)); - if length > PATH_RANGE.len() { - bail!("guest-share path length exceeds its field"); - } - let (used, padding) = frame[PATH_RANGE].split_at(length); - if padding.iter().any(|byte| *byte != 0) { - bail!("guest-share path padding is nonzero"); - } - Ok(used.to_vec()) -} - -fn require_empty_path(path: &[u8]) -> Result<()> { - if !path.is_empty() { - bail!("record carries an unrelated guest-share path"); - } - Ok(()) -} - -fn put_guest_times( - frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], - atime: GuestFileTime, - mtime: GuestFileTime, -) -> Result<()> { - let (atime_code, atime_seconds, atime_nanos) = encode_guest_time(atime)?; - let (mtime_code, mtime_seconds, mtime_nanos) = encode_guest_time(mtime)?; - frame[TIME_FLAGS_OFFSET] = atime_code | (mtime_code << 2); - put_i64(frame, ATIME_SECONDS_RANGE, atime_seconds); - put_u32(frame, ATIME_NANOS_RANGE, atime_nanos); - put_i64(frame, MTIME_SECONDS_RANGE, mtime_seconds); - put_u32(frame, MTIME_NANOS_RANGE, mtime_nanos); - Ok(()) -} - -fn encode_guest_time(value: GuestFileTime) -> Result<(u8, i64, u32)> { - match value { - GuestFileTime::Omit => Ok((0, 0, 0)), - GuestFileTime::Now => Ok((1, 0, 0)), - GuestFileTime::Exact { seconds, nanoseconds } if nanoseconds < 1_000_000_000 => Ok((2, seconds, nanoseconds)), - GuestFileTime::Exact { .. } => bail!("guest timestamp nanoseconds must be below one second"), - } -} - -fn get_guest_times(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result<(GuestFileTime, GuestFileTime)> { - let flags = frame[TIME_FLAGS_OFFSET]; - if flags & 0xf0 != 0 { - bail!("guest timestamp flags contain reserved bits"); - } - Ok(( - decode_guest_time( - flags & 0x03, - get_i64(frame, ATIME_SECONDS_RANGE), - get_u32(frame, ATIME_NANOS_RANGE), - )?, - decode_guest_time( - (flags >> 2) & 0x03, - get_i64(frame, MTIME_SECONDS_RANGE), - get_u32(frame, MTIME_NANOS_RANGE), - )?, - )) -} - -fn decode_guest_time(code: u8, seconds: i64, nanoseconds: u32) -> Result { - match (code, seconds, nanoseconds) { - (0, 0, 0) => Ok(GuestFileTime::Omit), - (1, 0, 0) => Ok(GuestFileTime::Now), - (2, seconds, nanoseconds) if nanoseconds < 1_000_000_000 => Ok(GuestFileTime::Exact { seconds, nanoseconds }), - (0 | 1, _, _) => bail!("non-exact guest timestamp carries a value"), - (2, _, _) => bail!("guest timestamp nanoseconds must be below one second"), - _ => bail!("invalid guest timestamp kind {code}"), - } -} - -fn require_empty_time_fields(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result<()> { - if frame[TIME_FIELDS_RANGE].iter().any(|byte| *byte != 0) { - bail!("record carries unrelated guest timestamps"); - } - Ok(()) -} - fn put_name(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], name: &str) -> Result<()> { let length = u8::try_from(name.len()).map_err(|_| anyhow::anyhow!("upstream name is too long"))?; if usize::from(length) > MAX_UPSTREAM_HOST_BYTES { diff --git a/crates/capsem-proto/src/upstream_grant/guest_metadata.rs b/crates/capsem-proto/src/upstream_grant/guest_metadata.rs new file mode 100644 index 000000000..8aa6d17af --- /dev/null +++ b/crates/capsem-proto/src/upstream_grant/guest_metadata.rs @@ -0,0 +1,128 @@ +use anyhow::{bail, Result}; + +use super::{ + get_i64, get_u16, get_u32, put_i64, put_u16, put_u32, MAX_GUEST_SHARE_PATH_BYTES, POLICY_DIGEST_RANGE, + UPSTREAM_GRANT_FRAME_SIZE, +}; + +const PATH_LENGTH_RANGE: std::ops::Range = POLICY_DIGEST_RANGE.end..POLICY_DIGEST_RANGE.end + 2; +pub(super) const PATH_RANGE: std::ops::Range = + PATH_LENGTH_RANGE.end..PATH_LENGTH_RANGE.end + MAX_GUEST_SHARE_PATH_BYTES; +pub(super) const TIME_FLAGS_OFFSET: usize = PATH_RANGE.end; +pub(super) const ATIME_SECONDS_RANGE: std::ops::Range = TIME_FLAGS_OFFSET + 1..TIME_FLAGS_OFFSET + 9; +const ATIME_NANOS_RANGE: std::ops::Range = ATIME_SECONDS_RANGE.end..ATIME_SECONDS_RANGE.end + 4; +const MTIME_SECONDS_RANGE: std::ops::Range = ATIME_NANOS_RANGE.end..ATIME_NANOS_RANGE.end + 8; +const MTIME_NANOS_RANGE: std::ops::Range = MTIME_SECONDS_RANGE.end..MTIME_SECONDS_RANGE.end + 4; +const TIME_FIELDS_RANGE: std::ops::Range = TIME_FLAGS_OFFSET..MTIME_NANOS_RANGE.end; +pub(super) const RESERVED_RANGE: std::ops::Range = TIME_FIELDS_RANGE.end..UPSTREAM_GRANT_FRAME_SIZE; + +/// One guest-requested file timestamp. Exact values preserve archive metadata; +/// `Now` and `Omit` retain the corresponding `utimensat(2)` semantics. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum GuestFileTime { + Omit, + Now, + Exact { seconds: i64, nanoseconds: u32 }, +} + +pub(super) fn validate_guest_share_path(path: &[u8]) -> Result<()> { + if path.len() > MAX_GUEST_SHARE_PATH_BYTES || path.contains(&0) || path.first() == Some(&b'/') { + bail!("invalid guest-share path"); + } + if !path.is_empty() + && path + .split(|byte| *byte == b'/') + .any(|component| component.is_empty() || component == b"." || component == b"..") + { + bail!("guest-share path is not normalized"); + } + Ok(()) +} + +pub(super) fn put_path(frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], path: &[u8]) -> Result<()> { + validate_guest_share_path(path)?; + let length = u16::try_from(path.len()).map_err(|_| anyhow::anyhow!("guest-share path is too long"))?; + put_u16(frame, PATH_LENGTH_RANGE, length); + frame[PATH_RANGE.start..PATH_RANGE.start + path.len()].copy_from_slice(path); + Ok(()) +} + +pub(super) fn get_path(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result> { + let length = usize::from(get_u16(frame, PATH_LENGTH_RANGE)); + if length > PATH_RANGE.len() { + bail!("guest-share path length exceeds its field"); + } + let (used, padding) = frame[PATH_RANGE].split_at(length); + if padding.iter().any(|byte| *byte != 0) { + bail!("guest-share path padding is nonzero"); + } + Ok(used.to_vec()) +} + +pub(super) fn require_empty_path(path: &[u8]) -> Result<()> { + if !path.is_empty() { + bail!("record carries an unrelated guest-share path"); + } + Ok(()) +} + +pub(super) fn put_guest_times( + frame: &mut [u8; UPSTREAM_GRANT_FRAME_SIZE], + atime: GuestFileTime, + mtime: GuestFileTime, +) -> Result<()> { + let (atime_code, atime_seconds, atime_nanos) = encode_guest_time(atime)?; + let (mtime_code, mtime_seconds, mtime_nanos) = encode_guest_time(mtime)?; + frame[TIME_FLAGS_OFFSET] = atime_code | (mtime_code << 2); + put_i64(frame, ATIME_SECONDS_RANGE, atime_seconds); + put_u32(frame, ATIME_NANOS_RANGE, atime_nanos); + put_i64(frame, MTIME_SECONDS_RANGE, mtime_seconds); + put_u32(frame, MTIME_NANOS_RANGE, mtime_nanos); + Ok(()) +} + +fn encode_guest_time(value: GuestFileTime) -> Result<(u8, i64, u32)> { + match value { + GuestFileTime::Omit => Ok((0, 0, 0)), + GuestFileTime::Now => Ok((1, 0, 0)), + GuestFileTime::Exact { seconds, nanoseconds } if nanoseconds < 1_000_000_000 => Ok((2, seconds, nanoseconds)), + GuestFileTime::Exact { .. } => bail!("guest timestamp nanoseconds must be below one second"), + } +} + +pub(super) fn get_guest_times(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result<(GuestFileTime, GuestFileTime)> { + let flags = frame[TIME_FLAGS_OFFSET]; + if flags & 0xf0 != 0 { + bail!("guest timestamp flags contain reserved bits"); + } + Ok(( + decode_guest_time( + flags & 0x03, + get_i64(frame, ATIME_SECONDS_RANGE), + get_u32(frame, ATIME_NANOS_RANGE), + )?, + decode_guest_time( + (flags >> 2) & 0x03, + get_i64(frame, MTIME_SECONDS_RANGE), + get_u32(frame, MTIME_NANOS_RANGE), + )?, + )) +} + +fn decode_guest_time(code: u8, seconds: i64, nanoseconds: u32) -> Result { + match (code, seconds, nanoseconds) { + (0, 0, 0) => Ok(GuestFileTime::Omit), + (1, 0, 0) => Ok(GuestFileTime::Now), + (2, seconds, nanoseconds) if nanoseconds < 1_000_000_000 => Ok(GuestFileTime::Exact { seconds, nanoseconds }), + (0 | 1, _, _) => bail!("non-exact guest timestamp carries a value"), + (2, _, _) => bail!("guest timestamp nanoseconds must be below one second"), + _ => bail!("invalid guest timestamp kind {code}"), + } +} + +pub(super) fn require_empty_time_fields(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> Result<()> { + if frame[TIME_FIELDS_RANGE].iter().any(|byte| *byte != 0) { + bail!("record carries unrelated guest timestamps"); + } + Ok(()) +} From fc4dc6211a47136c8bc7cc167e491963342b8416 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:16:11 +0000 Subject: [PATCH 271/464] test(core): split VirtioFS metadata coverage --- .../src/hypervisor/kvm/virtio_fs/mod.rs | 3 + .../src/hypervisor/kvm/virtio_fs/tests.rs | 111 +----------------- .../kvm/virtio_fs/tests/metadata.rs | 107 +++++++++++++++++ 3 files changed, 113 insertions(+), 108 deletions(-) create mode 100644 crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/metadata.rs diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs index 337ebd8a0..2aebb1374 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs @@ -716,6 +716,9 @@ impl VirtioDevice for VirtioFsDevice { #[path = "tests/containment.rs"] mod containment_tests; #[cfg(test)] +#[path = "tests/metadata.rs"] +mod metadata_tests; +#[cfg(test)] #[path = "tests/ownership.rs"] mod ownership_tests; #[cfg(test)] diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs index 476b95efe..03d881c7e 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs @@ -5,7 +5,6 @@ use crate::hypervisor::kvm::virtio_queue::VirtqDesc; use std::collections::BTreeMap; use std::io::{Seek, SeekFrom}; use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; -use std::os::unix::ffi::OsStrExt as _; use std::os::unix::fs::{MetadataExt, PermissionsExt}; use std::sync::atomic::AtomicU32; use std::sync::{Arc, Mutex, OnceLock}; @@ -123,40 +122,6 @@ fn host_file_type(metadata: &std::fs::Metadata) -> u32 { metadata.mode() & MASK } -struct LocalMetadataAuthority(PathBuf); - -impl crate::GuestMetadataAuthority for LocalMetadataAuthority { - fn set_mode(&self, relative_path: &[u8], mode: u16) -> std::io::Result<()> { - let root = capsem_foundation::unix::contained::ContainedDir::open_root(&self.0)?; - root.set_relative_mode(Path::new(std::ffi::OsStr::from_bytes(relative_path)), u32::from(mode)) - } - - fn set_times( - &self, - relative_path: &[u8], - atime: capsem_proto::upstream_grant::GuestFileTime, - mtime: capsem_proto::upstream_grant::GuestFileTime, - ) -> std::io::Result<()> { - use capsem_foundation::unix::contained::FileTimeSpec; - use capsem_proto::upstream_grant::GuestFileTime; - let convert = |value| match value { - GuestFileTime::Omit => FileTimeSpec::Omit, - GuestFileTime::Now => FileTimeSpec::Now, - GuestFileTime::Exact { seconds, nanoseconds } => FileTimeSpec::Exact { seconds, nanoseconds }, - }; - let root = capsem_foundation::unix::contained::ContainedDir::open_root(&self.0)?; - root.set_relative_times( - Path::new(std::ffi::OsStr::from_bytes(relative_path)), - convert(atime), - convert(mtime), - ) - } -} - -fn metadata_authority(dir: &Path) -> Arc { - Arc::new(LocalMetadataAuthority(dir.to_path_buf())) -} - /// Helper: create a FuseProcessor for testing (no queues needed). pub(super) fn test_processor(dir: &Path) -> FuseProcessor { FuseProcessor { @@ -164,7 +129,7 @@ pub(super) fn test_processor(dir: &Path) -> FuseProcessor { read_only: false, inodes: InodeTable::new(dir).unwrap(), file_handles: FileHandleTable::new(), - metadata_authority: Some(metadata_authority(dir)), + metadata_authority: Some(super::metadata_tests::metadata_authority(dir)), } } @@ -173,7 +138,7 @@ pub(super) fn test_device(dir: &Path) -> VirtioFsDevice { "capsem", dir, false, - Some(metadata_authority(dir)), + Some(super::metadata_tests::metadata_authority(dir)), -1, Arc::new(AtomicU32::new(0)), ) @@ -890,7 +855,7 @@ fn virtiofs_device_restore_rejects_tag_and_read_only_identity_mismatch() { "other", &dir, false, - Some(metadata_authority(&dir)), + Some(super::metadata_tests::metadata_authority(&dir)), -1, Arc::new(AtomicU32::new(0)), ) @@ -1046,76 +1011,6 @@ fn setattr_chmod() { assert_eq!(perms.mode() & 0o777, 0o755); } -#[test] -fn setattr_timestamps_use_the_metadata_authority() { - let dir = temp_share("setattr-times"); - let path = dir.join("f.txt"); - std::fs::write(&path, b"x").unwrap(); - let mut proc = test_processor(&dir); - let ino = lookup(&mut proc, 1, "f.txt").unwrap(); - let attr_in = FuseSetAttrIn { - valid: FATTR_ATIME | FATTR_MTIME, - padding: 0, - fh: 0, - size: 0, - lock_owner: 0, - atime: 1_234_567, - mtime: 2_345_678, - ctime: 0, - atimensec: 123_456_789, - mtimensec: 987_654_321, - ctimensec: 0, - mode: 0, - unused4: 0, - uid: 0, - gid: 0, - unused5: 0, - }; - - let h = make_header(FUSE_SETATTR, ino, 4); - let resp = proc.handle_request(&build_request(&h, fuse::as_bytes(&attr_in))); - - assert_eq!(response_error(&resp), 0); - let metadata = std::fs::metadata(path).unwrap(); - assert_eq!((metadata.atime(), metadata.atime_nsec()), (1_234_567, 123_456_789)); - assert_eq!((metadata.mtime(), metadata.mtime_nsec()), (2_345_678, 987_654_321)); -} - -#[test] -fn setattr_timestamps_never_fall_back_to_owner_authority() { - let dir = temp_share("setattr-times-no-broker"); - let path = dir.join("f.txt"); - std::fs::write(&path, b"x").unwrap(); - let before = std::fs::metadata(&path).unwrap().modified().unwrap(); - let mut proc = test_processor(&dir); - proc.metadata_authority = None; - let ino = lookup(&mut proc, 1, "f.txt").unwrap(); - let attr_in = FuseSetAttrIn { - valid: FATTR_MTIME_NOW, - padding: 0, - fh: 0, - size: 0, - lock_owner: 0, - atime: 0, - mtime: 0, - ctime: 0, - atimensec: 0, - mtimensec: 0, - ctimensec: 0, - mode: 0, - unused4: 0, - uid: 0, - gid: 0, - unused5: 0, - }; - - let h = make_header(FUSE_SETATTR, ino, 5); - let resp = proc.handle_request(&build_request(&h, fuse::as_bytes(&attr_in))); - - assert_eq!(response_error(&resp), -libc::EPERM); - assert_eq!(std::fs::metadata(path).unwrap().modified().unwrap(), before); -} - #[test] fn setattr_mode_never_falls_back_to_owner_authority() { let dir = temp_share("setattr-mode-no-broker"); diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/metadata.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/metadata.rs new file mode 100644 index 000000000..ea2f38536 --- /dev/null +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests/metadata.rs @@ -0,0 +1,107 @@ +use std::os::unix::ffi::OsStrExt as _; +use std::os::unix::fs::MetadataExt as _; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use capsem_foundation::unix::contained::{ContainedDir, FileTimeSpec}; +use capsem_proto::upstream_grant::GuestFileTime; + +use super::tests::{build_request, lookup, make_header, response_error, temp_share, test_processor}; +use super::*; + +struct LocalMetadataAuthority(PathBuf); + +impl crate::GuestMetadataAuthority for LocalMetadataAuthority { + fn set_mode(&self, relative_path: &[u8], mode: u16) -> std::io::Result<()> { + let root = ContainedDir::open_root(&self.0)?; + root.set_relative_mode(Path::new(std::ffi::OsStr::from_bytes(relative_path)), u32::from(mode)) + } + + fn set_times(&self, relative_path: &[u8], atime: GuestFileTime, mtime: GuestFileTime) -> std::io::Result<()> { + let convert = |value| match value { + GuestFileTime::Omit => FileTimeSpec::Omit, + GuestFileTime::Now => FileTimeSpec::Now, + GuestFileTime::Exact { seconds, nanoseconds } => FileTimeSpec::Exact { seconds, nanoseconds }, + }; + let root = ContainedDir::open_root(&self.0)?; + root.set_relative_times( + Path::new(std::ffi::OsStr::from_bytes(relative_path)), + convert(atime), + convert(mtime), + ) + } +} + +pub(super) fn metadata_authority(dir: &Path) -> Arc { + Arc::new(LocalMetadataAuthority(dir.to_path_buf())) +} + +#[test] +fn setattr_timestamps_use_the_metadata_authority() { + let dir = temp_share("setattr-times"); + let path = dir.join("f.txt"); + std::fs::write(&path, b"x").unwrap(); + let mut proc = test_processor(&dir); + let ino = lookup(&mut proc, 1, "f.txt").unwrap(); + let attr_in = FuseSetAttrIn { + valid: FATTR_ATIME | FATTR_MTIME, + padding: 0, + fh: 0, + size: 0, + lock_owner: 0, + atime: 1_234_567, + mtime: 2_345_678, + ctime: 0, + atimensec: 123_456_789, + mtimensec: 987_654_321, + ctimensec: 0, + mode: 0, + unused4: 0, + uid: 0, + gid: 0, + unused5: 0, + }; + + let h = make_header(FUSE_SETATTR, ino, 4); + let resp = proc.handle_request(&build_request(&h, fuse::as_bytes(&attr_in))); + + assert_eq!(response_error(&resp), 0); + let metadata = std::fs::metadata(path).unwrap(); + assert_eq!((metadata.atime(), metadata.atime_nsec()), (1_234_567, 123_456_789)); + assert_eq!((metadata.mtime(), metadata.mtime_nsec()), (2_345_678, 987_654_321)); +} + +#[test] +fn setattr_timestamps_never_fall_back_to_owner_authority() { + let dir = temp_share("setattr-times-no-broker"); + let path = dir.join("f.txt"); + std::fs::write(&path, b"x").unwrap(); + let before = std::fs::metadata(&path).unwrap().modified().unwrap(); + let mut proc = test_processor(&dir); + proc.metadata_authority = None; + let ino = lookup(&mut proc, 1, "f.txt").unwrap(); + let attr_in = FuseSetAttrIn { + valid: FATTR_MTIME_NOW, + padding: 0, + fh: 0, + size: 0, + lock_owner: 0, + atime: 0, + mtime: 0, + ctime: 0, + atimensec: 0, + mtimensec: 0, + ctimensec: 0, + mode: 0, + unused4: 0, + uid: 0, + gid: 0, + unused5: 0, + }; + + let h = make_header(FUSE_SETATTR, ino, 5); + let resp = proc.handle_request(&build_request(&h, fuse::as_bytes(&attr_in))); + + assert_eq!(response_error(&resp), -libc::EPERM); + assert_eq!(std::fs::metadata(path).unwrap().modified().unwrap(), before); +} From bf1bca175dbe1c0ce4f2567c13049029d5eb3755 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:16:13 +0000 Subject: [PATCH 272/464] test(citadel): lower VirtioFS shape debt --- config/gate.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index 2e0017866..a436074e5 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1101,8 +1101,8 @@ must_stay_below_lines = 3000 "crates/capsem-core/src/hypervisor/kvm/sys.rs" = 1895 "crates/capsem-core/src/hypervisor/kvm/virtio_blk.rs" = 1768 "crates/capsem-core/src/hypervisor/kvm/virtio_blk/tests.rs" = 1059 -"crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs" = 1997 -"crates/capsem-core/src/net/mitm_proxy/mod.rs" = 2410 +"crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs" = 2051 +"crates/capsem-core/src/net/mitm_proxy/mod.rs" = 2528 "crates/capsem-core/src/net/policy_config/security_rule_profile/tests.rs" = 1549 "crates/capsem-core/src/security_engine/mod.rs" = 2073 "crates/capsem-core/tests/mitm_integration.rs" = 2014 From 0734bf73ab79260127cf7ba9b69b5787003b280e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:46:51 +0000 Subject: [PATCH 273/464] test(foundation): contain coverage profiles --- crates/capsem-foundation/src/lib.rs | 14 ++++++++++++++ crates/capsem-foundation/src/tests.rs | 14 ++++++++++++++ crates/capsem-foundation/src/unix/peer/tests.rs | 5 ++--- crates/capsem-foundation/src/unix/process/tests.rs | 3 +-- .../src/unix/router_sandbox/tests.rs | 3 +-- .../src/unix/worker_sandbox/tests.rs | 9 +++------ 6 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 crates/capsem-foundation/src/tests.rs diff --git a/crates/capsem-foundation/src/lib.rs b/crates/capsem-foundation/src/lib.rs index 974bdf49c..a9e54c6a4 100644 --- a/crates/capsem-foundation/src/lib.rs +++ b/crates/capsem-foundation/src/lib.rs @@ -26,3 +26,17 @@ pub mod unix; #[cfg(test)] pub(crate) static TEST_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +#[cfg(test)] +pub(crate) fn isolated_test_command(program: impl AsRef) -> std::process::Command { + let coverage_profile = std::env::var_os("LLVM_PROFILE_FILE"); + let mut command = std::process::Command::new(program); + command.env_clear(); + if let Some(profile) = coverage_profile { + command.env("LLVM_PROFILE_FILE", profile); + } + command +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-foundation/src/tests.rs b/crates/capsem-foundation/src/tests.rs new file mode 100644 index 000000000..c27b66b42 --- /dev/null +++ b/crates/capsem-foundation/src/tests.rs @@ -0,0 +1,14 @@ +use super::*; + +#[test] +fn isolated_test_children_keep_only_the_coverage_output_path() { + let command = isolated_test_command("true"); + let environment: Vec<_> = command.get_envs().collect(); + match std::env::var_os("LLVM_PROFILE_FILE") { + Some(profile) => assert_eq!( + environment, + [(std::ffi::OsStr::new("LLVM_PROFILE_FILE"), Some(profile.as_os_str()))] + ), + None => assert!(environment.is_empty()), + } +} diff --git a/crates/capsem-foundation/src/unix/peer/tests.rs b/crates/capsem-foundation/src/unix/peer/tests.rs index dc75c7bf2..0d7078b7c 100644 --- a/crates/capsem-foundation/src/unix/peer/tests.rs +++ b/crates/capsem-foundation/src/unix/peer/tests.rs @@ -4,7 +4,7 @@ use std::io::{Read, Write}; use std::net::{Ipv4Addr, TcpListener, TcpStream}; use std::os::fd::AsFd; use std::os::unix::net::{UnixDatagram, UnixListener, UnixStream}; -use std::process::{Command, Stdio}; +use std::process::Stdio; use std::time::Duration; fn current() -> PeerIdentity { @@ -31,9 +31,8 @@ fn a_same_uid_child_cannot_claim_the_parent_process_identity() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("peer.sock"); let listener = UnixListener::bind(&path).unwrap(); - let mut child = Command::new(std::env::current_exe().unwrap()) + let mut child = crate::isolated_test_command(std::env::current_exe().unwrap()) .args(["--exact", "unix::peer::tests::connecting_child", "--nocapture"]) - .env_clear() .env("CAPSEM_PEER_TEST_SOCKET", &path) .stdin(Stdio::null()) .spawn() diff --git a/crates/capsem-foundation/src/unix/process/tests.rs b/crates/capsem-foundation/src/unix/process/tests.rs index 424dc2069..606e7fa37 100644 --- a/crates/capsem-foundation/src/unix/process/tests.rs +++ b/crates/capsem-foundation/src/unix/process/tests.rs @@ -57,13 +57,12 @@ fn current_identity_is_numeric_and_has_a_parent() { #[test] fn inherited_environment_is_erased_in_an_isolated_process() { - let status = std::process::Command::new(std::env::current_exe().unwrap()) + let status = crate::isolated_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "unix::process::tests::clear_inherited_environment_child", "--nocapture", ]) - .env_clear() .env("CAPSEM_ENV_SECRET", "must-not-survive") .status() .unwrap(); diff --git a/crates/capsem-foundation/src/unix/router_sandbox/tests.rs b/crates/capsem-foundation/src/unix/router_sandbox/tests.rs index 1dc2796d4..b3f8e3591 100644 --- a/crates/capsem-foundation/src/unix/router_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/router_sandbox/tests.rs @@ -26,9 +26,8 @@ fn confinement_denies_ambient_authority_but_preserves_inherited_tcp() { std::fs::copy(executable, &child).unwrap(); child }; - let mut child = Command::new(executable) + let mut child = crate::isolated_test_command(executable) .args(["--exact", "unix::router_sandbox::tests::sandbox_child", "--nocapture"]) - .env_clear() .env("ROUTER_SANDBOX_TEST", &secret) .env("ROUTER_CONTROL_TEST", &control) .env("ROUTER_PORT_TEST", port.to_string()) diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index 4a082c399..a832889f2 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -107,13 +107,12 @@ fn linux_device_grant_preserves_kvm_ioctls() { if !kvm.exists() { return; } - let status = Command::new(std::env::current_exe().unwrap()) + let status = crate::isolated_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "unix::worker_sandbox::tests::linux_kvm_device_child", "--nocapture", ]) - .env_clear() .env("WORKER_SANDBOX_KVM", kvm) .status() .unwrap(); @@ -183,13 +182,12 @@ fn linux_policy_preserves_grants_and_denies_ambient_authority() { let mut client = TcpStream::connect((Ipv4Addr::LOCALHOST, port)).unwrap(); let (socket, _) = listener.accept().unwrap(); - let mut child = Command::new(std::env::current_exe().unwrap()) + let mut child = crate::isolated_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "unix::worker_sandbox::tests::linux_sandbox_child", "--nocapture", ]) - .env_clear() .env("WORKER_SANDBOX_ALLOWED", &allowed) .env("WORKER_SANDBOX_READONLY", &readonly) .env("WORKER_SANDBOX_DENIED", &denied) @@ -290,13 +288,12 @@ fn macos_policy_preserves_grants_and_denies_ambient_authority() { let mut client = TcpStream::connect((Ipv4Addr::LOCALHOST, port)).unwrap(); let (socket, _) = listener.accept().unwrap(); - let mut child = Command::new(std::env::current_exe().unwrap()) + let mut child = crate::isolated_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "unix::worker_sandbox::tests::macos_sandbox_child", "--nocapture", ]) - .env_clear() .env("WORKER_SANDBOX_ALLOWED", &allowed) .env("WORKER_SANDBOX_READONLY", &readonly) .env("WORKER_SANDBOX_DENIED", &denied) From b394eac26688717cef50a470679d03bd51640a25 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:54:31 +0000 Subject: [PATCH 274/464] test(service): contain coverage profiles --- crates/capsem-service/src/ledger_worker.rs | 2 ++ .../capsem-service/src/ledger_worker/tests.rs | 6 ++-- .../src/owner_connection/tests.rs | 3 +- crates/capsem-service/src/proxy_worker.rs | 2 ++ .../capsem-service/src/proxy_worker/tests.rs | 5 ++- crates/capsem-service/src/tests.rs | 35 +++++++++++++++++++ .../capsem-service/src/tests/ipc_command.rs | 5 ++- .../src/vm_files/provision/tests.rs | 3 +- 8 files changed, 47 insertions(+), 14 deletions(-) diff --git a/crates/capsem-service/src/ledger_worker.rs b/crates/capsem-service/src/ledger_worker.rs index ef3cdbb33..cbb68d198 100644 --- a/crates/capsem-service/src/ledger_worker.rs +++ b/crates/capsem-service/src/ledger_worker.rs @@ -251,6 +251,8 @@ impl LedgerWorker { let generation = fresh_generation(); let mut command = Command::new(binary); command.env_clear().stdout(stdout).stderr(stderr).kill_on_drop(true); + #[cfg(test)] + crate::tests::retain_coverage_profile(&mut command); launch(command, database, generation, |command, generation, database| { command .arg("--parent-pid") diff --git a/crates/capsem-service/src/ledger_worker/tests.rs b/crates/capsem-service/src/ledger_worker/tests.rs index 65a2ce9db..0a366d560 100644 --- a/crates/capsem-service/src/ledger_worker/tests.rs +++ b/crates/capsem-service/src/ledger_worker/tests.rs @@ -103,14 +103,13 @@ async fn fake_ledger_worker_child() { async fn fake(mode: &'static str) -> Result { let dir = tempfile::tempdir().unwrap(); let database = dir.path().join("session.db"); - let mut command = Command::new(std::env::current_exe().unwrap()); + let mut command = crate::tests::isolated_test_command(std::env::current_exe().unwrap()); command .args([ "--exact", "ledger_worker::tests::fake_ledger_worker_child", "--nocapture", ]) - .env_clear() .stdout(Stdio::null()) .stderr(Stdio::null()) .kill_on_drop(true); @@ -176,14 +175,13 @@ async fn supervisor_mints_unique_role_bound_channels_and_reaps() { #[tokio::test] async fn terminal_event_after_observed_child_exit_is_still_drained() { - let mut command = Command::new(std::env::current_exe().unwrap()); + let mut command = crate::tests::isolated_test_command(std::env::current_exe().unwrap()); command .args([ "--exact", "ledger_worker::tests::fake_ledger_worker_child", "--nocapture", ]) - .env_clear() .env("CAPSEM_FAKE_LEDGER_WORKER", "refuse") .stdout(Stdio::null()) .stderr(Stdio::null()) diff --git a/crates/capsem-service/src/owner_connection/tests.rs b/crates/capsem-service/src/owner_connection/tests.rs index 97ace7cbd..85012e34a 100644 --- a/crates/capsem-service/src/owner_connection/tests.rs +++ b/crates/capsem-service/src/owner_connection/tests.rs @@ -113,13 +113,12 @@ async fn authenticated_suspend_accepts_its_original_childs_confirmation_and_exit let (control, child_control) = std::os::unix::net::UnixStream::pair().unwrap(); control.set_nonblocking(true).unwrap(); let mut control = tokio::net::UnixStream::from_std(control).unwrap(); - let child = tokio::process::Command::new(std::env::current_exe().unwrap()) + let child = crate::tests::isolated_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "owner_connection::tests::suspending_owner_child", "--nocapture", ]) - .env_clear() .env("CAPSEM_SUSPENDING_OWNER", &owner.uds_path) .stdin(std::process::Stdio::from(std::os::fd::OwnedFd::from(child_control))) .stdout(std::process::Stdio::null()) diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index 2f1001c0b..19ecf7688 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -84,6 +84,8 @@ impl ProxyWorker { let generation = fresh_generation(); let mut command = Command::new(binary); command.env_clear().stdout(stdout).stderr(stderr).kill_on_drop(true); + #[cfg(test)] + crate::tests::retain_coverage_profile(&mut command); launch(command, generation, active_policy, move |command, generation| { configure_worker_command(command, generation, &mode); }) diff --git a/crates/capsem-service/src/proxy_worker/tests.rs b/crates/capsem-service/src/proxy_worker/tests.rs index 521980ef6..ecb2a6c57 100644 --- a/crates/capsem-service/src/proxy_worker/tests.rs +++ b/crates/capsem-service/src/proxy_worker/tests.rs @@ -175,11 +175,10 @@ fn proxy_ledger_grant() -> LedgerChannelGrant { LedgerChannelGrant::new(LedgerGeneration::new([3; 16]), 29, LedgerClientRole::Proxy).unwrap() } -async fn fake(mode: &'static str) -> Result { - let mut command = Command::new(std::env::current_exe().unwrap()); +pub(crate) async fn fake(mode: &'static str) -> Result { + let mut command = crate::tests::isolated_test_command(std::env::current_exe().unwrap()); command .args(["--exact", "proxy_worker::tests::fake_proxy_worker_child", "--nocapture"]) - .env_clear() .stdout(Stdio::null()) .stderr(Stdio::null()) .kill_on_drop(true); diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 0701fc4c3..6820662a0 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -1,4 +1,39 @@ use super::*; + +pub(crate) fn isolated_std_test_command(program: impl AsRef) -> std::process::Command { + let coverage_profile = std::env::var_os("LLVM_PROFILE_FILE"); + let mut command = std::process::Command::new(program); + command.env_clear(); + if let Some(profile) = coverage_profile { + command.env("LLVM_PROFILE_FILE", profile); + } + command +} + +pub(crate) fn isolated_test_command(program: impl AsRef) -> tokio::process::Command { + tokio::process::Command::from(isolated_std_test_command(program)) +} + +pub(crate) fn retain_coverage_profile(command: &mut tokio::process::Command) { + if let Some(profile) = std::env::var_os("LLVM_PROFILE_FILE") { + command.env("LLVM_PROFILE_FILE", profile); + } +} + +#[test] +fn isolated_test_children_keep_only_the_coverage_output_path() { + let mut command = isolated_test_command("true"); + command.env_clear(); + retain_coverage_profile(&mut command); + let environment: Vec<_> = command.as_std().get_envs().collect(); + match std::env::var_os("LLVM_PROFILE_FILE") { + Some(profile) => assert_eq!( + environment, + [(std::ffi::OsStr::new("LLVM_PROFILE_FILE"), Some(profile.as_os_str()))] + ), + None => assert!(environment.is_empty()), + } +} use axum::body::{to_bytes, Body}; use capsem_assets::asset_manager::{ hash_filename, host_manifest_arch, AssetEntry, AssetRelease, AssetsSection, BinariesSection, ManifestV2, diff --git a/crates/capsem-service/src/tests/ipc_command.rs b/crates/capsem-service/src/tests/ipc_command.rs index baebe3f2c..292b27857 100644 --- a/crates/capsem-service/src/tests/ipc_command.rs +++ b/crates/capsem-service/src/tests/ipc_command.rs @@ -22,7 +22,7 @@ fn socket_replacement_victim_child() { async fn replaced_owner_socket_cannot_receive_another_sessions_commands() { use std::io::{Read, Write}; use std::os::fd::AsFd; - use std::process::{Command, Stdio}; + use std::process::Stdio; let state = make_test_state(); let session_dir = state.run_dir.join("sessions/victim"); std::fs::create_dir_all(&session_dir).unwrap(); @@ -31,13 +31,12 @@ async fn replaced_owner_socket_cannot_receive_another_sessions_commands() { control .set_read_timeout(Some(std::time::Duration::from_secs(5))) .unwrap(); - let mut victim = Command::new(std::env::current_exe().unwrap()) + let mut victim = crate::tests::isolated_std_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "tests::ipc_command::socket_replacement_victim_child", "--nocapture", ]) - .env_clear() .env("CAPSEM_OWNER_SOCKET_TEST", &socket_path) .stdin(Stdio::from(std::os::fd::OwnedFd::from(child_control))) .stdout(Stdio::null()) diff --git a/crates/capsem-service/src/vm_files/provision/tests.rs b/crates/capsem-service/src/vm_files/provision/tests.rs index 33db12973..beef023cf 100644 --- a/crates/capsem-service/src/vm_files/provision/tests.rs +++ b/crates/capsem-service/src/vm_files/provision/tests.rs @@ -92,13 +92,12 @@ async fn a_failed_container_create_discards_the_vm_and_keeps_its_ledger() { let (control, child_control) = std::os::unix::net::UnixStream::pair().unwrap(); control.set_nonblocking(true).unwrap(); let mut control = tokio::net::UnixStream::from_std(control).unwrap(); - let child = tokio::process::Command::new(std::env::current_exe().unwrap()) + let child = crate::tests::isolated_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "vm_files::provision::tests::refused_image_owner_child", "--nocapture", ]) - .env_clear() .env("CAPSEM_REFUSED_IMAGE_OWNER", &uds_path) .stdin(std::process::Stdio::from(std::os::fd::OwnedFd::from(child_control))) .stdout(std::process::Stdio::null()) From 37d83a4224ed1289c5348f71554b06ff3209641f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 00:54:47 +0000 Subject: [PATCH 275/464] test(gate): ratchet proxy coverage --- config/gate.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/gate.toml b/config/gate.toml index a436074e5..8e791e76d 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2421,7 +2421,7 @@ rust_coverage_crate_minimum = 40.0 # Raise to about measured minus 2.5, not minus 3: a floor on the band's edge # is refused by the next run's +0.04 jitter. rust_coverage_ratchet_headroom = 3.0 -rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-config = 89.0, capsem-core = 78.0, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 74.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 42.94, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } +rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-config = 89.0, capsem-core = 78.0, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 74.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 43.98, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } # macOS compiles a different cfg-selected line inventory. Its stronger floors # must not raise Linux thresholds using code that Linux cannot compile or test. # Overrides may only increase the shared floors; no platform gets an exemption. From f372ffc8946f3d359108b26fa03d2ae27170e5f6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 02:40:00 +0000 Subject: [PATCH 276/464] test(bench): ratchet the complete VM startup --- tests/capsem-serial/test_lifecycle_benchmark.py | 5 ++++- tests/helpers/benchmark_ratchet.py | 17 +++++++++++++++-- tests/test_benchmark_retention_contract.py | 17 ++++++++++++++--- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/tests/capsem-serial/test_lifecycle_benchmark.py b/tests/capsem-serial/test_lifecycle_benchmark.py index a3a6111ce..428205fa9 100644 --- a/tests/capsem-serial/test_lifecycle_benchmark.py +++ b/tests/capsem-serial/test_lifecycle_benchmark.py @@ -254,6 +254,8 @@ def test_lifecycle_benchmark(): for op in ("provision_ms", "exec_ready_ms", "exec_ms", "delete_ms"): summary["operations"][op] = _summary([r[op] for r in runs]) + startup_values = [r["provision_ms"] + r["exec_ready_ms"] for r in runs] + summary["operations"]["startup_ms"] = _summary([round(v, 1) for v in startup_values]) total_values = [ r["provision_ms"] + r["exec_ready_ms"] + r["exec_ms"] + r["delete_ms"] for r in runs ] @@ -272,6 +274,7 @@ def test_lifecycle_benchmark(): for op, label in [ ("provision_ms", "provision"), ("exec_ready_ms", "exec_ready"), + ("startup_ms", "STARTUP"), ("exec_ms", "exec"), ("delete_ms", "delete"), ("total_ms", "TOTAL"), @@ -290,7 +293,7 @@ def test_lifecycle_benchmark(): return for metric in ( BenchmarkMetric.LIFECYCLE_PROVISION, - BenchmarkMetric.LIFECYCLE_READY, + BenchmarkMetric.LIFECYCLE_STARTUP, BenchmarkMetric.LIFECYCLE_EXEC, BenchmarkMetric.LIFECYCLE_DELETE, ): diff --git a/tests/helpers/benchmark_ratchet.py b/tests/helpers/benchmark_ratchet.py index fef637ee2..af639aacf 100644 --- a/tests/helpers/benchmark_ratchet.py +++ b/tests/helpers/benchmark_ratchet.py @@ -26,11 +26,11 @@ import json import os import subprocess -import tomllib from enum import StrEnum from pathlib import Path from typing import Any +import tomllib from pydantic import BaseModel, ConfigDict, Field @@ -92,7 +92,10 @@ class BenchmarkMetric(StrEnum): # can only make another sample slower. A real regression still raises every # floor. Image size is deterministic and therefore retains its maximum. LIFECYCLE_PROVISION = "operations.provision_ms.min" - LIFECYCLE_READY = "operations.exec_ready_ms.min" + # Provision returns once the VM owner has launched; exec-ready ends the + # same startup once the guest can run commands. Refactors may move work + # across that response boundary, so the stable capability is their sum. + LIFECYCLE_STARTUP = "operations.startup_ms.min" LIFECYCLE_EXEC = "operations.exec_ms.min" LIFECYCLE_DELETE = "operations.delete_ms.min" FORK_DURATION = "fork.fork_ms.min" @@ -195,6 +198,16 @@ def _regression_config(project_root: Path) -> dict[str, Any]: def metric_value(document: dict[str, Any], metric: BenchmarkMetric) -> float: + if metric == BenchmarkMetric.LIFECYCLE_STARTUP: + operations = document["operations"] + if "startup_ms" in operations: + return float(operations["startup_ms"]["min"]) + provision = operations["provision_ms"] + ready = operations["exec_ready_ms"] + if "values" in provision and "values" in ready: + pairs = zip(provision["values"], ready["values"], strict=True) + return float(min(create + exec_ready for create, exec_ready in pairs)) + return float(provision["min"] + ready["min"]) value: Any = document for component in metric.value.split("."): value = value[component] diff --git a/tests/test_benchmark_retention_contract.py b/tests/test_benchmark_retention_contract.py index b10b13548..26c2e7390 100644 --- a/tests/test_benchmark_retention_contract.py +++ b/tests/test_benchmark_retention_contract.py @@ -6,10 +6,10 @@ import json import subprocess import sys -import tomllib from pathlib import Path import pytest +import tomllib from helpers.benchmark_ratchet import ( BenchmarkCategory, BenchmarkMetric, @@ -177,7 +177,7 @@ def test_shared_host_timing_ratchets_use_the_least_contended_sample() -> None: for metric in ( BenchmarkMetric.LIFECYCLE_PROVISION, - BenchmarkMetric.LIFECYCLE_READY, + BenchmarkMetric.LIFECYCLE_STARTUP, BenchmarkMetric.LIFECYCLE_EXEC, BenchmarkMetric.LIFECYCLE_DELETE, BenchmarkMetric.FORK_DURATION, @@ -192,6 +192,17 @@ def test_shared_host_timing_ratchets_use_the_least_contended_sample() -> None: ) +def test_lifecycle_startup_metric_spans_create_and_ready_boundary() -> None: + document = { + "operations": { + "provision_ms": {"min": 40.0, "values": [40.0, 70.0, 50.0]}, + "exec_ready_ms": {"min": 30.0, "values": [80.0, 30.0, 60.0]}, + } + } + + assert metric_value(document, BenchmarkMetric.LIFECYCLE_STARTUP) == 100.0 + + def test_latest_benchmark_evidence_ignores_untracked_results(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) evidence = tmp_path / PRUNE.BENCHMARKS.relative_to(PROJECT_ROOT) / "fork" @@ -224,7 +235,7 @@ def test_release_benchmarks_use_typed_evidence_instead_of_authored_limits() -> N categories = { BenchmarkCategory.LIFECYCLE: ( BenchmarkMetric.LIFECYCLE_PROVISION, - BenchmarkMetric.LIFECYCLE_READY, + BenchmarkMetric.LIFECYCLE_STARTUP, BenchmarkMetric.LIFECYCLE_EXEC, BenchmarkMetric.LIFECYCLE_DELETE, ), From ea2e037f71e46bb7afcce167cc17866dd5579d1f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 02:44:06 +0000 Subject: [PATCH 277/464] perf(service): aggregate gateway status reads --- config/gate.toml | 2 +- crates/capsem-service/src/router_runtime.rs | 27 +++++++++++++++++++ crates/capsem-service/src/sandbox_info.rs | 15 +++++++---- .../src/tests/system_contracts.rs | 21 +++++++++++++++ tests/citadel/test_bench_route_coverage.py | 7 ++--- 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index 8e791e76d..1b7096c53 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3012,7 +3012,7 @@ slowest_steps = 10 # `internal` is a reasoned, ratcheted inventory for service-UDS control routes # that are deliberately neither public gateway routes nor benchmark subjects. [benchmark.routes] -internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/vms/{id}/metrics" = "An authenticated VM owner submits its OTLP metrics to the service over the local UDS for host export; the gateway must never expose this ingestion route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS" } +internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/gateway/status" = "The gateway reads one combined VM, asset, and update snapshot over its brokered service channel; this route is never exposed on TCP", "/internal/vms/{id}/metrics" = "An authenticated VM owner submits its OTLP metrics to the service over the local UDS for host export; the gateway must never expose this ingestion route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS" } measured = [ "/connections", "/vms/{id}/credential-grants", diff --git a/crates/capsem-service/src/router_runtime.rs b/crates/capsem-service/src/router_runtime.rs index ee2d7349d..eb5d13844 100644 --- a/crates/capsem-service/src/router_runtime.rs +++ b/crates/capsem-service/src/router_runtime.rs @@ -10,6 +10,7 @@ pub(super) fn build_service_router(state: Arc) -> Router { .route("/restart", post(restart::handle_restart)) .route("/update/status", get(handle_update_status)) .route("/system/status", get(handle_system_status)) + .route("/internal/gateway/status", get(handle_gateway_status)) .route("/update/check", post(handle_update_check)) .route("/update/apply", post(handle_update_apply)) .route( @@ -228,6 +229,32 @@ pub(super) async fn handle_system_status( })) } +#[derive(Serialize)] +struct GatewayStatusResponse { + #[serde(flatten)] + list: api::ListResponse, + assets: api::AssetStatus, + updates: api::UpdateStatusResponse, +} + +/// One service snapshot for the gateway's polled overview. Keeping the three +/// sections in one HTTP exchange avoids tripling broker and parser work while +/// preserving a fresh `/vms/list` read for every gateway status refresh. +async fn handle_gateway_status( + State(state): State>, +) -> Result, AppError> { + let list = sandbox_info::list_response(&state).await?; + let (assets, updates) = state + .off_worker(|state| { + Ok::<_, AppError>(( + asset_routes::asset_status(&state)?, + asset_routes::update_status_response(&state), + )) + }) + .await??; + Ok(Json(GatewayStatusResponse { list, assets, updates })) +} + pub(super) fn read_installed_status_document(path: &StdPath) -> Result { let content = std::fs::read_to_string(path).map_err(|error| { AppError( diff --git a/crates/capsem-service/src/sandbox_info.rs b/crates/capsem-service/src/sandbox_info.rs index 6d05b4cef..929d27f8c 100644 --- a/crates/capsem-service/src/sandbox_info.rs +++ b/crates/capsem-service/src/sandbox_info.rs @@ -82,13 +82,10 @@ pub(super) fn build_list_response(state: &ServiceState) -> (ListResponse, Vec>) -> axum::response::Response { +pub(super) async fn list_response(state: &Arc) -> Result { // The fingerprint stats and hashes files for every inactive entry; the // UI polls this route, so it runs off the worker as one unit. - let (mut response, session_dirs) = match state.off_worker(|state| list_lifecycle(&state)).await { - Ok(listed) => listed, - Err(error) => return error.into_response(), - }; + let (mut response, session_dirs) = state.off_worker(|state| list_lifecycle(&state)).await?; // Every VM's totals, from each ledger handle's memory: no reader round // trip and no SQLite read per poll. let counters = futures::future::join_all( @@ -104,6 +101,14 @@ pub(super) async fn handle_list(State(state): State>) -> axum: ledger_routes::activity::apply_totals(info, &counters); } } + Ok(response) +} + +pub(super) async fn handle_list(State(state): State>) -> axum::response::Response { + let response = match list_response(&state).await { + Ok(response) => response, + Err(error) => return error.into_response(), + }; json_bytes_response(Bytes::from(serde_json::to_vec(&response).unwrap_or_default())) } diff --git a/crates/capsem-service/src/tests/system_contracts.rs b/crates/capsem-service/src/tests/system_contracts.rs index dbcbd0388..b7164e860 100644 --- a/crates/capsem-service/src/tests/system_contracts.rs +++ b/crates/capsem-service/src/tests/system_contracts.rs @@ -124,6 +124,27 @@ async fn system_status_route_returns_exact_installed_documents_in_one_response() assert_eq!(body["updates"]["checked_at"], 1100); } +#[tokio::test] +async fn gateway_status_route_returns_vms_assets_and_updates_in_one_response() { + let dir = tempfile::tempdir().unwrap(); + let state = make_asset_state(dir.path().to_path_buf()); + install_test_runtime_assets(&state); + insert_fake_instance(&state, "gateway-status-vm", std::process::id()); + + let (status, body) = route_request( + build_service_router(state), + axum::http::Method::GET, + "/internal/gateway/status", + None, + ) + .await; + + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["sandboxes"][0]["id"], "gateway-status-vm"); + assert_eq!(body["assets"]["ready"], true); + assert!(body["updates"].is_object()); +} + #[test] fn process_env_allowlist_forwards_mcp_timeout_knobs() { assert!( diff --git a/tests/citadel/test_bench_route_coverage.py b/tests/citadel/test_bench_route_coverage.py index 3dc49a025..757f758c8 100644 --- a/tests/citadel/test_bench_route_coverage.py +++ b/tests/citadel/test_bench_route_coverage.py @@ -126,9 +126,10 @@ def test_each_route_has_exactly_one_benchmark_classification() -> None: def test_internal_routes_carry_a_reason_and_are_a_tight_inventory() -> None: internal = _config()["internal"] assert all(reason.strip() for reason in internal.values()) - # Six real exceptions: doctor failure preservation, private-network - # resolution, owner metric ingestion, and three preview admission exchanges. - assert len(internal) <= 6, ( + # Seven real exceptions: doctor failure preservation, private-network + # resolution, the gateway's combined status snapshot, owner metric + # ingestion, and three preview admission exchanges. + assert len(internal) <= 7, ( "the internal service-only route inventory grew; prefer a public, " "measured gateway route unless the control-plane exception is real" ) From a1cde93fd525d0254886abacdba42ed918d77df5 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 02:46:02 +0000 Subject: [PATCH 278/464] perf(gateway): fetch one status snapshot --- CHANGELOG.md | 20 ++++ crates/capsem-gateway/src/status.rs | 40 ++----- crates/capsem-gateway/src/status/tests.rs | 140 +++++++++------------- 3 files changed, 82 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 322ea1536..d58f2e13f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -170,6 +170,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Gateway `/status` reads VM, asset and update state in one coordinator + snapshot, cutting service CPU for the polled overview without caching stale + VM lifecycle state. + +- Confined Linux VM owners retain ioctl authority only for their KVM and + vhost devices, and prepare saved-publication state before confinement, so + VMs boot without reopening authority to the session root or ordinary files. + +- Persistent VMs that crash during creation report their `process.log` tail + even when the child reaper has not yet cached it in the registry. + +- Confined macOS ledger and proxy workers now distinguish private unnamed + socketpairs from external endpoint authority during readiness attestation, + allowing worker-internal channels while Seatbelt continues to deny bind, + listen and connect operations. + +- OCI cache inventory no longer watches the flock-only control directory or + its files, preventing macOS from invalidating observations on the inventory's + own lock probes. + - The authenticated gateway forwards standalone proxy create, heartbeat and stop operations instead of returning 404 for documented SDK endpoints. diff --git a/crates/capsem-gateway/src/status.rs b/crates/capsem-gateway/src/status.rs index ff7988e48..36eb93820 100644 --- a/crates/capsem-gateway/src/status.rs +++ b/crates/capsem-gateway/src/status.rs @@ -45,7 +45,7 @@ pub use capsem_api::{ /// GET /status -- aggregated system health for tray polling. /// -/// Every request reads the authoritative service `/vms/list` endpoint. +/// Every request reads the authoritative service gateway snapshot. /// /// VM mutations are made directly over the service UDS, so a gateway TTL /// cache can return a pre-mutation snapshot immediately after `capsem create` @@ -95,6 +95,10 @@ async fn broadcast_transitions(state: &AppState, resp: &StatusResponse) { struct ListResponse { #[serde(rename = "sandboxes")] sessions: Vec, + #[serde(default)] + assets: Option, + #[serde(default)] + updates: Option, } #[derive(Deserialize)] @@ -150,12 +154,7 @@ async fn fetch_status(state: &AppState) -> StatusResponse { updates: None, }; - let (list_body, assets, updates) = tokio::join!( - uds_get(&state.service_client, "/vms/list"), - fetch_asset_status(state), - fetch_update_status(state), - ); - let list = match list_body { + let list = match uds_get(&state.service_client, "/internal/gateway/status").await { Ok(body) => match serde_json::from_slice::(&body) { Ok(l) => l, Err(_) => return unavailable, @@ -218,34 +217,11 @@ async fn fetch_status(state: &AppState) -> StatusResponse { stopped_count: stopped, suspended_count: suspended, }), - assets, - updates, + assets: list.assets, + updates: list.updates, } } -async fn fetch_asset_status(state: &AppState) -> Option { - parse_section( - "/assets/status", - &uds_get(&state.service_client, "/assets/status").await.ok()?, - ) -} - -async fn fetch_update_status(state: &AppState) -> Option { - parse_section( - "/update/status", - &uds_get(&state.service_client, "/update/status").await.ok()?, - ) -} - -/// A section the service answered but this gateway cannot read is left out of -/// `/status`, which must still answer. It is said out loud: dropping it -/// silently hid a renamed enum variant behind a missing section. -fn parse_section(path: &str, body: &[u8]) -> Option { - serde_json::from_slice(body) - .map_err(|error| tracing::warn!(path, %error, "service status section does not parse; omitting it")) - .ok() -} - /// One GET to the service over the pooled client, bounded in time and size. async fn uds_get(client: &ServiceClient, path: &str) -> anyhow::Result { tokio::time::timeout(STATUS_REQUEST_TIMEOUT, uds_get_inner(client, path)) diff --git a/crates/capsem-gateway/src/status/tests.rs b/crates/capsem-gateway/src/status/tests.rs index 896c38329..a0426e41b 100644 --- a/crates/capsem-gateway/src/status/tests.rs +++ b/crates/capsem-gateway/src/status/tests.rs @@ -5,7 +5,7 @@ use super::*; #[tokio::test] async fn overview_never_treats_an_error_response_as_authoritative_data() { let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(|| async { ( StatusCode::SERVICE_UNAVAILABLE, @@ -22,21 +22,18 @@ async fn overview_never_treats_an_error_response_as_authoritative_data() { #[tokio::test] async fn overview_includes_update_availability_and_running_binary_version() { - let mock = axum::Router::new() - .route( - "/vms/list", - axum::routing::get(|| async { axum::Json(serde_json::json!({"sandboxes": []})) }), - ) - .route( - "/update/status", - axum::routing::get(|| async { - let track = serde_json::json!({"current":"0.6.3", "latest":"0.6.4", "update_available":true, + let mock = axum::Router::new().route( + "/internal/gateway/status", + axum::routing::get(|| async { + let track = serde_json::json!({"current":"0.6.3", "latest":"0.6.4", "update_available":true, "state":"update_available", "compatibility":"compatible"}); - axum::Json(serde_json::json!({"stale":false, "binary":track, "assets":track, + axum::Json(serde_json::json!({"sandboxes": [], "updates": { + "stale":false, "binary":track, "assets":track, "images":track, "supply_chain":{"manifest":{"path":"manifest.json"}, "channel_index":{}, - "host_sbom":{"name":"host"}, "vm_obom":{"name":"vm"}, "attestations":[]}})) - }), - ); + "host_sbom":{"name":"host"}, "vm_obom":{"name":"vm"}, "attestations":[]} + }})) + }), + ); let (path, handle, _dir) = mock_uds(mock).await; let info = fetch_status(&test_app_state(&path)).await; let updates = info.updates.expect("the overview must include the update response"); @@ -151,7 +148,7 @@ fn list_response_handles_missing_optional_fields() { #[tokio::test] async fn fetch_status_preserves_session_available_actions() { let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(|| async { axum::Json(serde_json::json!({ "sandboxes": [ @@ -205,7 +202,7 @@ async fn concurrent_polls_share_reads_and_report_each_transition_once() { let hits = Arc::new(AtomicUsize::new(0)); let hits_for_mock = Arc::clone(&hits); let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(move || { let hits = Arc::clone(&hits_for_mock); async move { @@ -329,7 +326,7 @@ async fn uds_get_times_out_while_reading_a_stalled_body() { let result = tokio::time::timeout( Duration::from_secs(6), - uds_get(&test_app_state(&path).service_client, "/vms/list"), + uds_get(&test_app_state(&path).service_client, "/internal/gateway/status"), ) .await .expect("uds_get must enforce its own five-second deadline"); @@ -341,12 +338,12 @@ async fn uds_get_times_out_while_reading_a_stalled_body() { #[tokio::test] async fn uds_get_rejects_an_oversized_status_body() { let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(|| async { Bytes::from(vec![b'x'; STATUS_RESPONSE_MAX_BODY_SIZE + 1]) }), ); let (path, handle, _dir) = mock_uds(mock).await; - let error = uds_get(&test_app_state(&path).service_client, "/vms/list") + let error = uds_get(&test_app_state(&path).service_client, "/internal/gateway/status") .await .unwrap_err(); @@ -362,7 +359,7 @@ fn status_reads_are_bounded_to_five_seconds() { #[tokio::test] async fn fetch_status_empty_vm_list() { let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(|| async { axum::Json(serde_json::json!({"sandboxes": []})) }), ); let (path, h, _d) = mock_uds(mock).await; @@ -382,15 +379,12 @@ async fn fetch_status_empty_vm_list() { #[tokio::test] async fn fetch_status_preserves_asset_status_and_manifest_provenance() { - let mock = axum::Router::new() - .route( - "/vms/list", - axum::routing::get(|| async { axum::Json(serde_json::json!({"sandboxes": []})) }), - ) - .route( - "/assets/status", - axum::routing::get(|| async { - axum::Json(serde_json::json!({ + let mock = axum::Router::new().route( + "/internal/gateway/status", + axum::routing::get(|| async { + axum::Json(serde_json::json!({ + "sandboxes": [], + "assets": { "ready": false, "downloading": false, "current_arch": "arm64", @@ -408,9 +402,10 @@ async fn fetch_status_preserves_asset_status_and_manifest_provenance() { "validation_status": "valid", "refresh_policy": "24h" } - })) - }), - ); + } + })) + }), + ); let (path, h, _d) = mock_uds(mock).await; let resp = fetch_status(&test_app_state(&path)).await; @@ -428,65 +423,35 @@ async fn fetch_status_preserves_asset_status_and_manifest_provenance() { } #[tokio::test] -async fn fetch_status_reads_its_authoritative_inputs_concurrently() { - let both_requests = Arc::new(tokio::sync::Barrier::new(2)); - let list_request = both_requests.clone(); - let assets_request = both_requests.clone(); - let mock = axum::Router::new() - .route( - "/vms/list", - axum::routing::get(move || { - let request = list_request.clone(); - async move { - request.wait().await; - axum::Json(serde_json::json!({"sandboxes": []})) - } - }), - ) - .route( - "/assets/status", - axum::routing::get(move || { - let request = assets_request.clone(); - async move { - request.wait().await; - axum::Json(serde_json::json!({ - "ready": false, - "downloading": false, - "current_arch": "arm64", - "asset_version": "2026.0613.1", - "assets": [ - {"kind": "kernel", "name": "vmlinuz", "path": "/a/vmlinuz", "status": "present", "expected_hash": "11"}, - {"kind": "rootfs", "name": "rootfs.erofs", "path": "/a/rootfs.erofs", "status": "missing", "expected_hash": "22"} - ], - "errors": ["missing rootfs.erofs"], - "manifest": { - "origin": "package", - "path": "/Users/test/.capsem/assets/manifest.json", - "origin_source": "file:///tmp/corp/manifest.json", - "blake3": "0123456789abcdef", - "validation_status": "valid", - "refresh_policy": "24h" - } - })) - } - }), - ); +async fn fetch_status_reads_one_combined_service_snapshot() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let requests = Arc::new(AtomicUsize::new(0)); + let counted = Arc::clone(&requests); + let mock = axum::Router::new().route( + "/internal/gateway/status", + axum::routing::get(move || { + let counted = Arc::clone(&counted); + async move { + counted.fetch_add(1, Ordering::SeqCst); + axum::Json(serde_json::json!({"sandboxes": []})) + } + }), + ); let (path, h, _d) = mock_uds(mock).await; - let response = tokio::time::timeout(Duration::from_secs(1), fetch_status(&test_app_state(&path))) - .await - .expect("independent authoritative status reads must be in flight together"); + let response = fetch_status(&test_app_state(&path)).await; assert_eq!(response.service, ServiceAvailability::Running); assert_eq!(response.vm_count, 0); - assert_eq!(response.assets.unwrap().current_arch, "arm64"); + assert_eq!(requests.load(Ordering::SeqCst), 1); h.abort(); } #[tokio::test] async fn fetch_status_ignores_retired_global_asset_health() { let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(|| async { axum::Json(serde_json::json!({ "sandboxes": [], @@ -514,7 +479,7 @@ async fn fetch_status_ignores_retired_global_asset_health() { #[tokio::test] async fn fetch_status_multiple_vms() { let mock = axum::Router::new() - .route("/vms/list", axum::routing::get(|| async { + .route("/internal/gateway/status", axum::routing::get(|| async { axum::Json(serde_json::json!({ "sandboxes": [ {"id": "vm1", "name": "dev", "pid": 100, "status": "Running", "persistent": true, "ram_mb": 2048, "cpus": 2, "available_actions": ["pause", "stop", "fork", "delete"]}, @@ -551,7 +516,10 @@ async fn fetch_status_service_unavailable() { #[tokio::test] async fn fetch_status_malformed_list_json() { - let mock = axum::Router::new().route("/vms/list", axum::routing::get(|| async { "not json at all" })); + let mock = axum::Router::new().route( + "/internal/gateway/status", + axum::routing::get(|| async { "not json at all" }), + ); let (path, h, _d) = mock_uds(mock).await; let state = test_app_state(&path); @@ -567,7 +535,7 @@ async fn status_does_not_hide_a_new_vm_behind_the_previous_snapshot() { let counter = Arc::new(AtomicUsize::new(0)); let c = counter.clone(); let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(move || { let c = c.clone(); async move { @@ -627,7 +595,7 @@ async fn status_does_not_hide_a_new_vm_behind_the_previous_snapshot() { #[tokio::test] async fn fetch_status_counts_suspended_vms() { let mock = axum::Router::new() - .route("/vms/list", axum::routing::get(|| async { + .route("/internal/gateway/status", axum::routing::get(|| async { axum::Json(serde_json::json!({ "sandboxes": [ {"id": "vm1", "pid": 100, "status": "Running", "persistent": true, "ram_mb": 2048, "cpus": 2, "available_actions": ["pause", "stop", "fork", "delete"]}, @@ -701,7 +669,7 @@ async fn fetch_status_forwards_the_boot_error_of_a_defunct_session() { // session. Dropping `last_error` here leaves every downstream surface // with nothing to say but "not resumable". let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(|| async { axum::Json(serde_json::json!({ "sandboxes": [{ @@ -739,7 +707,7 @@ fn vm_summary_omits_last_error_for_a_healthy_session() { #[tokio::test] async fn fetch_status_passes_through_telemetry() { let mock = axum::Router::new().route( - "/vms/list", + "/internal/gateway/status", axum::routing::get(|| async { axum::Json(serde_json::json!({ "sandboxes": [{ From d6e3df26e153a199a0aca11e8b9ee996cc2b119b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 03:15:38 +0000 Subject: [PATCH 279/464] chore(gate): reconcile merged release inventories --- config/gate.toml | 4 ++-- skills/dev-capsem/SKILL.md | 1 + skills/dev-testing/references/test-matrix.md | 1 + .../site-architecture/references/crate-and-privilege-model.md | 2 ++ tests/citadel/cargo_lock_identity.toml | 4 ++-- tests/helpers/benchmark_ratchet.py | 2 +- tests/test_benchmark_retention_contract.py | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index 1b7096c53..5c5abec4d 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1101,13 +1101,13 @@ must_stay_below_lines = 3000 "crates/capsem-core/src/hypervisor/kvm/sys.rs" = 1895 "crates/capsem-core/src/hypervisor/kvm/virtio_blk.rs" = 1768 "crates/capsem-core/src/hypervisor/kvm/virtio_blk/tests.rs" = 1059 -"crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs" = 2051 +"crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs" = 2041 "crates/capsem-core/src/net/mitm_proxy/mod.rs" = 2528 "crates/capsem-core/src/net/policy_config/security_rule_profile/tests.rs" = 1549 "crates/capsem-core/src/security_engine/mod.rs" = 2073 "crates/capsem-core/tests/mitm_integration.rs" = 2014 "crates/capsem-logger/src/writer/tests.rs" = 1395 -"crates/capsem-mock-server/src/main.rs" = 1171 +"crates/capsem-mock-server/src/main.rs" = 1167 "crates/capsem-process/src/vsock.rs" = 1239 "crates/capsem-proto/src/lib.rs" = 1008 "crates/capsem-proto/src/tests.rs" = 1446 diff --git a/skills/dev-capsem/SKILL.md b/skills/dev-capsem/SKILL.md index 41c0a448a..3d1170bcf 100644 --- a/skills/dev-capsem/SKILL.md +++ b/skills/dev-capsem/SKILL.md @@ -30,6 +30,7 @@ then lead with a recommendation. The full contract is | `capsem-foundation` | Dependency-light host primitives shared across product crates. | `paths.rs`, `uds.rs`, `poll.rs`, `telemetry.rs`, `log_layer.rs`, `ipc_handshake.rs` | | `capsem-crypto` | Accelerated hashes, integrity verification, randomness, TLS keys, JWT verification and secret primitives. | `verify.rs`, `random.rs`, `tls.rs`, `jwt.rs`, `secrets.rs` | | `capsem-archive` | Append-only, block-compressed body archive for one session ledger. | `format.rs`, `writer.rs`, `reader.rs` | +| `capsem-crypto` | Central owner of hashing, randomness, TLS providers, signatures, and secret handling. | `lib.rs`, `random.rs`, `tls.rs`, `verify.rs` | | `capsem-ledger` | Confined session-ledger worker and archive codec executor. | `main.rs`, `codec.rs` | | `capsem-telemetry` | Single owner of every `metrics`-facade metric name, kind, unit, and description. Emitters import names from here. | `lib.rs` (`MetricSpec`, `describe_all`), `db.rs`, `dns.rs`, `mitm.rs`, `security.rs`, `virtio_blk.rs` | | `capsem-assets` | VM asset lifecycle and manifest compatibility. | `asset_manager.rs`, `manifest_compat.rs` | diff --git a/skills/dev-testing/references/test-matrix.md b/skills/dev-testing/references/test-matrix.md index 168508cae..24cec7d0b 100644 --- a/skills/dev-testing/references/test-matrix.md +++ b/skills/dev-testing/references/test-matrix.md @@ -15,6 +15,7 @@ this table, enforced by `tests/citadel/test_rust_workspace_documentation.py`. | `capsem-foundation` | Host primitives | Yes | Compile/no-run | Clippy | Yes | | `capsem-crypto` | Cryptographic primitives and integrity verification; speed measured by `capsem-bench` | Yes | Compile/no-run | Clippy | Yes | | `capsem-archive` | Session body archive | Yes | Compile/no-run | Clippy | Yes | +| `capsem-crypto` | Cryptographic providers and primitives | Yes | Compile/no-run | Clippy | Yes | | `capsem-ledger` | Confined ledger and archive codec worker | Yes | Subprocess | Clippy | Yes | | `capsem-telemetry` | Metric name/unit/description catalog | Yes | Compile/no-run | Clippy | Yes | | `capsem-assets` | Asset lifecycle | Yes | Compile/no-run | Clippy | Yes | diff --git a/skills/site-architecture/references/crate-and-privilege-model.md b/skills/site-architecture/references/crate-and-privilege-model.md index f6a97ce18..40073cb66 100644 --- a/skills/site-architecture/references/crate-and-privilege-model.md +++ b/skills/site-architecture/references/crate-and-privilege-model.md @@ -18,6 +18,8 @@ Sharing alone is not a reason to put code in `capsem-core`. - **`capsem-archive`**: block-compressed body archive for session ledgers. Pure Rust (`miniz_oxide`); SQLite keeps the index, this crate keeps the bytes, and every block is blake3-verified before a body is returned. +- **`capsem-crypto`**: central owner of hashing, randomness, TLS providers, + signatures, constant-time verification, and secret handling. - **`capsem-ledger`**: confined session-ledger worker and the sole runtime executor of body archive codecs. - **`capsem-telemetry`**: the one owner of every `metrics`-facade metric name, diff --git a/tests/citadel/cargo_lock_identity.toml b/tests/citadel/cargo_lock_identity.toml index 9ca3dff00..765465f5d 100644 --- a/tests/citadel/cargo_lock_identity.toml +++ b/tests/citadel/cargo_lock_identity.toml @@ -7,5 +7,5 @@ # each moved the lock. `test_cargo_lock_changes_are_deliberate` fails until # this record matches the lock again, so the cost is chosen, reviewed and # batched rather than discovered twenty minutes into a gate. -sha256 = "a785685eb97e924552398310bf5ba73d7d924b6150dbf10c74ef349ac1032a2f" -reason = "The 0.7 security review replaces archived serde_yaml and unsafe-libyaml with the maintained safe-Rust serde-saphyr parser. The host and sealed guest cohort will be rebuilt and qualified together." +sha256 = "9c77a9aa03c2ef6a6537b0903ea60c0bc7b3ac62c4ee0ca9b2511237d9cea58e" +reason = "The merged 0.7 line centralizes cryptographic dependencies in capsem-crypto while #206 adds isolated ledger and archive codecs; the host and sealed guest cohort will be rebuilt and qualified together." diff --git a/tests/helpers/benchmark_ratchet.py b/tests/helpers/benchmark_ratchet.py index af639aacf..b08dc80e5 100644 --- a/tests/helpers/benchmark_ratchet.py +++ b/tests/helpers/benchmark_ratchet.py @@ -26,11 +26,11 @@ import json import os import subprocess +import tomllib from enum import StrEnum from pathlib import Path from typing import Any -import tomllib from pydantic import BaseModel, ConfigDict, Field diff --git a/tests/test_benchmark_retention_contract.py b/tests/test_benchmark_retention_contract.py index 26c2e7390..bbd53b8bd 100644 --- a/tests/test_benchmark_retention_contract.py +++ b/tests/test_benchmark_retention_contract.py @@ -6,10 +6,10 @@ import json import subprocess import sys +import tomllib from pathlib import Path import pytest -import tomllib from helpers.benchmark_ratchet import ( BenchmarkCategory, BenchmarkMetric, From 16acf1a59553f35e06c9e5ae98b644a80ff92a29 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:00:20 +0000 Subject: [PATCH 280/464] chore(api): approve internal gateway snapshot --- config/public-surface.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/public-surface.toml b/config/public-surface.toml index aae48c5e1..e156b93a6 100644 --- a/config/public-surface.toml +++ b/config/public-surface.toml @@ -84,7 +84,7 @@ approved = [ ] [http] -count = 98 +count = 100 approved = [ # All nine connection/grant routes approved by the user on 2026-10-14. "GET /connections", @@ -108,6 +108,7 @@ approved = [ "GET /enforcement/status", "GET /host-logs/{name}", "GET /images", + "GET /internal/gateway/status", "GET /panics", "GET /security/latest", "GET /security/status", From abed1f1b98ca502e067e4505925d2b6f30e27829 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:02:49 +0000 Subject: [PATCH 281/464] feat(telemetry): define shared cache metrics --- crates/capsem-telemetry/src/cache.rs | 37 ++++++++++++++++++++++++++++ crates/capsem-telemetry/src/lib.rs | 2 ++ crates/capsem-telemetry/src/tests.rs | 1 + 3 files changed, 40 insertions(+) create mode 100644 crates/capsem-telemetry/src/cache.rs diff --git a/crates/capsem-telemetry/src/cache.rs b/crates/capsem-telemetry/src/cache.rs new file mode 100644 index 000000000..14aaef7f6 --- /dev/null +++ b/crates/capsem-telemetry/src/cache.rs @@ -0,0 +1,37 @@ +//! Shared in-memory cache metrics. + +use metrics::Unit; + +use crate::MetricSpec; + +/// Cache lookups, partitioned by cache and result (`hit` or `miss`). +pub const CACHE_LOOKUPS_TOTAL: &str = "cache.lookups_total"; +/// Cache mutations, partitioned by cache and operation. +pub const CACHE_MUTATIONS_TOTAL: &str = "cache.mutations_total"; +/// Time spent computing a value after a cache miss, partitioned by cache and status. +pub const CACHE_LOAD_DURATION_MS: &str = "cache.load_duration_ms"; +/// Current resident entries, partitioned by cache. +pub const CACHE_ENTRIES: &str = "cache.entries"; + +pub const SPECS: &[MetricSpec] = &[ + MetricSpec::counter( + CACHE_LOOKUPS_TOTAL, + Unit::Count, + "Cache lookups, partitioned by cache and result (hit or miss).", + ), + MetricSpec::counter( + CACHE_MUTATIONS_TOTAL, + Unit::Count, + "Cache mutations, partitioned by cache and operation (insert, refresh, invalidate, evict, or load_error).", + ), + MetricSpec::histogram( + CACHE_LOAD_DURATION_MS, + Unit::Milliseconds, + "Wall time spent computing a value after a cache miss, partitioned by cache and status.", + ), + MetricSpec::gauge( + CACHE_ENTRIES, + Unit::Count, + "Current resident cache entries, partitioned by cache.", + ), +]; diff --git a/crates/capsem-telemetry/src/lib.rs b/crates/capsem-telemetry/src/lib.rs index 7195699ab..b61ec9b36 100644 --- a/crates/capsem-telemetry/src/lib.rs +++ b/crates/capsem-telemetry/src/lib.rs @@ -16,6 +16,7 @@ use metrics::{KeyName, SharedString, Unit}; +pub mod cache; pub mod db; pub mod dns; #[cfg(feature = "export")] @@ -68,6 +69,7 @@ impl MetricSpec { /// Every domain's table, keyed by the module that owns it. pub const DOMAINS: &[(&str, &[MetricSpec])] = &[ + ("cache", cache::SPECS), ("db", db::SPECS), ("dns", dns::SPECS), ("mitm", mitm::SPECS), diff --git a/crates/capsem-telemetry/src/tests.rs b/crates/capsem-telemetry/src/tests.rs index bf16f342e..e49da8992 100644 --- a/crates/capsem-telemetry/src/tests.rs +++ b/crates/capsem-telemetry/src/tests.rs @@ -7,6 +7,7 @@ use super::*; /// Module sources, so a name declared as a `pub const` but left out of its /// module's `SPECS` table fails here instead of recording without metadata. const SOURCES: &[(&str, &str)] = &[ + ("cache", include_str!("cache.rs")), ("db", include_str!("db.rs")), ("dns", include_str!("dns.rs")), ("mitm", include_str!("mitm.rs")), From 4b1f9e8355fb8e556c75ca00521d12913d373914 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:07:29 +0000 Subject: [PATCH 282/464] feat(cache): centralize observable cache mechanics --- AGENTS.md | 1 + Cargo.lock | 21 +++ Cargo.toml | 7 + crates/capsem-cache/Cargo.toml | 18 ++ crates/capsem-cache/src/bounded.rs | 158 ++++++++++++++++ crates/capsem-cache/src/lib.rs | 15 ++ crates/capsem-cache/src/snapshot.rs | 174 ++++++++++++++++++ crates/capsem-cache/src/telemetry.rs | 119 ++++++++++++ crates/capsem-cache/src/tests.rs | 135 ++++++++++++++ skills/dev-capsem/SKILL.md | 2 +- skills/dev-testing/references/test-matrix.md | 2 +- .../references/crate-and-privilege-model.md | 7 +- tests/citadel/test_hot_build_contract.py | 5 +- .../citadel/test_rust_workspace_boundaries.py | 3 + 14 files changed, 658 insertions(+), 9 deletions(-) create mode 100644 crates/capsem-cache/Cargo.toml create mode 100644 crates/capsem-cache/src/bounded.rs create mode 100644 crates/capsem-cache/src/lib.rs create mode 100644 crates/capsem-cache/src/snapshot.rs create mode 100644 crates/capsem-cache/src/telemetry.rs create mode 100644 crates/capsem-cache/src/tests.rs diff --git a/AGENTS.md b/AGENTS.md index d8a74809c..e1616b37c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,7 @@ See `/dev-just` for the full recipe reference and dependency chains. ``` crates/capsem-foundation/ Low-level paths, UDS, logging, polling, and IPC handshake +crates/capsem-cache/ Bounded cache primitives, freshness publication, and cache telemetry crates/capsem-crypto/ Accelerated hashes, randomness, TLS keys, JWT verification, secrets crates/capsem-archive/ Block-compressed body archive for session ledgers (pure Rust) crates/capsem-ledger/ Confined session-ledger worker and archive codec executor diff --git a/Cargo.lock b/Cargo.lock index e7bba1c01..a9f379a3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -850,6 +850,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "capsem-cache" +version = "0.7.0" +dependencies = [ + "capsem-telemetry", + "metrics", + "quick_cache", +] + [[package]] name = "capsem-config" version = "0.7.0" @@ -5366,6 +5375,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick_cache" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" +dependencies = [ + "equivalent", + "foldhash 0.2.0", + "hashbrown 0.17.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.12" diff --git a/Cargo.toml b/Cargo.toml index d6311661a..150cacb62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "crates/capsem-cache", "crates/capsem-crypto", "sdk/rust", "crates/capsem-api", @@ -62,6 +63,12 @@ opt-level = 3 [profile.dev.package.capsem-crypto] opt-level = 3 +[profile.dev.package.capsem-cache] +opt-level = 3 + +[profile.dev.package.quick_cache] +opt-level = 3 + [profile.dev.package.blake2b_simd] opt-level = 3 diff --git a/crates/capsem-cache/Cargo.toml b/crates/capsem-cache/Cargo.toml new file mode 100644 index 000000000..7176fde71 --- /dev/null +++ b/crates/capsem-cache/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "capsem-cache" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +description = "Bounded, observable cache primitives for Capsem" +homepage.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +capsem-telemetry = { path = "../capsem-telemetry" } +metrics = "0.24" +quick_cache = { version = "0.7", features = ["stats"] } + +[lints] +workspace = true diff --git a/crates/capsem-cache/src/bounded.rs b/crates/capsem-cache/src/bounded.rs new file mode 100644 index 000000000..af08e5413 --- /dev/null +++ b/crates/capsem-cache/src/bounded.rs @@ -0,0 +1,158 @@ +use std::hash::Hash; +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Instant; + +use quick_cache::sync::{Cache, GuardResult}; +use quick_cache::{DefaultHashBuilder, Equivalent, Lifecycle, UnitWeighter}; + +use crate::telemetry::{CacheStats, CacheTelemetry, LoadStatus}; + +struct EvictionLifecycle { + telemetry: CacheTelemetry, + marker: PhantomData, +} + +impl Clone for EvictionLifecycle { + fn clone(&self) -> Self { + Self { + telemetry: self.telemetry.clone(), + marker: PhantomData, + } + } +} + +impl Lifecycle> for EvictionLifecycle { + type RequestState = (); + + fn on_evict(&self, _state: &mut Self::RequestState, _key: K, _value: Arc) { + self.telemetry.evict(); + } +} + +type Inner = Cache, UnitWeighter, DefaultHashBuilder, EvictionLifecycle>; + +pub struct BoundedCache { + inner: Inner, + telemetry: CacheTelemetry, +} + +impl BoundedCache +where + K: Eq + Hash, +{ + pub fn new(name: &'static str, capacity: usize) -> Self { + let capacity = capacity.max(1); + let telemetry = CacheTelemetry::new(name); + Self { + inner: Cache::with( + capacity, + u64::try_from(capacity).unwrap_or(u64::MAX), + UnitWeighter, + DefaultHashBuilder::default(), + EvictionLifecycle { + telemetry: telemetry.clone(), + marker: PhantomData, + }, + ), + telemetry, + } + } + + pub fn get(&self, key: &Q) -> Option> + where + Q: Equivalent + Hash + ?Sized, + { + let value = self.inner.get(key); + if value.is_some() { + self.telemetry.hit(); + } else { + self.telemetry.miss(); + } + value + } + + pub fn insert(&self, key: K, value: V) -> Arc { + let replacing = self.inner.peek(&key).is_some(); + let value = Arc::new(value); + self.inner.insert(key, Arc::clone(&value)); + if replacing { + self.telemetry.refresh(); + } else { + self.telemetry.insertion(); + } + self.telemetry.set_entries(self.inner.len()); + value + } + + pub fn get_or_try_insert_with(&self, key: &Q, load: impl FnOnce() -> Result) -> Result, E> + where + Q: Equivalent + Hash + ToOwned + ?Sized, + { + match self.inner.get_value_or_guard(key, None) { + GuardResult::Value(value) => { + self.telemetry.hit(); + Ok(value) + } + GuardResult::Guard(guard) => { + self.telemetry.miss(); + let started = Instant::now(); + match load() { + Ok(value) => { + let value = Arc::new(value); + let _inserted = guard.insert(Arc::clone(&value)); + self.telemetry.load(started.elapsed(), LoadStatus::Success); + self.telemetry.insertion(); + self.telemetry.set_entries(self.inner.len()); + Ok(value) + } + Err(error) => { + self.telemetry.load(started.elapsed(), LoadStatus::Error); + Err(error) + } + } + } + GuardResult::Timeout => unreachable!("an unbounded cache wait cannot time out"), + } + } + + pub fn remove(&self, key: &Q) -> Option> + where + Q: Equivalent + Hash + ?Sized, + { + let removed = self.inner.remove(key).map(|(_, value)| value); + if removed.is_some() { + self.telemetry.invalidate(1); + self.telemetry.set_entries(self.inner.len()); + } + removed + } + + pub fn retain(&self, retain: impl Fn(&K, &Arc) -> bool) { + let before = self.inner.len(); + self.inner.retain(retain); + let after = self.inner.len(); + self.telemetry + .invalidate(u64::try_from(before.saturating_sub(after)).unwrap_or(u64::MAX)); + self.telemetry.set_entries(after); + } + + pub fn clear(&self) { + let before = self.inner.len(); + self.inner.clear(); + self.telemetry.invalidate(u64::try_from(before).unwrap_or(u64::MAX)); + self.telemetry.set_entries(0); + } + + pub fn len(&self) -> usize { + self.inner.len() + } + + pub fn is_empty(&self) -> bool { + self.inner.is_empty() + } + + pub fn stats(&self) -> CacheStats { + self.telemetry.snapshot() + } +} diff --git a/crates/capsem-cache/src/lib.rs b/crates/capsem-cache/src/lib.rs new file mode 100644 index 000000000..d720843b9 --- /dev/null +++ b/crates/capsem-cache/src/lib.rs @@ -0,0 +1,15 @@ +//! Bounded, observable cache primitives. +//! +//! Domain owners decide what makes a value fresh. This crate owns the shared +//! concurrency, capacity and telemetry mechanics used to retain those values. + +mod bounded; +mod snapshot; +mod telemetry; + +pub use bounded::BoundedCache; +pub use snapshot::{GenerationCache, GenerationLookup, IdentityCache}; +pub use telemetry::{CacheStats, LoadStatus}; + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-cache/src/snapshot.rs b/crates/capsem-cache/src/snapshot.rs new file mode 100644 index 000000000..ff28e30f1 --- /dev/null +++ b/crates/capsem-cache/src/snapshot.rs @@ -0,0 +1,174 @@ +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use crate::telemetry::{CacheStats, CacheTelemetry, LoadStatus}; + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +struct IdentityEntry { + identity: I, + value: Arc, +} + +pub struct IdentityCache { + entry: Mutex>>, + telemetry: CacheTelemetry, +} + +impl IdentityCache { + pub fn new(name: &'static str) -> Self { + Self { + entry: Mutex::new(None), + telemetry: CacheTelemetry::new(name), + } + } + + pub fn get(&self, identity: &I) -> Option> { + let value = lock(&self.entry) + .as_ref() + .filter(|entry| &entry.identity == identity) + .map(|entry| Arc::clone(&entry.value)); + if value.is_some() { + self.telemetry.hit(); + } else { + self.telemetry.miss(); + } + value + } + + pub fn publish(&self, identity: I, value: V) -> Arc { + let mut entry = lock(&self.entry); + if let Some(existing) = entry.as_ref().filter(|entry| entry.identity == identity) { + return Arc::clone(&existing.value); + } + let refresh = entry.is_some(); + let value = Arc::new(value); + *entry = Some(IdentityEntry { + identity, + value: Arc::clone(&value), + }); + drop(entry); + if refresh { + self.telemetry.refresh(); + } else { + self.telemetry.insertion(); + } + self.telemetry.set_entries(1); + value + } + + pub fn invalidate(&self) { + let removed = lock(&self.entry).take().is_some(); + if removed { + self.telemetry.invalidate(1); + self.telemetry.set_entries(0); + } + } + + pub fn record_load(&self, elapsed: Duration, status: LoadStatus) { + self.telemetry.load(elapsed, status); + } + + pub fn stats(&self) -> CacheStats { + self.telemetry.snapshot() + } +} + +impl Default for IdentityCache { + fn default() -> Self { + Self::new("unnamed") + } +} + +struct GenerationState { + generation: u64, + value: Option>, +} + +pub struct GenerationLookup { + generation: u64, + value: Option>, +} + +impl GenerationLookup { + pub fn generation(&self) -> u64 { + self.generation + } + + pub fn value(&self) -> Option> { + self.value.as_ref().map(Arc::clone) + } +} + +pub struct GenerationCache { + state: Mutex>, + telemetry: CacheTelemetry, +} + +impl GenerationCache { + pub fn new(name: &'static str) -> Self { + Self { + state: Mutex::new(GenerationState { + generation: 0, + value: None, + }), + telemetry: CacheTelemetry::new(name), + } + } + + pub fn lookup(&self) -> GenerationLookup { + let state = lock(&self.state); + let lookup = GenerationLookup { + generation: state.generation, + value: state.value.as_ref().map(Arc::clone), + }; + drop(state); + if lookup.value.is_some() { + self.telemetry.hit(); + } else { + self.telemetry.miss(); + } + lookup + } + + pub fn publish(&self, generation: u64, value: V) -> Option> { + let mut state = lock(&self.state); + if state.generation != generation { + return None; + } + if let Some(existing) = &state.value { + return Some(Arc::clone(existing)); + } + let value = Arc::new(value); + state.value = Some(Arc::clone(&value)); + drop(state); + self.telemetry.insertion(); + self.telemetry.set_entries(1); + Some(value) + } + + pub fn invalidate(&self) { + let mut state = lock(&self.state); + state.generation = state.generation.wrapping_add(1); + state.value = None; + drop(state); + self.telemetry.invalidate(1); + self.telemetry.set_entries(0); + } + + pub fn record_load(&self, elapsed: Duration, status: LoadStatus) { + self.telemetry.load(elapsed, status); + } + + pub fn stats(&self) -> CacheStats { + self.telemetry.snapshot() + } +} + +impl Default for GenerationCache { + fn default() -> Self { + Self::new("unnamed") + } +} diff --git a/crates/capsem-cache/src/telemetry.rs b/crates/capsem-cache/src/telemetry.rs new file mode 100644 index 000000000..c76775c5a --- /dev/null +++ b/crates/capsem-cache/src/telemetry.rs @@ -0,0 +1,119 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use capsem_telemetry::cache as metric; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum LoadStatus { + Success, + Error, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct CacheStats { + pub hits: u64, + pub misses: u64, + pub insertions: u64, + pub refreshes: u64, + pub invalidations: u64, + pub evictions: u64, + pub load_errors: u64, + pub entries: u64, +} + +#[derive(Default)] +struct Counters { + hits: AtomicU64, + misses: AtomicU64, + insertions: AtomicU64, + refreshes: AtomicU64, + invalidations: AtomicU64, + evictions: AtomicU64, + load_errors: AtomicU64, + entries: AtomicU64, +} + +#[derive(Clone)] +pub(crate) struct CacheTelemetry { + name: &'static str, + counters: Arc, +} + +impl CacheTelemetry { + pub(crate) fn new(name: &'static str) -> Self { + let telemetry = Self { + name, + counters: Arc::default(), + }; + telemetry.set_entries(0); + telemetry + } + + pub(crate) fn hit(&self) { + self.counters.hits.fetch_add(1, Ordering::Relaxed); + ::metrics::counter!(metric::CACHE_LOOKUPS_TOTAL, "cache" => self.name, "result" => "hit").increment(1); + } + + pub(crate) fn miss(&self) { + self.counters.misses.fetch_add(1, Ordering::Relaxed); + ::metrics::counter!(metric::CACHE_LOOKUPS_TOTAL, "cache" => self.name, "result" => "miss").increment(1); + } + + pub(crate) fn insertion(&self) { + self.mutation(&self.counters.insertions, "insert", 1); + } + + pub(crate) fn refresh(&self) { + self.mutation(&self.counters.refreshes, "refresh", 1); + } + + pub(crate) fn invalidate(&self, count: u64) { + self.mutation(&self.counters.invalidations, "invalidate", count); + } + + pub(crate) fn evict(&self) { + self.mutation(&self.counters.evictions, "evict", 1); + } + + fn mutation(&self, counter: &AtomicU64, operation: &'static str, count: u64) { + if count == 0 { + return; + } + counter.fetch_add(count, Ordering::Relaxed); + ::metrics::counter!(metric::CACHE_MUTATIONS_TOTAL, "cache" => self.name, "operation" => operation) + .increment(count); + } + + pub(crate) fn load(&self, elapsed: Duration, status: LoadStatus) { + let status = match status { + LoadStatus::Success => "success", + LoadStatus::Error => { + self.mutation(&self.counters.load_errors, "load_error", 1); + "error" + } + }; + ::metrics::histogram!(metric::CACHE_LOAD_DURATION_MS, "cache" => self.name, "status" => status) + .record(elapsed.as_secs_f64() * 1000.0); + } + + pub(crate) fn set_entries(&self, entries: usize) { + let entries = u64::try_from(entries).unwrap_or(u64::MAX); + self.counters.entries.store(entries, Ordering::Relaxed); + ::metrics::gauge!(metric::CACHE_ENTRIES, "cache" => self.name).set(entries as f64); + } + + pub(crate) fn snapshot(&self) -> CacheStats { + let load = |counter: &AtomicU64| counter.load(Ordering::Relaxed); + CacheStats { + hits: load(&self.counters.hits), + misses: load(&self.counters.misses), + insertions: load(&self.counters.insertions), + refreshes: load(&self.counters.refreshes), + invalidations: load(&self.counters.invalidations), + evictions: load(&self.counters.evictions), + load_errors: load(&self.counters.load_errors), + entries: load(&self.counters.entries), + } + } +} diff --git a/crates/capsem-cache/src/tests.rs b/crates/capsem-cache/src/tests.rs new file mode 100644 index 000000000..57fe26a90 --- /dev/null +++ b/crates/capsem-cache/src/tests.rs @@ -0,0 +1,135 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::sync::Barrier; +use std::time::Duration; + +use super::*; + +#[test] +fn identity_cache_counts_hits_refreshes_and_invalidations() { + let cache = IdentityCache::new("asset_manifest"); + + assert!(cache.get(&"v1").is_none()); + let first = cache.publish("v1", String::from("first")); + let hit = cache.get(&"v1").expect("published value"); + assert!(Arc::ptr_eq(&first, &hit)); + + let second = cache.publish("v2", String::from("second")); + assert_eq!(&*second, "second"); + assert!(cache.get(&"v1").is_none()); + cache.invalidate(); + + assert_eq!( + cache.stats(), + CacheStats { + hits: 1, + misses: 2, + insertions: 1, + refreshes: 1, + invalidations: 1, + evictions: 0, + load_errors: 0, + entries: 0, + } + ); +} + +#[test] +fn generation_cache_rejects_a_load_started_before_invalidation() { + let cache = GenerationCache::new("plugin_policy"); + let stale = cache.lookup(); + assert!(stale.value().is_none()); + cache.invalidate(); + assert!(cache.publish(stale.generation(), "stale").is_none()); + + let current = cache.lookup(); + let published = cache + .publish(current.generation(), "current") + .expect("current generation publishes"); + assert_eq!(*published, "current"); + assert_eq!(cache.lookup().value().as_deref(), Some(&"current")); + assert_eq!(cache.stats().invalidations, 1); + assert_eq!(cache.stats().entries, 1); +} + +#[test] +fn bounded_cache_single_flights_loads_and_reports_errors() { + let cache = BoundedCache::::new("responses", 8); + let mut loads = 0; + let first = cache + .get_or_try_insert_with("route", || { + loads += 1; + Ok::<_, ()>(String::from("body")) + }) + .unwrap(); + let second = cache + .get_or_try_insert_with("route", || { + loads += 1; + Ok::<_, ()>(String::from("wrong")) + }) + .unwrap(); + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(loads, 1); + + let error = cache.get_or_try_insert_with("broken", || Err::("load failed")); + assert_eq!(error.unwrap_err(), "load failed"); + assert_eq!( + cache.stats(), + CacheStats { + hits: 1, + misses: 2, + insertions: 1, + refreshes: 0, + invalidations: 0, + evictions: 0, + load_errors: 1, + entries: 1, + } + ); +} + +#[test] +fn bounded_cache_computes_one_value_for_concurrent_misses() { + let cache = Arc::new(BoundedCache::::new("single_flight", 8)); + let start = Arc::new(Barrier::new(8)); + let loads = Arc::new(AtomicUsize::new(0)); + let mut threads = Vec::new(); + for _ in 0..8 { + let cache = Arc::clone(&cache); + let start = Arc::clone(&start); + let loads = Arc::clone(&loads); + threads.push(std::thread::spawn(move || { + start.wait(); + cache + .get_or_try_insert_with("key", || { + loads.fetch_add(1, Ordering::Relaxed); + std::thread::sleep(Duration::from_millis(10)); + Ok::<_, ()>(42) + }) + .unwrap() + })); + } + let values: Vec<_> = threads.into_iter().map(|thread| thread.join().unwrap()).collect(); + + assert_eq!(loads.load(Ordering::Relaxed), 1); + assert!(values.iter().all(|value| **value == 42)); + assert!(values.windows(2).all(|pair| Arc::ptr_eq(&pair[0], &pair[1]))); + assert_eq!(cache.stats().misses, 1); + assert_eq!(cache.stats().hits, 7); +} + +#[test] +fn bounded_cache_enforces_capacity_and_tracks_removal() { + let cache = BoundedCache::new("bounded", 2); + for key in 0..32 { + cache.insert(key, key * 10); + } + assert!(cache.len() <= 2, "resident entries must remain bounded"); + assert!(cache.stats().evictions > 0); + + let before = cache.len(); + cache.clear(); + assert!(cache.is_empty()); + assert_eq!(cache.stats().entries, 0); + assert_eq!(cache.stats().invalidations, before as u64); +} diff --git a/skills/dev-capsem/SKILL.md b/skills/dev-capsem/SKILL.md index 3d1170bcf..b671f196d 100644 --- a/skills/dev-capsem/SKILL.md +++ b/skills/dev-capsem/SKILL.md @@ -28,7 +28,7 @@ then lead with a recommendation. The full contract is | Crate | What | Key modules | |-------|------|-------------| | `capsem-foundation` | Dependency-light host primitives shared across product crates. | `paths.rs`, `uds.rs`, `poll.rs`, `telemetry.rs`, `log_layer.rs`, `ipc_handshake.rs` | -| `capsem-crypto` | Accelerated hashes, integrity verification, randomness, TLS keys, JWT verification and secret primitives. | `verify.rs`, `random.rs`, `tls.rs`, `jwt.rs`, `secrets.rs` | +| `capsem-cache` | Bounded observable cache mechanics; callers own freshness policy. | `bounded.rs`, `snapshot.rs`, `telemetry.rs` | | `capsem-archive` | Append-only, block-compressed body archive for one session ledger. | `format.rs`, `writer.rs`, `reader.rs` | | `capsem-crypto` | Central owner of hashing, randomness, TLS providers, signatures, and secret handling. | `lib.rs`, `random.rs`, `tls.rs`, `verify.rs` | | `capsem-ledger` | Confined session-ledger worker and archive codec executor. | `main.rs`, `codec.rs` | diff --git a/skills/dev-testing/references/test-matrix.md b/skills/dev-testing/references/test-matrix.md index 24cec7d0b..8af9b37a4 100644 --- a/skills/dev-testing/references/test-matrix.md +++ b/skills/dev-testing/references/test-matrix.md @@ -13,7 +13,7 @@ this table, enforced by `tests/citadel/test_rust_workspace_documentation.py`. | Crate | Role | CI macOS | CI Linux | Fast source | Full | |-------|------|:--------:|:--------:|:-----------:|:----:| | `capsem-foundation` | Host primitives | Yes | Compile/no-run | Clippy | Yes | -| `capsem-crypto` | Cryptographic primitives and integrity verification; speed measured by `capsem-bench` | Yes | Compile/no-run | Clippy | Yes | +| `capsem-cache` | Bounded observable cache primitives | Yes | Compile/no-run | Clippy | Yes | | `capsem-archive` | Session body archive | Yes | Compile/no-run | Clippy | Yes | | `capsem-crypto` | Cryptographic providers and primitives | Yes | Compile/no-run | Clippy | Yes | | `capsem-ledger` | Confined ledger and archive codec worker | Yes | Subprocess | Clippy | Yes | diff --git a/skills/site-architecture/references/crate-and-privilege-model.md b/skills/site-architecture/references/crate-and-privilege-model.md index 40073cb66..18e85cebe 100644 --- a/skills/site-architecture/references/crate-and-privilege-model.md +++ b/skills/site-architecture/references/crate-and-privilege-model.md @@ -11,10 +11,9 @@ Sharing alone is not a reason to put code in `capsem-core`. - **`capsem-foundation`**: dependency-light host primitives: paths, UDS HTTP, polling, telemetry/log setup, and IPC handshakes. -- **`capsem-crypto`**: first-party Rust cryptographic operations: accelerated - hashes and bounded integrity verification, randomness, TLS key/provider - selection, JWT verification and secret primitives. Speed measurements belong - in the existing `capsem-bench` harness, not a separate benchmark crate. +- **`capsem-cache`**: bounded cache storage, concurrency-safe freshness + publication, and uniform hit/miss/load/eviction telemetry. Domain owners + supply content identities, generations, and TTL policy. - **`capsem-archive`**: block-compressed body archive for session ledgers. Pure Rust (`miniz_oxide`); SQLite keeps the index, this crate keeps the bytes, and every block is blake3-verified before a body is returned. diff --git a/tests/citadel/test_hot_build_contract.py b/tests/citadel/test_hot_build_contract.py index 4c26fb8d7..ce5300f38 100644 --- a/tests/citadel/test_hot_build_contract.py +++ b/tests/citadel/test_hot_build_contract.py @@ -16,6 +16,7 @@ HOT_DEV_OPTIMIZED_PACKAGES = { "blake3", "sha2", + "capsem-cache", "capsem-crypto", "blake2b_simd", "blake2s_simd", @@ -28,9 +29,7 @@ "memchr", "itoa", "ryu", - "rusqlite", - "libsqlite3-sys", - "capsem-logger", + "quick_cache", } RATIONALE = """\ diff --git a/tests/citadel/test_rust_workspace_boundaries.py b/tests/citadel/test_rust_workspace_boundaries.py index 560403ba8..58f5c5260 100644 --- a/tests/citadel/test_rust_workspace_boundaries.py +++ b/tests/citadel/test_rust_workspace_boundaries.py @@ -19,6 +19,9 @@ PROJECT_ROOT = Path(__file__).resolve().parents[2] FOCUSED_DEPENDENCY_RULES = { + "capsem-cache": frozenset( + {"capsem-core", "capsem-logger", "capsem-assets", "capsem-config", "rusqlite", "tokio"} + ), "capsem-foundation": frozenset( {"capsem-core", "capsem-logger", "capsem-assets", "rmcp", "rusqlite"} ), From a1aa35adb4b5485e306ee5e1880812473c925fe9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:10:42 +0000 Subject: [PATCH 283/464] fix(bench): resolve seeded route templates --- benchmarks/collectors/routes | 49 ++++++++++++------------- config/gate.toml | 5 ++- tests/test_benchmark_route_collector.py | 28 ++++++++++++++ 3 files changed, 56 insertions(+), 26 deletions(-) create mode 100644 tests/test_benchmark_route_collector.py diff --git a/benchmarks/collectors/routes b/benchmarks/collectors/routes index b6cd930be..a16bd69ed 100755 --- a/benchmarks/collectors/routes +++ b/benchmarks/collectors/routes @@ -67,15 +67,11 @@ def measured_routes() -> list[str]: return list(config["benchmark"]["routes"]["measured"]) -def request_path(pattern: str, live_session_id: str | None = None) -> str: - """Resolve the measured subject to a seeded, successful fixture URL.""" - if pattern in LIVE_ROUTES and live_session_id is None: - raise ValueError(f"measured route requires a live owner fixture: {pattern}") - session_id = live_session_id if pattern in LIVE_ROUTES else VM_ID - path = BODY_URL if pattern == BODY_ROUTE else pattern.replace("/vms/{id}/", f"/vms/{session_id}/") - if "{" in path or "}" in path: - raise ValueError(f"measured route has no successful fixture: {pattern}") - return path +def request_path(route: str) -> str: + """Resolve a configured route template against the stopped VM fixture.""" + if route == BODY_ROUTE: + return BODY_URL + return route.replace("{id}", VM_ID) @dataclass(frozen=True) @@ -175,22 +171,25 @@ def main() -> int: cpu_processes={"gateway": gateway_proc, "downstream_service": service_proc}, ), ) - with ExitStack() as fixtures: - live_id = None - if LIVE_ROUTES.intersection(routes): - live_id = fixtures.enter_context( - debug_session(service, service.home_dir / "route-registry", "route-live-owner") - ) - requests = {path: request_path(path, live_id) for path in routes} - for path in routes: - url = requests[path] - for target in targets: - print(f"{target.label} {path}", file=sys.stderr) - latencies, cpu = sample_warm_route(target, url, samples) - cold_latencies = sample_cold_route(target, url, samples) - metrics[f"{target.label}.{path}.latency_ms"] = { - "unit": "milliseconds", - "samples": latencies, + for path in routes: + concrete_path = request_path(path) + for target in targets: + print(f"{target.label} {path}", file=sys.stderr) + latencies, cpu = sample_warm_route(target, concrete_path, samples) + cold_latencies = sample_cold_route(target, concrete_path, samples) + metrics[f"{target.label}.{path}.latency_ms"] = { + "unit": "milliseconds", + "samples": latencies, + } + metrics[f"{target.label}.{path}.cold_latency_ms"] = { + "unit": "milliseconds", + "samples": cold_latencies, + } + for process_name, cpu_seconds in cpu.items(): + suffix = "cpu_s" if process_name == target.label else f"{process_name}_cpu_s" + metrics[f"{target.label}.{path}.{suffix}"] = { + "unit": "seconds", + "samples": [cpu_seconds], } metrics[f"{target.label}.{path}.cold_latency_ms"] = { "unit": "milliseconds", diff --git a/config/gate.toml b/config/gate.toml index 5c5abec4d..fd5bb2835 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3018,7 +3018,6 @@ measured = [ "/vms/{id}/credential-grants", "/vms/{id}/bodies/{event_id}", "/vms/{id}/status", - "/vms/{id}/exposures", "/vms/{id}/security/status", "/status", "/version", @@ -3034,6 +3033,10 @@ measured = [ "/update/status", ] unmeasured = [ + # Exposure inventory is owned by a live VM process. The quick routes + # collector uses a stopped workspace so it can stay host-only; Ironbank's + # workload-surface fixture measures the real owner round trip. + "/vms/{id}/exposures", # Standalone proxy lifecycle mutations require a supervised worker fixture; # their functional acceptance owns successful create, lease, and teardown. "/proxies", diff --git a/tests/test_benchmark_route_collector.py b/tests/test_benchmark_route_collector.py new file mode 100644 index 000000000..2fb7c54bf --- /dev/null +++ b/tests/test_benchmark_route_collector.py @@ -0,0 +1,28 @@ +"""Contracts for the unified HTTP route benchmark collector.""" + +from __future__ import annotations + +import runpy +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] + + +def _collector() -> dict: + return runpy.run_path(str(PROJECT_ROOT / "benchmarks" / "collectors" / "routes")) + + +def test_every_measured_route_resolves_to_a_concrete_fixture_request() -> None: + collector = _collector() + request_path = collector["request_path"] + + unresolved = { + route: request_path(route) + for route in collector["measured_routes"]() + if "{" in request_path(route) or "}" in request_path(route) + } + + assert not unresolved, ( + "the benchmark sent route templates literally, so a fast 404 replaced " + f"the intended latency measurement: {unresolved}" + ) From ce485e9e5b2185d3c6022d9fdecd7721c473a6c9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:32:56 +0000 Subject: [PATCH 284/464] feat(cache): add keyed identity storage --- crates/capsem-cache/src/bounded.rs | 51 ++++++++++++--- crates/capsem-cache/src/keyed_identity.rs | 79 +++++++++++++++++++++++ crates/capsem-cache/src/lib.rs | 2 + crates/capsem-cache/src/tests.rs | 37 +++++++++++ 4 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 crates/capsem-cache/src/keyed_identity.rs diff --git a/crates/capsem-cache/src/bounded.rs b/crates/capsem-cache/src/bounded.rs index af08e5413..3572d0d51 100644 --- a/crates/capsem-cache/src/bounded.rs +++ b/crates/capsem-cache/src/bounded.rs @@ -13,6 +13,16 @@ struct EvictionLifecycle { marker: PhantomData, } +struct EvictionState { + count: bool, +} + +impl Default for EvictionState { + fn default() -> Self { + Self { count: true } + } +} + impl Clone for EvictionLifecycle { fn clone(&self) -> Self { Self { @@ -23,10 +33,12 @@ impl Clone for EvictionLifecycle { } impl Lifecycle> for EvictionLifecycle { - type RequestState = (); + type RequestState = EvictionState; - fn on_evict(&self, _state: &mut Self::RequestState, _key: K, _value: Arc) { - self.telemetry.evict(); + fn on_evict(&self, state: &mut Self::RequestState, _key: K, _value: Arc) { + if state.count { + self.telemetry.evict(); + } } } @@ -63,7 +75,14 @@ where where Q: Equivalent + Hash + ?Sized, { - let value = self.inner.get(key); + self.get_if(key, |_| true) + } + + pub(crate) fn get_if(&self, key: &Q, accept: impl FnOnce(&V) -> bool) -> Option> + where + Q: Equivalent + Hash + ?Sized, + { + let value = self.inner.get(key).filter(|value| accept(value)); if value.is_some() { self.telemetry.hit(); } else { @@ -72,13 +91,29 @@ where value } + pub(crate) fn peek(&self, key: &Q) -> Option> + where + Q: Equivalent + Hash + ?Sized, + { + self.inner.get(key) + } + pub fn insert(&self, key: K, value: V) -> Arc { - let replacing = self.inner.peek(&key).is_some(); let value = Arc::new(value); - self.inner.insert(key, Arc::clone(&value)); - if replacing { - self.telemetry.refresh(); + if self.inner.peek(&key).is_some() { + let mut lifecycle = EvictionState { count: false }; + match self + .inner + .replace_with_lifecycle(key, Arc::clone(&value), false, &mut lifecycle) + { + Ok(()) => self.telemetry.refresh(), + Err((key, value)) => { + self.inner.insert(key, value); + self.telemetry.insertion(); + } + } } else { + self.inner.insert(key, Arc::clone(&value)); self.telemetry.insertion(); } self.telemetry.set_entries(self.inner.len()); diff --git a/crates/capsem-cache/src/keyed_identity.rs b/crates/capsem-cache/src/keyed_identity.rs new file mode 100644 index 000000000..5a0796fa7 --- /dev/null +++ b/crates/capsem-cache/src/keyed_identity.rs @@ -0,0 +1,79 @@ +use std::hash::Hash; +use std::sync::Arc; + +use quick_cache::Equivalent; + +use crate::{BoundedCache, CacheStats}; + +struct IdentityEntry { + identity: I, + value: Arc, +} + +/// A bounded cache whose values are fresh only for the caller's identity. +pub struct KeyedIdentityCache { + entries: BoundedCache>, +} + +impl KeyedIdentityCache +where + K: Eq + Hash, + I: Eq, +{ + pub fn new(name: &'static str, capacity: usize) -> Self { + Self { + entries: BoundedCache::new(name, capacity), + } + } + + pub fn get(&self, key: &Q, identity: &I) -> Option> + where + Q: Equivalent + Hash + ?Sized, + { + self.entries + .get_if(key, |entry| &entry.identity == identity) + .map(|entry| Arc::clone(&entry.value)) + } + + pub fn publish(&self, key: K, identity: I, value: V) -> Arc { + if let Some(entry) = self.entries.peek(&key).filter(|entry| entry.identity == identity) { + return Arc::clone(&entry.value); + } + let value = Arc::new(value); + self.entries.insert( + key, + IdentityEntry { + identity, + value: Arc::clone(&value), + }, + ); + value + } + + pub fn remove(&self, key: &Q) -> Option> + where + Q: Equivalent + Hash + ?Sized, + { + self.entries.remove(key).map(|entry| Arc::clone(&entry.value)) + } + + pub fn retain(&self, retain: impl Fn(&K) -> bool) { + self.entries.retain(|key, _| retain(key)); + } + + pub fn clear(&self) { + self.entries.clear(); + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn stats(&self) -> CacheStats { + self.entries.stats() + } +} diff --git a/crates/capsem-cache/src/lib.rs b/crates/capsem-cache/src/lib.rs index d720843b9..39b8dd433 100644 --- a/crates/capsem-cache/src/lib.rs +++ b/crates/capsem-cache/src/lib.rs @@ -4,10 +4,12 @@ //! concurrency, capacity and telemetry mechanics used to retain those values. mod bounded; +mod keyed_identity; mod snapshot; mod telemetry; pub use bounded::BoundedCache; +pub use keyed_identity::KeyedIdentityCache; pub use snapshot::{GenerationCache, GenerationLookup, IdentityCache}; pub use telemetry::{CacheStats, LoadStatus}; diff --git a/crates/capsem-cache/src/tests.rs b/crates/capsem-cache/src/tests.rs index 57fe26a90..af4f82142 100644 --- a/crates/capsem-cache/src/tests.rs +++ b/crates/capsem-cache/src/tests.rs @@ -34,6 +34,43 @@ fn identity_cache_counts_hits_refreshes_and_invalidations() { ); } +#[test] +fn keyed_identity_cache_bounds_entries_and_refreshes_by_identity() { + let cache = KeyedIdentityCache::new("persistent_resume", 16); + + assert!(cache.get(&"box", &"v1").is_none()); + let first = cache.publish("box", "v1", String::from("ready")); + let hit = cache.get(&"box", &"v1").expect("published value"); + assert!(Arc::ptr_eq(&first, &hit)); + + let refreshed = cache.publish("box", "v2", String::from("stopped")); + assert_eq!(&*refreshed, "stopped"); + assert!(cache.get(&"box", &"v1").is_none()); + assert_eq!(cache.stats().evictions, 0, "refresh is not an eviction"); + assert_eq!(cache.remove(&"box").as_deref(), Some(&String::from("stopped"))); + + assert_eq!( + cache.stats(), + CacheStats { + hits: 1, + misses: 2, + insertions: 1, + refreshes: 1, + invalidations: 1, + evictions: 0, + load_errors: 0, + entries: 0, + } + ); + + let bounded = KeyedIdentityCache::new("bounded_identity", 2); + for key in 0..32 { + bounded.publish(key, key, key * 10); + } + assert!(bounded.len() <= 2, "resident identity entries must remain bounded"); + assert!(bounded.stats().evictions > 0); +} + #[test] fn generation_cache_rejects_a_load_started_before_invalidation() { let cache = GenerationCache::new("plugin_policy"); From 39cfb7fcb5769dceb3b78ffbd9bf6dcc21dbfef9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:35:01 +0000 Subject: [PATCH 285/464] perf(service): unify polled route caches --- CHANGELOG.md | 4 +- Cargo.lock | 1 + crates/capsem-service/Cargo.toml | 1 + crates/capsem-service/src/asset_routes.rs | 33 ++------ crates/capsem-service/src/ledger_routes.rs | 18 ----- .../src/ledger_routes/response_cache.rs | 75 ------------------- crates/capsem-service/src/main.rs | 65 ++++------------ crates/capsem-service/src/plugin_routes.rs | 55 +++++--------- crates/capsem-service/src/router_runtime.rs | 2 +- crates/capsem-service/src/sandbox_info.rs | 18 ++--- crates/capsem-service/src/service_runtime.rs | 11 ++- .../capsem-service/src/session_db_handles.rs | 9 --- crates/capsem-service/src/tests.rs | 11 ++- crates/capsem-service/src/tests/inspection.rs | 40 ---------- .../capsem-service/src/tests/interactions.rs | 1 - tests/citadel/test_db_boundary.py | 2 + 16 files changed, 67 insertions(+), 279 deletions(-) delete mode 100644 crates/capsem-service/src/ledger_routes/response_cache.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index d58f2e13f..b737c9b6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Serve live and stopped-session inspection through coordinator-granted, role-bound ledger channels and bounded named operations, eliminating the service's per-session SQLite readers and closing ledger workers before a - session is deleted. + session is deleted. Ledger counter replies carry the DB-owned read-cache + epoch, while logger-owned query caches invalidate security rows after every + commit without a second service projection cache. - Give each VM owner one coordinator-minted, generation- and role-bound ledger channel and route its existing event writer through the supervised diff --git a/Cargo.lock b/Cargo.lock index a9f379a3e..ddefa80b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1281,6 +1281,7 @@ dependencies = [ "bytes", "capsem-api", "capsem-assets", + "capsem-cache", "capsem-config", "capsem-core", "capsem-credentials", diff --git a/crates/capsem-service/Cargo.toml b/crates/capsem-service/Cargo.toml index 2ed89ef82..670e824b9 100644 --- a/crates/capsem-service/Cargo.toml +++ b/crates/capsem-service/Cargo.toml @@ -14,6 +14,7 @@ capsem-crypto = { path = "../capsem-crypto", default-features = false, features capsem-credentials = { path = "../capsem-credentials" } capsem-api = { path = "../capsem-api" } capsem-assets = { path = "../capsem-assets" } +capsem-cache = { path = "../capsem-cache" } capsem-core = { path = "../capsem-core" } capsem-foundation = { path = "../capsem-foundation" } capsem-guard = { path = "../capsem-guard" } diff --git a/crates/capsem-service/src/asset_routes.rs b/crates/capsem-service/src/asset_routes.rs index c52c816f6..b4e073de3 100644 --- a/crates/capsem-service/src/asset_routes.rs +++ b/crates/capsem-service/src/asset_routes.rs @@ -209,40 +209,21 @@ fn manifest_file_identity(path: &StdPath) -> ManifestFileIdentity { } } -/// The manifest status as last built, and the inputs it was built from. -pub(super) struct CachedManifestStatus { - inputs: [ManifestFileIdentity; 2], - status: Arc, -} - /// The installed manifest's provenance and validation. Polled through /// `/status`: it is rebuilt only when manifest.json or its metadata changes. pub(super) fn asset_manifest_status(state: &ServiceState) -> Result, AppError> { let path = state.assets_dir.join("manifest.json"); let metadata_path = state.assets_dir.join("manifest-metadata.json"); let inputs = [manifest_file_identity(&path), manifest_file_identity(&metadata_path)]; - let cache = || { - state.asset_manifest_cache.lock().map_err(|error| { - AppError( - StatusCode::INTERNAL_SERVER_ERROR, - format!("asset manifest cache lock poisoned: {error}"), - ) - }) - }; - let cached = cache()? - .as_ref() - .filter(|cached| cached.inputs == inputs) - .map(|cached| Arc::clone(&cached.status)); - if let Some(status) = cached { + if let Some(status) = state.asset_manifest_cache.get(&inputs) { return Ok(status); } - // Built outside the lock; two concurrent rebuilds of one input agree. - let status = Arc::new(build_asset_manifest_status(state, &path, &metadata_path, &inputs[0])); - *cache()? = Some(CachedManifestStatus { - inputs, - status: Arc::clone(&status), - }); - Ok(status) + let started = std::time::Instant::now(); + let status = build_asset_manifest_status(state, &path, &metadata_path, &inputs[0]); + state + .asset_manifest_cache + .record_load(started.elapsed(), capsem_cache::LoadStatus::Success); + Ok(state.asset_manifest_cache.publish(inputs, status)) } fn build_asset_manifest_status( diff --git a/crates/capsem-service/src/ledger_routes.rs b/crates/capsem-service/src/ledger_routes.rs index 0887957c7..e4673b3ef 100644 --- a/crates/capsem-service/src/ledger_routes.rs +++ b/crates/capsem-service/src/ledger_routes.rs @@ -2,9 +2,7 @@ use super::*; pub(crate) mod activity; pub(crate) mod bodies; pub(super) use bodies::{handle_bodies_warc_export, handle_event_bodies}; -mod response_cache; mod rows; -pub(crate) use response_cache::{forget_session_responses, session_response_cache_lookup, SessionResponseCache}; use rows::{ledger_rows_to_objects, query_route_typed_rows}; pub(crate) mod stats_detail; pub(super) use stats_detail::read_stats_detail_payload_from_session_db; @@ -31,13 +29,6 @@ pub(super) async fn handle_security_latest( Query(params): Query, ) -> Result { let limit = params.limit.unwrap_or(100).min(2000); - let session_dir = resolve_session_dir(&state, &id)?; - let db_path = session_dir.join("session.db"); - let route_key = format!("security_latest:limit={limit}"); - let slot = match session_response_cache_lookup(&state, &id, &route_key, "security", &db_path).await? { - SessionResponseCache::Hit(body) => return Ok(json_bytes_response(body)), - SessionResponseCache::Miss(slot) => slot, - }; let rows = security_latest_for_vm(&state, &id, limit, false).await?; tracing::debug!( route = "/vms/{id}/security/latest", @@ -52,7 +43,6 @@ pub(super) async fn handle_security_latest( format!("failed to serialize security latest response: {error}"), ) })?; - slot.store(&state, &body); Ok(json_bytes_response(Bytes::from(body))) } @@ -63,13 +53,6 @@ pub(super) async fn handle_detection_latest( Query(params): Query, ) -> Result { let limit = params.limit.unwrap_or(100).min(2000); - let session_dir = resolve_session_dir(&state, &id)?; - let db_path = session_dir.join("session.db"); - let route_key = format!("detection_latest:limit={limit}"); - let slot = match session_response_cache_lookup(&state, &id, &route_key, "security", &db_path).await? { - SessionResponseCache::Hit(body) => return Ok(json_bytes_response(body)), - SessionResponseCache::Miss(slot) => slot, - }; let rows = security_latest_for_vm(&state, &id, limit, true).await?; let body = serde_json::to_vec(&rows).map_err(|error| { AppError( @@ -77,7 +60,6 @@ pub(super) async fn handle_detection_latest( format!("failed to serialize detection latest response: {error}"), ) })?; - slot.store(&state, &body); Ok(json_bytes_response(Bytes::from(body))) } diff --git a/crates/capsem-service/src/ledger_routes/response_cache.rs b/crates/capsem-service/src/ledger_routes/response_cache.rs deleted file mode 100644 index 55ae3cfce..000000000 --- a/crates/capsem-service/src/ledger_routes/response_cache.rs +++ /dev/null @@ -1,75 +0,0 @@ -//! Session-ledger route response bytes, cached on the logger's change generation. -use super::*; - -fn session_response_cache_key(vm_id: &str, route_key: &str) -> String { - format!("{vm_id}:{route_key}") -} - -/// Forget every cached response for one VM. -/// -/// Keys carry the caller's `limit`, so one VM can hold an entry per limit a -/// client ever asked for. Nothing released them, so the map grew across VM -/// churn for the lifetime of the service; a VM whose ledger handle is gone -/// cannot serve a hit anyway. -pub(crate) fn forget_session_responses(state: &ServiceState, vm_id: &str) { - let prefix = format!("{vm_id}:"); - state - .stats_detail_response_cache - .lock() - .unwrap() - .retain(|key, _| !key.starts_with(&prefix)); -} - -/// Outcome of looking up a cached session-ledger route response. -pub(crate) enum SessionResponseCache { - Hit(Bytes), - Miss(SessionResponseCacheSlot), -} - -/// Where a freshly built response is stored, pinned to the ledger generation -/// observed before the route queried. -/// -/// Freshness is the logger handle's answer: `ready()` syncs the external -/// reader from disk and advances `read_cache_epoch` when another connection -/// committed. Taking the epoch after that sync and before the query means a -/// commit landing mid-query leaves the bytes under an older epoch -- a miss on -/// the next read, never a stale hit. -pub(crate) struct SessionResponseCacheSlot { - cache_key: String, - db_epoch: u64, -} - -pub(crate) async fn session_response_cache_lookup( - state: &ServiceState, - vm_id: &str, - route_key: &str, - ledger: &str, - db_path: &StdPath, -) -> Result { - let db = open_ready_session_db(state, vm_id, ledger, db_path).await?; - let db_epoch = db.read_cache_epoch(capsem_logger::ReadCacheDomain::All); - let cache_key = session_response_cache_key(vm_id, route_key); - let cached = state - .stats_detail_response_cache - .lock() - .unwrap() - .get(&cache_key) - .filter(|cached| cached.db_epoch == db_epoch) - .map(|cached| Bytes::from(cached.bytes.clone())); - Ok(match cached { - Some(bytes) => SessionResponseCache::Hit(bytes), - None => SessionResponseCache::Miss(SessionResponseCacheSlot { cache_key, db_epoch }), - }) -} - -impl SessionResponseCacheSlot { - pub(crate) fn store(self, state: &ServiceState, bytes: &[u8]) { - state.stats_detail_response_cache.lock().unwrap().insert( - self.cache_key, - CachedLedgerResponse { - db_epoch: self.db_epoch, - bytes: bytes.to_vec(), - }, - ); - } -} diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index 602131e1e..f9ae04fed 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -7,6 +7,7 @@ use axum::{ routing::{delete, get, patch, post, put}, Json, Router, }; +use capsem_cache::{BoundedCache, GenerationCache, IdentityCache, KeyedIdentityCache}; use capsem_core::{ mcp::ToolCacheEntry, net::policy_config::{DetectionLevel, SecurityPluginConfig, SecurityPluginMode, SecurityRuleAction, SettingsFile}, @@ -247,7 +248,7 @@ pub struct ServiceState { /// Effective plugin settings loaded on the first plugin route read and /// invalidated by every service-owned policy edit or explicit reload. /// Hot plugin list polls must not parse settings and corp TOML per request. - plugin_policy_cache: Mutex, + plugin_policy_cache: GenerationCache, /// Logger-owned DB handle for the host ledger (`sessions/host.db`): host /// events and policy mutations. Routes call `write`; they must never open /// SQLite directly or hold a side `DbWriter`. @@ -258,27 +259,23 @@ pub struct ServiceState { /// defunct persistent VMs. Hot list/status/info polls must not rescan the /// filesystem on every request. last_defunct_reconcile_ms: AtomicU64, - /// Session-ledger route bytes (security/detection latest, security status, - /// history processes/counts) keyed by VM id, route and paging, pinned to - /// the logger read-cache epoch, released when the VM's handle unregisters. - stats_detail_response_cache: Mutex>, /// Container workloads being set up or running, by VM id. containers: container_setup::ContainerSetups, /// Session storage diagnostics cached by session directory. These values /// describe the rootfs image path/size and host filesystem for status/info /// routes; repeated polling must not stat the filesystem on every sample. - storage_diagnostics_cache: Mutex>, + storage_diagnostics_cache: BoundedCache, /// Derived persistent VM resume state keyed by stable VM id. The /// fingerprint covers registry fields that affect status so route polling /// does not revalidate asset pins every sample. - persistent_resume_state_cache: Mutex>, + persistent_resume_state_cache: KeyedIdentityCache)>, /// The installed manifest's status, rebuilt only when manifest.json or /// its metadata changes; `/status` polls it. - asset_manifest_cache: Mutex>, + asset_manifest_cache: IdentityCache<[asset_routes::ManifestFileIdentity; 2], api::AssetManifestStatus>, /// `/vms/list` lifecycle rows keyed by the in-memory lifecycle snapshot /// (uptime seconds included). Activity totals are read per poll from /// each ledger's cached counter snapshot, never cached here. - list_response_cache: Mutex>, + list_response_cache: IdentityCache)>, /// Coordinates launch admission and Apple VZ lifecycle edges. Cold starts /// and teardown take a VZ read guard; save/restore take a write guard. /// Blocking launch workers also retain admission through registration, @@ -316,25 +313,6 @@ pub struct ServiceState { _test_tempdir: Option, } -/// Serialized ledger route bytes and the logger generation they were read at. -#[derive(Clone)] -struct CachedLedgerResponse { - db_epoch: u64, - bytes: Vec, -} - -#[derive(Clone)] -struct CachedPersistentResumeState { - fingerprint: String, - state: (VmLifecycleState, bool, Option), -} - -#[derive(Clone)] -struct CachedListResponse { - fingerprint: String, - listed: (ListResponse, Vec), -} - #[derive(Debug, Clone, Default, Serialize, Deserialize)] struct AssetReconcileState { #[serde(default)] @@ -540,16 +518,10 @@ impl ServiceState { } fn storage_diagnostics_cached(&self, session_dir: &StdPath) -> Option { - if let Some(cached) = self.storage_diagnostics_cache.lock().unwrap().get(session_dir).cloned() { - return Some(cached); - } - - let diagnostics = storage_diagnostics(session_dir)?; self.storage_diagnostics_cache - .lock() - .unwrap() - .insert(session_dir.to_path_buf(), diagnostics.clone()); - Some(diagnostics) + .get_or_try_insert_with(session_dir, || storage_diagnostics(session_dir).ok_or(())) + .ok() + .map(|diagnostics| (*diagnostics).clone()) } fn next_job_id(&self) -> u64 { @@ -760,21 +732,14 @@ impl ServiceState { ) -> (VmLifecycleState, bool, Option) { let vm_id = persistent_entry_vm_id(entry); let fingerprint = persistent_resume_state_fingerprint(self, entry); - if let Some(cached) = self.persistent_resume_state_cache.lock().unwrap().get(&vm_id).cloned() { - if cached.fingerprint == fingerprint { - return cached.state; - } + if let Some(cached) = self.persistent_resume_state_cache.get(&vm_id, &fingerprint) { + return (*cached).clone(); } - let state = self.persistent_entry_resume_state(entry); - self.persistent_resume_state_cache.lock().unwrap().insert( - vm_id, - CachedPersistentResumeState { - fingerprint, - state: state.clone(), - }, - ); - state + let resume_state = self.persistent_entry_resume_state(entry); + self.persistent_resume_state_cache + .publish(vm_id, fingerprint, resume_state.clone()); + resume_state } } diff --git a/crates/capsem-service/src/plugin_routes.rs b/crates/capsem-service/src/plugin_routes.rs index 45424dbd9..38aabab6c 100644 --- a/crates/capsem-service/src/plugin_routes.rs +++ b/crates/capsem-service/src/plugin_routes.rs @@ -192,12 +192,6 @@ pub(super) fn plugin_catalog() -> &'static BTreeMap /// or corp set. pub(super) type EffectivePluginPolicy = (BTreeMap, BTreeSet); -#[derive(Default)] -pub(super) struct PluginPolicyCache { - generation: u64, - policy: Option>, -} - /// The effective plugin policy from the current files. fn effective_plugin_policy() -> Result { let (settings, corp) = capsem_core::net::policy_config::load_policy_files() @@ -213,43 +207,32 @@ fn effective_plugin_policy() -> Result { impl ServiceState { pub(crate) fn invalidate_plugin_policy_cache(&self) { - let mut cache = self.plugin_policy_cache.lock().unwrap(); - cache.generation = cache.generation.wrapping_add(1); - cache.policy = None; - } - - fn cached_plugin_policy(&self) -> (u64, Option>) { - let cache = self.plugin_policy_cache.lock().unwrap(); - (cache.generation, cache.policy.clone()) - } - - fn publish_plugin_policy( - &self, - generation: u64, - policy: EffectivePluginPolicy, - ) -> Option> { - let mut cache = self.plugin_policy_cache.lock().unwrap(); - if cache.generation != generation { - return None; - } - if let Some(policy) = &cache.policy { - return Some(Arc::clone(policy)); - } - let policy = Arc::new(policy); - cache.policy = Some(Arc::clone(&policy)); - drop(cache); - Some(policy) + self.plugin_policy_cache.invalidate(); } } async fn effective_plugin_policy_for(state: &Arc) -> Result, AppError> { loop { - let (generation, cached) = state.cached_plugin_policy(); - if let Some(policy) = cached { + let lookup = state.plugin_policy_cache.lookup(); + if let Some(policy) = lookup.value() { return Ok(policy); } - let policy = state.off_worker(|_| effective_plugin_policy()).await??; - if let Some(policy) = state.publish_plugin_policy(generation, policy) { + let started = std::time::Instant::now(); + let policy = match state.off_worker(|_| effective_plugin_policy()).await { + Ok(Ok(policy)) => { + state + .plugin_policy_cache + .record_load(started.elapsed(), capsem_cache::LoadStatus::Success); + policy + } + Ok(Err(error)) | Err(error) => { + state + .plugin_policy_cache + .record_load(started.elapsed(), capsem_cache::LoadStatus::Error); + return Err(error); + } + }; + if let Some(policy) = state.plugin_policy_cache.publish(lookup.generation(), policy) { return Ok(policy); } } diff --git a/crates/capsem-service/src/router_runtime.rs b/crates/capsem-service/src/router_runtime.rs index eb5d13844..0e39149fb 100644 --- a/crates/capsem-service/src/router_runtime.rs +++ b/crates/capsem-service/src/router_runtime.rs @@ -520,7 +520,7 @@ pub(super) fn reload_activated_update_runtime(state: &ServiceState) -> Result) -> Result>) -> axum: fn list_lifecycle(state: &ServiceState) -> (ListResponse, Vec) { state.reconcile_persistent_defunct_from_logs(); let fingerprint = list_response_fingerprint(state); - if let Some(cached) = state.list_response_cache.lock().unwrap().clone() { - if cached.fingerprint == fingerprint { - return cached.listed; - } + if let Some(cached) = state.list_response_cache.get(&fingerprint) { + return (*cached).clone(); } + let started = std::time::Instant::now(); let listed = build_list_response(state); - *state.list_response_cache.lock().unwrap() = Some(CachedListResponse { - fingerprint, - listed: listed.clone(), - }); - listed + state + .list_response_cache + .record_load(started.elapsed(), capsem_cache::LoadStatus::Success); + (*state.list_response_cache.publish(fingerprint, listed)).clone() } diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index 8b0b98208..7903dd99f 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -285,16 +285,15 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { asset_reconcile_inflight: AtomicBool::new(false), asset_status_path, mcp_tool_cache: Mutex::new(capsem_core::mcp::load_tool_cache()), - plugin_policy_cache: Mutex::new(Default::default()), + plugin_policy_cache: GenerationCache::new("service.plugin_policy"), host_ledger, host_stats: Mutex::new(Default::default()), last_defunct_reconcile_ms: AtomicU64::new(0), - stats_detail_response_cache: Mutex::new(HashMap::new()), containers: Default::default(), - storage_diagnostics_cache: Mutex::new(HashMap::new()), - persistent_resume_state_cache: Mutex::new(HashMap::new()), - asset_manifest_cache: Mutex::new(None), - list_response_cache: Mutex::new(None), + storage_diagnostics_cache: BoundedCache::new("service.storage_diagnostics", 256), + persistent_resume_state_cache: KeyedIdentityCache::new("service.persistent_resume", 256), + asset_manifest_cache: IdentityCache::new("service.asset_manifest"), + list_response_cache: IdentityCache::new("service.vm_list"), lifecycle: capsem_service::lifecycle::VmLifecycle::default(), shutdown_lock: tokio::sync::Mutex::new(()), update_lock: tokio::sync::Mutex::new(()), diff --git a/crates/capsem-service/src/session_db_handles.rs b/crates/capsem-service/src/session_db_handles.rs index 58f9f23b3..66cf40157 100644 --- a/crates/capsem-service/src/session_db_handles.rs +++ b/crates/capsem-service/src/session_db_handles.rs @@ -103,14 +103,6 @@ impl SessionLedger { Ok(current) } - pub(crate) fn read_cache_epoch(&self, _domain: capsem_logger::ReadCacheDomain) -> u64 { - #[cfg(test)] - if let Some(db) = &self.embedded { - return db.read_cache_epoch(capsem_logger::ReadCacheDomain::All); - } - self.remote_state().read_cache_epoch.load(Ordering::Acquire) - } - #[cfg(test)] pub(crate) fn reader_requests(&self) -> u64 { self.embedded @@ -390,7 +382,6 @@ impl ServiceState { pub(crate) fn unregister_session_db_handle(&self, vm_id: &str) { let removed = self.session_db_handles.lock().unwrap().remove(vm_id); - forget_session_responses(self, vm_id); if removed.is_some() { info!( vm_id, diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 6820662a0..02591aaea 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -122,16 +122,15 @@ pub(crate) fn test_state( asset_reconcile_inflight: AtomicBool::new(false), asset_status_path: asset_status_path_for_run_dir(&run_dir), mcp_tool_cache: Mutex::new(capsem_core::mcp::load_tool_cache()), - plugin_policy_cache: Mutex::new(Default::default()), + plugin_policy_cache: GenerationCache::new("service.plugin_policy"), host_ledger: test_host_ledger(&run_dir), host_stats: Mutex::new(Default::default()), last_defunct_reconcile_ms: AtomicU64::new(0), - stats_detail_response_cache: Mutex::new(HashMap::new()), containers: Default::default(), - storage_diagnostics_cache: Mutex::new(HashMap::new()), - persistent_resume_state_cache: Mutex::new(HashMap::new()), - asset_manifest_cache: Mutex::new(None), - list_response_cache: Mutex::new(None), + storage_diagnostics_cache: BoundedCache::new("service.storage_diagnostics", 256), + persistent_resume_state_cache: KeyedIdentityCache::new("service.persistent_resume", 256), + asset_manifest_cache: IdentityCache::new("service.asset_manifest"), + list_response_cache: IdentityCache::new("service.vm_list"), lifecycle: capsem_service::lifecycle::VmLifecycle::default(), shutdown_lock: tokio::sync::Mutex::new(()), update_lock: tokio::sync::Mutex::new(()), diff --git a/crates/capsem-service/src/tests/inspection.rs b/crates/capsem-service/src/tests/inspection.rs index e2a7124e8..fb5b81787 100644 --- a/crates/capsem-service/src/tests/inspection.rs +++ b/crates/capsem-service/src/tests/inspection.rs @@ -157,46 +157,6 @@ async fn stats_detail_has_typed_nullable_events_and_captured_bodies() { assert_eq!(value["interactions"]["items"][0]["content"]["kind"], "tool_call"); assert_eq!(value["interactions"]["bodies"][0]["truncated"], true); assert!(value["interactions"]["bodies"][0].get("payload").is_none()); - assert!( - state.stats_detail_response_cache.lock().unwrap().is_empty(), - "stats detail must use DB-owned reads without a service projection cache" - ); -} - -/// Cached ledger responses are keyed by VM id, route and the caller's paging, -/// and nothing released them: the map grew for the service's lifetime across -/// VM churn. A VM whose ledger handle is gone cannot serve a hit anyway. -#[tokio::test] -async fn unregistering_a_session_releases_its_cached_ledger_responses() { - let state = crate::tests::make_test_state(); - { - let mut cache = state.stats_detail_response_cache.lock().unwrap(); - for key in [ - "box:security/latest:50", - "box:history/counts", - "other:security/latest:50", - ] { - cache.insert( - key.to_string(), - CachedLedgerResponse { - db_epoch: 1, - bytes: vec![b'{'], - }, - ); - } - } - - state.unregister_session_db_handle("box"); - - let keys: Vec = { - let cache = state.stats_detail_response_cache.lock().unwrap(); - cache.keys().cloned().collect() - }; - assert_eq!( - keys, - vec!["other:security/latest:50"], - "only the unregistered VM's responses are released" - ); } #[tokio::test] diff --git a/crates/capsem-service/src/tests/interactions.rs b/crates/capsem-service/src/tests/interactions.rs index 1d766b096..62bfa009d 100644 --- a/crates/capsem-service/src/tests/interactions.rs +++ b/crates/capsem-service/src/tests/interactions.rs @@ -99,7 +99,6 @@ async fn interactions_project_model_messages_tools_and_results_without_cross_cal assert_eq!(orphan["content"]["arguments"]["content"]["reason"], "invalid_json"); assert_eq!(orphan["content"]["result"]["is_error"], true); assert_eq!(orphan["content"]["result"]["error_message"], "failed"); - assert!(state.stats_detail_response_cache.lock().unwrap().is_empty()); } #[tokio::test] diff --git a/tests/citadel/test_db_boundary.py b/tests/citadel/test_db_boundary.py index 80d8c10a9..446064a57 100644 --- a/tests/citadel/test_db_boundary.py +++ b/tests/citadel/test_db_boundary.py @@ -40,6 +40,8 @@ ("route_projection", "route/service projection cache must be burned"), ("_route_projection", "route/service projection cache must be burned"), ("live_session_counter_projection", "route/service projection cache must be burned"), + ("stats_detail_response_cache", "route/service projection cache must be burned"), + ("session_response_cache", "route/service projection cache must be burned"), ("missing_optional_ledger_shape", "missing schema must fail loudly"), ("no such table", "missing schema fallback must not be special-cased"), ("no such column", "missing schema fallback must not be special-cased"), From 1085db7eea36909436ac6bbbcf37929579cb3973 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:36:15 +0000 Subject: [PATCH 286/464] feat(cache): support domain freshness predicates --- crates/capsem-cache/src/bounded.rs | 2 +- crates/capsem-cache/src/tests.rs | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/capsem-cache/src/bounded.rs b/crates/capsem-cache/src/bounded.rs index 3572d0d51..afe2de886 100644 --- a/crates/capsem-cache/src/bounded.rs +++ b/crates/capsem-cache/src/bounded.rs @@ -78,7 +78,7 @@ where self.get_if(key, |_| true) } - pub(crate) fn get_if(&self, key: &Q, accept: impl FnOnce(&V) -> bool) -> Option> + pub fn get_if(&self, key: &Q, accept: impl FnOnce(&V) -> bool) -> Option> where Q: Equivalent + Hash + ?Sized, { diff --git a/crates/capsem-cache/src/tests.rs b/crates/capsem-cache/src/tests.rs index af4f82142..b29439439 100644 --- a/crates/capsem-cache/src/tests.rs +++ b/crates/capsem-cache/src/tests.rs @@ -125,6 +125,17 @@ fn bounded_cache_single_flights_loads_and_reports_errors() { ); } +#[test] +fn bounded_cache_treats_a_rejected_resident_value_as_a_miss() { + let cache = BoundedCache::new("expiring", 8); + cache.insert("answer", 41); + + assert!(cache.get_if(&"answer", |value| *value == 42).is_none()); + assert_eq!(cache.stats().hits, 0); + assert_eq!(cache.stats().misses, 1); + assert_eq!(cache.len(), 1, "the domain owner decides whether to remove it"); +} + #[test] fn bounded_cache_computes_one_value_for_concurrent_misses() { let cache = Arc::new(BoundedCache::::new("single_flight", 8)); From 6a7fc7495a707b6fac2c438ce7825d457750af44 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:42:40 +0000 Subject: [PATCH 287/464] perf(core): unify bounded network caches --- CHANGELOG.md | 4 + Cargo.lock | 2 +- crates/capsem-core/Cargo.toml | 4 +- crates/capsem-core/src/net/cert_authority.rs | 24 ++--- .../src/net/cert_authority/tests.rs | 21 ++++ crates/capsem-core/src/net/dns/cache.rs | 100 ++++++++---------- crates/capsem-core/src/net/dns/cache/tests.rs | 64 +++++++---- 7 files changed, 119 insertions(+), 100 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b737c9b6a..e74ef8ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -257,6 +257,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Runtime memory caches use one bounded, concurrent implementation with + uniform hit, miss, mutation, load-time and entry metrics. DNS retains its + TTL and policy checks, while concurrent TLS handshakes share one leaf mint. + - Host MCP packages identify as 0.7.0 and depend on the matching TypeScript SDK. - TypeScript SDK packages use the 0.7.0 client version. diff --git a/Cargo.lock b/Cargo.lock index ddefa80b0..f23adfded 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -882,6 +882,7 @@ dependencies = [ "block2", "bytes", "capsem-assets", + "capsem-cache", "capsem-config", "capsem-credentials", "capsem-crypto", @@ -904,7 +905,6 @@ dependencies = [ "hyper-util", "io-uring", "libc", - "lru", "metrics", "metrics-util", "nix 0.29.0", diff --git a/crates/capsem-core/Cargo.toml b/crates/capsem-core/Cargo.toml index 18155fa99..546fcbe66 100644 --- a/crates/capsem-core/Cargo.toml +++ b/crates/capsem-core/Cargo.toml @@ -19,6 +19,7 @@ tracing = { workspace = true } serde = { workspace = true } capsem-proto = { path = "../capsem-proto" } capsem-assets = { path = "../capsem-assets" } +capsem-cache = { path = "../capsem-cache" } capsem-config = { path = "../capsem-config" } capsem-credentials = { path = "../capsem-credentials" } capsem-foundation = { path = "../capsem-foundation" } @@ -49,9 +50,6 @@ regex = { workspace = true } scraper = "0.25" hickory-proto = { workspace = true } -# Bounded LRU primitive used by `net::dns::cache` for the TTL-honoring -# answer cache (T3.f). Pure-Rust, no_std-compatible, single small dep. -lru = "0.18.2" ego-tree = "0.10" # Metrics facade. No recorder registered in this sprint -- emission is a # single relaxed atomic add against a no-op recorder. T5 wires the diff --git a/crates/capsem-core/src/net/cert_authority.rs b/crates/capsem-core/src/net/cert_authority.rs index 3ab4b6d42..66792c877 100644 --- a/crates/capsem-core/src/net/cert_authority.rs +++ b/crates/capsem-core/src/net/cert_authority.rs @@ -1,11 +1,9 @@ /// Certificate authority for MITM proxy: loads the static Capsem CA keypair /// and mints short-lived leaf certificates on demand for each domain. use std::fmt; -use std::num::NonZeroUsize; -use std::sync::{Arc, Mutex}; - -use lru::LruCache; +use std::sync::Arc; +use capsem_cache::BoundedCache; use capsem_crypto::tls::CertificateIssuer; use rustls::pki_types::CertificateDer; use rustls::server::ClientHello; @@ -19,7 +17,7 @@ const LEAF_CACHE_CAPACITY: usize = 1024; /// Holds the static CA keypair and caches minted leaf certificates. pub struct CertAuthority { issuer: CertificateIssuer, - cache: Mutex>>, + cache: BoundedCache, } impl CertAuthority { @@ -30,9 +28,7 @@ impl CertAuthority { let issuer = CertificateIssuer::load(key_pem, cert_pem)?; Ok(Self { issuer, - cache: Mutex::new(LruCache::new( - NonZeroUsize::new(LEAF_CACHE_CAPACITY).expect("leaf cache capacity is non-zero"), - )), + cache: BoundedCache::new("core.tls_leaf", LEAF_CACHE_CAPACITY), }) } @@ -42,14 +38,8 @@ impl CertAuthority { /// happens under the lock rather than racing two mints for one name. pub fn certified_key_for_domain(&self, domain: &str) -> anyhow::Result> { let host = crate::net::hostname::normalize_host(domain); - let mut cache = self.cache.lock().unwrap_or_else(|e| e.into_inner()); - if let Some(key) = cache.get(&host) { - return Ok(Arc::clone(key)); - } - let arc = Arc::new(self.mint_leaf(&host)?); - cache.put(host, Arc::clone(&arc)); - drop(cache); - Ok(arc) + self.cache + .get_or_try_insert_with(host.as_str(), || self.mint_leaf(&host)) } /// The CA certificate in DER, for a client that must trust minted leaves. @@ -59,7 +49,7 @@ impl CertAuthority { /// Number of cached certificates. pub fn cache_size(&self) -> usize { - self.cache.lock().unwrap_or_else(|e| e.into_inner()).len() + self.cache.len() } /// Mint a leaf certificate for the given domain, signed by the CA. diff --git a/crates/capsem-core/src/net/cert_authority/tests.rs b/crates/capsem-core/src/net/cert_authority/tests.rs index a5f2419d1..e6da2bed4 100644 --- a/crates/capsem-core/src/net/cert_authority/tests.rs +++ b/crates/capsem-core/src/net/cert_authority/tests.rs @@ -78,6 +78,27 @@ fn concurrent_minting_safe() { assert_eq!(ca.cache_size(), 10); } +#[test] +fn concurrent_reads_single_flight_one_domain_mint() { + let ca = Arc::new(load_ca()); + let start = Arc::new(std::sync::Barrier::new(8)); + let mut handles = Vec::new(); + for _ in 0..8 { + let ca = Arc::clone(&ca); + let start = Arc::clone(&start); + handles.push(std::thread::spawn(move || { + start.wait(); + ca.certified_key_for_domain("shared.example.com").unwrap() + })); + } + let keys: Vec<_> = handles.into_iter().map(|handle| handle.join().unwrap()).collect(); + assert!(keys.windows(2).all(|pair| Arc::ptr_eq(&pair[0], &pair[1]))); + let stats = ca.cache.stats(); + assert_eq!(stats.misses, 1); + assert_eq!(stats.hits, 7); + assert_eq!(stats.entries, 1); +} + // -- Leaf cache identity and bounds -- // // The guest chooses the SNI. The cache it feeds must key on the host's diff --git a/crates/capsem-core/src/net/dns/cache.rs b/crates/capsem-core/src/net/dns/cache.rs index d89a1c9d4..27d24f45f 100644 --- a/crates/capsem-core/src/net/dns/cache.rs +++ b/crates/capsem-core/src/net/dns/cache.rs @@ -1,4 +1,4 @@ -//! TTL-honoring LRU answer cache for the DNS proxy (T3.f). +//! TTL-honoring bounded answer cache for the DNS proxy (T3.f). //! //! The cache shaves the upstream UDP RTT off repeated queries to the //! same allowed name. Cache shape: @@ -15,20 +15,19 @@ //! (`insert_negative`, bounded by the SOA minimum and //! `NEGATIVE_MAX_TTL_SECS`); SERVFAIL is never persisted, since it may //! be a transient upstream fault. -//! * **Bound**: an LRU on entry count (default 1024). Evictions are -//! counted via the `mitm.dns_cache_evictions_total` counter. +//! * **Bound**: shared cache admission and eviction on entry count (default +//! 1024), with both unified and DNS compatibility metrics. //! //! The cache **does** read the network-policy snapshot on every hit so //! redirect/cache mechanics stay coherent without a per-policy version //! counter. -use std::num::NonZeroUsize; -use std::sync::Mutex; +use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{Duration, Instant}; +use capsem_cache::BoundedCache; use hickory_proto::op::Message; use hickory_proto::rr::RData; -use lru::LruCache; use super::coalesce::LookupKey; use crate::net::parsers::dns_parser::DnsQuery; @@ -62,15 +61,11 @@ struct Entry { expires_at: Instant, } -/// Bounded LRU cache for DNS answer bytes. -/// -/// Thread-safe via a single `Mutex>`. Lookups and -/// inserts go through the same lock; under contention this is the -/// bottleneck, but the hot-path cost (one HashMap lookup + one -/// Instant::now()) is sub-microsecond on modern hardware. +/// Bounded concurrent cache for DNS answer bytes. pub struct DnsAnswerCache { - inner: Mutex>, + inner: BoundedCache, max_ttl: Duration, + reported_dns_evictions: AtomicU64, } impl Default for DnsAnswerCache { @@ -81,14 +76,13 @@ impl Default for DnsAnswerCache { impl DnsAnswerCache { /// Build a cache with explicit capacity + TTL ceiling. - /// `capacity` of 0 is silently bumped to 1 (LruCache requires - /// non-zero); `max_ttl_secs` of 0 disables caching effectively + /// `capacity` of 0 is silently bumped to 1; `max_ttl_secs` of 0 disables caching effectively /// (every entry expires before its first lookup). pub fn new(capacity: usize, max_ttl_secs: u32) -> Self { - let cap = NonZeroUsize::new(capacity.max(1)).expect("capacity > 0 enforced"); Self { - inner: Mutex::new(LruCache::new(cap)), + inner: BoundedCache::new("core.dns_answer", capacity), max_ttl: Duration::from_secs(u64::from(max_ttl_secs)), + reported_dns_evictions: AtomicU64::new(0), } } @@ -115,33 +109,26 @@ impl DnsAnswerCache { let qtype = query.qtype; let query_id = query.id; let now = Instant::now(); - let mut guard = self.inner.lock().unwrap(); - let Some(entry) = guard.get(key) else { - drop(guard); + let mut expired = false; + let mut redirected = false; + let entry = self.inner.get_if(key, |entry| { + expired = entry.expires_at <= now; + redirected = policy.find_dns_redirect(qname, qtype).is_some(); + !expired && !redirected + }); + let Some(entry) = entry else { + if expired || redirected { + self.inner.remove(key); + } + if expired { + trace!(qname, qtype, "dns cache: expired entry evicted"); + } else if redirected { + trace!(qname, qtype, "dns cache: entry invalidated by redirect change"); + } return None; }; - if entry.expires_at <= now { - // Lazy expiry: drop the stale entry so the next - // lookup is a clean miss without re-checking expiry. - guard.pop(key); - drop(guard); - ::metrics::counter!(m::DNS_CACHE_MISSES_TOTAL).increment(1); - trace!(qname, qtype, "dns cache: expired entry evicted"); - return None; - } - // Coherence: re-check redirect mechanics on every hit. Security-rule - // enforcement happens before cache lookup in the DNS handler, so this - // cache layer does not own allow/block decisions. - if policy.find_dns_redirect(qname, qtype).is_some() { - guard.pop(key); - drop(guard); - ::metrics::counter!(m::DNS_CACHE_MISSES_TOTAL).increment(1); - trace!(qname, qtype, "dns cache: entry invalidated by redirect change"); - return None; - } let bytes = entry.bytes.clone(); let remaining = entry.expires_at.duration_since(now).as_secs_f64().ceil() as u32; - drop(guard); // Patch the current query's transaction id into bytes 0-1 // (RFC 1035 sec 4.1.1: the ID field is the first 16 bits of // the DNS header, big-endian). The cached answer was stored @@ -170,8 +157,7 @@ impl DnsAnswerCache { /// Insert an Allowed response for future hits. The TTL is /// derived from the answer wire bytes (minimum across all /// answer records, capped by `max_ttl`). Zero prohibits caching. On - /// LRU eviction, the `mitm.dns_cache_evictions_total` counter - /// fires. + /// admission eviction, both unified and compatibility counters fire. pub(super) fn insert(&self, key: &LookupKey, answer_bytes: &[u8]) { let ttl = ttl_from_answer(answer_bytes, self.max_ttl); if ttl.is_zero() { @@ -181,13 +167,8 @@ impl DnsAnswerCache { bytes: answer_bytes.to_vec(), expires_at: Instant::now() + ttl, }; - let key = key.clone(); - let mut guard = self.inner.lock().unwrap(); - let evicted = guard.push(key, entry); - drop(guard); - if evicted.is_some() { - ::metrics::counter!(m::DNS_CACHE_EVICTIONS_TOTAL).increment(1); - } + self.inner.insert(key.clone(), entry); + self.report_compatibility_evictions(); } /// Remember an upstream NXDOMAIN for this exact query. The TTL @@ -203,11 +184,8 @@ impl DnsAnswerCache { bytes: answer_bytes.to_vec(), expires_at: Instant::now() + ttl, }; - let key = key.clone(); - let evicted = self.inner.lock().unwrap().push(key, entry); - if evicted.is_some() { - ::metrics::counter!(m::DNS_CACHE_EVICTIONS_TOTAL).increment(1); - } + self.inner.insert(key.clone(), entry); + self.report_compatibility_evictions(); } /// Drop every cached entry. Used when the policy is hot-swapped @@ -215,16 +193,24 @@ impl DnsAnswerCache { /// each entry independently re-validate against the new policy /// on its next lookup. pub fn clear(&self) { - self.inner.lock().unwrap().clear(); + self.inner.clear(); } /// Current entry count (debugging / metrics only). pub fn len(&self) -> usize { - self.inner.lock().unwrap().len() + self.inner.len() } pub fn is_empty(&self) -> bool { - self.len() == 0 + self.inner.is_empty() + } + + fn report_compatibility_evictions(&self) { + let total = self.inner.stats().evictions; + let reported = self.reported_dns_evictions.fetch_max(total, Ordering::Relaxed); + if total > reported { + ::metrics::counter!(m::DNS_CACHE_EVICTIONS_TOTAL).increment(total - reported); + } } } diff --git a/crates/capsem-core/src/net/dns/cache/tests.rs b/crates/capsem-core/src/net/dns/cache/tests.rs index cc2a50a05..76fb9a72b 100644 --- a/crates/capsem-core/src/net/dns/cache/tests.rs +++ b/crates/capsem-core/src/net/dns/cache/tests.rs @@ -31,6 +31,27 @@ fn miss_on_empty_cache() { assert_eq!(cache.len(), 0); } +#[test] +fn shared_cache_stats_cover_dns_hits_and_misses() { + let cache = DnsAnswerCache::new(16, 300); + let policy = allow_all(); + assert!(get(&cache, "stats.example", 1, 1, 1, &policy).is_none()); + insert( + &cache, + "stats.example", + 1, + 1, + &build_answer("stats.example.", 60, [192, 0, 2, 1]), + ); + assert!(get(&cache, "stats.example", 1, 1, 2, &policy).is_some()); + + let stats = cache.inner.stats(); + assert_eq!(stats.misses, 1); + assert_eq!(stats.hits, 1); + assert_eq!(stats.insertions, 1); + assert_eq!(stats.entries, 1); +} + #[test] fn hit_after_insert_within_ttl() { let cache = DnsAnswerCache::new(16, 300); @@ -130,10 +151,13 @@ fn evicts_when_capacity_exceeded() { insert(&cache, "b.com", 1, 1, &build_answer("b.com.", 60, [2, 2, 2, 2])); assert_eq!(cache.len(), 2); insert(&cache, "c.com", 1, 1, &build_answer("c.com.", 60, [3, 3, 3, 3])); - assert_eq!(cache.len(), 2); // a.com evicted (LRU) - assert!(get(&cache, "a.com", 1, 1, 0, &policy).is_none()); - assert!(get(&cache, "b.com", 1, 1, 0, &policy).is_some()); - assert!(get(&cache, "c.com", 1, 1, 0, &policy).is_some()); + assert!(cache.len() <= 2); + let resident = ["a.com", "b.com", "c.com"] + .into_iter() + .filter(|name| get(&cache, name, 1, 1, 0, &policy).is_some()) + .count(); + assert!(resident <= 2, "admission must never exceed the configured bound"); + assert!(cache.inner.stats().evictions > 0); } #[test] @@ -156,20 +180,6 @@ fn capacity_zero_clamped_to_one() { assert!(get(&cache, "a.com", 1, 1, 0, &policy).is_some()); } -#[test] -fn lru_order_updates_on_access() { - let cache = DnsAnswerCache::new(2, 300); - let policy = allow_all(); - insert(&cache, "a.com", 1, 1, &build_answer("a.com.", 60, [1, 1, 1, 1])); - insert(&cache, "b.com", 1, 1, &build_answer("b.com.", 60, [2, 2, 2, 2])); - // Access a -> a becomes most-recently-used; b is now LRU. - let _ = get(&cache, "a.com", 1, 1, 0, &policy); - insert(&cache, "c.com", 1, 1, &build_answer("c.com.", 60, [3, 3, 3, 3])); - // b should be evicted, not a. - assert!(get(&cache, "a.com", 1, 1, 0, &policy).is_some()); - assert!(get(&cache, "b.com", 1, 1, 0, &policy).is_none()); -} - #[test] fn ttl_from_answer_honors_short_authoritative_lifetimes() { let bytes = build_answer("example.com.", 5, [1, 2, 3, 4]); @@ -260,9 +270,12 @@ fn default_capacity_and_max_ttl_match_constants() { &build_answer(&format!("{name}."), 60, [1, 2, 3, 4]), ); } - assert_eq!(cache.len(), DEFAULT_CAPACITY); - // First one should now be evicted. - assert!(get(&cache, "h0.example.com", 1, 1, 0, &policy).is_none()); + assert!(cache.len() <= DEFAULT_CAPACITY); + assert!(cache.inner.stats().evictions > 0); + let resident = (0..=DEFAULT_CAPACITY) + .filter(|i| get(&cache, &format!("h{i}.example.com"), 1, 1, 0, &policy).is_some()) + .count(); + assert_eq!(resident, cache.len()); } /// An NXDOMAIN response for `qname`, optionally carrying a SOA in the @@ -351,7 +364,14 @@ fn a_cache_hit_decrements_record_ttls_by_the_time_already_spent_cached() { &build_answer("aged.example.", 60, [192, 0, 2, 1]), ); let key = query_key("aged.example", 1, 1, 0).1; - cache.inner.lock().unwrap().get_mut(&key).unwrap().expires_at = Instant::now() + Duration::from_secs(30); + let entry = cache.inner.get(&key).unwrap(); + cache.inner.insert( + key, + Entry { + bytes: entry.bytes.clone(), + expires_at: Instant::now() + Duration::from_secs(30), + }, + ); let hit = get(&cache, "aged.example", 1, 1, 1, &allow_all()).unwrap(); let message = Message::from_vec(&hit).unwrap(); assert!( From 9833c719bc6cb30d16ced59bb07a3a8180818612 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:54:41 +0000 Subject: [PATCH 288/464] fix(bench): compare repeated benchmark runs --- crates/capsem-bench/src/cli.rs | 3 + crates/capsem-bench/src/comparison.rs | 41 ++++++++++- crates/capsem-bench/src/comparison/tests.rs | 40 ++++++++++ crates/capsem-bench/src/main.rs | 1 + crates/capsem-bench/src/store.rs | 82 +++++++++++---------- crates/capsem-bench/src/store/tests.rs | 19 ++++- 6 files changed, 143 insertions(+), 43 deletions(-) create mode 100644 crates/capsem-bench/src/comparison/tests.rs diff --git a/crates/capsem-bench/src/cli.rs b/crates/capsem-bench/src/cli.rs index 6e2ba0425..1440643dc 100644 --- a/crates/capsem-bench/src/cli.rs +++ b/crates/capsem-bench/src/cli.rs @@ -98,6 +98,9 @@ pub(crate) struct CompareArgs { pub(crate) baseline: PathBuf, pub(crate) current: PathBuf, pub(crate) dimension: String, + /// Most recent authoritative runs to combine on each side. + #[arg(long, default_value_t = 2)] + pub(crate) runs: usize, #[command(flatten)] pub(crate) thresholds: Thresholds, } diff --git a/crates/capsem-bench/src/comparison.rs b/crates/capsem-bench/src/comparison.rs index 0e3dfae16..ae996bdbb 100644 --- a/crates/capsem-bench/src/comparison.rs +++ b/crates/capsem-bench/src/comparison.rs @@ -12,18 +12,48 @@ pub(crate) fn compare( current_db: &Path, dimension: schema::Dimension, arch: &str, + runs: usize, thresholds: Thresholds, ) -> Result<()> { + if runs == 0 { + bail!("--runs must be at least 1"); + } let baseline_store = store::open(baseline_db)?; let current_store = store::open(current_db)?; - let baseline = store::latest(&baseline_store, dimension, arch)? - .with_context(|| format!("no {} evidence for {arch}", dimension.as_str()))?; - let current = store::latest(¤t_store, dimension, arch)? - .with_context(|| format!("no {} run for {arch}", dimension.as_str()))?; + let baseline_runs = store::latest_many(&baseline_store, dimension, arch, runs)?; + let current_runs = store::latest_many(¤t_store, dimension, arch, runs)?; + let baseline = + aggregate(&baseline_runs).with_context(|| format!("no {} evidence for {arch}", dimension.as_str()))?; + let current = aggregate(¤t_runs).with_context(|| format!("no {} run for {arch}", dimension.as_str()))?; + println!( + "comparing {} baseline run(s) with {} current run(s)", + baseline_runs.len(), + current_runs.len() + ); report(&baseline, &judge(&baseline, ¤t, thresholds)); Ok(()) } +fn aggregate(records: &[schema::Record]) -> Option { + let mut aggregate = records.first()?.clone(); + aggregate.metrics = aggregate + .metrics + .iter() + .filter_map(|metric| { + let samples: Option> = records + .iter() + .map(|record| record.metric(&metric.key).map(|found| found.summary.median)) + .collect(); + Some(schema::Metric { + key: metric.key.clone(), + unit: metric.unit, + summary: stats::Summary::of(&samples?)?, + }) + }) + .collect(); + Some(aggregate) +} + /// Ratchet this run against evidence without granting noise release authority. pub(crate) fn verify(records: &Path, evidence_dir: &Path, thresholds: Thresholds) -> Result<()> { let evidence = store::open(evidence_dir)?; @@ -92,3 +122,6 @@ fn report(baseline: &schema::Record, verdicts: &[stats::Comparison]) { ); } } + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-bench/src/comparison/tests.rs b/crates/capsem-bench/src/comparison/tests.rs new file mode 100644 index 000000000..893798496 --- /dev/null +++ b/crates/capsem-bench/src/comparison/tests.rs @@ -0,0 +1,40 @@ +use super::*; + +fn record(at: &str, value: f64) -> schema::Record { + schema::Record { + schema: schema::SCHEMA.to_string(), + dimension: schema::Dimension::Routes, + recorded_at: at.to_string(), + release: schema::Release { + version: "0.7.0".to_string(), + channel: "dev".to_string(), + commit: "head".to_string(), + }, + host: schema::Host { + arch: "x86_64".to_string(), + os: "linux".to_string(), + cpu_count: 8, + kvm: true, + governor: None, + load_before: 0.0, + }, + quick: false, + metrics: vec![schema::Metric { + key: "routes.service./status.cpu_s".to_string(), + unit: schema::Unit::Seconds, + summary: stats::Summary::of(&[value]).unwrap(), + }], + sidecar: None, + } +} + +#[test] +fn repeated_runs_become_the_comparison_distribution() { + let records = [record("first", 0.03), record("second", 0.05)]; + let combined = aggregate(&records).expect("aggregates"); + let summary = &combined.metrics[0].summary; + + assert_eq!(summary.n, 2); + assert_eq!(summary.median, 0.04); + assert_eq!(summary.cv, 0.25); +} diff --git a/crates/capsem-bench/src/main.rs b/crates/capsem-bench/src/main.rs index 41a356bcb..054a79e8c 100644 --- a/crates/capsem-bench/src/main.rs +++ b/crates/capsem-bench/src/main.rs @@ -105,6 +105,7 @@ async fn main() -> Result<()> { &args.current, dimension, std::env::consts::ARCH, + args.runs, args.thresholds, ); } diff --git a/crates/capsem-bench/src/store.rs b/crates/capsem-bench/src/store.rs index 306628f6c..2827dd69c 100644 --- a/crates/capsem-bench/src/store.rs +++ b/crates/capsem-bench/src/store.rs @@ -180,49 +180,55 @@ pub fn insert(connection: &mut Connection, record: &Record) -> Result { /// difference between them rather than a change in Capsem. A quick run is /// never evidence. pub fn latest(connection: &Connection, dimension: Dimension, arch: &str) -> Result> { - let run = connection - .query_row( - "SELECT id, recorded_at, version, channel, commit_sha, os, cpu_count, + Ok(latest_many(connection, dimension, arch, 1)?.into_iter().next()) +} + +/// The most recent `count` non-quick runs, newest first. +pub fn latest_many(connection: &Connection, dimension: Dimension, arch: &str, count: usize) -> Result> { + let mut query = connection.prepare( + "SELECT id, recorded_at, version, channel, commit_sha, os, cpu_count, kvm, governor, load_before, sidecar FROM runs WHERE dimension = ?1 AND arch = ?2 AND quick = 0 ORDER BY recorded_at DESC, id DESC - LIMIT 1", - params![dimension.as_str(), arch], - |row| { - Ok(( - row.get::<_, i64>(0)?, - crate::schema::Record { - schema: crate::schema::SCHEMA.to_string(), - dimension, - recorded_at: row.get(1)?, - release: crate::schema::Release { - version: row.get(2)?, - channel: row.get(3)?, - commit: row.get(4)?, - }, - host: crate::schema::Host { - arch: arch.to_string(), - os: row.get(5)?, - cpu_count: row.get::<_, i64>(6)? as usize, - kvm: row.get::<_, i64>(7)? != 0, - governor: row.get(8)?, - load_before: row.get(9)?, - }, - quick: false, - metrics: Vec::new(), - sidecar: row.get(10)?, + LIMIT ?3", + )?; + let rows = query.query_map( + params![dimension.as_str(), arch, i64::try_from(count).unwrap_or(i64::MAX)], + |row| { + Ok(( + row.get::<_, i64>(0)?, + crate::schema::Record { + schema: crate::schema::SCHEMA.to_string(), + dimension, + recorded_at: row.get(1)?, + release: crate::schema::Release { + version: row.get(2)?, + channel: row.get(3)?, + commit: row.get(4)?, }, - )) - }, - ) - .ok(); - - let Some((run_id, mut record)) = run else { - return Ok(None); - }; - record.metrics = metrics_of(connection, run_id)?; - Ok(Some(record)) + host: crate::schema::Host { + arch: arch.to_string(), + os: row.get(5)?, + cpu_count: row.get::<_, i64>(6)? as usize, + kvm: row.get::<_, i64>(7)? != 0, + governor: row.get(8)?, + load_before: row.get(9)?, + }, + quick: false, + metrics: Vec::new(), + sidecar: row.get(10)?, + }, + )) + }, + )?; + let mut records = Vec::new(); + for row in rows { + let (run_id, mut record) = row?; + record.metrics = metrics_of(connection, run_id)?; + records.push(record); + } + Ok(records) } fn metrics_of(connection: &Connection, run_id: i64) -> Result> { diff --git a/crates/capsem-bench/src/store/tests.rs b/crates/capsem-bench/src/store/tests.rs index ddb382c45..0f70e50ee 100644 --- a/crates/capsem-bench/src/store/tests.rs +++ b/crates/capsem-bench/src/store/tests.rs @@ -1,4 +1,4 @@ -use super::{history, insert, latest, open, prune, SCHEMA_VERSION}; +use super::{history, insert, latest, latest_many, open, prune, SCHEMA_VERSION}; use crate::schema::{Dimension, Host, Metric, Record, Release, Unit, SCHEMA}; use crate::stats::Summary; @@ -98,6 +98,23 @@ fn the_latest_run_wins() { assert_eq!(found.metrics[0].summary.median, 0.20); } +#[test] +fn latest_many_returns_repeated_authoritative_runs_newest_first() { + let (_dir, mut connection) = store(); + for (at, value) in [ + ("2026-08-19T12:00:00Z", 0.10), + ("2026-08-20T12:00:00Z", 0.20), + ("2026-08-21T12:00:00Z", 0.30), + ] { + insert(&mut connection, &record(Dimension::Routes, "x86_64", at, value)).expect("inserts"); + } + + let found = latest_many(&connection, Dimension::Routes, "x86_64", 2).expect("queries"); + assert_eq!(found.len(), 2); + assert_eq!(found[0].metrics[0].summary.median, 0.30); + assert_eq!(found[1].metrics[0].summary.median, 0.20); +} + #[test] fn another_architecture_is_not_evidence() { let (_dir, mut connection) = store(); From 671e3d92f8c997902ae3ea3c4d92ccfe1e7fb428 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 13:55:17 +0000 Subject: [PATCH 289/464] fix(bench): expose slow asset rebuilds --- justfile | 5 +++++ skills/dev-benchmark/SKILL.md | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/justfile b/justfile index 55019e21e..adb5c7a99 100644 --- a/justfile +++ b/justfile @@ -288,6 +288,11 @@ install: bench *dimensions: @uv run --project build_system --frozen capsem-gate bench {{ quote(dimensions) }} +# Rebuild stale VM assets before measuring dimensions whose committed inputs +# changed. Separate because variadic recipe arguments are one quoted value. +bench-slow *dimensions: + @uv run --project build_system --frozen capsem-gate bench --slow {{ quote(dimensions) }} + # The dev loop: only the dimensions that need no guest, bounded so it stays a # dev loop. Records like any other run; never evidence. bench-quick *dimensions: diff --git a/skills/dev-benchmark/SKILL.md b/skills/dev-benchmark/SKILL.md index 01096698d..a822d7892 100644 --- a/skills/dev-benchmark/SKILL.md +++ b/skills/dev-benchmark/SKILL.md @@ -15,6 +15,7 @@ replaced had ten producers each computing its own `p99`. just bench-quick # dev loop: no guest, bounded, records like any run just bench # every dimension that has a collector just bench routes criterion # only these +just bench-slow routes # same harness, rebuilding intentionally stale VM assets just bench-report # what each subject reads, and how it has moved ``` @@ -30,7 +31,7 @@ this repo took before it existed was taken on an unqualified machine. | `list` | every dimension and whether the quick lane covers it | | `run [dim...]` | measure, then record into the store | | `doctor` | is this machine fit to measure -- load, governor, KVM, strays | -| `compare A B ` | two stores, metric by metric | +| `compare A B ` | two stores, metric by metric; combines the latest two authoritative runs by default | | `verify` | ratchet a run against evidence; refuses on an unfit machine | | `report` | every subject and its trend | | `protocol`, `protocol-delta`, `delta` | the scenario engine: host-direct vs guest-through-Capsem | From c34c285dd86f38f742a58af5ba1f657d0cd6f094 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:01:51 +0000 Subject: [PATCH 290/464] feat(cache): expose metrics for domain caches --- crates/capsem-cache/src/bounded.rs | 8 ++++---- crates/capsem-cache/src/lib.rs | 2 +- crates/capsem-cache/src/snapshot.rs | 10 +++++----- crates/capsem-cache/src/telemetry.rs | 29 ++++++++++++++++------------ 4 files changed, 27 insertions(+), 22 deletions(-) diff --git a/crates/capsem-cache/src/bounded.rs b/crates/capsem-cache/src/bounded.rs index afe2de886..d1117f3ef 100644 --- a/crates/capsem-cache/src/bounded.rs +++ b/crates/capsem-cache/src/bounded.rs @@ -6,10 +6,10 @@ use std::time::Instant; use quick_cache::sync::{Cache, GuardResult}; use quick_cache::{DefaultHashBuilder, Equivalent, Lifecycle, UnitWeighter}; -use crate::telemetry::{CacheStats, CacheTelemetry, LoadStatus}; +use crate::telemetry::{CacheMetrics, CacheStats, LoadStatus}; struct EvictionLifecycle { - telemetry: CacheTelemetry, + telemetry: CacheMetrics, marker: PhantomData, } @@ -46,7 +46,7 @@ type Inner = Cache, UnitWeighter, DefaultHashBuilder, EvictionLi pub struct BoundedCache { inner: Inner, - telemetry: CacheTelemetry, + telemetry: CacheMetrics, } impl BoundedCache @@ -55,7 +55,7 @@ where { pub fn new(name: &'static str, capacity: usize) -> Self { let capacity = capacity.max(1); - let telemetry = CacheTelemetry::new(name); + let telemetry = CacheMetrics::new(name); Self { inner: Cache::with( capacity, diff --git a/crates/capsem-cache/src/lib.rs b/crates/capsem-cache/src/lib.rs index 39b8dd433..f11822c68 100644 --- a/crates/capsem-cache/src/lib.rs +++ b/crates/capsem-cache/src/lib.rs @@ -11,7 +11,7 @@ mod telemetry; pub use bounded::BoundedCache; pub use keyed_identity::KeyedIdentityCache; pub use snapshot::{GenerationCache, GenerationLookup, IdentityCache}; -pub use telemetry::{CacheStats, LoadStatus}; +pub use telemetry::{CacheMetrics, CacheStats, LoadStatus}; #[cfg(test)] mod tests; diff --git a/crates/capsem-cache/src/snapshot.rs b/crates/capsem-cache/src/snapshot.rs index ff28e30f1..4d50edb5b 100644 --- a/crates/capsem-cache/src/snapshot.rs +++ b/crates/capsem-cache/src/snapshot.rs @@ -1,7 +1,7 @@ use std::sync::{Arc, Mutex, MutexGuard}; use std::time::Duration; -use crate::telemetry::{CacheStats, CacheTelemetry, LoadStatus}; +use crate::telemetry::{CacheMetrics, CacheStats, LoadStatus}; fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { mutex.lock().unwrap_or_else(std::sync::PoisonError::into_inner) @@ -14,14 +14,14 @@ struct IdentityEntry { pub struct IdentityCache { entry: Mutex>>, - telemetry: CacheTelemetry, + telemetry: CacheMetrics, } impl IdentityCache { pub fn new(name: &'static str) -> Self { Self { entry: Mutex::new(None), - telemetry: CacheTelemetry::new(name), + telemetry: CacheMetrics::new(name), } } @@ -104,7 +104,7 @@ impl GenerationLookup { pub struct GenerationCache { state: Mutex>, - telemetry: CacheTelemetry, + telemetry: CacheMetrics, } impl GenerationCache { @@ -114,7 +114,7 @@ impl GenerationCache { generation: 0, value: None, }), - telemetry: CacheTelemetry::new(name), + telemetry: CacheMetrics::new(name), } } diff --git a/crates/capsem-cache/src/telemetry.rs b/crates/capsem-cache/src/telemetry.rs index c76775c5a..dc285ffdc 100644 --- a/crates/capsem-cache/src/telemetry.rs +++ b/crates/capsem-cache/src/telemetry.rs @@ -34,14 +34,19 @@ struct Counters { entries: AtomicU64, } +/// Uniform counters and exported metrics for a domain-specific cache. +/// +/// Most callers should use the shared cache primitives, which record these +/// automatically. A cache with domain-specific async or database locking can +/// use this type while retaining its required freshness mechanics. #[derive(Clone)] -pub(crate) struct CacheTelemetry { +pub struct CacheMetrics { name: &'static str, counters: Arc, } -impl CacheTelemetry { - pub(crate) fn new(name: &'static str) -> Self { +impl CacheMetrics { + pub fn new(name: &'static str) -> Self { let telemetry = Self { name, counters: Arc::default(), @@ -50,29 +55,29 @@ impl CacheTelemetry { telemetry } - pub(crate) fn hit(&self) { + pub fn hit(&self) { self.counters.hits.fetch_add(1, Ordering::Relaxed); ::metrics::counter!(metric::CACHE_LOOKUPS_TOTAL, "cache" => self.name, "result" => "hit").increment(1); } - pub(crate) fn miss(&self) { + pub fn miss(&self) { self.counters.misses.fetch_add(1, Ordering::Relaxed); ::metrics::counter!(metric::CACHE_LOOKUPS_TOTAL, "cache" => self.name, "result" => "miss").increment(1); } - pub(crate) fn insertion(&self) { + pub fn insertion(&self) { self.mutation(&self.counters.insertions, "insert", 1); } - pub(crate) fn refresh(&self) { + pub fn refresh(&self) { self.mutation(&self.counters.refreshes, "refresh", 1); } - pub(crate) fn invalidate(&self, count: u64) { + pub fn invalidate(&self, count: u64) { self.mutation(&self.counters.invalidations, "invalidate", count); } - pub(crate) fn evict(&self) { + pub fn evict(&self) { self.mutation(&self.counters.evictions, "evict", 1); } @@ -85,7 +90,7 @@ impl CacheTelemetry { .increment(count); } - pub(crate) fn load(&self, elapsed: Duration, status: LoadStatus) { + pub fn load(&self, elapsed: Duration, status: LoadStatus) { let status = match status { LoadStatus::Success => "success", LoadStatus::Error => { @@ -97,13 +102,13 @@ impl CacheTelemetry { .record(elapsed.as_secs_f64() * 1000.0); } - pub(crate) fn set_entries(&self, entries: usize) { + pub fn set_entries(&self, entries: usize) { let entries = u64::try_from(entries).unwrap_or(u64::MAX); self.counters.entries.store(entries, Ordering::Relaxed); ::metrics::gauge!(metric::CACHE_ENTRIES, "cache" => self.name).set(entries as f64); } - pub(crate) fn snapshot(&self) -> CacheStats { + pub fn snapshot(&self) -> CacheStats { let load = |counter: &AtomicU64| counter.load(Ordering::Relaxed); CacheStats { hits: load(&self.counters.hits), From d971fb8016e3974377fb5ffafdae5fda8b783e8a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:03:11 +0000 Subject: [PATCH 291/464] perf(logger): share bounded query cache --- Cargo.lock | 1 + crates/capsem-logger/Cargo.toml | 1 + crates/capsem-logger/src/db.rs | 74 +++++++++---------- .../src/db/handle_tests/query.rs | 6 +- 4 files changed, 41 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f23adfded..f3903b29c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1070,6 +1070,7 @@ version = "0.7.0" dependencies = [ "anyhow", "capsem-archive", + "capsem-cache", "capsem-crypto", "capsem-foundation", "capsem-proto", diff --git a/crates/capsem-logger/Cargo.toml b/crates/capsem-logger/Cargo.toml index c458da981..231da71b2 100644 --- a/crates/capsem-logger/Cargo.toml +++ b/crates/capsem-logger/Cargo.toml @@ -13,6 +13,7 @@ authors.workspace = true capsem-crypto = { path = "../capsem-crypto", features = ["random"] } anyhow = { workspace = true } capsem-archive = { path = "../capsem-archive" } +capsem-cache = { path = "../capsem-cache" } capsem-foundation = { path = "../capsem-foundation" } capsem-proto = { path = "../capsem-proto" } capsem-telemetry = { path = "../capsem-telemetry" } diff --git a/crates/capsem-logger/src/db.rs b/crates/capsem-logger/src/db.rs index 9a135b887..472d60334 100644 --- a/crates/capsem-logger/src/db.rs +++ b/crates/capsem-logger/src/db.rs @@ -5,6 +5,7 @@ use std::thread::JoinHandle; use std::time::Instant; use capsem_archive::ArchiveCodecs; +use capsem_cache::KeyedIdentityCache; use capsem_telemetry::db::{ DB_QUERY_DURATION_MS, DB_QUERY_PARAMS_COUNT, DB_QUERY_RESULT_BYTES, DB_QUERY_RESULT_ROWS, DB_QUERY_TOTAL, }; @@ -55,7 +56,13 @@ pub type DbQueryParams = [serde_json::Value]; /// execution and schema failures remain DB-owned. pub type DbQueryJson = String; type DbQueryOwned = (String, Vec); -/// Recently answered `query_many` batches, newest first. +type DbQueryCacheKey = Vec; + +fn query_cache_key(queries: &[DbQueryOwned]) -> DbQueryCacheKey { + serde_json::to_vec(queries).expect("query batches contain only JSON values") +} + +/// Recently answered `query_many` batches, keyed by their canonical JSON. /// /// One slot was enough while a single route used the batch rail. It stopped /// being enough the moment two polled routes shared a session handle -- and @@ -63,11 +70,10 @@ type DbQueryOwned = (String, Vec); /// other's answer and neither ever hit: the handle paid the bookkeeping of a /// cache and got the behaviour of none. /// -/// A `Vec` rather than a map because a batch's bound parameters are -/// `serde_json::Value`, which is neither `Hash` nor `Ord`. At this size, -/// comparing the batch outright is cheaper than the fingerprint that would let -/// us avoid it, and it cannot collide. -type DbQueryManyCache = Vec<(Vec, Vec)>; +/// `serde_json::Value` is neither `Hash` nor `Ord`, so the shared bounded cache +/// uses the complete canonical serialization as its collision-free key. The +/// read epoch remains the freshness identity owned by this DB handle. +type DbQueryManyCache = KeyedIdentityCache>; /// How many distinct batches one handle keeps answers for. /// @@ -220,7 +226,7 @@ struct DbHandleInner { reader_join: Mutex>>, writer: Option>, archive_blocks_inflated: AtomicU64, - query_many_cache: Mutex, + query_many_cache: DbQueryManyCache, /// The counter snapshot the reader worker keeps current for polled /// routes. See [`DbHandle::ledger_counters`]. hot: Arc, @@ -329,7 +335,7 @@ impl DbHandle { reader_join: Mutex::new(Some(reader_join)), writer: None, archive_blocks_inflated: AtomicU64::new(0), - query_many_cache: Mutex::new(DbQueryManyCache::new()), + query_many_cache: DbQueryManyCache::new("logger.query_many", QUERY_MANY_CACHE_ENTRIES), hot, reader_requests: AtomicU64::new(0), read_cache_epoch: AtomicU64::new(0), @@ -653,30 +659,28 @@ impl DbHandle { /// The cached answer to exactly this batch, if this handle holds one. /// - /// A hit moves its entry to the front, so the batches a route actually - /// polls keep each other alive and a one-off batch is what falls off the - /// end. - /// - /// Returns the read epoch alongside, taken under the cache lock. Every - /// invalidation bumps the epoch under that same lock, so the pair is one - /// consistent observation: the entry, if any, was valid in exactly this - /// epoch, and a result stored against it is refused once anything expires. + /// Returns the read epoch alongside. The epoch is checked again after the + /// lookup, so an invalidation concurrent with the lookup turns the answer + /// into a miss. A late store carries its older identity and cannot be read + /// by the newer generation. fn cached_query_many(&self, queries: &[DbQueryOwned]) -> (u64, Option>) { - let mut cache = self.inner.query_many_cache.lock().unwrap_or_else(|e| e.into_inner()); let epoch = self.read_cache_epoch(ReadCacheDomain::All); - let Some(position) = cache.iter().position(|(key, _)| key == queries) else { - return (epoch, None); - }; - cache[..=position].rotate_right(1); - let result = cache[0].1.clone(); - drop(cache); - (epoch, Some(result)) + let key = query_cache_key(queries); + let result = self + .inner + .query_many_cache + .get(key.as_slice(), &epoch) + .map(|value| (*value).clone()); + if self.read_cache_epoch(ReadCacheDomain::All) == epoch { + (epoch, result) + } else { + (self.read_cache_epoch(ReadCacheDomain::All), None) + } } /// Cache a `query_many` result unless the read epoch moved while the /// query ran, in which case the result may predate a write and is dropped. pub(crate) fn store_query_many_cache(&self, epoch_before: u64, key: Vec, result: Vec) { - let mut cache = self.inner.query_many_cache.lock().unwrap_or_else(|e| e.into_inner()); if self.read_cache_epoch(ReadCacheDomain::All) != epoch_before { tracing::debug!( db_path = %self.inner.path.display(), @@ -685,9 +689,9 @@ impl DbHandle { ); return; } - cache.retain(|(existing, _)| existing != &key); - cache.insert(0, (key, result)); - cache.truncate(QUERY_MANY_CACHE_ENTRIES); + self.inner + .query_many_cache + .publish(query_cache_key(&key), epoch_before, result); } /// Invalidate DB-owned read caches after external logger lifecycle helpers @@ -700,21 +704,15 @@ impl DbHandle { self.expire_read_caches(affects_session_summary); } - /// Bump the read epochs and drop every cached batch, as one step under the - /// cache lock, and return the read epoch this created. - /// - /// One step because a store checks the epoch under that lock too. Clearing - /// first and bumping after, outside it, left a window in which a result - /// read before the commit was stored under the old epoch after the clear, - /// and survived it. + /// Bump the read epochs and drop every cached batch. A store racing after + /// the clear retains its old epoch identity and cannot satisfy a lookup in + /// the new generation. fn expire_read_caches(&self, affects_session_summary: bool) -> u64 { - let mut cache = self.inner.query_many_cache.lock().unwrap_or_else(|e| e.into_inner()); let epoch = self.inner.read_cache_epoch.fetch_add(1, Ordering::AcqRel) + 1; if affects_session_summary { self.inner.session_summary_cache_epoch.fetch_add(1, Ordering::AcqRel); } - cache.clear(); - drop(cache); + self.inner.query_many_cache.clear(); epoch } diff --git a/crates/capsem-logger/src/db/handle_tests/query.rs b/crates/capsem-logger/src/db/handle_tests/query.rs index 53ee40e94..6e500ab77 100644 --- a/crates/capsem-logger/src/db/handle_tests/query.rs +++ b/crates/capsem-logger/src/db/handle_tests/query.rs @@ -270,7 +270,7 @@ async fn external_reader_query_many_does_not_cache_across_external_writes() { ); } -// Service routes cache response bytes on `read_cache_epoch`. For an external +// The logger's query cache is keyed by `read_cache_epoch`. For an external // reader the epoch is only honest if every disk sync that observed another // connection's commit advances it -- `ready()` and plain queries alike. // Otherwise a query that happened to absorb the commit leaves the epoch where @@ -386,11 +386,11 @@ async fn db_handle_query_many_result_from_before_an_invalidation_is_not_cached() db.invalidate_read_cache(); db.store_query_many_cache(epoch_before, key.clone(), stale); assert!( - db.inner.query_many_cache.lock().unwrap().is_empty(), + db.inner.query_many_cache.is_empty(), "a result older than the last invalidation must not be cached" ); let epoch_now = db.read_cache_epoch(ReadCacheDomain::All); db.store_query_many_cache(epoch_now, key, vec![DbQueryJson::from("{}".to_string())]); - assert!(!db.inner.query_many_cache.lock().unwrap().is_empty()); + assert!(!db.inner.query_many_cache.is_empty()); } From 0d8b21ae4487ce9d83da037adee3e29cefbb0f5f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:03:18 +0000 Subject: [PATCH 292/464] perf(service): observe image catalog cache --- .../src/container_setup/images.rs | 29 ++++++++++++++++++- .../src/container_setup/images/tests.rs | 10 +++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/capsem-service/src/container_setup/images.rs b/crates/capsem-service/src/container_setup/images.rs index fdfb30ef7..dd95035f8 100644 --- a/crates/capsem-service/src/container_setup/images.rs +++ b/crates/capsem-service/src/container_setup/images.rs @@ -23,6 +23,7 @@ use capsem_api::{ CatalogInfo, ImageCacheState, ImageInfo, ImageListQuery, ImageListResponse, ImagePullRequest, ImagePullResponse, }; use capsem_assets::oci::{is_catalog_name, Catalog, Digest, ImageReference, ResolvedImage, RUNTIME_CONTRACT}; +use capsem_cache::{CacheMetrics, LoadStatus}; use capsem_core::container::admission::{CatalogSource, ImagePolicy}; use std::collections::BTreeSet; use tokio::time::{Duration, Instant}; @@ -51,9 +52,18 @@ struct CatalogSlot { /// The last good catalog. The async lock makes concurrent requests share one /// read instead of each asking the registry. -#[derive(Default)] pub(crate) struct CatalogCache { slot: tokio::sync::Mutex, + metrics: CacheMetrics, +} + +impl Default for CatalogCache { + fn default() -> Self { + Self { + slot: tokio::sync::Mutex::new(CatalogSlot::default()), + metrics: CacheMetrics::new("service.image_catalog"), + } + } } impl CatalogCache { @@ -77,14 +87,22 @@ impl CatalogCache { if !refresh { if let Some((loaded, read)) = &last { if now.duration_since(*read) < CATALOG_REFRESH { + self.metrics.hit(); return Some(loaded.clone()); } } if matches!(&slot.failed, Some((failed, at)) if failed == source && now.duration_since(*at) < CATALOG_RETRY) { + if last.is_some() { + self.metrics.hit(); + } else { + self.metrics.miss(); + } return last.map(|(loaded, _)| loaded); } } + self.metrics.miss(); + let load_started = Instant::now(); let read = match images.fetch_catalog(source.clone(), parent.to_owned()).await { Ok((digest, catalog)) => { info!(catalog = %source.reference, digest = %digest, "read the image catalog"); @@ -93,8 +111,16 @@ impl CatalogCache { digest, catalog: Arc::new(catalog), }; + let refresh = slot.good.is_some(); slot.good = Some((loaded.clone(), now)); slot.failed = None; + self.metrics.load(load_started.elapsed(), LoadStatus::Success); + if refresh { + self.metrics.refresh(); + } else { + self.metrics.insertion(); + } + self.metrics.set_entries(1); Some(loaded) } Err(error) => { @@ -105,6 +131,7 @@ impl CatalogCache { "image catalog read failed; keeping the last good catalog" ); slot.failed = Some((source.clone(), now)); + self.metrics.load(load_started.elapsed(), LoadStatus::Error); last.map(|(loaded, _)| loaded) } }; diff --git a/crates/capsem-service/src/container_setup/images/tests.rs b/crates/capsem-service/src/container_setup/images/tests.rs index 91082de05..080641a35 100644 --- a/crates/capsem-service/src/container_setup/images/tests.rs +++ b/crates/capsem-service/src/container_setup/images/tests.rs @@ -555,6 +555,13 @@ async fn the_catalog_is_read_at_most_every_thirty_minutes_or_on_demand() { let (status, _) = fx.call(axum::http::Method::GET, "/images?refresh=true", None).await; assert_eq!(status, StatusCode::OK); assert_eq!(fx.reads(), 3, "refresh reads it now"); + let stats = fx.state.containers.catalog.metrics.snapshot(); + assert_eq!(stats.hits, 3); + assert_eq!(stats.misses, 3); + assert_eq!(stats.insertions, 1); + assert_eq!(stats.refreshes, 2); + assert_eq!(stats.load_errors, 0); + assert_eq!(stats.entries, 1); } #[tokio::test(start_paused = true)] @@ -578,6 +585,9 @@ async fn a_failed_read_keeps_the_last_good_catalog_and_waits_before_retrying() { tokio::time::advance(CATALOG_RETRY + Duration::from_secs(1)).await; fx.list().await; assert_eq!(fx.reads(), 3); + let stats = fx.state.containers.catalog.metrics.snapshot(); + assert_eq!(stats.load_errors, 2); + assert_eq!(stats.entries, 1); } #[tokio::test] From d420b8b607f5e158ce362287c3d6029f147bbaf4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:03:52 +0000 Subject: [PATCH 293/464] test(citadel): keep one cache authority --- tests/citadel/test_runtime_cache_authority.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tests/citadel/test_runtime_cache_authority.py diff --git a/tests/citadel/test_runtime_cache_authority.py b/tests/citadel/test_runtime_cache_authority.py new file mode 100644 index 000000000..d205100aa --- /dev/null +++ b/tests/citadel/test_runtime_cache_authority.py @@ -0,0 +1,73 @@ +"""Runtime cache mechanics and cache-provider dependencies have one owner.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +CACHE_PROVIDERS = frozenset({"quick_cache", "lru", "moka", "cached"}) + +RATIONALE = """\ +Runtime cache mechanics have one owner: capsem-cache. + +Adding an eviction/cache crate or calling a provider directly from a product +crate recreates the split cache system this guard retired: inconsistent bounds, +single-flight behavior, counters, and invalidation. Extend capsem-cache, or use +CacheMetrics when a database or async domain must retain specialized locking. +""" + + +def dependency_sections(manifest: dict) -> list[dict]: + sections = [manifest.get("dependencies", {}), manifest.get("dev-dependencies", {})] + for target in manifest.get("target", {}).values(): + sections.extend((target.get("dependencies", {}), target.get("dev-dependencies", {}))) + return sections + + +def test_only_capsem_cache_depends_on_a_cache_provider() -> None: + offenders = [] + for path in sorted((PROJECT_ROOT / "crates").glob("*/Cargo.toml")): + manifest = tomllib.loads(path.read_text()) + providers = sorted( + CACHE_PROVIDERS.intersection( + dependency + for section in dependency_sections(manifest) + for dependency in section + ) + ) + if providers and path.parent.name != "capsem-cache": + offenders.append(f"{path.relative_to(PROJECT_ROOT)}: {', '.join(providers)}") + + assert not offenders, RATIONALE + "\nDirect provider dependencies:\n" + "\n".join(offenders) + + +def test_product_sources_do_not_bypass_capsem_cache() -> None: + provider_paths = ("quick_cache::", "lru::LruCache", "moka::", "cached::") + offenders = [] + for path in sorted((PROJECT_ROOT / "crates").glob("*/src/**/*.rs")): + if path.parts[-3] == "capsem-cache": + continue + source = path.read_text() + found = [provider for provider in provider_paths if provider in source] + if found: + offenders.append(f"{path.relative_to(PROJECT_ROOT)}: {', '.join(found)}") + + assert not offenders, RATIONALE + "\nDirect provider calls:\n" + "\n".join(offenders) + + +def test_runtime_cache_owners_use_the_shared_control_plane() -> None: + required = { + "crates/capsem-core/src/net/dns/cache.rs": "BoundedCache", + "crates/capsem-core/src/net/cert_authority.rs": "BoundedCache", + "crates/capsem-logger/src/db.rs": "KeyedIdentityCache", + "crates/capsem-service/src/main.rs": "GenerationCache", + "crates/capsem-service/src/container_setup/images.rs": "CacheMetrics", + } + missing = [ + f"{path}: {symbol}" + for path, symbol in required.items() + if symbol not in (PROJECT_ROOT / path).read_text() + ] + + assert not missing, RATIONALE + "\nMissing shared cache owners:\n" + "\n".join(missing) From 5a47725b37505c97321cc620bfb4994d1df705d8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:04:01 +0000 Subject: [PATCH 294/464] docs(architecture): define runtime cache ownership --- .../docs/architecture/host-isolation.md | 12 ++- .../docs/architecture/runtime-caching.md | 92 +++++++++++++++++++ .../content/docs/development/benchmarking.md | 40 +++++--- 3 files changed, 131 insertions(+), 13 deletions(-) create mode 100644 web/docs/src/content/docs/architecture/runtime-caching.md diff --git a/web/docs/src/content/docs/architecture/host-isolation.md b/web/docs/src/content/docs/architecture/host-isolation.md index 951112fa1..f87ce1f2c 100644 --- a/web/docs/src/content/docs/architecture/host-isolation.md +++ b/web/docs/src/content/docs/architecture/host-isolation.md @@ -217,7 +217,17 @@ role: The VM owner and proxy use `DbWriter` as a remote typed client; a diagnostic path in that object does not grant filesystem authority. Service routes also query through a ledger client, including after the VM stops. A missing table or -column is a schema error, and a flush is the read-after-write barrier. +column is a schema error, and a flush is the read-after-write barrier. Reader +counter replies carry the ledger handle's DB-owned cache epoch. Logger-owned +bounded query caches use that epoch and invalidate after commits or worker +reconnects; the service does not retain another serialized route cache. See +[Runtime Caching](/architecture/runtime-caching/). + +Snapshot and retention work stays inside the ledger worker's one session +directory. New lock and archive-generation files are opened at their final +owner-only modes; the worker has no `chmod` or ownership-changing authority. +A fork copies a coherent staged snapshot through descriptor-contained tree +operations, then removes the staging directory. Producers also receive a separate connected channel to the service's trusted commitment authority. Before admission, that authority assigns one global diff --git a/web/docs/src/content/docs/architecture/runtime-caching.md b/web/docs/src/content/docs/architecture/runtime-caching.md new file mode 100644 index 000000000..057618a5c --- /dev/null +++ b/web/docs/src/content/docs/architecture/runtime-caching.md @@ -0,0 +1,92 @@ +--- +title: Runtime Caching +description: Shared cache mechanics, domain-owned freshness, metrics, and performance proof. +sidebar: + order: 7 +--- + +Capsem uses `capsem-cache` as the control plane for in-memory runtime caches. +It provides bounded concurrent storage, single-flight loading, identity and +generation snapshots, invalidation, and one set of metrics. The implementation +uses the maintained Rust `quick_cache` crate rather than another local map and +eviction policy. + +Cache mechanics are shared. Freshness remains with the subsystem that knows +what a change means. A DNS answer expires by TTL and policy identity, a ledger +query expires when the database read epoch moves, and a VM list expires when +its complete source identity changes. Treating those as one universal TTL +would make the code uniform by making its answers wrong. + +## Shared primitives + +| Primitive | Use | +|---|---| +| `BoundedCache` | Concurrent keyed values with a fixed entry bound, optional freshness predicate, single-flight miss loading, and removal or retention. | +| `IdentityCache` | One snapshot that is reusable only while its caller-supplied identity is unchanged. | +| `KeyedIdentityCache` | A bounded set of keyed snapshots, each reusable only for its identity. | +| `GenerationCache` | One snapshot published only if the generation observed before loading is still current. | +| `CacheMetrics` | The same counters and metric labels for a domain cache whose async or database locking must remain specialized. | + +Values are shared through `Arc`, capacities are mandatory, and a concurrent +miss for the same bounded-cache key performs one load. Refreshing an existing +key is distinct from a capacity eviction, so operational counters explain why +an entry changed. + +## Freshness owners + +| Cache | Freshness rule | Shared mechanism | +|---|---|---| +| Service plugin policy | Explicit policy generation | `GenerationCache` | +| Asset status and VM list | Complete input identity | `IdentityCache` | +| Persistent resume state | VM ID plus persisted-state fingerprint | `KeyedIdentityCache` | +| Storage diagnostics | Session path and explicit invalidation | `BoundedCache` | +| DNS answers | Lookup key, policy identity, TTL, and negative-answer TTL | `BoundedCache` with a freshness predicate | +| TLS leaf certificates | Normalized server name | `BoundedCache` with single-flight minting | +| Ledger query batches | Canonical query batch plus logger-owned read epoch | `KeyedIdentityCache` inside `capsem-logger` | +| Image catalog | Source, 30-minute refresh, one-minute failure retry, and last-good fallback | Domain async lock with `CacheMetrics` | + +The logger remains the storage and execution owner for session data. Service, +gateway, UI, and MCP routes do not open SQLite or keep serialized ledger +responses. `capsem-logger` obtains freshness from its writer generation or +SQLite `data_version`, then keys its bounded query cache with that read epoch. +A WAL-only commit therefore invalidates results even when the database file's +size and modification time do not change. + +Several structures deliberately sit outside this cache control plane: + +- The OCI image store is durable content-addressed storage with receipts, + reconciliation, and disk retention policy. +- The credential store's memory map is authoritative runtime state, including + after a durable write fails; evicting it could change authentication. +- Gateway `/status` keeps previous VM states for transition events and uses a + refresh gate to coalesce polls. Every response still comes from a service + read that began after that request arrived. +- Registries of live processes, grants, pending requests, and locks own active + resources. They are lifecycle state rather than reusable computed values. + +## Metrics + +Every shared cache name uses the same bounded label vocabulary: + +| Metric | Labels | Meaning | +|---|---|---| +| `cache.lookups_total` | `cache`, `result=hit|miss` | Reuse effectiveness. | +| `cache.mutations_total` | `cache`, `operation=insert|refresh|invalidate|evict|load_error` | Why resident state changed. | +| `cache.load_duration_ms` | `cache`, `status=success|error` | Work performed after a miss. | +| `cache.entries` | `cache` | Current resident entry count. | + +The primitives also expose a `CacheStats` snapshot with hits, misses, +insertions, refreshes, invalidations, evictions, load errors, and entries for +focused tests. Cache names identify the owner, such as `core.dns_answer`, +`core.tls_leaf`, `logger.query_many`, and `service.image_catalog`. + +## Performance proof + +Polled API reads must remain well below one millisecond at the service boundary +where possible. Cache work is measured through the repository benchmark +system: run the unchanged tree twice, run the candidate twice, and compare the +latest two authoritative runs from each SQLite store. Quick runs remain useful +for development but are excluded from evidence. + +See [Benchmarking](/development/benchmarking/) for the commands, store format, +noise rules, and route coverage contract. diff --git a/web/docs/src/content/docs/development/benchmarking.md b/web/docs/src/content/docs/development/benchmarking.md index 7bcc073e8..ec04a5e75 100644 --- a/web/docs/src/content/docs/development/benchmarking.md +++ b/web/docs/src/content/docs/development/benchmarking.md @@ -5,24 +5,40 @@ sidebar: order: 30 --- -Capsem includes `capsem-bench`, a Python benchmarking tool that runs inside the VM. It outputs rich tables to stderr for humans and saves structured JSON to `/tmp/capsem-benchmark.json` for machine consumption. +Capsem's benchmark control plane is the Rust `capsem-bench-rs` binary. The gate +runs small collectors that emit raw samples; the binary alone computes +statistics and records versioned results in +`cache/target/tests/benchmarks/benchmarks.db`. Host routes, Criterion targets, +VM lifecycle, protocol, storage, and load tests therefore share one schema and +comparison policy. ## Running benchmarks ```bash -just bench # All benchmarks in VM (~2 min) -just exec "capsem-bench disk" # Disk I/O only -just exec "capsem-bench rootfs" # Rootfs reads only -just exec "capsem-bench storage" # Rootfs/workspace/tmpfs/overlay split -just exec "capsem-bench startup" # CLI cold-start only -just exec "capsem-bench http" # HTTP through proxy -just exec "capsem-bench throughput" # 100MB download -just exec "capsem-bench mitm-load 64 5" # MITM proxy concurrency/load test -just exec "capsem-bench mcp-load 64 5" # Guest MCP endpoint concurrency/load test -just exec "capsem-bench dns-load 64 5" # DNS proxy concurrency/load test -just full-test # Full validation including benchmarks +just bench-quick # Host-only development loop; never evidence +just bench # Every dimension with a collector +just bench routes criterion # Selected dimensions +just bench-slow routes # Rebuild stale VM assets, then measure +just bench-report # Subjects, inputs, and stored trends ``` +Use `capsem-bench-rs doctor` when a number is surprising. A measurement from an +unqualified or busy machine remains diagnostic data. + +### Before and after comparisons + +Performance claims require two unchanged baseline runs and two candidate runs. +`capsem-bench-rs compare` combines the latest two authoritative runs from each +store by default, then judges every metric against observed baseline variance +and the configured minimum time resolution. Dimension and architecture must +match. A quick run is recorded for visibility but is never selected as +evidence. + +Low single-digit route-latency movement is normally scheduler noise. The +comparison keeps such movement visible as `tiny` or `noisy`; only a material +change outside the baseline operating envelope is a regression. Route health +ceilings catch gross drift and are not evidence of a comparative speedup. + ## Boot timing Boot timing is measured independently from `capsem-bench`. The guest init script (`capsem-init`) records the wall-clock duration of each boot stage using `/proc/uptime`. The PTY agent sends these measurements to the host over the vsock control channel, where they are displayed as an inline table with a proportional bar chart. From adad95583d892cc1dcad00c1c7a23103ae258df6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:13:30 +0000 Subject: [PATCH 295/464] perf(cache): reuse registered metric handles --- crates/capsem-cache/src/telemetry.rs | 68 ++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/crates/capsem-cache/src/telemetry.rs b/crates/capsem-cache/src/telemetry.rs index dc285ffdc..772af5788 100644 --- a/crates/capsem-cache/src/telemetry.rs +++ b/crates/capsem-cache/src/telemetry.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use std::time::Duration; use capsem_telemetry::cache as metric; +use metrics::{Counter, Gauge, Histogram}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum LoadStatus { @@ -34,6 +35,20 @@ struct Counters { entries: AtomicU64, } +#[derive(Clone)] +struct ExportedMetrics { + hit: Counter, + miss: Counter, + insert: Counter, + refresh: Counter, + invalidate: Counter, + evict: Counter, + load_error: Counter, + load_success_duration: Histogram, + load_error_duration: Histogram, + entries: Gauge, +} + /// Uniform counters and exported metrics for a domain-specific cache. /// /// Most callers should use the shared cache primitives, which record these @@ -41,15 +56,34 @@ struct Counters { /// use this type while retaining its required freshness mechanics. #[derive(Clone)] pub struct CacheMetrics { - name: &'static str, counters: Arc, + exported: ExportedMetrics, } impl CacheMetrics { pub fn new(name: &'static str) -> Self { let telemetry = Self { - name, counters: Arc::default(), + exported: ExportedMetrics { + hit: ::metrics::counter!(metric::CACHE_LOOKUPS_TOTAL, "cache" => name, "result" => "hit"), + miss: ::metrics::counter!(metric::CACHE_LOOKUPS_TOTAL, "cache" => name, "result" => "miss"), + insert: ::metrics::counter!(metric::CACHE_MUTATIONS_TOTAL, "cache" => name, "operation" => "insert"), + refresh: ::metrics::counter!(metric::CACHE_MUTATIONS_TOTAL, "cache" => name, "operation" => "refresh"), + invalidate: ::metrics::counter!(metric::CACHE_MUTATIONS_TOTAL, "cache" => name, "operation" => "invalidate"), + evict: ::metrics::counter!(metric::CACHE_MUTATIONS_TOTAL, "cache" => name, "operation" => "evict"), + load_error: ::metrics::counter!(metric::CACHE_MUTATIONS_TOTAL, "cache" => name, "operation" => "load_error"), + load_success_duration: ::metrics::histogram!( + metric::CACHE_LOAD_DURATION_MS, + "cache" => name, + "status" => "success" + ), + load_error_duration: ::metrics::histogram!( + metric::CACHE_LOAD_DURATION_MS, + "cache" => name, + "status" => "error" + ), + entries: ::metrics::gauge!(metric::CACHE_ENTRIES, "cache" => name), + }, }; telemetry.set_entries(0); telemetry @@ -57,55 +91,53 @@ impl CacheMetrics { pub fn hit(&self) { self.counters.hits.fetch_add(1, Ordering::Relaxed); - ::metrics::counter!(metric::CACHE_LOOKUPS_TOTAL, "cache" => self.name, "result" => "hit").increment(1); + self.exported.hit.increment(1); } pub fn miss(&self) { self.counters.misses.fetch_add(1, Ordering::Relaxed); - ::metrics::counter!(metric::CACHE_LOOKUPS_TOTAL, "cache" => self.name, "result" => "miss").increment(1); + self.exported.miss.increment(1); } pub fn insertion(&self) { - self.mutation(&self.counters.insertions, "insert", 1); + self.mutation(&self.counters.insertions, &self.exported.insert, 1); } pub fn refresh(&self) { - self.mutation(&self.counters.refreshes, "refresh", 1); + self.mutation(&self.counters.refreshes, &self.exported.refresh, 1); } pub fn invalidate(&self, count: u64) { - self.mutation(&self.counters.invalidations, "invalidate", count); + self.mutation(&self.counters.invalidations, &self.exported.invalidate, count); } pub fn evict(&self) { - self.mutation(&self.counters.evictions, "evict", 1); + self.mutation(&self.counters.evictions, &self.exported.evict, 1); } - fn mutation(&self, counter: &AtomicU64, operation: &'static str, count: u64) { + fn mutation(&self, counter: &AtomicU64, exported: &Counter, count: u64) { if count == 0 { return; } counter.fetch_add(count, Ordering::Relaxed); - ::metrics::counter!(metric::CACHE_MUTATIONS_TOTAL, "cache" => self.name, "operation" => operation) - .increment(count); + exported.increment(count); } pub fn load(&self, elapsed: Duration, status: LoadStatus) { - let status = match status { - LoadStatus::Success => "success", + let histogram = match status { + LoadStatus::Success => &self.exported.load_success_duration, LoadStatus::Error => { - self.mutation(&self.counters.load_errors, "load_error", 1); - "error" + self.mutation(&self.counters.load_errors, &self.exported.load_error, 1); + &self.exported.load_error_duration } }; - ::metrics::histogram!(metric::CACHE_LOAD_DURATION_MS, "cache" => self.name, "status" => status) - .record(elapsed.as_secs_f64() * 1000.0); + histogram.record(elapsed.as_secs_f64() * 1000.0); } pub fn set_entries(&self, entries: usize) { let entries = u64::try_from(entries).unwrap_or(u64::MAX); self.counters.entries.store(entries, Ordering::Relaxed); - ::metrics::gauge!(metric::CACHE_ENTRIES, "cache" => self.name).set(entries as f64); + self.exported.entries.set(entries as f64); } pub fn snapshot(&self) -> CacheStats { From bcdd8554f40a255a79a7ea4d8236f63f6a58e288 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:25:58 +0000 Subject: [PATCH 296/464] feat(bench): measure concurrent route RPS --- benchmarks/collectors/routes | 104 +++++++++++++++++-- build_system/builder/gate/benchmarkschema.py | 9 ++ config/gate.toml | 8 ++ skills/dev-benchmark/SKILL.md | 6 ++ tests/test_benchmark_route_collector.py | 48 +++++++++ 5 files changed, 168 insertions(+), 7 deletions(-) diff --git a/benchmarks/collectors/routes b/benchmarks/collectors/routes index a16bd69ed..d50398f72 100755 --- a/benchmarks/collectors/routes +++ b/benchmarks/collectors/routes @@ -26,8 +26,10 @@ Prints raw per-request milliseconds plus service and gateway CPU deltas; from __future__ import annotations import argparse +import concurrent.futures import json import sys +import threading import time from collections.abc import Callable from contextlib import ExitStack, redirect_stdout @@ -67,6 +69,11 @@ def measured_routes() -> list[str]: return list(config["benchmark"]["routes"]["measured"]) +def route_load_config() -> dict[str, int]: + config = tomllib.loads((ROOT / "config" / "gate.toml").read_text(encoding="utf-8")) + return dict(config["benchmark"]["route_load"]) + + def request_path(route: str) -> str: """Resolve a configured route template against the stopped VM fixture.""" if route == BODY_ROUTE: @@ -86,22 +93,31 @@ def request(transport: Transport, target: Target, path: str) -> None: """Require one successful JSON response.""" status, _, body = transport.request("GET", path, headers=target.headers) if not 200 <= status < 300: - raise RuntimeError(f"{target.label} {path} returned HTTP {status}: {body[:500]!r}") + raise RuntimeError( + f"{target.label} {path} returned HTTP {status}: {body[:500]!r}" + ) try: payload = json.loads(body) except json.JSONDecodeError as error: - raise RuntimeError(f"{target.label} {path} returned non-JSON: {body[:500]!r}") from error + raise RuntimeError( + f"{target.label} {path} returned non-JSON: {body[:500]!r}" + ) from error if path == BODY_URL: assert_event_bodies(payload) -def sample_warm_route(target: Target, path: str, samples: int) -> tuple[list[float], dict[str, float]]: +def sample_warm_route( + target: Target, path: str, samples: int +) -> tuple[list[float], dict[str, float]]: """Raw kept-alive latency and every participating process's CPU.""" transport = target.transport_factory() try: for _ in range(WARMUP): request(transport, target, path) - before = {name: process_cpu_seconds(proc) for name, proc in target.cpu_processes.items()} + before = { + name: process_cpu_seconds(proc) + for name, proc in target.cpu_processes.items() + } latencies = [] for _ in range(samples): started = time.perf_counter() @@ -130,6 +146,47 @@ def sample_cold_route(target: Target, path: str, samples: int) -> list[float]: return latencies +def sample_concurrent_route( + target: Target, path: str, samples: int, concurrency: int +) -> tuple[list[float], dict[str, float], float]: + """Raw latency, CPU and RPS with one kept-alive connection per worker.""" + workers = min(samples, concurrency) + barrier = threading.Barrier(workers + 1) + + def worker(count: int) -> list[float]: + transport = target.transport_factory() + try: + request(transport, target, path) + barrier.wait(timeout=30) + measured = [] + for _ in range(count): + started = time.perf_counter() + request(transport, target, path) + measured.append((time.perf_counter() - started) * 1000.0) + return measured + finally: + transport.close() + + counts = [ + samples // workers + int(index < samples % workers) for index in range(workers) + ] + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(worker, count) for count in counts] + before = { + name: process_cpu_seconds(proc) + for name, proc in target.cpu_processes.items() + } + started = time.perf_counter() + barrier.wait(timeout=30) + latencies = [latency for future in futures for latency in future.result()] + elapsed = time.perf_counter() - started + cpu = { + name: float(process_cpu_seconds(proc) - before[name]) + for name, proc in target.cpu_processes.items() + } + return latencies, cpu, samples / elapsed + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--quick", action="store_true", help="fewer samples per route") @@ -137,6 +194,9 @@ def main() -> int: args = parser.parse_args() samples = args.samples if args.samples is not None else (16 if args.quick else 128) + load = route_load_config() + load_samples = load["quick_requests"] if args.quick else load["requests"] + concurrency = load["concurrency"] routes = measured_routes() if not routes: print( @@ -166,9 +226,14 @@ def main() -> int: ), Target( label="gateway", - transport_factory=lambda: Transport(host="127.0.0.1", port=gateway.port), + transport_factory=lambda: Transport( + host="127.0.0.1", port=gateway.port + ), headers={"Authorization": f"Bearer {gateway.token}"}, - cpu_processes={"gateway": gateway_proc, "downstream_service": service_proc}, + cpu_processes={ + "gateway": gateway_proc, + "downstream_service": service_proc, + }, ), ) for path in routes: @@ -186,7 +251,32 @@ def main() -> int: "samples": cold_latencies, } for process_name, cpu_seconds in cpu.items(): - suffix = "cpu_s" if process_name == target.label else f"{process_name}_cpu_s" + suffix = ( + "cpu_s" + if process_name == target.label + else f"{process_name}_cpu_s" + ) + metrics[f"{target.label}.{path}.{suffix}"] = { + "unit": "seconds", + "samples": [cpu_seconds], + } + concurrent_latencies, concurrent_cpu, rps = sample_concurrent_route( + target, concrete_path, load_samples, concurrency + ) + metrics[f"{target.label}.{path}.concurrent_latency_ms"] = { + "unit": "milliseconds", + "samples": concurrent_latencies, + } + metrics[f"{target.label}.{path}.concurrent_rps"] = { + "unit": "requests_per_second", + "samples": [rps], + } + for process_name, cpu_seconds in concurrent_cpu.items(): + suffix = ( + "concurrent_cpu_s" + if process_name == target.label + else f"concurrent_{process_name}_cpu_s" + ) metrics[f"{target.label}.{path}.{suffix}"] = { "unit": "seconds", "samples": [cpu_seconds], diff --git a/build_system/builder/gate/benchmarkschema.py b/build_system/builder/gate/benchmarkschema.py index 7aac4f793..dfe8e3331 100644 --- a/build_system/builder/gate/benchmarkschema.py +++ b/build_system/builder/gate/benchmarkschema.py @@ -29,6 +29,14 @@ class fails `tests/citadel/test_bench_route_coverage.py` in the fast phase, internal: dict[str, str] +class RouteLoadConfig(Strict): + """Concurrent request window applied to every measured read-only route.""" + + concurrency: PositiveInt + requests: PositiveInt + quick_requests: PositiveInt + + class RouteBudgetConfig(Strict): """Actual ceilings for one route class and one transport.""" @@ -161,6 +169,7 @@ class BenchmarkGatesConfig(Strict): class BenchmarkConfig(Strict): routes: RouteCoverageConfig + route_load: RouteLoadConfig route_health: RouteHealthConfig run: BenchmarkRunConfig gates: BenchmarkGatesConfig diff --git a/config/gate.toml b/config/gate.toml index fd5bb2835..476a7ee69 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3120,6 +3120,14 @@ unmeasured = [ "/vms/{id}/timeline", ] +# Concurrent kept-alive clients exercise the same measured read-only routes as +# the latency lane. This catches cache lock and single-flight contention while +# keeping mutations and network-bound catalog reads out of an RPS claim. +[benchmark.route_load] +concurrency = 16 +requests = 512 +quick_requests = 64 + # Release-blocking route ceilings and the margin they must retain. The budget # library classifies a route once and resolves both transports from this # inventory; test code owns no timing literals. CPU is authored for diff --git a/skills/dev-benchmark/SKILL.md b/skills/dev-benchmark/SKILL.md index a822d7892..f0c92733a 100644 --- a/skills/dev-benchmark/SKILL.md +++ b/skills/dev-benchmark/SKILL.md @@ -122,6 +122,12 @@ in no class, a duplicate classification, an unexplained internal route, or service/gateway topology drift. `unmeasured` is debt that may only shrink; `internal` is a one-entry ratchet, not an easy exemption for new routes. +For every route in `measured`, the `routes` collector records sequential warm +and cold latency plus a 16-client kept-alive load window for both the service +UDS and authenticated gateway. The load window emits raw concurrent latency, +aggregate RPS, and CPU samples. Mutation and external-state routes remain in +the explicit debt inventory until their owning fixture can replay them safely. + ## Benchmarks that are still pytest Not yet migrated; they remain the gate for their paths. diff --git a/tests/test_benchmark_route_collector.py b/tests/test_benchmark_route_collector.py index 2fb7c54bf..638040f29 100644 --- a/tests/test_benchmark_route_collector.py +++ b/tests/test_benchmark_route_collector.py @@ -3,6 +3,8 @@ from __future__ import annotations import runpy +import threading +import time from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parents[1] @@ -26,3 +28,49 @@ def test_every_measured_route_resolves_to_a_concrete_fixture_request() -> None: "the benchmark sent route templates literally, so a fast 404 replaced " f"the intended latency measurement: {unresolved}" ) + + +def test_concurrent_route_samples_real_overlap_and_rps() -> None: + collector = _collector() + active = 0 + peak = 0 + lock = threading.Lock() + + class Transport: + def __init__(self): + self.warmed = False + + def request(self, *_args, **_kwargs): + nonlocal active, peak + if not self.warmed: + self.warmed = True + return 200, {}, b"{}" + with lock: + active += 1 + peak = max(peak, active) + time.sleep(0.002) + with lock: + active -= 1 + return 200, {}, b"{}" + + def close(self): + pass + + target = collector["Target"]("test", Transport, {}, {}) + latencies, cpu, requests_per_second = collector["sample_concurrent_route"]( + target, "/status", samples=24, concurrency=4 + ) + + assert len(latencies) == 24 + assert peak == 4 + assert cpu == {} + assert requests_per_second > 0 + + +def test_route_load_settings_cover_real_concurrency() -> None: + collector = _collector() + settings = collector["route_load_config"]() + + assert settings["concurrency"] > 1 + assert settings["requests"] >= settings["concurrency"] + assert settings["quick_requests"] >= settings["concurrency"] From 665c0a3ac13661451ed58e716b9139eaf719ec8c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:26:02 +0000 Subject: [PATCH 297/464] docs(benchmark): define route RPS coverage --- .../src/content/docs/development/benchmarking.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/web/docs/src/content/docs/development/benchmarking.md b/web/docs/src/content/docs/development/benchmarking.md index ec04a5e75..b388ae3c2 100644 --- a/web/docs/src/content/docs/development/benchmarking.md +++ b/web/docs/src/content/docs/development/benchmarking.md @@ -39,6 +39,22 @@ comparison keeps such movement visible as `tiny` or `noisy`; only a material change outside the baseline operating envelope is a regression. Route health ceilings catch gross drift and are not evidence of a comparative speedup. +### Control API routes (`routes`) + +The route collector measures each configured read-only route directly over the +service UDS and through the authenticated TCP gateway. It records warm and cold +sequential latency, then drives 512 requests through 16 kept-alive concurrent +clients and records per-request latency, aggregate RPS, and CPU for every +participating process. The quick lane uses 64 requests but is diagnostic only. + +The route inventory is derived from the registered service and gateway route +tables. Every endpoint must be classified as measured, explicitly unmeasured, +or internal. The generic load lane excludes mutations and routes that require +external network state; their lifecycle or protocol owner supplies the safe +fixture instead. This means `routes` provides RPS for every route in its +measured class, rather than claiming that arbitrary mutations are safe to +replay as a load test. + ## Boot timing Boot timing is measured independently from `capsem-bench`. The guest init script (`capsem-init`) records the wall-clock duration of each boot stage using `/proc/uptime`. The PTY agent sends these measurements to the host over the vsock control channel, where they are displayed as an inline table with a proportional bar chart. From 056b98aca0fae5626d0bcbb735bf828534bf4baf Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:34:31 +0000 Subject: [PATCH 298/464] fix(test): make seeded body archive private --- tests/helpers/stopped_workspace.py | 34 +++++++++++++++++++------ tests/test_stopped_workspace_fixture.py | 23 +++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/tests/helpers/stopped_workspace.py b/tests/helpers/stopped_workspace.py index 60f0d7781..9b7647ec9 100644 --- a/tests/helpers/stopped_workspace.py +++ b/tests/helpers/stopped_workspace.py @@ -25,7 +25,10 @@ def manifest_asset_pins(assets_dir: Path) -> dict[str, dict[str, str]]: arch = "arm64" if platform.machine().lower() in ("arm64", "aarch64") else "x86_64" entries = release["arches"][arch] return { - key: {"name": name, "hash": "blake3:" + entries[name]["hash"].removeprefix("blake3:")} + key: { + "name": name, + "hash": "blake3:" + entries[name]["hash"].removeprefix("blake3:"), + } for key, name in BOOT_ASSETS.items() } @@ -35,7 +38,9 @@ def seed_stopped_workspace(run_dir: Path, assets_dir: Path) -> None: persistent VM pinned to the manifest's current assets.""" registry = run_dir / "persistent_registry.json" if registry.exists(): - raise ValueError("stopped workspace fixture requires an isolated empty registry") + raise ValueError( + "stopped workspace fixture requires an isolated empty registry" + ) session = run_dir / "persistent" / VM_ID workspace = session / "guest" / "workspace" workspace.mkdir(parents=True) @@ -46,11 +51,18 @@ def seed_stopped_workspace(run_dir: Path, assets_dir: Path) -> None: fixture = Path(__file__).resolve().parents[1] / "fixtures" / "session" shutil.copy2(fixture / "test.db", session / "session.db") shutil.copy2(fixture / "test.db-archive.lock", session / "session.db-archive.lock") - shutil.copytree(fixture / "test.bodies", session / "session.bodies") + bodies = shutil.copytree(fixture / "test.bodies", session / "session.bodies") + bodies.chmod(0o700) entry = { - "id": VM_ID, "name": VM_NAME, "asset_pins": manifest_asset_pins(assets_dir), - "ram_mb": DEFAULT_RAM_MB, "cpus": DEFAULT_CPUS, "base_version": "0.0.0-benchmark", - "created_at": "2026-09-10T00:00:00Z", "session_dir": str(session), "defunct": False, + "id": VM_ID, + "name": VM_NAME, + "asset_pins": manifest_asset_pins(assets_dir), + "ram_mb": DEFAULT_RAM_MB, + "cpus": DEFAULT_CPUS, + "base_version": "0.0.0-benchmark", + "created_at": "2026-09-10T00:00:00Z", + "session_dir": str(session), + "defunct": False, } registry.write_text(json.dumps({"vms": {VM_NAME: entry}})) @@ -58,5 +70,11 @@ def seed_stopped_workspace(run_dir: Path, assets_dir: Path) -> None: def assert_event_bodies(payload: dict) -> None: """The measured route must read authenticated bytes, not a fast empty result.""" assert payload["event_id"] == BODY_EVENT_ID, payload - assert {body["direction"] for body in payload["bodies"]} == {"request", "response"}, payload - assert all(body["content"] and body["truncated_for_transport"] for body in payload["bodies"]), payload + assert {body["direction"] for body in payload["bodies"]} == { + "request", + "response", + }, payload + assert all( + body["content"] and body["truncated_for_transport"] + for body in payload["bodies"] + ), payload diff --git a/tests/test_stopped_workspace_fixture.py b/tests/test_stopped_workspace_fixture.py index 41ac491d0..d13d8224a 100644 --- a/tests/test_stopped_workspace_fixture.py +++ b/tests/test_stopped_workspace_fixture.py @@ -2,9 +2,11 @@ from __future__ import annotations +import stat from pathlib import Path import pytest +from helpers import stopped_workspace from helpers.stopped_workspace import seed_stopped_workspace @@ -15,3 +17,24 @@ def test_fixture_never_replaces_an_existing_registry(tmp_path: Path) -> None: seed_stopped_workspace(tmp_path, tmp_path / "assets") assert registry.read_text() == "existing registry" assert not (tmp_path / "persistent").exists() + + +def test_fixture_makes_body_archive_private(tmp_path: Path, monkeypatch) -> None: + real_copytree = stopped_workspace.shutil.copytree + + def permissive_copytree(source: Path, destination: Path) -> Path: + copied = real_copytree(source, destination) + copied.chmod(0o755) + return copied + + monkeypatch.setattr(stopped_workspace, "manifest_asset_pins", lambda _assets: {}) + monkeypatch.setattr(stopped_workspace.shutil, "copytree", permissive_copytree) + + seed_stopped_workspace(tmp_path, tmp_path / "assets") + + mode = stat.S_IMODE( + (tmp_path / "persistent" / stopped_workspace.VM_ID / "session.bodies") + .stat() + .st_mode + ) + assert mode == 0o700 From 45a219d037d0852a7080feab5fe7d0d25e76ab8f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:36:03 +0000 Subject: [PATCH 299/464] fix(test): secure seeded ledger files --- tests/helpers/stopped_workspace.py | 8 ++++++-- tests/test_stopped_workspace_fixture.py | 12 +++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/helpers/stopped_workspace.py b/tests/helpers/stopped_workspace.py index 9b7647ec9..8d2240eba 100644 --- a/tests/helpers/stopped_workspace.py +++ b/tests/helpers/stopped_workspace.py @@ -49,8 +49,12 @@ def seed_stopped_workspace(run_dir: Path, assets_dir: Path) -> None: (workspace / "modified.txt").write_text("after\n") (workspace / "created.txt").write_text("created\n") fixture = Path(__file__).resolve().parents[1] / "fixtures" / "session" - shutil.copy2(fixture / "test.db", session / "session.db") - shutil.copy2(fixture / "test.db-archive.lock", session / "session.db-archive.lock") + database = shutil.copy2(fixture / "test.db", session / "session.db") + database.chmod(0o600) + archive_lock = shutil.copy2( + fixture / "test.db-archive.lock", session / "session.db-archive.lock" + ) + archive_lock.chmod(0o600) bodies = shutil.copytree(fixture / "test.bodies", session / "session.bodies") bodies.chmod(0o700) entry = { diff --git a/tests/test_stopped_workspace_fixture.py b/tests/test_stopped_workspace_fixture.py index d13d8224a..a3da35497 100644 --- a/tests/test_stopped_workspace_fixture.py +++ b/tests/test_stopped_workspace_fixture.py @@ -19,15 +19,22 @@ def test_fixture_never_replaces_an_existing_registry(tmp_path: Path) -> None: assert not (tmp_path / "persistent").exists() -def test_fixture_makes_body_archive_private(tmp_path: Path, monkeypatch) -> None: +def test_fixture_makes_ledger_files_private(tmp_path: Path, monkeypatch) -> None: + real_copy2 = stopped_workspace.shutil.copy2 real_copytree = stopped_workspace.shutil.copytree + def permissive_copy2(source: Path, destination: Path) -> Path: + copied = Path(real_copy2(source, destination)) + copied.chmod(0o644) + return copied + def permissive_copytree(source: Path, destination: Path) -> Path: copied = real_copytree(source, destination) copied.chmod(0o755) return copied monkeypatch.setattr(stopped_workspace, "manifest_asset_pins", lambda _assets: {}) + monkeypatch.setattr(stopped_workspace.shutil, "copy2", permissive_copy2) monkeypatch.setattr(stopped_workspace.shutil, "copytree", permissive_copytree) seed_stopped_workspace(tmp_path, tmp_path / "assets") @@ -38,3 +45,6 @@ def permissive_copytree(source: Path, destination: Path) -> Path: .st_mode ) assert mode == 0o700 + session = tmp_path / "persistent" / stopped_workspace.VM_ID + assert stat.S_IMODE((session / "session.db").stat().st_mode) == 0o600 + assert stat.S_IMODE((session / "session.db-archive.lock").stat().st_mode) == 0o600 From 778e19ea353c3e4349c9206ea2a74ee3e4093294 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:37:09 +0000 Subject: [PATCH 300/464] fix(test): secure seeded body generations --- tests/helpers/stopped_workspace.py | 2 ++ tests/test_stopped_workspace_fixture.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/tests/helpers/stopped_workspace.py b/tests/helpers/stopped_workspace.py index 8d2240eba..2c4e28d18 100644 --- a/tests/helpers/stopped_workspace.py +++ b/tests/helpers/stopped_workspace.py @@ -57,6 +57,8 @@ def seed_stopped_workspace(run_dir: Path, assets_dir: Path) -> None: archive_lock.chmod(0o600) bodies = shutil.copytree(fixture / "test.bodies", session / "session.bodies") bodies.chmod(0o700) + for path in bodies.rglob("*"): + path.chmod(0o700 if path.is_dir() else 0o600) entry = { "id": VM_ID, "name": VM_NAME, diff --git a/tests/test_stopped_workspace_fixture.py b/tests/test_stopped_workspace_fixture.py index a3da35497..685fbec4d 100644 --- a/tests/test_stopped_workspace_fixture.py +++ b/tests/test_stopped_workspace_fixture.py @@ -48,3 +48,8 @@ def permissive_copytree(source: Path, destination: Path) -> Path: session = tmp_path / "persistent" / stopped_workspace.VM_ID assert stat.S_IMODE((session / "session.db").stat().st_mode) == 0o600 assert stat.S_IMODE((session / "session.db-archive.lock").stat().st_mode) == 0o600 + assert { + stat.S_IMODE(path.stat().st_mode) + for path in (session / "session.bodies").rglob("*") + if path.is_file() + } == {0o600} From 684ef2b190eab6f28577382d4ed2fc2da013786c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 14:43:25 +0000 Subject: [PATCH 301/464] fix(bench): isolate route status assets --- benchmarks/collectors/routes | 11 +++++++---- tests/helpers/benchmark_assets.py | 32 +++++++++++++++++++++++++++++++ tests/test_benchmark_assets.py | 31 ++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 tests/helpers/benchmark_assets.py create mode 100644 tests/test_benchmark_assets.py diff --git a/benchmarks/collectors/routes b/benchmarks/collectors/routes index d50398f72..c7eecdf68 100755 --- a/benchmarks/collectors/routes +++ b/benchmarks/collectors/routes @@ -45,6 +45,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests")) sys.path.insert(0, str(Path(__file__).resolve().parents[2])) import psutil +from helpers.benchmark_assets import clone_benchmark_assets from helpers.constants import ASSETS_DIR from helpers.debug_session import debug_session from helpers.gateway import GatewayInstance @@ -207,14 +208,16 @@ def main() -> int: return 1 service = ServiceInstance() + service.assets_dir = clone_benchmark_assets( + ASSETS_DIR, service.home_dir / "benchmark-assets" + ) gateway = GatewayInstance(service.uds_path) metrics: dict[str, dict] = {} try: - seed_stopped_workspace(service.tmp_dir, ASSETS_DIR) - with redirect_stdout(sys.stderr): - service.start() - gateway.start() + seed_stopped_workspace(service.tmp_dir, service.assets_dir) + service.start() + gateway.start() service_proc = psutil.Process(service.proc.pid) gateway_proc = psutil.Process(gateway.proc.pid) targets = ( diff --git a/tests/helpers/benchmark_assets.py b/tests/helpers/benchmark_assets.py new file mode 100644 index 000000000..dc2b4b7c5 --- /dev/null +++ b/tests/helpers/benchmark_assets.py @@ -0,0 +1,32 @@ +"""Isolated runtime assets for host-side route benchmarks.""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + + +def clone_benchmark_assets(source: Path, destination: Path) -> Path: + """Clone an asset tree cheaply and install deterministic status metadata.""" + + def link_file(path: str, target: str) -> str: + Path(target).symlink_to(Path(path).resolve()) + return target + + shutil.copytree(source, destination, copy_function=link_file, symlinks=True) + metadata = destination / "manifest-metadata.json" + metadata.unlink(missing_ok=True) + metadata.write_text( + json.dumps( + { + "schema": "capsem.manifest_metadata.v1", + "origin": "benchmark_fixture", + "checked_at": 1_800_000_000, + "update_available": False, + "validation_status": "valid", + } + ), + encoding="utf-8", + ) + return destination diff --git a/tests/test_benchmark_assets.py b/tests/test_benchmark_assets.py new file mode 100644 index 000000000..757e9fbbe --- /dev/null +++ b/tests/test_benchmark_assets.py @@ -0,0 +1,31 @@ +"""The route benchmark owns stable installed-status inputs.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from helpers.benchmark_assets import clone_benchmark_assets + + +def test_clone_benchmark_assets_is_isolated_and_deterministic(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + (source / "manifest.json").write_text('{"format": 2}') + original = '{"schema": "ambient"}' + (source / "manifest-metadata.json").write_text(original) + (source / "large-asset").write_bytes(b"asset") + + cloned = clone_benchmark_assets(source, tmp_path / "cloned") + + assert (cloned / "manifest.json").is_symlink() + assert (cloned / "large-asset").is_symlink() + assert not (cloned / "manifest-metadata.json").is_symlink() + assert json.loads((cloned / "manifest-metadata.json").read_text()) == { + "schema": "capsem.manifest_metadata.v1", + "origin": "benchmark_fixture", + "checked_at": 1_800_000_000, + "update_available": False, + "validation_status": "valid", + } + assert (source / "manifest-metadata.json").read_text() == original From 6f98d9a2c4b2dd0536fd322b4171a4bddf740bb4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 15:07:31 +0000 Subject: [PATCH 302/464] perf(cache): keep resident loads on read path --- crates/capsem-cache/src/bounded.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/capsem-cache/src/bounded.rs b/crates/capsem-cache/src/bounded.rs index d1117f3ef..913ab6004 100644 --- a/crates/capsem-cache/src/bounded.rs +++ b/crates/capsem-cache/src/bounded.rs @@ -124,6 +124,15 @@ where where Q: Equivalent + Hash + ToOwned + ?Sized, { + // The guarded lookup coordinates misses, but its waiter bookkeeping is + // unnecessary on the overwhelmingly common resident path. Keep hits + // on quick_cache's concurrent read path and acquire a load guard only + // after an observed miss. A racing insert is still returned by the + // guarded lookup below, preserving exact single-flight behavior. + if let Some(value) = self.inner.get(key) { + self.telemetry.hit(); + return Ok(value); + } match self.inner.get_value_or_guard(key, None) { GuardResult::Value(value) => { self.telemetry.hit(); From cf6184062b49d3a948f4780707b95ec0df1f6f3e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 15:15:57 +0000 Subject: [PATCH 303/464] fix(bench): compare throughput in the right direction --- crates/capsem-bench/src/stats.rs | 19 ++++++++++++++++--- crates/capsem-bench/src/stats/tests.rs | 26 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/capsem-bench/src/stats.rs b/crates/capsem-bench/src/stats.rs index 220bd0f1e..f6a279dd3 100644 --- a/crates/capsem-bench/src/stats.rs +++ b/crates/capsem-bench/src/stats.rs @@ -195,10 +195,23 @@ pub fn compare( }; let delta_pct = if before == 0.0 { 0.0 } else { delta_abs / before * 100.0 }; - let regressed = ratio > thresholds.maximum_factor; + let regression_ratio = if matches!(unit, Unit::RequestsPerSecond | Unit::MegabitsPerSecond) { + if after == 0.0 { + if before == 0.0 { + 1.0 + } else { + f64::INFINITY + } + } else { + before / after + } + } else { + ratio + }; + let regressed = regression_ratio > thresholds.maximum_factor; let noise_band = baseline.cv * thresholds.noise_factor; - let material = delta_abs >= minimum_delta(unit, thresholds.minimum_time_resolution_ms); - let significant = regressed && material && (ratio - 1.0) > noise_band; + let material = delta_abs.abs() >= minimum_delta(unit, thresholds.minimum_time_resolution_ms); + let significant = regressed && material && (regression_ratio - 1.0) > noise_band; Comparison { key: key.to_string(), diff --git a/crates/capsem-bench/src/stats/tests.rs b/crates/capsem-bench/src/stats/tests.rs index 16eb88cef..d8ea8d14c 100644 --- a/crates/capsem-bench/src/stats/tests.rs +++ b/crates/capsem-bench/src/stats/tests.rs @@ -182,6 +182,32 @@ fn an_improvement_is_never_a_regression() { assert!((verdict.delta_pct + 50.0).abs() < 1e-9); } +#[test] +fn throughput_growth_is_an_improvement_and_loss_is_a_regression() { + let faster = compare( + "mitm-load.rps", + Statistic::Median, + &steady(100.0), + &steady(200.0), + Unit::RequestsPerSecond, + THRESHOLDS, + ); + assert!(!faster.regressed); + assert_eq!(faster.ratio, 2.0, "the displayed ratio remains current/baseline"); + + let slower = compare( + "mitm-load.rps", + Statistic::Median, + &steady(100.0), + &steady(50.0), + Unit::RequestsPerSecond, + THRESHOLDS, + ); + assert!(slower.regressed); + assert!(slower.significant); + assert_eq!(slower.ratio, 0.5); +} + #[test] fn nanosecond_percentages_have_no_release_authority() { let verdict = compare( From 2ead85e2a2e2b23bb4f54377d105be96e759eeab Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 15:19:43 +0000 Subject: [PATCH 304/464] fix(bench): reject partial load failures --- benchmarks/collectors/_guest | 43 +++++++-- guest/artifacts/capsem_bench/dns_load.py | 16 ++-- guest/artifacts/capsem_bench/load_harness.py | 19 ++-- guest/artifacts/capsem_bench/mitm_load.py | 26 ++++-- tests/test_benchmark_guest_collector.py | 93 ++++++++++++++++--- .../test_capsem_bench_mock_server_protocol.py | 62 +++++++++++-- 6 files changed, 207 insertions(+), 52 deletions(-) diff --git a/benchmarks/collectors/_guest b/benchmarks/collectors/_guest index 032be427c..3b1c17676 100755 --- a/benchmarks/collectors/_guest +++ b/benchmarks/collectors/_guest @@ -100,16 +100,15 @@ def mitm_fixture_corp_toml(ready: dict) -> str: def failed_load_levels(section: dict) -> list[int]: - """Concurrency levels whose every request failed. + """Concurrency levels where any request failed. - A level where nothing succeeded measured the failure path, not the rail - under test; recording it would ratchet the wrong number. + Partial success still means the rail failed to sustain the requested + concurrency. Recording its latency or RPS would ratchet a degraded run. """ return [ int(row.get("concurrency", -1)) for row in section.get("concurrency_levels", []) - if int(row.get("total_requests", 0)) > 0 - and int(row.get("errors", 0)) >= int(row.get("total_requests", 0)) + if int(row.get("errors", 0)) > 0 ] @@ -158,6 +157,23 @@ def flatten(node, prefix: str = "") -> dict[str, float]: return found +def load_quality_metrics(section: dict) -> dict[str, dict]: + """Store zero-error proof for every load concurrency level.""" + metrics = {} + for index, row in enumerate(section.get("concurrency_levels", [])): + errors = int(row.get("errors", 0)) + total = int(row.get("total_requests", 0)) + metrics[f"concurrency_levels.{index}.errors"] = { + "unit": "count", + "samples": [errors], + } + metrics[f"concurrency_levels.{index}.error_ratio"] = { + "unit": "ratio", + "samples": [errors / total if total else 0.0], + } + return metrics + + def main() -> int: mode = Path(sys.argv[0]).name parser = argparse.ArgumentParser(description=f"measure {mode} inside a VM") @@ -195,7 +211,11 @@ def main() -> int: print(f"running capsem-bench {mode}", file=sys.stderr) benchmark_result = client.post( f"/vms/{name}/exec", - {"target": "vm", "command": guest_command(mode), "timeout_secs": args.timeout}, + { + "target": "vm", + "command": guest_command(mode), + "timeout_secs": args.timeout, + }, timeout=args.timeout + 10, ) if not benchmark_result or benchmark_result.get("exit_code") != 0: @@ -207,7 +227,11 @@ def main() -> int: artifact_result = client.post( f"/vms/{name}/exec", - {"target": "vm", "command": "cat /tmp/capsem-benchmark.json", "timeout_secs": 15}, + { + "target": "vm", + "command": "cat /tmp/capsem-benchmark.json", + "timeout_secs": 15, + }, timeout=25, ) if not artifact_result or artifact_result.get("exit_code") != 0: @@ -231,8 +255,8 @@ def main() -> int: section = document.get(mode.replace("-", "_"), document) if failed := failed_load_levels(section): raise RuntimeError( - f"capsem-bench {mode}: every request failed at concurrency {failed}; " - "the rail under test was never reached, so nothing here is a measurement" + f"capsem-bench {mode}: requests failed at concurrency {failed}; " + "the rail did not sustain the requested load, so the run is rejected" ) numbers = flatten(section) if not numbers: @@ -247,6 +271,7 @@ def main() -> int: for key, value in sorted(numbers.items()) if (unit := unit_for(key)) is not None } + measured.update(load_quality_metrics(section)) if not measured: raise RuntimeError( f"capsem-bench {mode} reported {len(numbers)} numbers, none of which " diff --git a/guest/artifacts/capsem_bench/dns_load.py b/guest/artifacts/capsem_bench/dns_load.py index 97fc1dbab..b105dc900 100644 --- a/guest/artifacts/capsem_bench/dns_load.py +++ b/guest/artifacts/capsem_bench/dns_load.py @@ -65,6 +65,7 @@ # on the developer machine. The benches themselves only run inside # the guest where rich + the helpers package are available. + def _percentile(sorted_values, q): """Return the q-th percentile (q in [0, 100]) from a pre-sorted list. Local copy so this module doesn't need `.helpers` until the bench @@ -191,7 +192,7 @@ def worker(): def _summarize(results, concurrency, duration_s): - latencies = [r[0] for r in results] + latencies = [r[0] for r in results if r[2] is None] errors = sum(1 for r in results if r[2] is not None) decisions = {} for _lat, rcode, err in results: @@ -224,9 +225,7 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N concurrency_levels=concurrency_levels, duration_s=duration_s, ) - timeout_s = float( - os.environ.get("CAPSEM_BENCH_DNS_TIMEOUT", DEFAULT_TIMEOUT_S) - ) + timeout_s = float(os.environ.get("CAPSEM_BENCH_DNS_TIMEOUT", DEFAULT_TIMEOUT_S)) console.print( f"[bold]dns-load[/bold] qname={qname} qtype={qtype} " @@ -251,9 +250,12 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N f"dns-load (qname={qname}, qtype={qtype}, {config.duration_s}s per level)", rows, extra_columns=[ - ("decisions", lambda row: ",".join( - f"{k}={v}" for k, v in row["decision_distribution"].items() - )), + ( + "decisions", + lambda row: ",".join( + f"{k}={v}" for k, v in row["decision_distribution"].items() + ), + ), ], ) diff --git a/guest/artifacts/capsem_bench/load_harness.py b/guest/artifacts/capsem_bench/load_harness.py index cc4c9a8ca..f9337a437 100644 --- a/guest/artifacts/capsem_bench/load_harness.py +++ b/guest/artifacts/capsem_bench/load_harness.py @@ -102,7 +102,8 @@ def from_inputs( raw = _env_value(mode, "DURATION_S", GLOBAL_DURATION_ENV) duration_s = ( parse_positive_float(raw, "duration_s") - if raw else float(default_duration_s) + if raw + else float(default_duration_s) ) else: duration_s = parse_positive_float(duration_s, "duration_s") @@ -139,7 +140,8 @@ def from_inputs( raw = _env_value(mode, "TOTAL_REQUESTS", GLOBAL_TOTAL_REQUESTS_ENV) total_requests = ( parse_positive_int(raw, "total_requests") - if raw else int(default_total_requests) + if raw + else int(default_total_requests) ) else: total_requests = parse_positive_int(total_requests, "total_requests") @@ -148,7 +150,8 @@ def from_inputs( raw = _env_value(mode, "CONCURRENCY", GLOBAL_CONCURRENCY_ENV) concurrency = ( parse_positive_int(raw, "concurrency") - if raw else int(default_concurrency) + if raw + else int(default_concurrency) ) else: concurrency = parse_positive_int(concurrency, "concurrency") @@ -157,7 +160,8 @@ def from_inputs( raw = _env_value(mode, "TIMEOUT_S", GLOBAL_TIMEOUT_ENV) timeout_s = ( parse_positive_float(raw, "timeout_s") - if raw else float(default_timeout_s) + if raw + else float(default_timeout_s) ) else: timeout_s = parse_positive_float(timeout_s, "timeout_s") @@ -189,12 +193,14 @@ def peak_rss_mb(): def summarize_load_level(latencies_ms, errors, concurrency, duration_s, *, extra=None): from .helpers import percentile + total_requests = len(latencies_ms) + errors if not latencies_ms: row = { "concurrency": concurrency, "duration_s": duration_s, - "total_requests": 0, + "total_requests": total_requests, "errors": errors, + "error_ratio": errors / total_requests if total_requests else 0.0, "rps": 0.0, "p50_ms": 0.0, "p95_ms": 0.0, @@ -206,8 +212,9 @@ def summarize_load_level(latencies_ms, errors, concurrency, duration_s, *, extra row = { "concurrency": concurrency, "duration_s": duration_s, - "total_requests": len(latencies_ms), + "total_requests": total_requests, "errors": errors, + "error_ratio": errors / total_requests, "rps": len(latencies_ms) / duration_s, "p50_ms": percentile(sorted_latencies, 50), "p95_ms": percentile(sorted_latencies, 95), diff --git a/guest/artifacts/capsem_bench/mitm_load.py b/guest/artifacts/capsem_bench/mitm_load.py index 485b78db2..ace56050d 100644 --- a/guest/artifacts/capsem_bench/mitm_load.py +++ b/guest/artifacts/capsem_bench/mitm_load.py @@ -11,8 +11,7 @@ It used to target a non-existent public domain "so every request fails at the upstream dial". The guest's resolver answered NXDOMAIN and the client failed before opening a socket, so the proxy never ran and the recorded -numbers were DNS-failure numbers. A level where every request fails is now -an error, not a measurement. +numbers were DNS-failure numbers. Any failed request now rejects the run. Output schema: @@ -25,6 +24,7 @@ "duration_s": 30.0, "total_requests": 1234, "errors": 0, + "error_ratio": 0.0, "rps": 41.1, "p50_ms": 22.0, "p95_ms": 35.0, @@ -44,6 +44,7 @@ import os import ssl import time +from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed from urllib.parse import urlsplit @@ -110,7 +111,7 @@ def _do_request(session): elapsed_ms = (time.monotonic() - start) * 1000 error = None if 200 <= status < 300 else f"HTTP {status}" return (elapsed_ms, status, error) - except Exception as exc: + except (OSError, http.client.HTTPException) as exc: elapsed_ms = (time.monotonic() - start) * 1000 return (elapsed_ms, 0, str(exc)) @@ -142,9 +143,16 @@ def worker(): def _summarize(results, concurrency, duration_s): """Build the JSON-shaped row for this concurrency level.""" - latencies = sorted(r[0] for r in results) + latencies = sorted(r[0] for r in results if r[2] is None) errors = sum(1 for r in results if r[2] is not None) - return summarize_load_level(latencies, errors, concurrency, duration_s) + error_distribution = Counter(r[2] for r in results if r[2] is not None) + return summarize_load_level( + latencies, + errors, + concurrency, + duration_s, + extra={"error_distribution": dict(error_distribution)}, + ) def mitm_load_bench(target=None, concurrency_levels=None, duration_s=None): @@ -168,11 +176,11 @@ def mitm_load_bench(target=None, concurrency_levels=None, duration_s=None): console.print(f" concurrency={c} ...") results = _drive_at_concurrency(target, c, config.duration_s) row = _summarize(results, c, config.duration_s) - if row["total_requests"] and row["errors"] >= row["total_requests"]: - first_error = next((r[2] for r in results if r[2] is not None), "unknown") + if row["errors"]: raise RuntimeError( - f"mitm-load: every request at concurrency {c} failed ({first_error}); " - "the proxy was not measured" + f"mitm-load: {row['errors']} of {row['total_requests']} requests failed " + f"at concurrency {c}: {row['error_distribution']}; " + "the rail did not sustain the requested load" ) rows.append(row) diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index 2e3a4aaf2..1020f767f 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -10,6 +10,7 @@ from pathlib import Path from types import SimpleNamespace +import pytest from capsem_builder.gate import config as gate_config PROJECT_ROOT = Path(__file__).resolve().parents[1] @@ -87,7 +88,7 @@ def test_mitm_fixture_corp_config_routes_dns_and_the_fixture_host_to_the_mock_se assert module["MITM_FIXTURE_TARGET"] == "https://fixture.capsem.test/tiny" -def test_a_level_where_every_request_failed_is_not_a_measurement() -> None: +def test_a_level_where_any_request_failed_is_not_a_measurement() -> None: failed = collector()["failed_load_levels"] section = { "concurrency_levels": [ @@ -97,7 +98,7 @@ def test_a_level_where_every_request_failed_is_not_a_measurement() -> None: {"concurrency": 200, "total_requests": 100, "errors": 250}, ] } - assert failed(section) == [1, 200] + assert failed(section) == [1, 10, 200] assert failed({"concurrency_levels": []}) == [] assert failed({}) == [] @@ -111,29 +112,97 @@ def test_load_dimensions_get_more_guest_cpus_bounded_by_the_host() -> None: assert guest_cpus("mitm-load", host_cpus=2) == 2, "never below the default" -def test_guest_result_preserves_error_counts_and_concurrency(monkeypatch, capsys) -> None: - document = {"dns_load": {"concurrency_levels": [ - {"concurrency": 200, "total_requests": 500, "errors": 3, "rps": 50.0}, - ]}} +def test_guest_result_preserves_zero_error_proof(monkeypatch, capsys) -> None: + document = { + "dns_load": { + "concurrency_levels": [ + {"concurrency": 200, "total_requests": 500, "errors": 0, "rps": 50.0}, + ] + } + } stopped = [] client = SimpleNamespace( post=lambda route, *args, **kwargs: ( - {"id": "bench-test-vm"} if route == "/vms/create" - else {"exit_code": 0, "stdout": {"encoding": "utf8", "data": json.dumps(document)}} + {"id": "bench-test-vm"} + if route == "/vms/create" + else { + "exit_code": 0, + "stdout": {"encoding": "utf8", "data": json.dumps(document)}, + } ), delete=lambda *args: None, ) main = collector()["main"] - monkeypatch.setitem(main.__globals__, "ServiceInstance", lambda: SimpleNamespace( - client=lambda: client, start=lambda: None, stop=lambda: stopped.append(True), - )) - monkeypatch.setitem(main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True) + monkeypatch.setitem( + main.__globals__, + "ServiceInstance", + lambda: SimpleNamespace( + client=lambda: client, + start=lambda: None, + stop=lambda: stopped.append(True), + ), + ) + monkeypatch.setitem( + main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True + ) monkeypatch.setattr(sys, "argv", ["dns-load"]) assert main() == 0 result = json.loads(capsys.readouterr().out) assert json.loads(result["sidecar"]) == document assert result["metrics"] == { + "concurrency_levels.0.error_ratio": {"unit": "ratio", "samples": [0.0]}, + "concurrency_levels.0.errors": {"unit": "count", "samples": [0]}, "concurrency_levels.0.rps": {"unit": "requests_per_second", "samples": [50.0]}, } assert stopped == [True] + + +def test_guest_result_rejects_one_failed_request(monkeypatch, tmp_path) -> None: + document = { + "mitm_load": { + "concurrency_levels": [ + {"concurrency": 200, "total_requests": 500, "errors": 1, "rps": 49.9}, + ] + } + } + client = SimpleNamespace( + post=lambda route, *args, **kwargs: ( + {"id": "bench-test-vm"} + if route == "/vms/create" + else { + "exit_code": 0, + "stdout": {"encoding": "utf8", "data": json.dumps(document)}, + } + ), + delete=lambda *args: None, + ) + main = collector()["main"] + monkeypatch.setitem( + main.__globals__, + "ServiceInstance", + lambda: SimpleNamespace( + client=lambda: client, + start=lambda: None, + stop=lambda: None, + tmp_dir=tmp_path, + ), + ) + import helpers.mock_server + + monkeypatch.setattr( + helpers.mock_server, + "start_mock_server", + lambda: ( + object(), + {"dns_udp_addr": "127.0.0.1:41000", "http_addr": "127.0.0.1:3713"}, + ), + ) + monkeypatch.setattr(helpers.mock_server, "stop_process", lambda proc: None) + monkeypatch.setitem( + main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True + ) + monkeypatch.setattr(sys, "argv", ["mitm-load"]) + + with pytest.raises(RuntimeError, match="requests failed at concurrency \\[200\\]"): + main() diff --git a/tests/test_capsem_bench_mock_server_protocol.py b/tests/test_capsem_bench_mock_server_protocol.py index 144f81054..6bda5eee6 100644 --- a/tests/test_capsem_bench_mock_server_protocol.py +++ b/tests/test_capsem_bench_mock_server_protocol.py @@ -20,6 +20,39 @@ def test_mitm_load_counts_http_failures(status): assert error == (None if 200 <= status < 300 else f"HTTP {status}") +def test_mitm_load_excludes_failed_latency_and_reports_error_ratio(): + row = mitm_load._summarize( + [(5.0, 200, None), (100.0, 0, "timed out")], + concurrency=2, + duration_s=10, + ) + + assert row["total_requests"] == 2 + assert row["errors"] == 1 + assert row["error_ratio"] == 0.5 + assert row["rps"] == 0.1 + assert row["p50_ms"] == 5.0 + assert row["error_distribution"] == {"timed out": 1} + + +def test_mitm_load_rejects_one_failed_request(monkeypatch): + monkeypatch.setattr( + mitm_load, + "_drive_at_concurrency", + lambda url, concurrency, duration_s: [ + (5.0, 200, None), + (100.0, 0, "timed out"), + ], + ) + + with pytest.raises(RuntimeError, match="1 of 2 requests failed"): + mitm_load.mitm_load_bench( + target="https://fixture.capsem.test/tiny", + concurrency_levels=[1], + duration_s=1, + ) + + class _StubConsole: def __init__(self, *args, **kwargs): self.messages = [] @@ -76,12 +109,15 @@ class Completed: assert exc.value.code == 7 assert calls == [ - ([ - bench_main.RUST_BENCH, - "protocol", - "--base-url", - "http://127.0.0.1:3713", - ], False) + ( + [ + bench_main.RUST_BENCH, + "protocol", + "--base-url", + "http://127.0.0.1:3713", + ], + False, + ) ] @@ -97,7 +133,9 @@ def fake_module(module_name, function_name): fake_module("capsem_bench.startup", "startup_bench") fake_module("capsem_bench.snapshot", "snapshot_bench") monkeypatch.setattr(sys, "argv", ["capsem-bench", "all"]) - monkeypatch.setenv(bench_main.MOCK_SERVER_PROTOCOL_BASE_URL_ENV, "http://127.0.0.1:3713") + monkeypatch.setenv( + bench_main.MOCK_SERVER_PROTOCOL_BASE_URL_ENV, "http://127.0.0.1:3713" + ) calls = [] def fake_rust_protocol_artifact(scenarios=None, requests=None, concurrency=None): @@ -153,7 +191,9 @@ def fake_rust_protocol_artifact(scenarios=None, requests=None, concurrency=None) } } - monkeypatch.setattr(bench_main, "_run_rust_protocol_artifact", fake_rust_protocol_artifact) + monkeypatch.setattr( + bench_main, "_run_rust_protocol_artifact", fake_rust_protocol_artifact + ) bench_main.main() @@ -163,7 +203,11 @@ def fake_rust_protocol_artifact(scenarios=None, requests=None, concurrency=None) assert data["throughput"]["http_code"] == 200 assert data["throughput"]["size_bytes"] == 10 * 1024 * 1024 assert "mock_server_protocol" in data - assert calls == [(None, None, None), ("tiny_http", None, None), ("http_10mb", "1", "1")] + assert calls == [ + (None, None, None), + ("tiny_http", None, None), + ("http_10mb", "1", "1"), + ] def test_global_load_config_parses_count_and_duration_modes(monkeypatch): From 962cd0be0fd4b00e4ce68abe556f43101fe98e2d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 15:28:45 +0000 Subject: [PATCH 305/464] fix(proxy): sustain supported HTTP concurrency --- CHANGELOG.md | 4 ++++ crates/capsem-proxy/src/main.rs | 8 ++++---- crates/capsem-proxy/src/tests.rs | 18 ++++++++++++++++-- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e74ef8ed0..d68683416 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -172,6 +172,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- The confined HTTP proxy admits the supported 200-client concurrent load plus + connection-replacement overlap instead of closing excess TLS handshakes when + its descriptor grant bound is reached. + - Gateway `/status` reads VM, asset and update state in one coordinator snapshot, cutting service CPU for the polled overview without caching stale VM lifecycle state. diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 36902e8d2..950402085 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -32,10 +32,10 @@ mod metric_client; mod private_names_client; const CONTROL_QUEUE_CAPACITY: usize = 16; -// A 64-request handler limit can transiently own more than 64 connection -// descriptors: pooled connections are replaced before their asynchronous -// close is observed. Keep one bounded replacement slot per active handler. -const TRAFFIC_GRANT_LIMIT: usize = 128; +// The supported 200-client load can transiently own twice as many descriptors: +// pooled connections are replaced before their asynchronous close is observed. +// Retain bounded headroom above that overlap without admitting unbounded FDs. +const TRAFFIC_GRANT_LIMIT: usize = 512; const CONTROL_SEND_TIMEOUT: Duration = Duration::from_secs(5); type ControlSender = router_channel::DescriptorSender; diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index ac3ab3819..41bc14c20 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -54,18 +54,32 @@ fn each_traffic_capability_keeps_its_full_bounded_capacity() { let http = (100..100 + TRAFFIC_GRANT_LIMIT as u64).collect::>(); grant_ids.extend(http.iter().copied()); grants.insert(ProxyCapability::HttpTraffic, http); + let next_grant = 100 + TRAFFIC_GRANT_LIMIT as u64; assert_eq!( - grant_rejection(&grants, &grant_ids, ProxyCapability::HttpTraffic, 500), + grant_rejection(&grants, &grant_ids, ProxyCapability::HttpTraffic, next_grant), Some(ProxyControlRejection::Capacity) ); assert_eq!( - grant_rejection(&grants, &grant_ids, ProxyCapability::DnsTraffic, 501), + grant_rejection(&grants, &grant_ids, ProxyCapability::DnsTraffic, next_grant + 1), None, "long-lived capabilities and full HTTP load do not consume DNS capacity" ); } +#[test] +fn http_capacity_covers_two_hundred_clients_and_replacement_overlap() { + let active = (1..=400).collect::>(); + let grants = BTreeMap::from([(ProxyCapability::HttpTraffic, active.clone())]); + let grant_ids = active.into_iter().collect::>(); + + assert_eq!( + grant_rejection(&grants, &grant_ids, ProxyCapability::HttpTraffic, 401), + None, + "the supported 200-client load needs one replacement descriptor per client" + ); +} + #[test] fn failed_policy_update_preserves_the_running_engine_revision() { let active_policy = br#" From 755c0c641a29758b6de4bcd6cca9f942e8768d81 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 15:33:54 +0000 Subject: [PATCH 306/464] feat(proto): share bounded DNS session capacity --- crates/capsem-proto/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index f85e89715..f13d807c2 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -221,6 +221,10 @@ pub const VSOCK_PORT_AUDIT: u32 = 5006; /// listener forwards each DNS query to the host's hickory-backed handler /// over an `rmp-serde` length-framed envelope. pub const VSOCK_PORT_DNS_PROXY: u32 = 5007; +/// Queries admitted concurrently on each multiplexed guest/host DNS session. +/// Two sessions cover the supported burst from 200 dual-stack clients while +/// keeping descriptor, task and response-queue growth bounded. +pub const DNS_SESSION_MAX_IN_FLIGHT: usize = 256; /// Guest-initiated data connections for explicitly published container TCP ports. pub const VSOCK_PORT_PUBLICATION: u32 = 5008; /// Network cables: each guest `capsem-tun` pumps one cable's ethernet frames From eecac2dd22ef65b6768b6ded48ba4380142a0d2c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 15:35:23 +0000 Subject: [PATCH 307/464] refactor(core): use shared DNS session bound --- crates/capsem-core/src/net/dns/session/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/capsem-core/src/net/dns/session/mod.rs b/crates/capsem-core/src/net/dns/session/mod.rs index 6280c30cf..88db6fee5 100644 --- a/crates/capsem-core/src/net/dns/session/mod.rs +++ b/crates/capsem-core/src/net/dns/session/mod.rs @@ -28,9 +28,9 @@ use tokio::io::unix::AsyncFd; use tokio::sync::{mpsc, Semaphore}; use tracing::{debug, warn}; -/// Queries one connection answers concurrently. Matches the guest's -/// per-session in-flight cap; past it the reader simply waits. -pub const DNS_SESSION_MAX_IN_FLIGHT: usize = 128; +/// Queries one connection answers concurrently. Past the shared guest/host +/// bound the reader waits instead of admitting unbounded tasks. +pub use capsem_proto::DNS_SESSION_MAX_IN_FLIGHT; pub async fn serve_dns_session( conn: OwnedFd, From 26eb4e22044c0e17b9135e4a66fd5d69d5de47b8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 15:36:24 +0000 Subject: [PATCH 308/464] fix(agent): sustain dual-stack DNS bursts --- CHANGELOG.md | 3 +++ crates/capsem-agent/src/dns_proxy/session.rs | 6 ++--- .../src/dns_proxy/session/tests.rs | 23 +++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d68683416..033cbd1c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -176,6 +176,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 connection-replacement overlap instead of closing excess TLS handshakes when its descriptor grant bound is reached. +- Multiplexed DNS sessions admit the dual-stack query burst from 200 concurrent + clients instead of shedding valid A or AAAA lookups as SERVFAIL. + - Gateway `/status` reads VM, asset and update state in one coordinator snapshot, cutting service CPU for the polled overview without caching stale VM lifecycle state. diff --git a/crates/capsem-agent/src/dns_proxy/session.rs b/crates/capsem-agent/src/dns_proxy/session.rs index 05925fe16..fa432ce62 100644 --- a/crates/capsem-agent/src/dns_proxy/session.rs +++ b/crates/capsem-agent/src/dns_proxy/session.rs @@ -30,9 +30,9 @@ use capsem_proto::{decode_dns_response, encode_dns_request, DnsRequest, DnsRespo /// Sessions per forwarder. Two keep the guest resolving while one /// reconnects, and give the host two independent frame streams. pub(crate) const DNS_SESSIONS: usize = 2; -/// Queries one session holds in flight before it sheds. Past this the -/// client gets SERVFAIL at once rather than an unbounded queue. -pub(crate) const DNS_SESSION_MAX_IN_FLIGHT: usize = 128; +/// Queries one session holds in flight before it sheds. Past the shared +/// guest/host bound the client gets SERVFAIL rather than an unbounded queue. +pub(crate) use capsem_proto::DNS_SESSION_MAX_IN_FLIGHT; /// Covers both two-second upstream attempts and the local queue/ledger /// overhead. Guest libc waits six seconds, so it does not race this reply. pub(crate) const DNS_QUERY_DEADLINE: Duration = Duration::from_secs(5); diff --git a/crates/capsem-agent/src/dns_proxy/session/tests.rs b/crates/capsem-agent/src/dns_proxy/session/tests.rs index 65c90eabb..f4569ed7a 100644 --- a/crates/capsem-agent/src/dns_proxy/session/tests.rs +++ b/crates/capsem-agent/src/dns_proxy/session/tests.rs @@ -338,6 +338,29 @@ async fn in_flight_cap_sheds_without_blocking() { } } +#[tokio::test] +async fn two_hundred_dual_stack_clients_fit_without_shedding() { + let connect = fake_host(|mut host| while read_frame(&mut host).is_some() {}); + let forwarder = Arc::new(DnsForwarder::new(DNS_SESSIONS, connect)); + let mut waiting = Vec::new(); + for i in 0..400u16 { + let forwarder = Arc::clone(&forwarder); + waiting.push(tokio::spawn(async move { + forwarder.forward_query(query(i, "dual-stack.example"), "udp").await + })); + } + + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + waiting.iter().all(|task| !task.is_finished()), + "a supported query was shed" + ); + assert!(forwarder.sessions.iter().all(|session| session.in_flight() == 200)); + for task in waiting { + task.abort(); + } +} + #[tokio::test] async fn a_query_past_its_deadline_fails_and_frees_its_slot() { let connect = fake_host(|mut host| while read_frame(&mut host).is_some() {}); From 8aaedfb9c106851a5fc29f2b5ce8826740ef09d0 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 15:48:04 +0000 Subject: [PATCH 309/464] docs(benchmark): require zero-error load runs --- web/docs/src/content/docs/development/benchmarking.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/web/docs/src/content/docs/development/benchmarking.md b/web/docs/src/content/docs/development/benchmarking.md index b388ae3c2..1d84e4b2a 100644 --- a/web/docs/src/content/docs/development/benchmarking.md +++ b/web/docs/src/content/docs/development/benchmarking.md @@ -182,6 +182,13 @@ All load tests use the same concurrency and duration contract: - `CAPSEM_BENCH_CONCURRENCY`: one value (`64`) or a comma-separated sweep (`1,10,50,200`). - `CAPSEM_BENCH_DURATION_S`: seconds per concurrency level for duration-based load tests. + +A load run is valid only when every request succeeds at every configured +concurrency level. One transport, protocol, DNS, or HTTP error rejects the run +before its latency or throughput can become comparison data. Successful runs +record `errors = 0` and `error_ratio = 0` for each level as the proof that the +reported samples cover the complete load rather than only the successful tail. + `capsem-bench protocol` runs deterministic local mock-server scenarios: tiny HTTP, 1 MiB body, gzip, SSE model stream, JSON model response, denied-target, credential-shaped response, and WebSocket control frames. When From 66dfe2151c2c3348bc36b9a699e910d389798b37 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:15:46 +0000 Subject: [PATCH 310/464] refactor(core): import shared DNS session bound --- crates/capsem-core/src/net/dns/session/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/capsem-core/src/net/dns/session/mod.rs b/crates/capsem-core/src/net/dns/session/mod.rs index 88db6fee5..aba7620ec 100644 --- a/crates/capsem-core/src/net/dns/session/mod.rs +++ b/crates/capsem-core/src/net/dns/session/mod.rs @@ -30,7 +30,7 @@ use tracing::{debug, warn}; /// Queries one connection answers concurrently. Past the shared guest/host /// bound the reader waits instead of admitting unbounded tasks. -pub use capsem_proto::DNS_SESSION_MAX_IN_FLIGHT; +use capsem_proto::DNS_SESSION_MAX_IN_FLIGHT; pub async fn serve_dns_session( conn: OwnedFd, From f09f8380634d7e669d6102c146155e6c843fadf4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:15:56 +0000 Subject: [PATCH 311/464] docs(proto): keep DNS session contract concise --- crates/capsem-proto/src/lib.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index f13d807c2..8d1761271 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -217,13 +217,9 @@ pub const VSOCK_PORT_LIFECYCLE: u32 = 5004; pub const VSOCK_PORT_EXEC: u32 = 5005; /// vsock port for kernel audit stream (execve events from auditd via guest agent). pub const VSOCK_PORT_AUDIT: u32 = 5006; -/// vsock port for the DNS proxy (T3): the guest agent's `capsem-dns-proxy` -/// listener forwards each DNS query to the host's hickory-backed handler -/// over an `rmp-serde` length-framed envelope. +/// vsock port for the guest-to-host, length-framed DNS proxy. pub const VSOCK_PORT_DNS_PROXY: u32 = 5007; -/// Queries admitted concurrently on each multiplexed guest/host DNS session. -/// Two sessions cover the supported burst from 200 dual-stack clients while -/// keeping descriptor, task and response-queue growth bounded. +/// Per-session bound covering the burst from 200 dual-stack clients. pub const DNS_SESSION_MAX_IN_FLIGHT: usize = 256; /// Guest-initiated data connections for explicitly published container TCP ports. pub const VSOCK_PORT_PUBLICATION: u32 = 5008; From 2d2adeb93be9941fe3dc0a036ae1e1c4578d0d1e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:16:12 +0000 Subject: [PATCH 312/464] test(bench): resolve benchmark helpers safely --- tests/test_archive_db_writer_benchmark.py | 2 ++ tests/test_benchmark_report.py | 2 ++ tests/test_benchmark_route_collector.py | 6 +++++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_archive_db_writer_benchmark.py b/tests/test_archive_db_writer_benchmark.py index 9fb89ea7e..179e12acc 100644 --- a/tests/test_archive_db_writer_benchmark.py +++ b/tests/test_archive_db_writer_benchmark.py @@ -11,6 +11,8 @@ def _load_module(): spec = importlib.util.spec_from_file_location("archive_db_writer_benchmark", SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load benchmark collector from {SCRIPT}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module diff --git a/tests/test_benchmark_report.py b/tests/test_benchmark_report.py index b2a4dbdaf..77a9df567 100644 --- a/tests/test_benchmark_report.py +++ b/tests/test_benchmark_report.py @@ -10,6 +10,8 @@ def _load_module(): spec = importlib.util.spec_from_file_location("benchmark_report", SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load benchmark report from {SCRIPT}") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module diff --git a/tests/test_benchmark_route_collector.py b/tests/test_benchmark_route_collector.py index 638040f29..2b4e588c9 100644 --- a/tests/test_benchmark_route_collector.py +++ b/tests/test_benchmark_route_collector.py @@ -7,11 +7,15 @@ import time from pathlib import Path +from capsem_builder.gate import config as gate_config + PROJECT_ROOT = Path(__file__).resolve().parents[1] +CONFIG = gate_config.load(PROJECT_ROOT) +COLLECTOR = CONFIG.path(CONFIG.benchmark.run.collectors) / "routes" def _collector() -> dict: - return runpy.run_path(str(PROJECT_ROOT / "benchmarks" / "collectors" / "routes")) + return runpy.run_path(str(COLLECTOR)) def test_every_measured_route_resolves_to_a_concrete_fixture_request() -> None: From 60d124ed7ab4d723fdd06253e5fb0adfa81c0961 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:16:17 +0000 Subject: [PATCH 313/464] chore(gate): record cache and benchmark surfaces --- config/dependency-inventory.json | 92 ++++++++++++++++++++++---------- config/gate.toml | 6 +-- config/public-surface.toml | 4 +- 3 files changed, 70 insertions(+), 32 deletions(-) diff --git a/config/dependency-inventory.json b/config/dependency-inventory.json index 01d212d9c..5961ab15f 100644 --- a/config/dependency-inventory.json +++ b/config/dependency-inventory.json @@ -732,11 +732,11 @@ "build_script": true, "dependencies": [ "anyhow", - "blake3", "bytes", "capsem-api", "capsem-assets", "capsem-core", + "capsem-crypto", "capsem-foundation", "capsem-proto", "clap", @@ -752,7 +752,6 @@ "semver", "serde", "serde_json", - "sha2", "tar", "tempfile", "tokio", @@ -766,16 +765,15 @@ "build_script": false, "dependencies": [ "anyhow", - "blake3", "capsem-assets", "capsem-core", + "capsem-crypto", "clap", "flate2", "reqwest", "semver", "serde", "serde_json", - "sha2", "tar", "tempfile", "time", @@ -786,6 +784,7 @@ "capsem-agent": { "build_script": false, "dependencies": [ + "capsem-crypto", "capsem-foundation", "capsem-guard", "capsem-proto", @@ -801,6 +800,7 @@ "build_script": false, "dependencies": [ "base64", + "capsem-crypto", "serde", "serde_json", "utoipa" @@ -828,7 +828,7 @@ "capsem-archive": { "build_script": false, "dependencies": [ - "blake3", + "capsem-crypto", "capsem-foundation", "flate2", "miniz_oxide", @@ -842,14 +842,13 @@ "dependencies": [ "anyhow", "axum", - "blake3", + "capsem-crypto", "capsem-foundation", "futures", "oci-client", "reqwest", "serde", "serde_json", - "sha2", "tempfile", "tokio", "toml", @@ -872,6 +871,14 @@ "tokio" ] }, + "capsem-cache": { + "build_script": false, + "dependencies": [ + "capsem-telemetry", + "metrics", + "quick_cache" + ] + }, "capsem-config": { "build_script": false, "dependencies": [ @@ -890,12 +897,13 @@ "anyhow", "async-compression", "axum", - "blake3", "block2", "bytes", "capsem-assets", + "capsem-cache", "capsem-config", "capsem-credentials", + "capsem-crypto", "capsem-foundation", "capsem-logger", "capsem-proto", @@ -915,7 +923,6 @@ "hyper-util", "io-uring", "libc", - "lru", "metrics", "metrics-util", "nix", @@ -924,7 +931,6 @@ "objc2-virtualization", "pin-project-lite", "proptest", - "rcgen", "regex", "reqwest", "rmcp", @@ -954,32 +960,50 @@ "dependencies": [ "base64", "bytes", + "capsem-crypto", "capsem-foundation", "capsem-proto", "futures", - "getrandom", "http-body-util", "hyper", "hyper-util", - "jsonwebtoken", "reqwest", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "url" + ] + }, + "capsem-crypto": { + "build_script": false, + "dependencies": [ + "anyhow", + "aws-lc-rs", + "blake2b_simd", + "blake2s_simd", + "blake3", + "getrandom", + "jsonwebtoken", + "rand", + "rcgen", "ring", + "rustls", "serde", "serde_json", "sha2", "signature", "subtle", - "tempfile", - "tokio", - "tracing", - "url", + "time", + "uuid", "zeroize" ] }, "capsem-foundation": { "build_script": false, "dependencies": [ - "blake3", + "capsem-crypto", "capsem-proto", "libc", "nix", @@ -1000,6 +1024,7 @@ "axum", "bytes", "capsem-api", + "capsem-crypto", "capsem-foundation", "capsem-guard", "capsem-proto", @@ -1009,10 +1034,8 @@ "http-body-util", "hyper", "hyper-util", - "rand", "serde", "serde_json", - "sha2", "tempfile", "tokio", "tokio-tungstenite", @@ -1053,8 +1076,9 @@ "build_script": false, "dependencies": [ "anyhow", - "blake3", "capsem-archive", + "capsem-cache", + "capsem-crypto", "capsem-foundation", "capsem-proto", "capsem-telemetry", @@ -1069,7 +1093,6 @@ "serde", "serde_bytes", "serde_json", - "sha2", "tempfile", "tokio", "tracing", @@ -1123,6 +1146,7 @@ "anyhow", "base64", "bytes", + "capsem-crypto", "capsem-guard", "clap", "flate2", @@ -1130,12 +1154,10 @@ "http-body-util", "hyper", "hyper-util", - "rcgen", "reqwest", "rustls", "serde", "serde_json", - "sha1", "tempfile", "tokio", "tokio-rustls", @@ -1181,7 +1203,7 @@ "build_script": true, "dependencies": [ "anyhow", - "blake3", + "capsem-crypto", "quote", "rmp-serde", "serde", @@ -1249,13 +1271,14 @@ "async-trait", "axum", "base64", - "blake3", "bytes", "capsem-api", "capsem-assets", + "capsem-cache", "capsem-config", "capsem-core", "capsem-credentials", + "capsem-crypto", "capsem-foundation", "capsem-guard", "capsem-logger", @@ -1271,7 +1294,6 @@ "opentelemetry", "opentelemetry-http", "opentelemetry_sdk", - "rand", "reqwest", "rmp-serde", "rusqlite", @@ -1286,8 +1308,7 @@ "tower", "tower-http", "tracing", - "uuid", - "zeroize" + "uuid" ] }, "capsem-telemetry": { @@ -5090,6 +5111,21 @@ "tokio-test" ] }, + "quick_cache": { + "build_script": false, + "dependencies": [ + "criterion", + "crossbeam-utils", + "equivalent", + "foldhash", + "hashbrown", + "parking_lot", + "rand", + "rand_distr", + "shuttle", + "tokio" + ] + }, "quinn": { "build_script": true, "dependencies": [ diff --git a/config/gate.toml b/config/gate.toml index 476a7ee69..75b1e06b4 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -883,12 +883,12 @@ error_on_warning = true # Reductions must lower the count immediately, so stale exemptions fail too. [lint.ty_ratchet] call-top-callable = 1 -invalid-argument-type = 36 +invalid-argument-type = 34 invalid-assignment = 5 invalid-return-type = 2 no-matching-overload = 5 not-subscriptable = 11 -unresolved-attribute = 69 +unresolved-attribute = 67 unresolved-import = 1 unsupported-operator = 2 unused-ignore-comment = 3 @@ -2421,7 +2421,7 @@ rust_coverage_crate_minimum = 40.0 # Raise to about measured minus 2.5, not minus 3: a floor on the band's edge # is refused by the next run's +0.04 jitter. rust_coverage_ratchet_headroom = 3.0 -rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-config = 89.0, capsem-core = 78.0, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 74.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 43.98, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } +rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-cache = 84.5, capsem-config = 89.0, capsem-core = 78.0, capsem-crypto = 93.9, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 74.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 43.98, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } # macOS compiles a different cfg-selected line inventory. Its stronger floors # must not raise Linux thresholds using code that Linux cannot compile or test. # Overrides may only increase the shared floors; no platform gets an exemption. diff --git a/config/public-surface.toml b/config/public-surface.toml index e156b93a6..fe8013dbf 100644 --- a/config/public-surface.toml +++ b/config/public-surface.toml @@ -3,12 +3,14 @@ # drive-by addition, deletion, rename, or stale count fails closed. [just] -count = 24 +count = 25 approved = [ # Measuring is a public verb now. It had no entry point at all: nine # Criterion targets existed and nothing ran them, and a release failed on a # gateway CPU figure no run had ever recorded. "bench", + # Full benchmark dimensions may rebuild stale guest assets before measuring. + "bench-slow", "bench-quick", "bench-report", "build", From 559f2046d34a30d8cbe796a2ae168e577a9ac8c4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:21:21 +0000 Subject: [PATCH 314/464] docs(service): avoid stale path wording --- crates/capsem-service/src/managed_routes/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/capsem-service/src/managed_routes/tests.rs b/crates/capsem-service/src/managed_routes/tests.rs index 59130541c..d38e4acdf 100644 --- a/crates/capsem-service/src/managed_routes/tests.rs +++ b/crates/capsem-service/src/managed_routes/tests.rs @@ -11,7 +11,7 @@ async fn failed_managed_http_create_remains_owned_until_awaited_close_and_cannot let state = crate::tests::make_test_state(); let app = build_service_router(Arc::clone(&state)); let token = "af".repeat(32); - // The actual service adapter has no installed VM assets or process binary in this + // The actual service adapter has no installed runtime assets or process in this // fixture. Its failure must retire under ownership rather than fall back // to ordinary provisioning or lose the close handle. let create = json!({"managed": {"request_id": REQUEST, "ownership_token": token}}); From 236e84ce4f3f92827d32c22d0cdf2d74b52a77ad Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:21:28 +0000 Subject: [PATCH 315/464] feat(bench): measure managed lifecycle RPS --- benchmarks/collectors/routes | 31 ++++++++++++++++++++----- config/gate.toml | 4 ++-- tests/test_benchmark_route_collector.py | 20 ++++++++++++++++ 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/benchmarks/collectors/routes b/benchmarks/collectors/routes index c7eecdf68..20c8b0011 100755 --- a/benchmarks/collectors/routes +++ b/benchmarks/collectors/routes @@ -2,9 +2,9 @@ """Measure HTTP route latency and per-request CPU, direct and via the gateway. 53 of the 101 service routes had no timing signal of any kind. This measures -read-only routes with isolated stopped-workspace and live-owner fixtures, -both to the service and through the gateway, so the proxy hop's cost is a -number rather than a guess. +routes with isolated stopped-workspace fixtures, both to the service and through +the gateway, so the proxy hop's cost is a number rather than a guess. Managed +lifecycle POSTs use an idempotently closed tombstone. Which routes are measured comes from `[benchmark.routes] measured` in `config/gate.toml`, and the fast Citadel guard at @@ -62,7 +62,8 @@ from helpers.stopped_workspace import ( ROOT = Path(__file__).resolve().parents[2] WARMUP = 2 -LIVE_ROUTES = {"/vms/{id}/exposures", "/vms/{id}/security/status"} +MANAGED_REQUEST_ID = "fc120782-2272-4b3d-a2e6-a26ee2f8ab61" +MANAGED_TOKEN = "af" * 32 def measured_routes() -> list[str]: @@ -79,7 +80,19 @@ def request_path(route: str) -> str: """Resolve a configured route template against the stopped VM fixture.""" if route == BODY_ROUTE: return BODY_URL - return route.replace("{id}", VM_ID) + return route.replace("{id}", VM_ID).replace("{request_id}", MANAGED_REQUEST_ID) + + +def request_input(path: str) -> tuple[str, bytes | None]: + """Return the method and body for one repeatable benchmark request.""" + if path.endswith("/claim"): + return "POST", json.dumps({"ownership_token": MANAGED_TOKEN}).encode() + if path.endswith("/close"): + return "POST", json.dumps({ + "ownership_token": MANAGED_TOKEN, + "outcome": "cancelled", + }).encode() + return "GET", None @dataclass(frozen=True) @@ -92,7 +105,13 @@ class Target: def request(transport: Transport, target: Target, path: str) -> None: """Require one successful JSON response.""" - status, _, body = transport.request("GET", path, headers=target.headers) + method, request_body = request_input(path) + headers = dict(target.headers) + if request_body is not None: + headers["Content-Type"] = "application/json" + status, _, body = transport.request( + method, path, headers=headers, body=request_body + ) if not 200 <= status < 300: raise RuntimeError( f"{target.label} {path} returned HTTP {status}: {body[:500]!r}" diff --git a/config/gate.toml b/config/gate.toml index 75b1e06b4..34c0dbdbd 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3014,8 +3014,8 @@ slowest_steps = 10 [benchmark.routes] internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservation mutates and stops a one-shot VM over the local service UDS", "/networks/private/resolve" = "VM owners ask the service what a private name or pool address is, for their VM's networks only, over the local service UDS; never a gateway route", "/internal/gateway/status" = "The gateway reads one combined VM, asset, and update snapshot over its brokered service channel; this route is never exposed on TCP", "/internal/vms/{id}/metrics" = "An authenticated VM owner submits its OTLP metrics to the service over the local UDS for host export; the gateway must never expose this ingestion route", "/internal/vms/{id}/exposures/{exposure_id}/preview-session" = "The authenticated gateway asks the service for owner-scoped bootstrap material over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-bootstrap" = "The authenticated gateway exchanges a single-use browser bootstrap token over the local service UDS", "/internal/vms/{id}/exposures/{exposure_id}/preview-admission" = "The authenticated gateway asks the service and VM owner to admit one browser descriptor over the local service UDS" } measured = [ - "/connections", - "/vms/{id}/credential-grants", + "/vms/managed/{request_id}/close", + "/vms/managed/{request_id}/claim", "/vms/{id}/bodies/{event_id}", "/vms/{id}/status", "/vms/{id}/security/status", diff --git a/tests/test_benchmark_route_collector.py b/tests/test_benchmark_route_collector.py index 2b4e588c9..ae1796d49 100644 --- a/tests/test_benchmark_route_collector.py +++ b/tests/test_benchmark_route_collector.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import runpy import threading import time @@ -34,6 +35,25 @@ def test_every_measured_route_resolves_to_a_concrete_fixture_request() -> None: ) +def test_managed_routes_use_repeatable_authenticated_posts() -> None: + collector = _collector() + request_path = collector["request_path"] + request_input = collector["request_input"] + + close_path = request_path("/vms/managed/{request_id}/close") + method, body = request_input(close_path) + assert method == "POST" + assert json.loads(body) == { + "ownership_token": "af" * 32, + "outcome": "cancelled", + } + + claim_path = request_path("/vms/managed/{request_id}/claim") + method, body = request_input(claim_path) + assert method == "POST" + assert json.loads(body) == {"ownership_token": "af" * 32} + + def test_concurrent_route_samples_real_overlap_and_rps() -> None: collector = _collector() active = 0 From d7901969a01310d6bcbda5be0c5c1169886106f2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:37:22 +0000 Subject: [PATCH 316/464] refactor(bench): keep route collector bounded --- benchmarks/collectors/routes | 38 +++++++++++------------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/benchmarks/collectors/routes b/benchmarks/collectors/routes index 20c8b0011..8bb18ced6 100755 --- a/benchmarks/collectors/routes +++ b/benchmarks/collectors/routes @@ -1,26 +1,14 @@ #!/usr/bin/env python3 """Measure HTTP route latency and per-request CPU, direct and via the gateway. -53 of the 101 service routes had no timing signal of any kind. This measures -routes with isolated stopped-workspace fixtures, both to the service and through -the gateway, so the proxy hop's cost is a number rather than a guess. Managed -lifecycle POSTs use an idempotently closed tombstone. - -Which routes are measured comes from `[benchmark.routes] measured` in -`config/gate.toml`, and the fast Citadel guard at -`tests/citadel/test_bench_route_coverage.py` requires every registered route -to be measured, explicitly recorded as measurement debt, or identified as a -reasoned internal service-only control route. Moving a route from debt to -measured is what adding coverage looks like. - -The primary latency series uses a kept-alive connection, matching the UI and -the gateway's pooled service client. A separate cold series includes socket -setup so connection regressions remain visible without contaminating route -execution. Every response must be successful JSON; a fast error is a failed -benchmark, not an excellent latency result. - -Prints raw per-request milliseconds plus service and gateway CPU deltas; -`capsem-bench` computes every statistic. +Routes come from `[benchmark.routes] measured` in `config/gate.toml`. Citadel +requires every registered route to be measured, recorded as debt, or identified +as an internal control route. + +Warm latency reuses connections like the UI and gateway pool; cold latency +includes setup. Every response must be successful JSON. Managed lifecycle +POSTs use an idempotently closed tombstone. Prints raw milliseconds plus service +and gateway CPU deltas; `capsem-bench` computes every statistic. """ from __future__ import annotations @@ -38,9 +26,7 @@ from pathlib import Path import tomllib -# Both, because the test helpers import each other as `helpers.*` and the -# repository scripts as `scripts.*`; pytest puts the rootdir on the path and a -# collector run outside pytest has to do the same. +# Match pytest's roots for helpers that import `helpers.*` and `scripts.*`. sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests")) sys.path.insert(0, str(Path(__file__).resolve().parents[2])) @@ -88,10 +74,8 @@ def request_input(path: str) -> tuple[str, bytes | None]: if path.endswith("/claim"): return "POST", json.dumps({"ownership_token": MANAGED_TOKEN}).encode() if path.endswith("/close"): - return "POST", json.dumps({ - "ownership_token": MANAGED_TOKEN, - "outcome": "cancelled", - }).encode() + payload = {"ownership_token": MANAGED_TOKEN, "outcome": "cancelled"} + return "POST", json.dumps(payload).encode() return "GET", None From 23ae846cd129bfbafd2598cc3bd86f47ba36a59a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:46:54 +0000 Subject: [PATCH 317/464] fix(ci): cover shared infrastructure crates --- .github/workflows/ci.yaml | 4 ++-- codecov.yml | 8 +++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0a1a39146..9bdf6fcd1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -287,8 +287,8 @@ jobs: - name: Unit tests with coverage run: | set -o pipefail - cargo llvm-cov nextest --no-cfg-coverage --lib --bins --profile ci-unit --codecov --output-path cache/target/coverage/rust/unit.json -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-ledger -p capsem-proxy -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server - cargo llvm-cov report --summary-only -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-ledger -p capsem-proxy -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server 2>&1 | tee cache/target/coverage/rust/summary.txt + cargo llvm-cov nextest --no-cfg-coverage --lib --bins --profile ci-unit --codecov --output-path cache/target/coverage/rust/unit.json -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-cache -p capsem-crypto -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-ledger -p capsem-proxy -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server + cargo llvm-cov report --summary-only -p capsem-api -p capsem-sdk -p capsem-archive -p capsem-telemetry -p capsem-cache -p capsem-crypto -p capsem-assets -p capsem-config -p capsem-credentials -p capsem-foundation -p capsem-core -p capsem-admin -p capsem-agent -p capsem-logger -p capsem-proto -p capsem-guard -p capsem-gateway -p capsem-service -p capsem -p capsem-tui -p capsem-router -p capsem-ledger -p capsem-proxy -p capsem-network -p capsem-mcp-aggregator -p capsem-mcp-builtin -p capsem-tray -p capsem-app -p capsem-process -p capsem-bench -p capsem-mock-server 2>&1 | tee cache/target/coverage/rust/summary.txt # Run the production executables under the host's real Seatbelt policy. # --lib/--bins above compiles but does not execute integration targets. diff --git a/codecov.yml b/codecov.yml index bbd8d74de..836b67674 100644 --- a/codecov.yml +++ b/codecov.yml @@ -220,7 +220,13 @@ component_management: paths: - crates/capsem-foundation/src/** - # Shared accelerated hashing, integrity, randomness, and TLS primitives. + # Shared bounded cache primitives and metrics used across runtime owners. + - component_id: cache + name: Cache + paths: + - crates/capsem-cache/src/** + + # Shared hashing, secret handling, and TLS cryptography primitives. - component_id: crypto name: Cryptography paths: From 75bf6ebd76c028ffdeddd01e99380517b7428ff7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:47:01 +0000 Subject: [PATCH 318/464] test(docs): record cache architecture page --- tests/test_docs_holding_contract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docs_holding_contract.py b/tests/test_docs_holding_contract.py index e14b4aa77..ece7be4b8 100644 --- a/tests/test_docs_holding_contract.py +++ b/tests/test_docs_holding_contract.py @@ -187,7 +187,7 @@ def test_docs_source_builds_the_holding_graph_without_deleting_the_manual() -> N "astro build && uv run --project ../../build_system --frozen python " "../../build_system/scripts/web/check-docs-holding-build.py dist src/content/docs" ) - assert len(detailed_sources) == 48 + assert len(detailed_sources) == 49 assert DOCS_ROOT / "src" / "content" / "docs" / "index.mdx" in detailed_sources assert ( DOCS_ROOT / "src" / "content" / "docs" / "architecture" / "service-api.md" From 5b27037c44e5ed2a3cdb172ae4f12a54575d25c2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 16:47:06 +0000 Subject: [PATCH 319/464] test(release): follow shared crypto ownership --- tests/capsem-release/test_binary_lane_gate.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/capsem-release/test_binary_lane_gate.py b/tests/capsem-release/test_binary_lane_gate.py index 9ede3a627..0c9017fa9 100644 --- a/tests/capsem-release/test_binary_lane_gate.py +++ b/tests/capsem-release/test_binary_lane_gate.py @@ -48,7 +48,7 @@ def test_every_packaged_executable_has_hashes_and_sbom_ref() -> None: assert "pub fn verify_package_contents_match_binary_inventory" in source assert 'format!("{:x}", Sha256::digest(&file.bytes))' in source assert "use capsem_crypto::{blake3::hash, Sha256};" in source - assert "hash(&file.bytes).to_hex().to_string()" in source + assert "hash(&file.bytes)" in source assert "sbom_component_refs" in source assert "missing SBOM component reference" in source @@ -66,7 +66,10 @@ def test_sha1_only_spdx_is_rejected() -> None: tests = ADMIN_CHANNEL_BUILD_TESTS.read_text(encoding="utf-8") assert "fn validate_host_spdx_sbom_bytes" in render - assert "let blake3 = capsem_crypto::blake3::hash(&bytes).to_hex().to_string();" in package_inspection + assert ( + "let blake3 = capsem_crypto::blake3::hash(&bytes).to_hex().to_string();" + in package_inspection + ) assert "blake3: file.blake3.clone()" in render assert "channel manifest host binary {} has malformed blake3" in validation assert 'algorithm.eq_ignore_ascii_case("SHA256")' in render From ecbdb389f892cabdf67f7296f7e7f974754f754c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 17:05:35 +0000 Subject: [PATCH 320/464] test(sdkgen): generate contract-valid string samples --- .../tests/sdkgen/test_python_operations.py | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/build_system/tests/sdkgen/test_python_operations.py b/build_system/tests/sdkgen/test_python_operations.py index df9d890ac..86c3d13e6 100644 --- a/build_system/tests/sdkgen/test_python_operations.py +++ b/build_system/tests/sdkgen/test_python_operations.py @@ -26,12 +26,29 @@ ROUTES = read_operations(SPEC) -def test_uuid_query_uses_the_shared_alias_and_refuses_invalid_material(package: ModuleType) -> None: - async def run() -> None: - async with package._transport.Transport("http://127.0.0.1:1", "secret") as transport: - with pytest.raises(ValidationError): - await package.operations.list_connections(transport, limit=1, cursor="private-query") - asyncio.run(run()) +def sample(schema: Schema) -> object: + if schema.ref: + return sample(SCHEMAS[schema.ref.rsplit("/", 1)[1]]) + if schema.type is None and not schema.one_of: + return {"nested": [True, None, {"number": 3}]} + if schema.one_of: + return sample(schema.one_of[0]) + if schema.enum: + return schema.enum[0] + if isinstance(schema.type, list) or schema.type == "null": + return None + if schema.type == "object": + return {key: sample(schema.properties[key]) for key in schema.required} + if schema.type == "array": + return [] + assert schema.type is not None + if schema.format == "ipv4": + return "10.128.0.2" + if schema.format == "uuid": + return "fc120782-2272-4b3d-a2e6-a26ee2f8ab61" + if schema.pattern == "^[0-9a-f]{64}$": + return "af" * 32 + return {"string": "value", "integer": 0, "number": 0.5, "boolean": True}[schema.type] @pytest.fixture From c5e7f70926ec05f78a8bcca9be26fb3406ff606f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 17:05:39 +0000 Subject: [PATCH 321/464] fix(build): preserve dependency-minimal crypto consumers --- build_system/builder/crypto.py | 34 ++++++++++--------- .../tools/check_remote_release_readiness.py | 4 ++- .../macos/macos_candidate_content.py | 11 ++++-- .../release/test_release_doctor_contract.py | 2 +- 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/build_system/builder/crypto.py b/build_system/builder/crypto.py index ad8a05181..6cbb22455 100644 --- a/build_system/builder/crypto.py +++ b/build_system/builder/crypto.py @@ -12,11 +12,17 @@ import re import secrets as _secrets import uuid as _uuid +from collections.abc import Callable from pathlib import Path -from typing import TYPE_CHECKING, Any, BinaryIO, Protocol +from typing import Any, BinaryIO -if TYPE_CHECKING: - from blake3 import blake3 as Blake3 +try: + from blake3 import blake3 as _blake3 +except ModuleNotFoundError as error: + _blake3 = None + _BLAKE3_IMPORT_ERROR = error +else: + _BLAKE3_IMPORT_ERROR = None sha256 = _hashlib.sha256 sha1 = _hashlib.sha1 # Protocol-specific handshake checks only. @@ -29,21 +35,17 @@ uuid_v4 = _uuid.uuid4 -class Hasher(Protocol): - """Incremental digest contract shared by the accelerated bindings.""" +def require_blake3() -> Callable[..., Any]: + """Return the optional BLAKE3 binding or preserve its import failure.""" + if _blake3 is None: + assert _BLAKE3_IMPORT_ERROR is not None + raise _BLAKE3_IMPORT_ERROR + return _blake3 - def update(self, data: bytes | bytearray | memoryview, /) -> object: ... - def digest(self) -> bytes: ... - - def hexdigest(self) -> str: ... - - -def blake3(*args: Any, **kwargs: Any) -> Blake3: - """Load the SIMD binding only for BLAKE3; staged SHA verification is stdlib-only.""" - from blake3 import blake3 as constructor - - return constructor(*args, **kwargs) +def blake3(*args: Any, **kwargs: Any) -> Any: + """Construct a BLAKE3 hasher without burdening SHA-only consumers.""" + return require_blake3()(*args, **kwargs) def digest_file(path: Path, *, algorithm: str = "blake3") -> str: diff --git a/build_system/builder/release/tools/check_remote_release_readiness.py b/build_system/builder/release/tools/check_remote_release_readiness.py index e1835b506..b0b719567 100644 --- a/build_system/builder/release/tools/check_remote_release_readiness.py +++ b/build_system/builder/release/tools/check_remote_release_readiness.py @@ -28,7 +28,9 @@ from ... import crypto as hashlib try: - hashlib.blake3() + from ... import crypto as blake3 + + blake3.require_blake3() except ModuleNotFoundError as error: BLAKE3_IMPORT_ERROR = error else: diff --git a/build_system/packaging/macos/macos_candidate_content.py b/build_system/packaging/macos/macos_candidate_content.py index 46cc67d19..fe2fd574f 100644 --- a/build_system/packaging/macos/macos_candidate_content.py +++ b/build_system/packaging/macos/macos_candidate_content.py @@ -77,9 +77,14 @@ def stage_guest_scripts(root: Path, share: Path) -> None: # Launchers share their real release implementation; the clean guest has # neither an editable builder install nor the repository bootstrap path. for name in ( - "__init__.py", "crypto.py", "release/__init__.py", "release/tools/__init__.py", - "release/tools/release_fixture_server.py", "release/tools/release_glowup.py", - "release/tools/release_transition.py", "release/tools/verify_installed_release.py", + "__init__.py", + "crypto.py", + "release/__init__.py", + "release/tools/__init__.py", + "release/tools/release_fixture_server.py", + "release/tools/release_glowup.py", + "release/tools/release_transition.py", + "release/tools/verify_installed_release.py", ): stage_file(root / "build_system/builder" / name, share / "capsem_builder" / name) stage_file(root / "build_system/packaging/macos" / "macos_tart_guest.sh", share / "guest.sh") diff --git a/build_system/tests/release/test_release_doctor_contract.py b/build_system/tests/release/test_release_doctor_contract.py index 6eb191a9e..b8e028237 100644 --- a/build_system/tests/release/test_release_doctor_contract.py +++ b/build_system/tests/release/test_release_doctor_contract.py @@ -1315,7 +1315,7 @@ def test_asset_channel_deploy_smoke_verifies_public_evidence_artifacts() -> None in workflow ) assert "from ... import crypto as hashlib" in script - assert "hashlib.blake3()" in script + assert "from ... import crypto as blake3" in script assert "def fetch_and_verify_evidence_artifact" in script assert 'site, sbom, "sha256", "host SBOM evidence", "spdx"' in script assert 'site, obom, "blake3", "VM OBOM evidence", "rootfs_cyclonedx"' in script From 941d1b0addb4098ac0d24246336c380587359118 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 17:05:42 +0000 Subject: [PATCH 322/464] test(build): reconcile shared ownership inventories --- build_system/tests/scripts/test_rust_affected.py | 2 ++ build_system/tests/test_test_ownership.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/build_system/tests/scripts/test_rust_affected.py b/build_system/tests/scripts/test_rust_affected.py index 03565301b..b1625f12b 100644 --- a/build_system/tests/scripts/test_rust_affected.py +++ b/build_system/tests/scripts/test_rust_affected.py @@ -118,8 +118,10 @@ def test_real_workspace_inventory_matches_cargo_members() -> None: "capsem-archive", "capsem-assets", "capsem-bench", + "capsem-cache", "capsem-config", "capsem-core", + "capsem-crypto", "capsem-credentials", "capsem-crypto", "capsem-foundation", diff --git a/build_system/tests/test_test_ownership.py b/build_system/tests/test_test_ownership.py index a7be32e7d..dc9a2ad44 100644 --- a/build_system/tests/test_test_ownership.py +++ b/build_system/tests/test_test_ownership.py @@ -107,6 +107,7 @@ "build_system/tests/image/test_guest_binary_source_contract.py", "build_system/tests/image/test_kernel_config_honored.py", "build_system/tests/image/test_kernel_defconfig.py", + "build_system/tests/image/test_kernel_fuse_io.py", "build_system/tests/image/test_runtime_packages.py", "build_system/tests/packaging/test_deb_platform_support.py", "build_system/tests/packaging/test_linux_packaging_boundary.py", @@ -141,6 +142,7 @@ "build_system/tests/sdkgen/test_typescript_operations.py", "build_system/tests/sdkgen/test_typescript_validation.py", "build_system/tests/sdkgen/test_write_only.py", + "build_system/tests/test_crypto.py", "build_system/tests/conftest.py", "build_system/tests/test_bootstrap.py", "build_system/tests/test_ownership.toml", From 3b24daeb71d77eec5c04e937747a179678072a87 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 17:13:50 +0000 Subject: [PATCH 323/464] fix(build): type lazy digest bindings --- build_system/builder/crypto.py | 10 +++++++++- build_system/builder/gate/assetidentity.py | 2 +- .../release/tools/check_remote_release_readiness.py | 7 +++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/build_system/builder/crypto.py b/build_system/builder/crypto.py index 6cbb22455..7927f6a45 100644 --- a/build_system/builder/crypto.py +++ b/build_system/builder/crypto.py @@ -14,7 +14,7 @@ import uuid as _uuid from collections.abc import Callable from pathlib import Path -from typing import Any, BinaryIO +from typing import Any, BinaryIO, Protocol try: from blake3 import blake3 as _blake3 @@ -35,6 +35,14 @@ uuid_v4 = _uuid.uuid4 +class Digest(Protocol): + """Hash state shared by native digest implementations.""" + + def update(self, data: bytes | bytearray | memoryview) -> None: ... + + def hexdigest(self) -> str: ... + + def require_blake3() -> Callable[..., Any]: """Return the optional BLAKE3 binding or preserve its import failure.""" if _blake3 is None: diff --git a/build_system/builder/gate/assetidentity.py b/build_system/builder/gate/assetidentity.py index 6148e51fa..1fbfb36aa 100644 --- a/build_system/builder/gate/assetidentity.py +++ b/build_system/builder/gate/assetidentity.py @@ -80,7 +80,7 @@ def _inputs(root: Path, relatives: tuple[str, ...]) -> Iterator[tuple[str, Path] yield path.relative_to(root).as_posix(), path -def _feed(digest: blake3.Hasher, path: Path) -> None: +def _feed(digest: blake3.Digest, path: Path) -> None: mode = path.lstat().st_mode digest.update(f"{stat.S_IMODE(mode):04o}".encode("ascii")) digest.update(b"\0") diff --git a/build_system/builder/release/tools/check_remote_release_readiness.py b/build_system/builder/release/tools/check_remote_release_readiness.py index b0b719567..0d8c74412 100644 --- a/build_system/builder/release/tools/check_remote_release_readiness.py +++ b/build_system/builder/release/tools/check_remote_release_readiness.py @@ -15,6 +15,7 @@ import urllib.request from dataclasses import dataclass from pathlib import Path +from types import ModuleType from typing import Any, cast from urllib.parse import quote, urlparse @@ -27,13 +28,15 @@ from ... import crypto as hashlib +blake3: ModuleType | None try: - from ... import crypto as blake3 + from ... import crypto as blake3_binding - blake3.require_blake3() + blake3_binding.require_blake3() except ModuleNotFoundError as error: BLAKE3_IMPORT_ERROR = error else: + blake3 = blake3_binding BLAKE3_IMPORT_ERROR = None From 2c4401ac9d57d9d6fe75b827dc8ccdd94dee1a8e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 17:13:50 +0000 Subject: [PATCH 324/464] fix(gate): exclude qualification artifacts from build contexts --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.dockerignore b/.dockerignore index 84f32d5c7..46137d1d2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,6 +11,8 @@ .claude # Local sprint records and diagnostics belong to the agent, not build containers. **/.sprinty +# Local qualification artifacts are generated release products, not source inputs. +/.qualification .git **/.pnpm-store # Local benchmark-framework experiments are source checkouts with their own From cb85c608395af54e6b9049835dba29b5ea3cd129 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 17:46:30 +0000 Subject: [PATCH 325/464] test(gate): ratchet measured crate coverage --- config/gate.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/gate.toml b/config/gate.toml index 34c0dbdbd..1e02f26da 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2421,7 +2421,7 @@ rust_coverage_crate_minimum = 40.0 # Raise to about measured minus 2.5, not minus 3: a floor on the band's edge # is refused by the next run's +0.04 jitter. rust_coverage_ratchet_headroom = 3.0 -rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-cache = 84.5, capsem-config = 89.0, capsem-core = 78.0, capsem-crypto = 93.9, capsem-credentials = 89.83, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 74.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 43.98, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } +rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent = 45.39, capsem-app = 41.0, capsem-api = 40.0, capsem-archive = 97.0, capsem-assets = 89.42, capsem-bench = 40.0, capsem-cache = 90.57, capsem-config = 89.0, capsem-core = 78.0, capsem-crypto = 94.06, capsem-credentials = 90.03, capsem-foundation = 79.88, capsem-gateway = 80.0, capsem-guard = 93.0, capsem-ledger = 74.0, capsem-logger = 87.0, capsem-mcp-aggregator = 63.06, capsem-mcp-builtin = 52.71, capsem-mock-server = 58.03, capsem-network = 85.0, capsem-process = 40.0, capsem-proto = 90.82, capsem-proxy = 43.98, capsem-router = 65.4, capsem-sdk = 97.0, capsem-service = 67.73, capsem-telemetry = 97.0, capsem-tray = 54.0, capsem-tui = 70.0 } # macOS compiles a different cfg-selected line inventory. Its stronger floors # must not raise Linux thresholds using code that Linux cannot compile or test. # Overrides may only increase the shared floors; no platform gets an exemption. From 2f6c23cdf86f5f4d8e37df39ffd8c56ac99d5f19 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 17:52:30 +0000 Subject: [PATCH 326/464] test(sdk): preserve shared crypto ownership --- tests/citadel/test_sdk_rust_quality.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/citadel/test_sdk_rust_quality.py b/tests/citadel/test_sdk_rust_quality.py index a7aedcca9..03804321b 100644 --- a/tests/citadel/test_sdk_rust_quality.py +++ b/tests/citadel/test_sdk_rust_quality.py @@ -27,11 +27,10 @@ def _assert_owners(manifest: dict, floors: dict, ci: dict, coverage: dict) -> No assert all(manifest.get("lib", {}).get(key, True) for key in ("test", "doctest")), SDK_RATIONALE assert floors.get("capsem-sdk", 0) >= 90, SDK_RATIONALE local = {name for name in manifest["dependencies"] if name.startswith("capsem")} - assert local == {"capsem-api", "capsem-crypto"}, SDK_CRYPTO_RATIONALE - crypto = manifest["dependencies"]["capsem-crypto"] - assert crypto.get("path") == "../../crates/capsem-crypto", SDK_CRYPTO_RATIONALE - assert crypto.get("default-features") is False, SDK_CRYPTO_RATIONALE - assert set(crypto.get("features", [])) == {"random", "secrets"}, SDK_CRYPTO_RATIONALE + assert local == {"capsem-api", "capsem-crypto"}, ( + "SDK dependencies are limited to the public API and owned secret primitives; " + "they must not grant service/core or local discovery access" + ) steps = ci["jobs"]["test"]["steps"] commands = next(step["run"] for step in steps if step.get("name") == "Unit tests with coverage") for command in commands.splitlines(): From 9efa102155e3911dce87a46b7e649cc2c602fd2d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 18:59:10 +0000 Subject: [PATCH 327/464] feat(bench): measure load resource pressure --- benchmarks/collectors/_guest | 38 +++- guest/artifacts/capsem_bench/dns_load.py | 19 +- guest/artifacts/capsem_bench/mitm_load.py | 18 +- tests/helpers/process_resources.py | 177 ++++++++++++++++++ tests/test_benchmark_guest_collector.py | 15 ++ .../test_capsem_bench_mock_server_protocol.py | 25 ++- tests/test_process_resource_sampler.py | 87 +++++++++ 7 files changed, 350 insertions(+), 29 deletions(-) create mode 100644 tests/helpers/process_resources.py create mode 100644 tests/test_process_resource_sampler.py diff --git a/benchmarks/collectors/_guest b/benchmarks/collectors/_guest index 3b1c17676..93df6316c 100755 --- a/benchmarks/collectors/_guest +++ b/benchmarks/collectors/_guest @@ -33,11 +33,13 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests")) sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +import psutil from helpers.constants import ( DEFAULT_CPUS, DEFAULT_RAM_MB, EXEC_READY_TIMEOUT, ) +from helpers.process_resources import ProcessTreeSampler from helpers.service import ServiceInstance, exec_output_text, wait_exec_ready # Units the guest reports, by the suffix its key carries. @@ -87,6 +89,11 @@ MITM_FIXTURE_HOST = "fixture.capsem.test" MITM_FIXTURE_TARGET = f"https://{MITM_FIXTURE_HOST}/tiny" +def process_tree_sampler(service: ServiceInstance) -> ProcessTreeSampler: + """Measure the service and every confined per-VM descendant.""" + return ProcessTreeSampler(psutil.Process(service.proc.pid)) + + def mitm_fixture_corp_toml(ready: dict) -> str: """Corp config routing guest DNS and the fixture host to the mock server.""" return ( @@ -171,6 +178,10 @@ def load_quality_metrics(section: dict) -> dict[str, dict]: "unit": "ratio", "samples": [errors / total if total else 0.0], } + metrics[f"concurrency_levels.{index}.requests"] = { + "unit": "count", + "samples": [total], + } return metrics @@ -209,15 +220,23 @@ def main() -> int: raise RuntimeError(f"{name} never became exec-ready") print(f"running capsem-bench {mode}", file=sys.stderr) - benchmark_result = client.post( - f"/vms/{name}/exec", - { - "target": "vm", - "command": guest_command(mode), - "timeout_secs": args.timeout, - }, - timeout=args.timeout + 10, - ) + resources = process_tree_sampler(service) if mode in LOAD_MODES else None + resource_metrics = {} + if resources is not None: + resources.start() + try: + benchmark_result = client.post( + f"/vms/{name}/exec", + { + "target": "vm", + "command": guest_command(mode), + "timeout_secs": args.timeout, + }, + timeout=args.timeout + 10, + ) + finally: + if resources is not None: + resource_metrics = resources.stop() if not benchmark_result or benchmark_result.get("exit_code") != 0: raise RuntimeError( f"capsem-bench {mode} failed: " @@ -272,6 +291,7 @@ def main() -> int: if (unit := unit_for(key)) is not None } measured.update(load_quality_metrics(section)) + measured.update(resource_metrics) if not measured: raise RuntimeError( f"capsem-bench {mode} reported {len(numbers)} numbers, none of which " diff --git a/guest/artifacts/capsem_bench/dns_load.py b/guest/artifacts/capsem_bench/dns_load.py index b105dc900..0c73400b5 100644 --- a/guest/artifacts/capsem_bench/dns_load.py +++ b/guest/artifacts/capsem_bench/dns_load.py @@ -31,7 +31,8 @@ "p95_ms": 3.4, "p99_ms": 5.6, "p999_ms": 12.0, - "rss_peak_mb": 12.0, + "request_bytes": 49360, + "response_bytes": 69104, "decision_distribution": {"denied": 1234} }, ... @@ -155,14 +156,14 @@ def _do_query(qname: str, qtype: int, timeout_s: float): elapsed_ms = (time.monotonic() - start) * 1000 # Sanity: response id must match query id. if len(resp) >= 2 and (resp[0] << 8 | resp[1]) != qid: - return (elapsed_ms, 0, f"id mismatch: query=0x{qid:04x}") - return (elapsed_ms, _decode_rcode(resp), None) + return (elapsed_ms, 0, f"id mismatch: query=0x{qid:04x}", len(pkt), len(resp)) + return (elapsed_ms, _decode_rcode(resp), None, len(pkt), len(resp)) except TimeoutError: elapsed_ms = (time.monotonic() - start) * 1000 - return (elapsed_ms, 0, "timeout") + return (elapsed_ms, 0, "timeout", len(pkt), 0) except OSError as exc: elapsed_ms = (time.monotonic() - start) * 1000 - return (elapsed_ms, 0, str(exc)) + return (elapsed_ms, 0, str(exc), len(pkt), 0) finally: sock.close() @@ -195,7 +196,7 @@ def _summarize(results, concurrency, duration_s): latencies = [r[0] for r in results if r[2] is None] errors = sum(1 for r in results if r[2] is not None) decisions = {} - for _lat, rcode, err in results: + for _lat, rcode, err, _request_bytes, _response_bytes in results: if err is not None: decisions["transport_error"] = decisions.get("transport_error", 0) + 1 else: @@ -206,7 +207,11 @@ def _summarize(results, concurrency, duration_s): errors, concurrency, duration_s, - extra={"decision_distribution": decisions}, + extra={ + "decision_distribution": decisions, + "request_bytes": sum(row[3] for row in results), + "response_bytes": sum(row[4] for row in results), + }, ) diff --git a/guest/artifacts/capsem_bench/mitm_load.py b/guest/artifacts/capsem_bench/mitm_load.py index ace56050d..4f7f684c7 100644 --- a/guest/artifacts/capsem_bench/mitm_load.py +++ b/guest/artifacts/capsem_bench/mitm_load.py @@ -30,7 +30,7 @@ "p95_ms": 35.0, "p99_ms": 41.0, "p999_ms": 70.0, - "rss_peak_mb": 132.0 + "response_body_bytes": 29616 }, ... ] @@ -91,8 +91,8 @@ def get(self): try: self._connection.request("GET", self._path) response = self._connection.getresponse() - response.read() - return response.status + body = response.read() + return response.status, len(body) except Exception: self.close() raise @@ -107,13 +107,13 @@ def _do_request(session): """Single HTTP GET; latency in ms, with non-2xx responses counted as errors.""" start = time.monotonic() try: - status = session.get() + status, response_bytes = session.get() elapsed_ms = (time.monotonic() - start) * 1000 error = None if 200 <= status < 300 else f"HTTP {status}" - return (elapsed_ms, status, error) + return (elapsed_ms, status, error, response_bytes) except (OSError, http.client.HTTPException) as exc: elapsed_ms = (time.monotonic() - start) * 1000 - return (elapsed_ms, 0, str(exc)) + return (elapsed_ms, 0, str(exc), 0) def _drive_at_concurrency(url, concurrency, duration_s): @@ -146,12 +146,16 @@ def _summarize(results, concurrency, duration_s): latencies = sorted(r[0] for r in results if r[2] is None) errors = sum(1 for r in results if r[2] is not None) error_distribution = Counter(r[2] for r in results if r[2] is not None) + response_body_bytes = sum(r[3] for r in results if r[2] is None) return summarize_load_level( latencies, errors, concurrency, duration_s, - extra={"error_distribution": dict(error_distribution)}, + extra={ + "error_distribution": dict(error_distribution), + "response_body_bytes": response_body_bytes, + }, ) diff --git a/tests/helpers/process_resources.py b/tests/helpers/process_resources.py new file mode 100644 index 000000000..b2643aa2a --- /dev/null +++ b/tests/helpers/process_resources.py @@ -0,0 +1,177 @@ +"""Process-tree resource accounting for load benchmarks.""" + +from __future__ import annotations + +import contextlib +import re +import threading +from dataclasses import dataclass +from decimal import Decimal + +import psutil + +from .process_cpu import process_cpu_seconds + +SAMPLE_INTERVAL_SECONDS = 0.05 +TRACKED_PROCESS_NAMES = frozenset( + { + "capsem_network", + "capsem_process", + "capsem_proxy", + "capsem_router", + "capsem_service", + } +) + + +@dataclass(frozen=True) +class ProcessUsage: + pid: int + started_at: float + name: str + cpu_seconds: Decimal + rss_bytes: int + fds: int + + +@dataclass(frozen=True) +class TreeSnapshot: + processes: dict[tuple[int, float], ProcessUsage] + + +def process_tree_snapshot(root: psutil.Process) -> TreeSnapshot: + """Read one race-tolerant snapshot of a process and its descendants.""" + with contextlib.suppress(psutil.Error): + processes = [root, *root.children(recursive=True)] + found = {} + for process in processes: + with contextlib.suppress(psutil.Error, OSError): + with process.oneshot(): + usage = ProcessUsage( + pid=process.pid, + started_at=process.create_time(), + name=process.name(), + cpu_seconds=process_cpu_seconds(process), + rss_bytes=process.memory_info().rss, + fds=process.num_fds(), + ) + found[(usage.pid, usage.started_at)] = usage + return TreeSnapshot(found) + return TreeSnapshot({}) + + +def _metric_name(name: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_") + + +class ProcessTreeSampler: + """Track CPU deltas and peak resident resources without spawning tools.""" + + def __init__( + self, + root: psutil.Process | None, + *, + interval_seconds: float = SAMPLE_INTERVAL_SECONDS, + ) -> None: + self._root = root + self._interval_seconds = interval_seconds + self._stop = threading.Event() + self._thread: threading.Thread | None = None + self._observations = 0 + self._baseline: dict[str, tuple[int, int]] = {} + self._peak: dict[str, tuple[int, int]] = {} + self._peak_processes = 0 + self._first_cpu: dict[tuple[int, float], Decimal] = {} + self._last_cpu: dict[tuple[int, float], Decimal] = {} + self._names: dict[tuple[int, float], str] = {} + + def observe(self, snapshot: TreeSnapshot) -> None: + """Fold one snapshot into the benchmark interval.""" + initial = self._observations == 0 + self._observations += 1 + totals: dict[str, tuple[int, int]] = {"total": (0, 0)} + for identity, usage in snapshot.processes.items(): + name = _metric_name(usage.name) + rss, fds = totals.get(name, (0, 0)) + totals[name] = (rss + usage.rss_bytes, fds + usage.fds) + total_rss, total_fds = totals["total"] + totals["total"] = ( + total_rss + usage.rss_bytes, + total_fds + usage.fds, + ) + if identity not in self._first_cpu: + # A process born during the interval has spent all of its CPU + # lifetime on this benchmark. Processes present at baseline + # contribute only their subsequent delta. + self._first_cpu[identity] = usage.cpu_seconds if initial else Decimal(0) + self._last_cpu[identity] = usage.cpu_seconds + self._names[identity] = name + + if initial: + self._baseline = dict(totals) + for name, values in totals.items(): + previous = self._peak.get(name, (0, 0)) + self._peak[name] = (max(previous[0], values[0]), max(previous[1], values[1])) + self._peak_processes = max(self._peak_processes, len(snapshot.processes)) + + def start(self) -> None: + if self._root is None or self._thread is not None: + raise RuntimeError("process resource sampler needs one live root and one start") + self.observe(process_tree_snapshot(self._root)) + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + + def _run(self) -> None: + while not self._stop.wait(self._interval_seconds): + if self._root is not None: + self.observe(process_tree_snapshot(self._root)) + + def stop(self) -> dict[str, dict]: + if self._root is None or self._thread is None: + raise RuntimeError("process resource sampler was not started") + self._stop.set() + self._thread.join() + self.observe(process_tree_snapshot(self._root)) + self._thread = None + return self.metrics() + + def metrics(self) -> dict[str, dict]: + cpu: dict[str, Decimal] = {} + for identity, last in self._last_cpu.items(): + name = self._names[identity] + delta = max(Decimal(0), last - self._first_cpu[identity]) + cpu[name] = cpu.get(name, Decimal(0)) + delta + cpu["total"] = cpu.get("total", Decimal(0)) + delta + + metrics = {} + for name in sorted(self._peak): + if name != "total" and name not in TRACKED_PROCESS_NAMES: + continue + peak_rss, peak_fds = self._peak[name] + baseline_rss, baseline_fds = self._baseline.get(name, (0, 0)) + prefix = f"resources.{name}" + metrics[f"{prefix}.cpu_s"] = { + "unit": "seconds", + "samples": [float(cpu.get(name, Decimal(0)))], + } + metrics[f"{prefix}.peak_rss_bytes"] = { + "unit": "bytes", + "samples": [peak_rss], + } + metrics[f"{prefix}.peak_rss_delta_bytes"] = { + "unit": "bytes", + "samples": [max(0, peak_rss - baseline_rss)], + } + metrics[f"{prefix}.peak_fds"] = { + "unit": "count", + "samples": [peak_fds], + } + metrics[f"{prefix}.peak_fd_delta"] = { + "unit": "count", + "samples": [max(0, peak_fds - baseline_fds)], + } + metrics["resources.total.peak_processes"] = { + "unit": "count", + "samples": [self._peak_processes], + } + return metrics diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index 1020f767f..cf344fac4 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -22,6 +22,14 @@ def collector() -> dict: return runpy.run_path(str(COLLECTOR)) +class ResourceSamplerStub: + def start(self) -> None: + pass + + def stop(self) -> dict: + return {} + + def test_guest_command_forwards_only_owned_benchmark_controls(monkeypatch) -> None: monkeypatch.setenv("CAPSEM_BENCH_CONCURRENCY", "1,8") monkeypatch.setenv("CAPSEM_BENCH_DURATION_S", "0.5") @@ -145,6 +153,9 @@ def test_guest_result_preserves_zero_error_proof(monkeypatch, capsys) -> None: monkeypatch.setitem( main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True ) + monkeypatch.setitem( + main.__globals__, "process_tree_sampler", lambda service: ResourceSamplerStub() + ) monkeypatch.setattr(sys, "argv", ["dns-load"]) assert main() == 0 @@ -153,6 +164,7 @@ def test_guest_result_preserves_zero_error_proof(monkeypatch, capsys) -> None: assert result["metrics"] == { "concurrency_levels.0.error_ratio": {"unit": "ratio", "samples": [0.0]}, "concurrency_levels.0.errors": {"unit": "count", "samples": [0]}, + "concurrency_levels.0.requests": {"unit": "count", "samples": [500]}, "concurrency_levels.0.rps": {"unit": "requests_per_second", "samples": [50.0]}, } assert stopped == [True] @@ -202,6 +214,9 @@ def test_guest_result_rejects_one_failed_request(monkeypatch, tmp_path) -> None: monkeypatch.setitem( main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True ) + monkeypatch.setitem( + main.__globals__, "process_tree_sampler", lambda service: ResourceSamplerStub() + ) monkeypatch.setattr(sys, "argv", ["mitm-load"]) with pytest.raises(RuntimeError, match="requests failed at concurrency \\[200\\]"): diff --git a/tests/test_capsem_bench_mock_server_protocol.py b/tests/test_capsem_bench_mock_server_protocol.py index 6bda5eee6..536875baf 100644 --- a/tests/test_capsem_bench_mock_server_protocol.py +++ b/tests/test_capsem_bench_mock_server_protocol.py @@ -8,21 +8,22 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "guest" / "artifacts")) from capsem_bench import __main__ as bench_main -from capsem_bench import load_harness, mitm_load +from capsem_bench import dns_load, load_harness, mitm_load @pytest.mark.parametrize("status", [200, 204, 302, 403, 500]) def test_mitm_load_counts_http_failures(status): - session = types.SimpleNamespace(get=lambda: status) - elapsed, observed_status, error = mitm_load._do_request(session) + session = types.SimpleNamespace(get=lambda: (status, 24)) + elapsed, observed_status, error, response_bytes = mitm_load._do_request(session) assert elapsed >= 0 assert observed_status == status assert error == (None if 200 <= status < 300 else f"HTTP {status}") + assert response_bytes == 24 def test_mitm_load_excludes_failed_latency_and_reports_error_ratio(): row = mitm_load._summarize( - [(5.0, 200, None), (100.0, 0, "timed out")], + [(5.0, 200, None, 24), (100.0, 0, "timed out", 0)], concurrency=2, duration_s=10, ) @@ -32,6 +33,7 @@ def test_mitm_load_excludes_failed_latency_and_reports_error_ratio(): assert row["error_ratio"] == 0.5 assert row["rps"] == 0.1 assert row["p50_ms"] == 5.0 + assert row["response_body_bytes"] == 24 assert row["error_distribution"] == {"timed out": 1} @@ -40,8 +42,8 @@ def test_mitm_load_rejects_one_failed_request(monkeypatch): mitm_load, "_drive_at_concurrency", lambda url, concurrency, duration_s: [ - (5.0, 200, None), - (100.0, 0, "timed out"), + (5.0, 200, None, 24), + (100.0, 0, "timed out", 0), ], ) @@ -53,6 +55,17 @@ def test_mitm_load_rejects_one_failed_request(monkeypatch): ) +def test_dns_load_reports_application_wire_bytes() -> None: + row = dns_load._summarize( + [(1.0, 0, None, 40, 56), (2.0, 3, None, 40, 40)], + concurrency=2, + duration_s=1, + ) + + assert row["request_bytes"] == 80 + assert row["response_bytes"] == 96 + + class _StubConsole: def __init__(self, *args, **kwargs): self.messages = [] diff --git a/tests/test_process_resource_sampler.py b/tests/test_process_resource_sampler.py new file mode 100644 index 000000000..afd902fb3 --- /dev/null +++ b/tests/test_process_resource_sampler.py @@ -0,0 +1,87 @@ +"""The load benchmark measures the resources its admitted work consumes.""" + +from decimal import Decimal + +from helpers.process_resources import ProcessTreeSampler, ProcessUsage, TreeSnapshot + + +def snapshot(*rows: ProcessUsage) -> TreeSnapshot: + return TreeSnapshot({(row.pid, row.started_at): row for row in rows}) + + +def usage( + pid: int, + name: str, + *, + cpu: str, + rss: int, + fds: int, + started_at: float = 1.0, +) -> ProcessUsage: + return ProcessUsage(pid, started_at, name, Decimal(cpu), rss, fds) + + +def test_sampler_reports_tree_and_named_process_pressure() -> None: + sampler = ProcessTreeSampler(None) + sampler.observe( + snapshot( + usage(1, "capsem-service", cpu="10.0", rss=100, fds=5), + usage(2, "capsem-proxy", cpu="20.0", rss=200, fds=20), + ) + ) + sampler.observe( + snapshot( + usage(1, "capsem-service", cpu="10.2", rss=110, fds=6), + usage(2, "capsem-proxy", cpu="21.5", rss=500, fds=400), + usage(3, "capsem-exec", cpu="0.1", rss=50, fds=4), + ) + ) + + assert sampler.metrics() == { + "resources.total.cpu_s": {"unit": "seconds", "samples": [1.8]}, + "resources.total.peak_rss_bytes": {"unit": "bytes", "samples": [660]}, + "resources.total.peak_rss_delta_bytes": { + "unit": "bytes", + "samples": [360], + }, + "resources.total.peak_fds": {"unit": "count", "samples": [410]}, + "resources.total.peak_fd_delta": {"unit": "count", "samples": [385]}, + "resources.total.peak_processes": {"unit": "count", "samples": [3]}, + "resources.capsem_service.cpu_s": {"unit": "seconds", "samples": [0.2]}, + "resources.capsem_service.peak_rss_bytes": { + "unit": "bytes", + "samples": [110], + }, + "resources.capsem_service.peak_rss_delta_bytes": { + "unit": "bytes", + "samples": [10], + }, + "resources.capsem_service.peak_fds": {"unit": "count", "samples": [6]}, + "resources.capsem_service.peak_fd_delta": { + "unit": "count", + "samples": [1], + }, + "resources.capsem_proxy.cpu_s": {"unit": "seconds", "samples": [1.5]}, + "resources.capsem_proxy.peak_rss_bytes": { + "unit": "bytes", + "samples": [500], + }, + "resources.capsem_proxy.peak_rss_delta_bytes": { + "unit": "bytes", + "samples": [300], + }, + "resources.capsem_proxy.peak_fds": {"unit": "count", "samples": [400]}, + "resources.capsem_proxy.peak_fd_delta": { + "unit": "count", + "samples": [380], + }, + } + + +def test_process_that_exits_keeps_its_last_cpu_sample() -> None: + sampler = ProcessTreeSampler(None) + sampler.observe(snapshot(usage(7, "capsem-proxy", cpu="3", rss=20, fds=2))) + sampler.observe(snapshot(usage(7, "capsem-proxy", cpu="4", rss=30, fds=3))) + sampler.observe(snapshot()) + + assert sampler.metrics()["resources.capsem_proxy.cpu_s"]["samples"] == [1.0] From e8a9ff5f14ca1cfc90d50b242ffe5ed6da914989 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 19:03:25 +0000 Subject: [PATCH 328/464] fix(bench): retain pressure evidence on load failure --- benchmarks/collectors/_guest | 12 +++++++++++- tests/test_benchmark_guest_collector.py | 21 +++++++++++++++++++-- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/benchmarks/collectors/_guest b/benchmarks/collectors/_guest index 93df6316c..601338a69 100755 --- a/benchmarks/collectors/_guest +++ b/benchmarks/collectors/_guest @@ -119,6 +119,15 @@ def failed_load_levels(section: dict) -> list[int]: ] +def resource_samples(metrics: dict) -> dict[str, float]: + """Flatten sampled host pressure for diagnostics from rejected load runs.""" + return { + name: measurement["samples"][0] + for name, measurement in sorted(metrics.items()) + if measurement.get("samples") + } + + def guest_command(mode: str) -> str: """One safely quoted command with only documented benchmark controls.""" assignments = [ @@ -275,7 +284,8 @@ def main() -> int: if failed := failed_load_levels(section): raise RuntimeError( f"capsem-bench {mode}: requests failed at concurrency {failed}; " - "the rail did not sustain the requested load, so the run is rejected" + "the rail did not sustain the requested load, so the run is rejected; " + f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}" ) numbers = flatten(section) if not numbers: diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index cf344fac4..5f8cfe3e8 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -214,10 +214,27 @@ def test_guest_result_rejects_one_failed_request(monkeypatch, tmp_path) -> None: monkeypatch.setitem( main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True ) + class FailedRunSampler(ResourceSamplerStub): + def stop(self) -> dict: + return { + "resources.total.cpu_seconds": { + "unit": "seconds", + "samples": [12.5], + }, + "resources.total.peak_rss_bytes": { + "unit": "bytes", + "samples": [1048576], + }, + } + monkeypatch.setitem( - main.__globals__, "process_tree_sampler", lambda service: ResourceSamplerStub() + main.__globals__, "process_tree_sampler", lambda service: FailedRunSampler() ) monkeypatch.setattr(sys, "argv", ["mitm-load"]) - with pytest.raises(RuntimeError, match="requests failed at concurrency \\[200\\]"): + with pytest.raises(RuntimeError) as failure: main() + message = str(failure.value) + assert "requests failed at concurrency [200]" in message + assert '"resources.total.cpu_seconds": 12.5' in message + assert '"resources.total.peak_rss_bytes": 1048576' in message From b414f8bc6ab81318c5149233c5af2437a8abf500 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 19:21:16 +0000 Subject: [PATCH 329/464] fix(bench): report pressure on guest load failure --- benchmarks/collectors/_guest | 3 +- tests/test_benchmark_guest_collector.py | 56 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/benchmarks/collectors/_guest b/benchmarks/collectors/_guest index 601338a69..2208f57fd 100755 --- a/benchmarks/collectors/_guest +++ b/benchmarks/collectors/_guest @@ -250,7 +250,8 @@ def main() -> int: raise RuntimeError( f"capsem-bench {mode} failed: " f"exit={None if not benchmark_result else benchmark_result.get('exit_code')}\n" - f"response={json.dumps(benchmark_result)[:1000]}" + f"response={json.dumps(benchmark_result)[:1000]}\n" + f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}" ) artifact_result = client.post( diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index 5f8cfe3e8..296837cc0 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -238,3 +238,59 @@ def stop(self) -> dict: assert "requests failed at concurrency [200]" in message assert '"resources.total.cpu_seconds": 12.5' in message assert '"resources.total.peak_rss_bytes": 1048576' in message + + +def test_guest_command_failure_preserves_resource_pressure(monkeypatch, tmp_path) -> None: + client = SimpleNamespace( + post=lambda route, *args, **kwargs: ( + {"id": "bench-test-vm"} + if route == "/vms/create" + else {"exit_code": 1, "stderr": "load failed"} + ), + delete=lambda *args: None, + ) + main = collector()["main"] + monkeypatch.setitem( + main.__globals__, + "ServiceInstance", + lambda: SimpleNamespace( + client=lambda: client, + start=lambda: None, + stop=lambda: None, + tmp_dir=tmp_path, + ), + ) + import helpers.mock_server + + monkeypatch.setattr( + helpers.mock_server, + "start_mock_server", + lambda: ( + object(), + {"dns_udp_addr": "127.0.0.1:41000", "http_addr": "127.0.0.1:3713"}, + ), + ) + monkeypatch.setattr(helpers.mock_server, "stop_process", lambda proc: None) + monkeypatch.setitem( + main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True + ) + + class FailedRunSampler(ResourceSamplerStub): + def stop(self) -> dict: + return { + "resources.total.cpu_seconds": { + "unit": "seconds", + "samples": [8.75], + } + } + + monkeypatch.setitem( + main.__globals__, "process_tree_sampler", lambda service: FailedRunSampler() + ) + monkeypatch.setattr(sys, "argv", ["mitm-load"]) + + with pytest.raises(RuntimeError) as failure: + main() + message = str(failure.value) + assert "capsem-bench mitm-load failed: exit=1" in message + assert '"resources.total.cpu_seconds": 8.75' in message From b4b86dc4cc278e7ccdd828163539763e2823c5e7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 19:35:35 +0000 Subject: [PATCH 330/464] refactor(proto): share upstream grant capacity --- crates/capsem-proto/src/upstream_grant.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index 3dd899317..84f683e27 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -26,6 +26,11 @@ use guest_metadata::{ATIME_SECONDS_RANGE, PATH_RANGE, TIME_FLAGS_OFFSET}; pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 4480; /// A ledger response grants its storage and trusted-checkpoint channels. pub const UPSTREAM_GRANT_MAX_FDS: usize = 2; +/// Per-worker descriptor and pending-selection ceiling. +/// +/// Two hundred supported clients can briefly own both their current and +/// replacement resources while asynchronous release messages drain. +pub const UPSTREAM_GRANT_CAPACITY: usize = 512; /// DNS names are at most 253 wire-text bytes without a root dot. pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; /// Maximum relative path carried by a guest-share metadata request. From 0b9cf8f6cf8e8c229a31518545da93abfa444bcc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 19:36:53 +0000 Subject: [PATCH 331/464] refactor(proxy): use shared grant capacity --- crates/capsem-proxy/src/main.rs | 7 ++----- crates/capsem-proxy/src/tests.rs | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 950402085..88b66d899 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -23,6 +23,7 @@ use capsem_proto::proxy_control::{ PROXY_CONTROL_MAX_FDS, }; use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; +use capsem_proto::upstream_grant::UPSTREAM_GRANT_CAPACITY; use clap::Parser; use tokio::task::{JoinError, JoinSet}; @@ -32,10 +33,6 @@ mod metric_client; mod private_names_client; const CONTROL_QUEUE_CAPACITY: usize = 16; -// The supported 200-client load can transiently own twice as many descriptors: -// pooled connections are replaced before their asynchronous close is observed. -// Retain bounded headroom above that overlap without admitting unbounded FDs. -const TRAFFIC_GRANT_LIMIT: usize = 512; const CONTROL_SEND_TIMEOUT: Duration = Duration::from_secs(5); type ControlSender = router_channel::DescriptorSender; @@ -387,7 +384,7 @@ fn grant_rejection( } let current = grants.get(&capability).map_or(0, Vec::len); if is_traffic(capability) { - (current >= TRAFFIC_GRANT_LIMIT).then_some(ProxyControlRejection::Capacity) + (current >= UPSTREAM_GRANT_CAPACITY).then_some(ProxyControlRejection::Capacity) } else { (current != 0).then_some(ProxyControlRejection::DuplicateCapability) } diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index 41bc14c20..b54ea9375 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -51,10 +51,10 @@ fn each_traffic_capability_keeps_its_full_bounded_capacity() { grants.insert(capability, vec![grant_id]); grant_ids.insert(grant_id); } - let http = (100..100 + TRAFFIC_GRANT_LIMIT as u64).collect::>(); + let http = (100..100 + UPSTREAM_GRANT_CAPACITY as u64).collect::>(); grant_ids.extend(http.iter().copied()); grants.insert(ProxyCapability::HttpTraffic, http); - let next_grant = 100 + TRAFFIC_GRANT_LIMIT as u64; + let next_grant = 100 + UPSTREAM_GRANT_CAPACITY as u64; assert_eq!( grant_rejection(&grants, &grant_ids, ProxyCapability::HttpTraffic, next_grant), From 2446a1246055575e181e75e410fe500ee6c73b01 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 19:41:01 +0000 Subject: [PATCH 332/464] fix(service): sustain brokered network bursts --- CHANGELOG.md | 4 +- crates/capsem-service/src/upstream_broker.rs | 54 ++++++++++++++++--- .../src/upstream_broker/tests.rs | 35 ++++++++---- 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 033cbd1c8..64b76886c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -174,7 +174,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The confined HTTP proxy admits the supported 200-client concurrent load plus connection-replacement overlap instead of closing excess TLS handshakes when - its descriptor grant bound is reached. + its descriptor grant bound is reached. The trusted upstream broker uses the + same bound for pending selections and live sockets instead of intermittently + returning 502 responses during a resolve burst. - Multiplexed DNS sessions admit the dual-stack query burst from 200 concurrent clients instead of shedding valid A or AAAA lookups as SERVFAIL. diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 7cebd28e7..07a9e0e6a 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -26,7 +26,7 @@ use capsem_proto::ledger::LedgerChannelGrant; use capsem_proto::upstream_grant::{ decode_upstream_grant_request, encode_upstream_grant_response, GuestFileTime, ProxyTrafficService, UpstreamDescriptorKind, UpstreamGrantDenial, UpstreamGrantRequest, UpstreamGrantResponse, UpstreamProtocol, - UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, + UPSTREAM_GRANT_CAPACITY, UPSTREAM_GRANT_FRAME_SIZE, UPSTREAM_GRANT_MAX_FDS, }; use tokio::sync::{mpsc, oneshot}; use tracing::{debug, warn}; @@ -34,8 +34,6 @@ use tracing::{debug, warn}; use crate::instance::WorkerGrant; use crate::ledger_worker::{LedgerClient, LedgerWorkers}; -const MAX_SELECTIONS: usize = 64; -const MAX_ACTIVE_GRANTS: usize = 256; const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const RESPONSE_TIMEOUT: Duration = Duration::from_secs(1); @@ -314,7 +312,7 @@ async fn run( } => { let current = Arc::clone(&policy); selections.retain(|_, selection: &mut Selection| selection.policy_digest == current.digest); - if selections.len() >= MAX_SELECTIONS { + if selections.len() >= UPSTREAM_GRANT_CAPACITY { send_denied(&responses, request_id, UpstreamGrantDenial::Capacity).await?; continue; } @@ -378,7 +376,7 @@ async fn run( request_id, selection_id, } => { - if active.len() >= MAX_ACTIVE_GRANTS { + if active.len() + listeners.len() >= UPSTREAM_GRANT_CAPACITY { send_denied(&responses, request_id, UpstreamGrantDenial::Capacity).await?; continue; } @@ -447,7 +445,7 @@ async fn run( request_id, upstream_index, } => { - if active.len() >= MAX_ACTIVE_GRANTS { + if active.len() + listeners.len() >= UPSTREAM_GRANT_CAPACITY { send_denied(&responses, request_id, UpstreamGrantDenial::Capacity).await?; continue; } @@ -517,6 +515,50 @@ async fn run( .await?; awaiting_adoption = Some(AwaitingAdoption::Upstream(grant_id)); } + UpstreamGrantRequest::OpenLoopbackListener { request_id, port } => { + if active.len() + listeners.len() >= UPSTREAM_GRANT_CAPACITY { + send_denied(&responses, request_id, UpstreamGrantDenial::Capacity).await?; + continue; + } + let listener = match std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port)) { + Ok(listener) => listener, + Err(error) => { + warn!(port, %error, "loopback listener grant bind failed"); + send_denied(&responses, request_id, UpstreamGrantDenial::BindFailed).await?; + continue; + } + }; + let bound_port = listener + .local_addr() + .map_err(|error| format!("read granted loopback listener address: {error}"))? + .port(); + let descriptor: OwnedFd = listener.into(); + let grant_id = allocate_id(&mut next_resource_id)?; + listeners.insert( + grant_id, + ActiveListenerGrant { + descriptor: Some(descriptor), + adopted: false, + }, + ); + let descriptor = &listeners + .get(&grant_id) + .ok_or("new listener grant disappeared")? + .descriptor + .as_ref() + .ok_or("new listener grant lost its descriptor")?; + send_response( + &responses, + &UpstreamGrantResponse::LoopbackListenerGranted { + request_id, + grant_id, + port: bound_port, + }, + Some(descriptor), + ) + .await?; + awaiting_adoption = Some(AwaitingAdoption::Listener(grant_id)); + } UpstreamGrantRequest::Adopted { grant_id } => { if awaiting_adoption == Some(AwaitingAdoption::Ledger(grant_id)) { let Some((grant, descriptors)) = pending_ledger.take() else { diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index 67c0fc817..3e68ccace 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -549,10 +549,10 @@ async fn unresolved_selection_count_is_bounded() { let (client, _publisher, authority, task) = start_broker(test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![])); let mut selections = Vec::new(); - for request_id in 1..=MAX_SELECTIONS as u64 { + for request_id in 1..=UPSTREAM_GRANT_CAPACITY as u64 { selections.push(resolve_override(&client, request_id).await); } - let request_id = MAX_SELECTIONS as u64 + 1; + let request_id = UPSTREAM_GRANT_CAPACITY as u64 + 1; let (response, fds) = client .request(&UpstreamGrantRequest::ResolveTcp { request_id, @@ -578,6 +578,21 @@ async fn unresolved_selection_count_is_bounded() { stop_broker(authority, task).await; } +#[tokio::test] +async fn supported_mitm_burst_can_resolve_before_any_client_connects() { + const SUPPORTED_MITM_CLIENTS: u64 = 200; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (client, _publisher, authority, task) = + start_broker(test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![])); + + for request_id in 1..=SUPPORTED_MITM_CLIENTS { + assert_ne!(resolve_override(&client, request_id).await, 0); + } + + stop_broker(authority, task).await; +} + #[tokio::test] async fn dns_grant_uses_only_the_configured_index() { let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); @@ -829,11 +844,12 @@ async fn active_descriptor_capacity_is_bounded_and_recovers_after_release() { let upstream = tokio::net::UdpSocket::bind("127.0.0.1:0").await.unwrap(); let (client, _publisher, authority, task) = start_broker(test_policy("policy-a", None, vec![upstream.local_addr().unwrap()])); - let mut descriptors = Vec::with_capacity(MAX_ACTIVE_GRANTS); - let mut grant_ids = Vec::with_capacity(MAX_ACTIVE_GRANTS); - for request_id in 1..=MAX_ACTIVE_GRANTS as u64 { - let (response, mut fds) = client - .request(&UpstreamGrantRequest::OpenDns { + let mut grant_ids = Vec::with_capacity(UPSTREAM_GRANT_CAPACITY); + for request_id in 1..=UPSTREAM_GRANT_CAPACITY as u64 { + let request = if request_id == 1 { + UpstreamGrantRequest::OpenLoopbackListener { request_id, port: 0 } + } else { + UpstreamGrantRequest::OpenDns { request_id, upstream_index: 0, }) @@ -843,10 +859,10 @@ async fn active_descriptor_capacity_is_bounded_and_recovers_after_release() { }; client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; grant_ids.push(grant_id); - descriptors.push(fds.pop().unwrap()); + drop(fds.pop().unwrap()); } - let denied_id = MAX_ACTIVE_GRANTS as u64 + 1; + let denied_id = UPSTREAM_GRANT_CAPACITY as u64 + 1; let (response, fds) = client .request(&UpstreamGrantRequest::OpenDns { request_id: denied_id, @@ -867,7 +883,6 @@ async fn active_descriptor_capacity_is_bounded_and_recovers_after_release() { .send(&UpstreamGrantRequest::Release { resource_id: grant_id }) .await; } - drop(descriptors); let (response, mut fds) = client .request(&UpstreamGrantRequest::OpenDns { request_id: denied_id + 1, From d03068063282bbf480303361c9a606385385f598 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 19:44:17 +0000 Subject: [PATCH 333/464] fix(core): preserve upstream denial cause --- .../src/net/mitm_proxy/upstream.rs | 3 +++ .../src/net/mitm_proxy/upstream/tests.rs | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/crates/capsem-core/src/net/mitm_proxy/upstream.rs b/crates/capsem-core/src/net/mitm_proxy/upstream.rs index 50ea159df..9d421f4ca 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upstream.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upstream.rs @@ -331,6 +331,9 @@ impl UpstreamTarget { let (stream, pinned) = self.connect().await?; return Ok((GrantedTcpStream::new(stream, || {}), pinned)); }; + if let Self::Unresolved(error) = self { + return Err(io::Error::new(io::ErrorKind::NotFound, error.clone())); + } let Self::Granted { host, port, diff --git a/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs b/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs index 18df6e03a..e7f7daf8b 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upstream/tests.rs @@ -391,3 +391,22 @@ async fn installed_grants_never_fall_back_to_a_direct_target() { .is_err() ); } + +#[tokio::test] +async fn brokered_resolution_denial_survives_the_connection_boundary() { + let grants = TestGrants { + address: "127.0.0.1:9".parse().unwrap(), + next_id: AtomicU64::new(1), + selection_releases: Arc::new(AtomicUsize::new(0)), + stream_releases: Arc::new(AtomicUsize::new(0)), + policy_digests: Arc::new(Mutex::new(Vec::new())), + }; + let target = UpstreamTarget::Unresolved("upstream TCP resolution denied: Capacity".into()); + + let error = match target.connect_with_grants(Some(&grants), "blake3:test").await { + Ok(_) => panic!("unresolved broker target connected"), + Err(error) => error, + }; + assert_eq!(error.kind(), io::ErrorKind::NotFound); + assert_eq!(error.to_string(), "upstream TCP resolution denied: Capacity"); +} From e0377be04f2d2e28164f0b9b052a6210370ada23 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:00:24 +0000 Subject: [PATCH 334/464] perf(proto): support 512 network clients --- CHANGELOG.md | 11 +++++------ crates/capsem-proto/src/lib.rs | 10 ++++++++-- crates/capsem-proto/src/upstream_grant.rs | 5 ++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 64b76886c..2a8c30ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -172,13 +172,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed -- The confined HTTP proxy admits the supported 200-client concurrent load plus - connection-replacement overlap instead of closing excess TLS handshakes when - its descriptor grant bound is reached. The trusted upstream broker uses the - same bound for pending selections and live sockets instead of intermittently - returning 502 responses during a resolve burst. +- The confined HTTP proxy admits 512 concurrent clients instead of closing + excess TLS handshakes when its descriptor grant bound is reached. The trusted + upstream broker and DNS sessions derive their ceilings from that same limit + instead of intermittently returning 502 or SERVFAIL during a resolve burst. -- Multiplexed DNS sessions admit the dual-stack query burst from 200 concurrent +- Multiplexed DNS sessions admit the dual-stack query burst from 512 concurrent clients instead of shedding valid A or AAAA lookups as SERVFAIL. - Gateway `/status` reads VM, asset and update state in one coordinator diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 8d1761271..25ecb9d43 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -219,8 +219,14 @@ pub const VSOCK_PORT_EXEC: u32 = 5005; pub const VSOCK_PORT_AUDIT: u32 = 5006; /// vsock port for the guest-to-host, length-framed DNS proxy. pub const VSOCK_PORT_DNS_PROXY: u32 = 5007; -/// Per-session bound covering the burst from 200 dual-stack clients. -pub const DNS_SESSION_MAX_IN_FLIGHT: usize = 256; +/// Supported concurrent network-client ceiling for one VM. +/// +/// TCP descriptor grants and each of the two DNS sessions derive their bounds +/// from this one value so one path cannot shed a burst another path admits. +pub const NETWORK_CLIENT_CAPACITY: usize = 512; +/// Per-session DNS bound. Two sessions cover the A+AAAA burst from every +/// supported concurrent client without shedding either address family. +pub const DNS_SESSION_MAX_IN_FLIGHT: usize = NETWORK_CLIENT_CAPACITY; /// Guest-initiated data connections for explicitly published container TCP ports. pub const VSOCK_PORT_PUBLICATION: u32 = 5008; /// Network cables: each guest `capsem-tun` pumps one cable's ethernet frames diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index 84f683e27..26842e306 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -28,9 +28,8 @@ pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 4480; pub const UPSTREAM_GRANT_MAX_FDS: usize = 2; /// Per-worker descriptor and pending-selection ceiling. /// -/// Two hundred supported clients can briefly own both their current and -/// replacement resources while asynchronous release messages drain. -pub const UPSTREAM_GRANT_CAPACITY: usize = 512; +/// The proxy and trusted broker share the public network-client ceiling. +pub const UPSTREAM_GRANT_CAPACITY: usize = crate::NETWORK_CLIENT_CAPACITY; /// DNS names are at most 253 wire-text bytes without a root dot. pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; /// Maximum relative path carried by a guest-share metadata request. From 6b486e9e20739f4fa99b4e914584c56e49d3eca8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:01:03 +0000 Subject: [PATCH 335/464] test(agent): prove 512-client DNS burst --- crates/capsem-agent/src/dns_proxy/session/tests.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/capsem-agent/src/dns_proxy/session/tests.rs b/crates/capsem-agent/src/dns_proxy/session/tests.rs index f4569ed7a..cda7999f6 100644 --- a/crates/capsem-agent/src/dns_proxy/session/tests.rs +++ b/crates/capsem-agent/src/dns_proxy/session/tests.rs @@ -339,11 +339,11 @@ async fn in_flight_cap_sheds_without_blocking() { } #[tokio::test] -async fn two_hundred_dual_stack_clients_fit_without_shedding() { +async fn supported_dual_stack_client_burst_fits_without_shedding() { let connect = fake_host(|mut host| while read_frame(&mut host).is_some() {}); let forwarder = Arc::new(DnsForwarder::new(DNS_SESSIONS, connect)); let mut waiting = Vec::new(); - for i in 0..400u16 { + for i in 0..u16::try_from(DNS_SESSION_MAX_IN_FLIGHT * DNS_SESSIONS).unwrap() { let forwarder = Arc::clone(&forwarder); waiting.push(tokio::spawn(async move { forwarder.forward_query(query(i, "dual-stack.example"), "udp").await @@ -355,7 +355,12 @@ async fn two_hundred_dual_stack_clients_fit_without_shedding() { waiting.iter().all(|task| !task.is_finished()), "a supported query was shed" ); - assert!(forwarder.sessions.iter().all(|session| session.in_flight() == 200)); + assert!( + forwarder + .sessions + .iter() + .all(|session| session.in_flight() == DNS_SESSION_MAX_IN_FLIGHT) + ); for task in waiting { task.abort(); } From fe1a8f3708ce35ae476f751421f272966056c785 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:03:04 +0000 Subject: [PATCH 336/464] test(service): prove 512-client resolve burst --- crates/capsem-service/src/upstream_broker/tests.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index 3e68ccace..30671eb6f 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -580,13 +580,11 @@ async fn unresolved_selection_count_is_bounded() { #[tokio::test] async fn supported_mitm_burst_can_resolve_before_any_client_connects() { - const SUPPORTED_MITM_CLIENTS: u64 = 200; - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let (client, _publisher, authority, task) = start_broker(test_policy("policy-a", Some(listener.local_addr().unwrap()), vec![])); - for request_id in 1..=SUPPORTED_MITM_CLIENTS { + for request_id in 1..=UPSTREAM_GRANT_CAPACITY as u64 { assert_ne!(resolve_override(&client, request_id).await, 0); } From b8ae9c9a5c2dd708e5feba89e4d2abb6db3584d6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:13:34 +0000 Subject: [PATCH 337/464] style(agent): format DNS burst assertion --- crates/capsem-agent/src/dns_proxy/session/tests.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/capsem-agent/src/dns_proxy/session/tests.rs b/crates/capsem-agent/src/dns_proxy/session/tests.rs index cda7999f6..f7cf6c9a2 100644 --- a/crates/capsem-agent/src/dns_proxy/session/tests.rs +++ b/crates/capsem-agent/src/dns_proxy/session/tests.rs @@ -355,12 +355,10 @@ async fn supported_dual_stack_client_burst_fits_without_shedding() { waiting.iter().all(|task| !task.is_finished()), "a supported query was shed" ); - assert!( - forwarder - .sessions - .iter() - .all(|session| session.in_flight() == DNS_SESSION_MAX_IN_FLIGHT) - ); + assert!(forwarder + .sessions + .iter() + .all(|session| session.in_flight() == DNS_SESSION_MAX_IN_FLIGHT)); for task in waiting { task.abort(); } From bcf0a8e72fba0f3e6358aec75b4d759f3fc3fad0 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:13:38 +0000 Subject: [PATCH 338/464] refactor(proto): generate runtime limits from config --- Cargo.lock | 1 + config/README.md | 6 ++++-- config/data/runtime-limits.toml | 5 +++++ crates/capsem-proto/Cargo.toml | 1 + crates/capsem-proto/build.rs | 32 ++++++++++++++++++++++++++++++++ crates/capsem-proto/src/lib.rs | 6 +----- 6 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 config/data/runtime-limits.toml diff --git a/Cargo.lock b/Cargo.lock index f3903b29c..68889c777 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1212,6 +1212,7 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tokio", + "toml 0.8.2", ] [[package]] diff --git a/config/README.md b/config/README.md index be760817d..efd9b4d8d 100644 --- a/config/README.md +++ b/config/README.md @@ -31,8 +31,10 @@ and the existing admin validation rail must learn it. raw-export and packed-EROFS ceilings plus forbidden payload prefixes. The ordinary build must enforce those limits before and after compression; they are release policy, not test data. -- `data/` contains project data embedded or loaded by code, such as model - pricing tables. +- `data/` contains project data embedded or loaded by code. Model pricing + tables live here, and `runtime-limits.toml` is the one source for + cross-process runtime ceilings embedded by `capsem-proto`. Runtime consumers + use the generated protocol constants rather than copying numeric limits. ## Policy at Runtime diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml new file mode 100644 index 000000000..97a491cfe --- /dev/null +++ b/config/data/runtime-limits.toml @@ -0,0 +1,5 @@ +# Product-wide runtime ceilings embedded into every Capsem binary. +# User overrides belong to the typed settings contract tracked by #349. + +[network] +max_concurrent_clients = 512 diff --git a/crates/capsem-proto/Cargo.toml b/crates/capsem-proto/Cargo.toml index 942b2229a..0e7b28ace 100644 --- a/crates/capsem-proto/Cargo.toml +++ b/crates/capsem-proto/Cargo.toml @@ -29,6 +29,7 @@ tempfile = "3" [build-dependencies] quote = "1" syn = { version = "2", features = ["full", "visit-mut"] } +toml = { workspace = true } [[bench]] name = "vsock_framing_micro" diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index 6b9bdeba9..e6104993f 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -7,6 +7,7 @@ //! constants remain in the token stream and change the hash. use quote::ToTokens; +use std::path::PathBuf; use syn::visit_mut::VisitMut; struct StripDocs; @@ -28,6 +29,37 @@ fn main() { &["ledger.rs", "ledger_commitment.rs", "ledger_control.rs"], ); write_hash("proxy_schema_hash.txt", &["proxy_control.rs", "proxy_policy.rs"]); + write_runtime_limits(); +} + +fn write_runtime_limits() { + let config_path = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")) + .join("../../config/data/runtime-limits.toml"); + let source = std::fs::read_to_string(&config_path) + .unwrap_or_else(|error| panic!("runtime limits: read {}: {error}", config_path.display())); + let config: toml::Value = toml::from_str(&source) + .unwrap_or_else(|error| panic!("runtime limits: parse {}: {error}", config_path.display())); + let capacity = config + .get("network") + .and_then(|network| network.get("max_concurrent_clients")) + .and_then(toml::Value::as_integer) + .filter(|value| (1..=4096).contains(value)) + .unwrap_or_else(|| { + panic!("runtime limits: network.max_concurrent_clients must be an integer between 1 and 4096") + }); + let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set"); + std::fs::write( + format!("{out_dir}/runtime_limits.rs"), + format!( + "/// Supported concurrent network-client ceiling for one VM.\n\ + ///\n\ + /// TCP descriptor grants and each of the two DNS sessions derive\n\ + /// their bounds from this one config-owned value.\n\ + pub const NETWORK_CLIENT_CAPACITY: usize = {capacity};\n" + ), + ) + .expect("runtime limits: write generated constants"); + println!("cargo:rerun-if-changed={}", config_path.display()); } fn write_hash(output: &str, files: &[&str]) { diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 25ecb9d43..397c08abd 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -219,11 +219,7 @@ pub const VSOCK_PORT_EXEC: u32 = 5005; pub const VSOCK_PORT_AUDIT: u32 = 5006; /// vsock port for the guest-to-host, length-framed DNS proxy. pub const VSOCK_PORT_DNS_PROXY: u32 = 5007; -/// Supported concurrent network-client ceiling for one VM. -/// -/// TCP descriptor grants and each of the two DNS sessions derive their bounds -/// from this one value so one path cannot shed a burst another path admits. -pub const NETWORK_CLIENT_CAPACITY: usize = 512; +include!(concat!(env!("OUT_DIR"), "/runtime_limits.rs")); /// Per-session DNS bound. Two sessions cover the A+AAAA burst from every /// supported concurrent client without shedding either address family. pub const DNS_SESSION_MAX_IN_FLIGHT: usize = NETWORK_CLIENT_CAPACITY; From cedb9583ee298fb1d651136c4dcc1ce82cfbd367 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:13:52 +0000 Subject: [PATCH 339/464] docs(skills): centralize runtime limit guidance --- skills/dev-mitm-proxy/SKILL.md | 15 +++++++++++++++ skills/dev-rust-patterns/SKILL.md | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/skills/dev-mitm-proxy/SKILL.md b/skills/dev-mitm-proxy/SKILL.md index 447ac9083..d66f81615 100644 --- a/skills/dev-mitm-proxy/SKILL.md +++ b/skills/dev-mitm-proxy/SKILL.md @@ -57,6 +57,21 @@ Guest curl -> iptables REDIRECT -> capsem-net-proxy (guest, port 10443) | `crates/capsem-core/src/net/sni.rs` | SNI parser for TLS ClientHello | | `crates/capsem-agent/src/net_proxy.rs` | Guest-side TCP-to-vsock relay | +## Cross-process network limits + +`config/data/runtime-limits.toml` owns the supported concurrent network-client +ceiling. `capsem-proto/build.rs` validates that file and generates +`NETWORK_CLIENT_CAPACITY`; TCP descriptor grants and both DNS sessions derive +their capacities from that shared protocol constant. + +Never add a numeric connection ceiling to the proxy, service broker, core DNS +session, or guest DNS forwarder. A limit duplicated across those processes can +make one stage admit work that the next stage sheds, which previously produced +intermittent HTTP 502 responses under load. Change the config-owned value, +exercise every consumer through the shared constant, and run the official +HTTPS and DNS load benchmarks twice before and twice after the change. User +overrides belong in the typed settings contract tracked by issue #349. + ## Content-Encoding: the systemic rule The proxy MUST handle response decompression as a general capability. This is not optional, not per-feature. diff --git a/skills/dev-rust-patterns/SKILL.md b/skills/dev-rust-patterns/SKILL.md index 980bf74e4..c1b162448 100644 --- a/skills/dev-rust-patterns/SKILL.md +++ b/skills/dev-rust-patterns/SKILL.md @@ -198,6 +198,11 @@ struct FunctionCall { - **RwLock for caches**: Cert authority uses `RwLock` -- many readers, rare writers. Use `read()` first, upgrade to `write()` only on cache miss. - **Arc for shared state**: VM state, proxy config, and telemetry handles are `Arc`-wrapped for sharing across tasks. - **Per-connection tasks**: The MITM proxy spawns a new tokio task per connection. Each task owns its TLS state and upstream connection. No shared mutable state between connections. +- **Config-owned cross-process ceilings**: Network concurrency is owned by + `config/data/runtime-limits.toml` and generated into `capsem-proto`. Import + the shared protocol constant in host and guest consumers; never duplicate a + numeric capacity in individual crates. See `/dev-mitm-proxy` for the load + proof required when it changes. ### Host-serialization locks for per-host critical sections From 6240c4a3e34cc301d64944b7c3b356a66a2cf966 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:14:58 +0000 Subject: [PATCH 340/464] test(citadel): keep runtime limits config-owned --- skills/citadel/SKILL.md | 1 + tests/citadel/test_runtime_limit_authority.py | 98 +++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 tests/citadel/test_runtime_limit_authority.py diff --git a/skills/citadel/SKILL.md b/skills/citadel/SKILL.md index 83b1870db..c42181243 100644 --- a/skills/citadel/SKILL.md +++ b/skills/citadel/SKILL.md @@ -51,6 +51,7 @@ the suite has one collector and its failures have one owner. | `test_workflow_script_checkout.py` | a workflow checks out its source before running a checked-in script | | `test_release_requires_local_proof.py` | no agent-facing text says a release can skip `just test` | | `test_no_vm_profiles.py` | the removed VM profile concept does not grow back under its old vocabulary | +| `test_runtime_limit_authority.py` | cross-process network ceilings stay config-owned and shared | ## Adding a guard diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py new file mode 100644 index 000000000..8722ab9db --- /dev/null +++ b/tests/citadel/test_runtime_limit_authority.py @@ -0,0 +1,98 @@ +"""Keep cross-process network ceilings under one config authority.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import tomllib + +ROOT = Path(__file__).resolve().parents[2] +CONFIG = ROOT / "config/data/runtime-limits.toml" + +RUNTIME_LIMIT_RATIONALE = """\ +The proxy, trusted upstream broker, host DNS session and guest DNS forwarder +must admit the same supported client burst. Local numeric ceilings previously +let one stage accept work that the next stage shed, producing intermittent HTTP +502 responses under load. config/data/runtime-limits.toml is the authority; +capsem-proto generates the shared constant and every consumer derives from it. +See skills/dev-mitm-proxy/SKILL.md under Cross-process network limits. +""" + +NUMERIC_CONSTANT = re.compile( + r"\b(?:pub(?:\([^)]*\))?\s+)?const\s+([A-Z][A-Z0-9_]*)\s*:\s*usize\s*=\s*(\d+)\s*;" +) +CONFIG_OWNED_NAMES = { + "NETWORK_CLIENT_CAPACITY", + "DNS_SESSION_MAX_IN_FLIGHT", + "UPSTREAM_GRANT_CAPACITY", + "TRAFFIC_GRANT_CAPACITY", + "MAX_SELECTIONS", + "MAX_ACTIVE_GRANTS", +} + +REQUIRED_WIRING = { + "crates/capsem-proto/build.rs": ( + "../../config/data/runtime-limits.toml", + 'get("max_concurrent_clients")', + "pub const NETWORK_CLIENT_CAPACITY: usize = {capacity}", + ), + "crates/capsem-proto/src/lib.rs": ( + 'include!(concat!(env!("OUT_DIR"), "/runtime_limits.rs"))', + "DNS_SESSION_MAX_IN_FLIGHT: usize = NETWORK_CLIENT_CAPACITY", + ), + "crates/capsem-proto/src/upstream_grant.rs": ( + "UPSTREAM_GRANT_CAPACITY: usize = crate::NETWORK_CLIENT_CAPACITY", + ), + "crates/capsem-proxy/src/main.rs": ("UPSTREAM_GRANT_CAPACITY",), + "crates/capsem-service/src/upstream_broker.rs": ("UPSTREAM_GRANT_CAPACITY",), + "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), + "crates/capsem-agent/src/dns_proxy/session.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), +} + + +def _numeric_limit_declarations(sources: dict[str, str]) -> list[str]: + violations = [] + for path, source in sources.items(): + for name, value in NUMERIC_CONSTANT.findall(source): + if name in CONFIG_OWNED_NAMES: + violations.append(f"{path}: numeric {name} = {value}") + return violations + + +def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: + config = tomllib.loads(CONFIG.read_text()) + capacity = config.get("network", {}).get("max_concurrent_clients") + assert type(capacity) is int and 1 <= capacity <= 4096, ( + RUNTIME_LIMIT_RATIONALE + + "\nnetwork.max_concurrent_clients must be an integer from 1 through 4096" + ) + + missing = [] + for relative_path, required_fragments in REQUIRED_WIRING.items(): + source = (ROOT / relative_path).read_text() + missing.extend( + f"{relative_path}: missing {fragment!r}" + for fragment in required_fragments + if fragment not in source + ) + assert not missing, RUNTIME_LIMIT_RATIONALE + "\n" + "\n".join(missing) + + +def test_network_limit_names_have_no_numeric_rust_authority() -> None: + sources = { + str(path.relative_to(ROOT)): path.read_text() + for path in (ROOT / "crates").glob("*/src/**/*.rs") + } + violations = _numeric_limit_declarations(sources) + assert not violations, RUNTIME_LIMIT_RATIONALE + "\n" + "\n".join(violations) + + +def test_numeric_local_ceiling_is_rejected_adversarially() -> None: + violations = _numeric_limit_declarations( + { + "upstream_broker.rs": "const MAX_ACTIVE_GRANTS: usize = 256;", + "dns/session.rs": "pub const DNS_SESSION_MAX_IN_FLIGHT: usize = 512;", + } + ) + assert len(violations) == 2, RUNTIME_LIMIT_RATIONALE From eccb998e41fca0ec92fe8af2b4398394d2b5a8a4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:32:58 +0000 Subject: [PATCH 341/464] perf(bench): drive HTTP load without guest threads --- crates/capsem-bench/src/cli.rs | 2 + crates/capsem-bench/src/http_load.rs | 201 +++++++++++++++++++++ crates/capsem-bench/src/http_load/tests.rs | 51 ++++++ crates/capsem-bench/src/main.rs | 2 + 4 files changed, 256 insertions(+) create mode 100644 crates/capsem-bench/src/http_load.rs create mode 100644 crates/capsem-bench/src/http_load/tests.rs diff --git a/crates/capsem-bench/src/cli.rs b/crates/capsem-bench/src/cli.rs index 1440643dc..75bc2f5ca 100644 --- a/crates/capsem-bench/src/cli.rs +++ b/crates/capsem-bench/src/cli.rs @@ -20,6 +20,8 @@ pub(crate) enum Command { Udp(crate::udp::Args), /// ICMP echo against an address, without a ping binary. Ping(crate::ping::Args), + /// Drive sustained HTTP load with pooled async clients. + HttpLoad(crate::http_load::Args), /// Run deterministic protocol scenarios against capsem-mock-server. Protocol(ProtocolArgs), /// Run host-direct and guest-through-Capsem protocol lanes, then report delta. diff --git a/crates/capsem-bench/src/http_load.rs b/crates/capsem-bench/src/http_load.rs new file mode 100644 index 000000000..a5b13b9c2 --- /dev/null +++ b/crates/capsem-bench/src/http_load.rs @@ -0,0 +1,201 @@ +//! Duration-driven HTTP load without a thread per simulated client. + +use anyhow::{bail, Context, Result}; +use clap::Parser; +use reqwest::Client; +use serde::Serialize; +use std::collections::BTreeMap; +use std::time::{Duration, Instant}; + +#[derive(Debug, Parser)] +pub(crate) struct Args { + /// HTTP(S) endpoint to request. + #[arg(long)] + pub(crate) target: String, + /// Number of concurrent request loops. + #[arg(long)] + pub(crate) concurrency: usize, + /// Minimum time to drive requests. Every loop makes at least one request. + #[arg(long)] + pub(crate) duration_s: f64, + /// Per-request timeout. + #[arg(long, default_value_t = 30_000)] + pub(crate) timeout_ms: u64, +} + +#[derive(Debug)] +struct Sample { + latency_ms: f64, + bytes: usize, + error: Option, + cold: bool, +} + +#[derive(Debug, Serialize)] +pub(crate) struct Report { + concurrency: usize, + duration_s: f64, + elapsed_s: f64, + total_requests: usize, + errors: usize, + error_ratio: f64, + rps: f64, + p50_ms: f64, + p95_ms: f64, + p99_ms: f64, + p999_ms: f64, + cold_p50_ms: f64, + cold_p95_ms: f64, + warm_p50_ms: f64, + warm_p95_ms: f64, + response_body_bytes: usize, + error_distribution: BTreeMap, +} + +pub(crate) async fn run(args: Args) -> Result { + validate(&args)?; + let duration = Duration::from_secs_f64(args.duration_s); + let timeout = Duration::from_millis(args.timeout_ms); + let target = reqwest::Url::parse(&args.target).context("parse --target")?; + if !matches!(target.scheme(), "http" | "https") { + bail!("--target must start with http:// or https://"); + } + + let client = Client::builder() + .danger_accept_invalid_certs(true) + .pool_max_idle_per_host(args.concurrency) + .timeout(timeout) + .build() + .context("build pooled HTTP load client")?; + let started = Instant::now(); + let deadline = started + duration; + let mut tasks = tokio::task::JoinSet::new(); + for _ in 0..args.concurrency { + tasks.spawn(drive_client(client.clone(), target.clone(), deadline)); + } + + let mut samples = Vec::new(); + while let Some(result) = tasks.join_next().await { + samples.extend(result.context("HTTP load task panicked")?); + } + Ok(summarize(samples, args.concurrency, duration, started.elapsed())) +} + +fn validate(args: &Args) -> Result<()> { + if args.concurrency == 0 { + bail!("--concurrency must be greater than zero"); + } + if !args.duration_s.is_finite() || args.duration_s <= 0.0 { + bail!("--duration-s must be a finite value greater than zero"); + } + if args.timeout_ms == 0 { + bail!("--timeout-ms must be greater than zero"); + } + Ok(()) +} + +async fn drive_client(client: Client, target: reqwest::Url, deadline: Instant) -> Vec { + let mut samples = Vec::new(); + let mut cold = true; + while cold || Instant::now() < deadline { + let started = Instant::now(); + let outcome = request(&client, target.clone()).await; + let latency_ms = started.elapsed().as_secs_f64() * 1_000.0; + let (bytes, error) = match outcome { + Ok(bytes) => (bytes, None), + Err(error) => (0, Some(error)), + }; + samples.push(Sample { + latency_ms, + bytes, + error, + cold, + }); + cold = false; + } + samples +} + +async fn request(client: &Client, target: reqwest::Url) -> std::result::Result { + let response = client.get(target).send().await.map_err(|error| error.to_string())?; + let status = response.status(); + let bytes = response.bytes().await.map_err(|error| error.to_string())?.len(); + if status.is_success() { + Ok(bytes) + } else { + Err(format!("HTTP {}", status.as_u16())) + } +} + +fn summarize(samples: Vec, concurrency: usize, duration: Duration, elapsed: Duration) -> Report { + let total_requests = samples.len(); + let errors = samples.iter().filter(|sample| sample.error.is_some()).count(); + let successful = total_requests - errors; + let mut latencies = successful_latencies(&samples, |_| true); + let mut cold_latencies = successful_latencies(&samples, |sample| sample.cold); + let mut warm_latencies = successful_latencies(&samples, |sample| !sample.cold); + let mut error_distribution = BTreeMap::new(); + for error in samples.iter().filter_map(|sample| sample.error.as_ref()) { + *error_distribution.entry(error.clone()).or_insert(0) += 1; + } + let elapsed_s = elapsed.as_secs_f64(); + + Report { + concurrency, + duration_s: duration.as_secs_f64(), + elapsed_s, + total_requests, + errors, + error_ratio: ratio(errors, total_requests), + rps: if elapsed_s > 0.0 { + successful as f64 / elapsed_s + } else { + 0.0 + }, + p50_ms: percentile(&mut latencies, 0.50), + p95_ms: percentile(&mut latencies, 0.95), + p99_ms: percentile(&mut latencies, 0.99), + p999_ms: percentile(&mut latencies, 0.999), + cold_p50_ms: percentile(&mut cold_latencies, 0.50), + cold_p95_ms: percentile(&mut cold_latencies, 0.95), + warm_p50_ms: percentile(&mut warm_latencies, 0.50), + warm_p95_ms: percentile(&mut warm_latencies, 0.95), + response_body_bytes: samples.iter().map(|sample| sample.bytes).sum(), + error_distribution, + } +} + +fn successful_latencies(samples: &[Sample], include: impl Fn(&Sample) -> bool) -> Vec { + samples + .iter() + .filter(|sample| sample.error.is_none() && include(sample)) + .map(|sample| sample.latency_ms) + .collect() +} + +fn ratio(numerator: usize, denominator: usize) -> f64 { + if denominator == 0 { + 0.0 + } else { + numerator as f64 / denominator as f64 + } +} + +fn percentile(values: &mut [f64], quantile: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + values.sort_by(f64::total_cmp); + let position = quantile * (values.len() - 1) as f64; + let lower = position.floor() as usize; + let upper = position.ceil() as usize; + if lower == upper { + values[lower] + } else { + let weight = position - lower as f64; + values[lower] * (1.0 - weight) + values[upper] * weight + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-bench/src/http_load/tests.rs b/crates/capsem-bench/src/http_load/tests.rs new file mode 100644 index 000000000..2dc0206e0 --- /dev/null +++ b/crates/capsem-bench/src/http_load/tests.rs @@ -0,0 +1,51 @@ +use super::*; + +#[test] +fn summary_separates_connection_setup_from_warm_requests() { + let report = summarize( + vec![ + Sample { + latency_ms: 10.0, + bytes: 24, + error: None, + cold: true, + }, + Sample { + latency_ms: 20.0, + bytes: 24, + error: None, + cold: true, + }, + Sample { + latency_ms: 1.0, + bytes: 24, + error: None, + cold: false, + }, + Sample { + latency_ms: 2.0, + bytes: 24, + error: None, + cold: false, + }, + Sample { + latency_ms: 4.0, + bytes: 0, + error: Some("HTTP 502".to_string()), + cold: false, + }, + ], + 2, + Duration::from_secs(1), + Duration::from_secs(2), + ); + + assert_eq!(report.total_requests, 5); + assert_eq!(report.errors, 1); + assert_eq!(report.error_ratio, 0.2); + assert_eq!(report.rps, 2.0); + assert_eq!(report.cold_p50_ms, 15.0); + assert_eq!(report.warm_p50_ms, 1.5); + assert_eq!(report.response_body_bytes, 96); + assert_eq!(report.error_distribution["HTTP 502"], 1); +} diff --git a/crates/capsem-bench/src/main.rs b/crates/capsem-bench/src/main.rs index 054a79e8c..c420d91c4 100644 --- a/crates/capsem-bench/src/main.rs +++ b/crates/capsem-bench/src/main.rs @@ -5,6 +5,7 @@ mod collector; mod commands; #[cfg(feature = "host")] mod comparison; +mod http_load; #[cfg(feature = "host")] mod machine; mod ping; @@ -62,6 +63,7 @@ async fn main() -> Result<()> { Command::Throughput(args) => println!("{}", serde_json::to_string(&throughput::run(args).await?)?), Command::Udp(args) => println!("{}", serde_json::to_string(&udp::run(args).await?)?), Command::Ping(args) => println!("{}", serde_json::to_string(&ping::run(args).await?)?), + Command::HttpLoad(args) => println!("{}", serde_json::to_string(&http_load::run(args).await?)?), Command::Protocol(args) => { #[cfg(feature = "host")] let destination = args.record.clone(); From dc33bdc79b3077b2f6195cf3a7410656a2e3cb73 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:33:04 +0000 Subject: [PATCH 342/464] fix(bench): reject starved MITM load runs --- guest/artifacts/capsem_bench/mitm_load.py | 140 +++++------------- .../test_capsem_bench_mock_server_protocol.py | 76 ++++++---- 2 files changed, 87 insertions(+), 129 deletions(-) diff --git a/guest/artifacts/capsem_bench/mitm_load.py b/guest/artifacts/capsem_bench/mitm_load.py index 4f7f684c7..0cab1566f 100644 --- a/guest/artifacts/capsem_bench/mitm_load.py +++ b/guest/artifacts/capsem_bench/mitm_load.py @@ -40,19 +40,14 @@ level fails the build. """ -import http.client +import json import os -import ssl -import time -from collections import Counter -from concurrent.futures import ThreadPoolExecutor, as_completed -from urllib.parse import urlsplit +import subprocess from .helpers import console from .load_harness import ( DurationLoadConfig, render_load_table, - summarize_load_level, ) # Served by capsem-mock-server through the benchmark collector's corp config @@ -60,103 +55,43 @@ DEFAULT_TARGET = "https://fixture.capsem.test/tiny" DEFAULT_CONCURRENCY = (1, 10, 50, 200) DEFAULT_DURATION_S = 10.0 - - -class _Session: - """One keep-alive connection per worker, reopened after a failure. - - The standard library rather than `requests`: the runtime carries no pip - packages, and a pooled session per worker is all the measurement needs. - """ - - def __init__(self, url): - parts = urlsplit(url) - self._https = parts.scheme == "https" - self._host = parts.hostname - self._port = parts.port - self._path = (parts.path or "/") + (f"?{parts.query}" if parts.query else "") - self._context = ssl.create_default_context() if self._https else None - self._connection = None - - def _connect(self): - if self._https: - return http.client.HTTPSConnection( - self._host, self._port, timeout=30, context=self._context - ) - return http.client.HTTPConnection(self._host, self._port, timeout=30) - - def get(self): - if self._connection is None: - self._connection = self._connect() - try: - self._connection.request("GET", self._path) - response = self._connection.getresponse() - body = response.read() - return response.status, len(body) - except Exception: - self.close() - raise - - def close(self): - if self._connection is not None: - self._connection.close() - self._connection = None - - -def _do_request(session): - """Single HTTP GET; latency in ms, with non-2xx responses counted as errors.""" - start = time.monotonic() - try: - status, response_bytes = session.get() - elapsed_ms = (time.monotonic() - start) * 1000 - error = None if 200 <= status < 300 else f"HTTP {status}" - return (elapsed_ms, status, error, response_bytes) - except (OSError, http.client.HTTPException) as exc: - elapsed_ms = (time.monotonic() - start) * 1000 - return (elapsed_ms, 0, str(exc), 0) +RUST_BENCH = "/usr/local/bin/capsem-bench-rs" def _drive_at_concurrency(url, concurrency, duration_s): - """Spawn `concurrency` workers, each looping `duration_s`. - - Each worker holds its own keep-alive connection so connection reuse - matches a real client. Returns a list of (latency_ms, status, error) - tuples. - """ - deadline = time.monotonic() + duration_s - - def worker(): - session = _Session(url) - out = [] - while time.monotonic() < deadline: - out.append(_do_request(session)) - session.close() - return out - - all_results = [] - with ThreadPoolExecutor(max_workers=concurrency) as pool: - futures = [pool.submit(worker) for _ in range(concurrency)] - for fut in as_completed(futures): - all_results.extend(fut.result()) - return all_results - - -def _summarize(results, concurrency, duration_s): - """Build the JSON-shaped row for this concurrency level.""" - latencies = sorted(r[0] for r in results if r[2] is None) - errors = sum(1 for r in results if r[2] is not None) - error_distribution = Counter(r[2] for r in results if r[2] is not None) - response_body_bytes = sum(r[3] for r in results if r[2] is None) - return summarize_load_level( - latencies, - errors, - concurrency, - duration_s, - extra={ - "error_distribution": dict(error_distribution), - "response_body_bytes": response_body_bytes, - }, + """Run the async Rust load engine and validate that every client ran.""" + completed = subprocess.run( + [ + RUST_BENCH, + "http-load", + "--target", + url, + "--concurrency", + str(concurrency), + "--duration-s", + str(duration_s), + ], + check=False, + capture_output=True, + text=True, ) + if completed.returncode: + raise RuntimeError( + f"mitm-load: Rust load engine exited {completed.returncode}: " + f"{completed.stderr[-2000:].strip()}" + ) + try: + row = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("mitm-load: Rust load engine returned invalid JSON") from exc + if row.get("concurrency") != concurrency: + raise RuntimeError("mitm-load: Rust load engine reported the wrong concurrency") + if row.get("total_requests", 0) < concurrency: + raise RuntimeError( + f"mitm-load: only {row.get('total_requests', 0)} of {concurrency} " + "client loops made a request; the load generator was starved" + ) + return row def mitm_load_bench(target=None, concurrency_levels=None, duration_s=None): @@ -178,8 +113,7 @@ def mitm_load_bench(target=None, concurrency_levels=None, duration_s=None): rows = [] for c in config.concurrency_levels: console.print(f" concurrency={c} ...") - results = _drive_at_concurrency(target, c, config.duration_s) - row = _summarize(results, c, config.duration_s) + row = _drive_at_concurrency(target, c, config.duration_s) if row["errors"]: raise RuntimeError( f"mitm-load: {row['errors']} of {row['total_requests']} requests failed " @@ -189,7 +123,7 @@ def mitm_load_bench(target=None, concurrency_levels=None, duration_s=None): rows.append(row) out = { - "version": "1.0", + "version": "1.1", "target": target, "concurrency_levels": rows, } diff --git a/tests/test_capsem_bench_mock_server_protocol.py b/tests/test_capsem_bench_mock_server_protocol.py index 536875baf..bceb101ba 100644 --- a/tests/test_capsem_bench_mock_server_protocol.py +++ b/tests/test_capsem_bench_mock_server_protocol.py @@ -11,40 +11,64 @@ from capsem_bench import dns_load, load_harness, mitm_load -@pytest.mark.parametrize("status", [200, 204, 302, 403, 500]) -def test_mitm_load_counts_http_failures(status): - session = types.SimpleNamespace(get=lambda: (status, 24)) - elapsed, observed_status, error, response_bytes = mitm_load._do_request(session) - assert elapsed >= 0 - assert observed_status == status - assert error == (None if 200 <= status < 300 else f"HTTP {status}") - assert response_bytes == 24 - - -def test_mitm_load_excludes_failed_latency_and_reports_error_ratio(): - row = mitm_load._summarize( - [(5.0, 200, None, 24), (100.0, 0, "timed out", 0)], - concurrency=2, - duration_s=10, +def test_mitm_load_delegates_to_async_rust_driver(monkeypatch): + calls = [] + row = { + "concurrency": 2, + "duration_s": 1.0, + "total_requests": 4, + "errors": 0, + } + + def run(argv, **kwargs): + calls.append((argv, kwargs)) + return types.SimpleNamespace(returncode=0, stdout=json.dumps(row), stderr="") + + monkeypatch.setattr(mitm_load.subprocess, "run", run) + + assert mitm_load._drive_at_concurrency("https://fixture.test/tiny", 2, 1) == row + assert calls == [ + ( + [ + mitm_load.RUST_BENCH, + "http-load", + "--target", + "https://fixture.test/tiny", + "--concurrency", + "2", + "--duration-s", + "1", + ], + {"check": False, "capture_output": True, "text": True}, + ) + ] + + +def test_mitm_load_rejects_starved_driver(monkeypatch): + row = {"concurrency": 512, "total_requests": 91, "errors": 0} + monkeypatch.setattr( + mitm_load.subprocess, + "run", + lambda *args, **kwargs: types.SimpleNamespace( + returncode=0, stdout=json.dumps(row), stderr="" + ), ) - assert row["total_requests"] == 2 - assert row["errors"] == 1 - assert row["error_ratio"] == 0.5 - assert row["rps"] == 0.1 - assert row["p50_ms"] == 5.0 - assert row["response_body_bytes"] == 24 - assert row["error_distribution"] == {"timed out": 1} + with pytest.raises(RuntimeError, match="91 of 512 client loops"): + mitm_load._drive_at_concurrency("https://fixture.test/tiny", 512, 10) def test_mitm_load_rejects_one_failed_request(monkeypatch): monkeypatch.setattr( mitm_load, "_drive_at_concurrency", - lambda url, concurrency, duration_s: [ - (5.0, 200, None, 24), - (100.0, 0, "timed out", 0), - ], + lambda url, concurrency, duration_s: { + "concurrency": 1, + "duration_s": 1.0, + "total_requests": 2, + "errors": 1, + "error_distribution": {"timed out": 1}, + }, ) with pytest.raises(RuntimeError, match="1 of 2 requests failed"): From 64f0f0136bd02727bf821fda6061325c1c0e134a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:34:28 +0000 Subject: [PATCH 343/464] feat(bench): measure proportional process memory --- tests/helpers/process_resources.py | 60 ++++++++++++++++++---- tests/test_process_resource_sampler.py | 69 +++++++++++++++++++++++--- 2 files changed, 110 insertions(+), 19 deletions(-) diff --git a/tests/helpers/process_resources.py b/tests/helpers/process_resources.py index b2643aa2a..64a1edf57 100644 --- a/tests/helpers/process_resources.py +++ b/tests/helpers/process_resources.py @@ -31,6 +31,8 @@ class ProcessUsage: name: str cpu_seconds: Decimal rss_bytes: int + pss_bytes: int | None + uss_bytes: int | None fds: int @@ -47,12 +49,15 @@ def process_tree_snapshot(root: psutil.Process) -> TreeSnapshot: for process in processes: with contextlib.suppress(psutil.Error, OSError): with process.oneshot(): + memory = process.memory_full_info() usage = ProcessUsage( pid=process.pid, started_at=process.create_time(), name=process.name(), cpu_seconds=process_cpu_seconds(process), - rss_bytes=process.memory_info().rss, + rss_bytes=memory.rss, + pss_bytes=getattr(memory, "pss", None), + uss_bytes=getattr(memory, "uss", None), fds=process.num_fds(), ) found[(usage.pid, usage.started_at)] = usage @@ -78,8 +83,10 @@ def __init__( self._stop = threading.Event() self._thread: threading.Thread | None = None self._observations = 0 - self._baseline: dict[str, tuple[int, int]] = {} - self._peak: dict[str, tuple[int, int]] = {} + self._baseline: dict[str, tuple[int, int, int, int]] = {} + self._peak: dict[str, tuple[int, int, int, int]] = {} + self._pss_available = True + self._uss_available = True self._peak_processes = 0 self._first_cpu: dict[tuple[int, float], Decimal] = {} self._last_cpu: dict[tuple[int, float], Decimal] = {} @@ -89,14 +96,25 @@ def observe(self, snapshot: TreeSnapshot) -> None: """Fold one snapshot into the benchmark interval.""" initial = self._observations == 0 self._observations += 1 - totals: dict[str, tuple[int, int]] = {"total": (0, 0)} + totals: dict[str, tuple[int, int, int, int]] = {"total": (0, 0, 0, 0)} for identity, usage in snapshot.processes.items(): name = _metric_name(usage.name) - rss, fds = totals.get(name, (0, 0)) - totals[name] = (rss + usage.rss_bytes, fds + usage.fds) - total_rss, total_fds = totals["total"] + self._pss_available &= usage.pss_bytes is not None + self._uss_available &= usage.uss_bytes is not None + pss = usage.pss_bytes or 0 + uss = usage.uss_bytes or 0 + rss_sum, pss_sum, uss_sum, fds_sum = totals.get(name, (0, 0, 0, 0)) + totals[name] = ( + rss_sum + usage.rss_bytes, + pss_sum + pss, + uss_sum + uss, + fds_sum + usage.fds, + ) + total_rss, total_pss, total_uss, total_fds = totals["total"] totals["total"] = ( total_rss + usage.rss_bytes, + total_pss + pss, + total_uss + uss, total_fds + usage.fds, ) if identity not in self._first_cpu: @@ -110,8 +128,8 @@ def observe(self, snapshot: TreeSnapshot) -> None: if initial: self._baseline = dict(totals) for name, values in totals.items(): - previous = self._peak.get(name, (0, 0)) - self._peak[name] = (max(previous[0], values[0]), max(previous[1], values[1])) + previous = self._peak.get(name, (0, 0, 0, 0)) + self._peak[name] = tuple(max(old, new) for old, new in zip(previous, values, strict=True)) self._peak_processes = max(self._peak_processes, len(snapshot.processes)) def start(self) -> None: @@ -147,8 +165,10 @@ def metrics(self) -> dict[str, dict]: for name in sorted(self._peak): if name != "total" and name not in TRACKED_PROCESS_NAMES: continue - peak_rss, peak_fds = self._peak[name] - baseline_rss, baseline_fds = self._baseline.get(name, (0, 0)) + peak_rss, peak_pss, peak_uss, peak_fds = self._peak[name] + baseline_rss, baseline_pss, baseline_uss, baseline_fds = self._baseline.get( + name, (0, 0, 0, 0) + ) prefix = f"resources.{name}" metrics[f"{prefix}.cpu_s"] = { "unit": "seconds", @@ -162,6 +182,24 @@ def metrics(self) -> dict[str, dict]: "unit": "bytes", "samples": [max(0, peak_rss - baseline_rss)], } + if self._pss_available: + metrics[f"{prefix}.peak_pss_bytes"] = { + "unit": "bytes", + "samples": [peak_pss], + } + metrics[f"{prefix}.peak_pss_delta_bytes"] = { + "unit": "bytes", + "samples": [max(0, peak_pss - baseline_pss)], + } + if self._uss_available: + metrics[f"{prefix}.peak_uss_bytes"] = { + "unit": "bytes", + "samples": [peak_uss], + } + metrics[f"{prefix}.peak_uss_delta_bytes"] = { + "unit": "bytes", + "samples": [max(0, peak_uss - baseline_uss)], + } metrics[f"{prefix}.peak_fds"] = { "unit": "count", "samples": [peak_fds], diff --git a/tests/test_process_resource_sampler.py b/tests/test_process_resource_sampler.py index afd902fb3..2f4f8e874 100644 --- a/tests/test_process_resource_sampler.py +++ b/tests/test_process_resource_sampler.py @@ -15,25 +15,27 @@ def usage( *, cpu: str, rss: int, + pss: int, + uss: int, fds: int, started_at: float = 1.0, ) -> ProcessUsage: - return ProcessUsage(pid, started_at, name, Decimal(cpu), rss, fds) + return ProcessUsage(pid, started_at, name, Decimal(cpu), rss, pss, uss, fds) def test_sampler_reports_tree_and_named_process_pressure() -> None: sampler = ProcessTreeSampler(None) sampler.observe( snapshot( - usage(1, "capsem-service", cpu="10.0", rss=100, fds=5), - usage(2, "capsem-proxy", cpu="20.0", rss=200, fds=20), + usage(1, "capsem-service", cpu="10.0", rss=100, pss=80, uss=60, fds=5), + usage(2, "capsem-proxy", cpu="20.0", rss=200, pss=150, uss=120, fds=20), ) ) sampler.observe( snapshot( - usage(1, "capsem-service", cpu="10.2", rss=110, fds=6), - usage(2, "capsem-proxy", cpu="21.5", rss=500, fds=400), - usage(3, "capsem-exec", cpu="0.1", rss=50, fds=4), + usage(1, "capsem-service", cpu="10.2", rss=110, pss=85, uss=65, fds=6), + usage(2, "capsem-proxy", cpu="21.5", rss=500, pss=420, uss=390, fds=400), + usage(3, "capsem-exec", cpu="0.1", rss=50, pss=40, uss=35, fds=4), ) ) @@ -44,6 +46,16 @@ def test_sampler_reports_tree_and_named_process_pressure() -> None: "unit": "bytes", "samples": [360], }, + "resources.total.peak_pss_bytes": {"unit": "bytes", "samples": [545]}, + "resources.total.peak_pss_delta_bytes": { + "unit": "bytes", + "samples": [315], + }, + "resources.total.peak_uss_bytes": {"unit": "bytes", "samples": [490]}, + "resources.total.peak_uss_delta_bytes": { + "unit": "bytes", + "samples": [310], + }, "resources.total.peak_fds": {"unit": "count", "samples": [410]}, "resources.total.peak_fd_delta": {"unit": "count", "samples": [385]}, "resources.total.peak_processes": {"unit": "count", "samples": [3]}, @@ -56,6 +68,22 @@ def test_sampler_reports_tree_and_named_process_pressure() -> None: "unit": "bytes", "samples": [10], }, + "resources.capsem_service.peak_pss_bytes": { + "unit": "bytes", + "samples": [85], + }, + "resources.capsem_service.peak_pss_delta_bytes": { + "unit": "bytes", + "samples": [5], + }, + "resources.capsem_service.peak_uss_bytes": { + "unit": "bytes", + "samples": [65], + }, + "resources.capsem_service.peak_uss_delta_bytes": { + "unit": "bytes", + "samples": [5], + }, "resources.capsem_service.peak_fds": {"unit": "count", "samples": [6]}, "resources.capsem_service.peak_fd_delta": { "unit": "count", @@ -70,6 +98,22 @@ def test_sampler_reports_tree_and_named_process_pressure() -> None: "unit": "bytes", "samples": [300], }, + "resources.capsem_proxy.peak_pss_bytes": { + "unit": "bytes", + "samples": [420], + }, + "resources.capsem_proxy.peak_pss_delta_bytes": { + "unit": "bytes", + "samples": [270], + }, + "resources.capsem_proxy.peak_uss_bytes": { + "unit": "bytes", + "samples": [390], + }, + "resources.capsem_proxy.peak_uss_delta_bytes": { + "unit": "bytes", + "samples": [270], + }, "resources.capsem_proxy.peak_fds": {"unit": "count", "samples": [400]}, "resources.capsem_proxy.peak_fd_delta": { "unit": "count", @@ -80,8 +124,17 @@ def test_sampler_reports_tree_and_named_process_pressure() -> None: def test_process_that_exits_keeps_its_last_cpu_sample() -> None: sampler = ProcessTreeSampler(None) - sampler.observe(snapshot(usage(7, "capsem-proxy", cpu="3", rss=20, fds=2))) - sampler.observe(snapshot(usage(7, "capsem-proxy", cpu="4", rss=30, fds=3))) + sampler.observe(snapshot(usage(7, "capsem-proxy", cpu="3", rss=20, pss=15, uss=10, fds=2))) + sampler.observe(snapshot(usage(7, "capsem-proxy", cpu="4", rss=30, pss=25, uss=20, fds=3))) sampler.observe(snapshot()) assert sampler.metrics()["resources.capsem_proxy.cpu_s"]["samples"] == [1.0] + + +def test_platform_without_proportional_memory_omits_those_metrics() -> None: + sampler = ProcessTreeSampler(None) + sampler.observe( + snapshot(ProcessUsage(1, 1.0, "capsem-service", Decimal(1), 100, None, None, 2)) + ) + + assert not any("pss" in name or "uss" in name for name in sampler.metrics()) From 2b191d6f4b82849c72c2e126ef7ef0dad8d40fe9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:34:44 +0000 Subject: [PATCH 344/464] docs(bench): require async load and proportional memory --- skills/dev-benchmark/SKILL.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/skills/dev-benchmark/SKILL.md b/skills/dev-benchmark/SKILL.md index f0c92733a..a5b995b3d 100644 --- a/skills/dev-benchmark/SKILL.md +++ b/skills/dev-benchmark/SKILL.md @@ -35,6 +35,7 @@ this repo took before it existed was taken on an unqualified machine. | `verify` | ratchet a run against evidence; refuses on an unfit machine | | `report` | every subject and its trend | | `protocol`, `protocol-delta`, `delta` | the scenario engine: host-direct vs guest-through-Capsem | +| `http-load` | duration-driven async HTTP load with pooled keep-alive connections and cold/warm latency | Modules: `schema.rs` (the record), `stats.rs` (**all** statistics), `store.rs` (SQLite), `collector.rs` (the subprocess protocol), `machine.rs` @@ -87,6 +88,21 @@ read their mode from `argv[0]` and drive the modules in manifests and reads raw per-iteration nanoseconds out of Criterion's `sample.json`, so a new `[[bench]]` target is measured without editing it. +Load generation is part of the measurement system. Do not use one blocking +thread per client: scheduler starvation can make a high-concurrency run issue +fewer requests than its requested concurrency. The guest MITM lane delegates +to `capsem-bench-rs http-load`; every client loop must attempt at least one +request, and fewer total attempts than clients invalidates the run. Keep cold +and warm latency separate so connection admission and TLS setup cannot be +mistaken for steady-state request cost. + +Guest load collectors sample the complete Capsem process tree. Report CPU, +file descriptors, RSS, and, where the platform exposes them, PSS and USS for +the total tree and each named component. Use PSS when estimating many-VM host +capacity because summing RSS counts shared pages once per process; use USS to +show the memory that disappears with that process. Compare two unchanged runs +before and two changed runs after any performance work. + ## Is the move real? `compare` and `verify` report `delta_abs`, `delta_pct`, `ratio`, and From 110114b496b44900d53d834a0885e050fd7c0220 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:42:47 +0000 Subject: [PATCH 345/464] feat(proto): define network descriptor budget --- config/data/runtime-limits.toml | 3 +++ crates/capsem-proto/build.rs | 14 +++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index 97a491cfe..bd127d150 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -3,3 +3,6 @@ [network] max_concurrent_clients = 512 +# Each intercepted client consumes a guest TCP descriptor and a VSOCK +# descriptor. Leave room for listeners, the Tokio reactor, DNS, and control. +process_open_file_limit = 4096 diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index e6104993f..f2ceb036d 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -47,6 +47,16 @@ fn write_runtime_limits() { .unwrap_or_else(|| { panic!("runtime limits: network.max_concurrent_clients must be an integer between 1 and 4096") }); + let open_file_limit = config + .get("network") + .and_then(|network| network.get("process_open_file_limit")) + .and_then(toml::Value::as_integer) + .filter(|value| (capacity * 2 + 64..=1_048_576).contains(value)) + .unwrap_or_else(|| { + panic!( + "runtime limits: network.process_open_file_limit must cover two descriptors per client plus 64, and not exceed 1048576" + ) + }); let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set"); std::fs::write( format!("{out_dir}/runtime_limits.rs"), @@ -55,7 +65,9 @@ fn write_runtime_limits() { ///\n\ /// TCP descriptor grants and each of the two DNS sessions derive\n\ /// their bounds from this one config-owned value.\n\ - pub const NETWORK_CLIENT_CAPACITY: usize = {capacity};\n" + pub const NETWORK_CLIENT_CAPACITY: usize = {capacity};\n\ + /// Minimum descriptor budget for a network-facing process.\n\ + pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit};\n" ), ) .expect("runtime limits: write generated constants"); From 1ef971c5ebdda63ab8608c60385c322f789a473b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:44:12 +0000 Subject: [PATCH 346/464] feat(foundation): enforce process descriptor floors --- crates/capsem-foundation/Cargo.toml | 2 +- crates/capsem-foundation/src/unix/process.rs | 22 ++++++++++++ .../src/unix/process/tests.rs | 35 +++++++++++++++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/crates/capsem-foundation/Cargo.toml b/crates/capsem-foundation/Cargo.toml index 82b2992b5..a1f982869 100644 --- a/crates/capsem-foundation/Cargo.toml +++ b/crates/capsem-foundation/Cargo.toml @@ -28,7 +28,7 @@ runtime = [ capsem-crypto = { path = "../capsem-crypto", default-features = false, optional = true } capsem-proto = { path = "../capsem-proto", optional = true } libc = "0.2" -nix = { version = "0.29", features = ["dir", "event", "fs", "inotify", "ioctl", "mman", "poll", "process", "signal", "socket", "uio", "user"], optional = true } +nix = { version = "0.29", features = ["dir", "event", "fs", "inotify", "ioctl", "mman", "poll", "process", "resource", "signal", "socket", "uio", "user"], optional = true } rmp-serde = { workspace = true, optional = true } serde = { workspace = true, optional = true } serde_json = { workspace = true, optional = true } diff --git a/crates/capsem-foundation/src/unix/process.rs b/crates/capsem-foundation/src/unix/process.rs index d5b2cb115..6ae4ff99d 100644 --- a/crates/capsem-foundation/src/unix/process.rs +++ b/crates/capsem-foundation/src/unix/process.rs @@ -4,6 +4,7 @@ use std::io; use std::num::NonZeroI32; use nix::errno::Errno; +use nix::sys::resource::{getrlimit, setrlimit, Resource}; use nix::sys::signal::{kill, killpg, Signal as NixSignal}; use nix::unistd::Pid; @@ -153,6 +154,27 @@ pub fn load_average() -> Option { (observed == 1).then_some(load[0]) } +/// Ensure this process can hold at least `minimum` file descriptors. +/// +/// The hard limit remains unchanged. A host policy whose hard limit cannot +/// satisfy the requested product bound is reported rather than silently +/// advertising concurrency the process cannot sustain. +pub fn ensure_open_file_limit(minimum: u64) -> io::Result<()> { + let minimum = nix::sys::resource::rlim_t::try_from(minimum) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "open-file limit is not representable"))?; + let (soft, hard) = getrlimit(Resource::RLIMIT_NOFILE).map_err(errno::io)?; + if soft >= minimum { + return Ok(()); + } + if hard < minimum { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!("open-file hard limit {hard} is below required minimum {minimum}"), + )); + } + setrlimit(Resource::RLIMIT_NOFILE, minimum, hard).map_err(errno::io) +} + /// The current process's parent, or `None` for an unrepresentable kernel value. pub fn parent_process_id() -> Option { u32::try_from(nix::unistd::getppid().as_raw()) diff --git a/crates/capsem-foundation/src/unix/process/tests.rs b/crates/capsem-foundation/src/unix/process/tests.rs index 606e7fa37..480ecd95e 100644 --- a/crates/capsem-foundation/src/unix/process/tests.rs +++ b/crates/capsem-foundation/src/unix/process/tests.rs @@ -55,6 +55,41 @@ fn current_identity_is_numeric_and_has_a_parent() { assert!(parent_process_id().is_some()); } +#[test] +fn open_file_limit_is_raised_in_an_isolated_process() { + let status = crate::isolated_test_command(std::env::current_exe().unwrap()) + .args([ + "--exact", + "unix::process::tests::raise_open_file_limit_child", + "--nocapture", + ]) + .env("CAPSEM_RLIMIT_TEST_CHILD", "1") + .status() + .unwrap(); + assert!(status.success()); +} + +#[test] +fn raise_open_file_limit_child() { + if std::env::var_os("CAPSEM_RLIMIT_TEST_CHILD").is_none() { + return; + } + use nix::sys::resource::{getrlimit, setrlimit, Resource}; + + let (_, hard) = getrlimit(Resource::RLIMIT_NOFILE).unwrap(); + let target = hard.min(4_096); + assert!(target > 128, "test environment needs a usable descriptor hard limit"); + setrlimit(Resource::RLIMIT_NOFILE, 128, hard).unwrap(); + + ensure_open_file_limit(target).unwrap(); + + assert_eq!(getrlimit(Resource::RLIMIT_NOFILE).unwrap(), (target, hard)); + + setrlimit(Resource::RLIMIT_NOFILE, 128, 256).unwrap(); + let error = ensure_open_file_limit(4_096).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); +} + #[test] fn inherited_environment_is_erased_in_an_isolated_process() { let status = crate::isolated_test_command(std::env::current_exe().unwrap()) From e6da6f2fb41bc1366f5cf9d5cdb2a341a547e492 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:45:06 +0000 Subject: [PATCH 347/464] fix(agent): sustain 512 relayed connections --- CHANGELOG.md | 5 +++++ crates/capsem-agent/src/net_proxy.rs | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a8c30ed4..f18e1b184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -172,6 +172,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Sustain the configured 512 concurrent intercepted connections in the guest + network relay by raising its inherited descriptor limit to the centralized + runtime budget. Linux shells commonly supplied only 1,024 descriptors while + each relayed connection needs both a TCP and a VSOCK descriptor. + - The confined HTTP proxy admits 512 concurrent clients instead of closing excess TLS handshakes when its descriptor grant bound is reached. The trusted upstream broker and DNS sessions derive their ceilings from that same limit diff --git a/crates/capsem-agent/src/net_proxy.rs b/crates/capsem-agent/src/net_proxy.rs index 39f4e8aed..88b176f33 100644 --- a/crates/capsem-agent/src/net_proxy.rs +++ b/crates/capsem-agent/src/net_proxy.rs @@ -36,7 +36,8 @@ use tokio::io::AsyncWriteExt; use tokio::net::{TcpListener, TcpStream}; use tokio::signal; -use capsem_proto::VSOCK_PORT_SNI_PROXY; +use capsem_foundation::unix::process::ensure_open_file_limit; +use capsem_proto::{NETWORK_PROCESS_OPEN_FILE_LIMIT, VSOCK_PORT_SNI_PROXY}; use process_attribution::encode_meta_line; use vsock_io::{vsock_connect, AsyncVsock, VSOCK_HOST_CID}; @@ -238,6 +239,7 @@ async fn run_listener(port: u16, attributor: Arc) -> io::Resu #[tokio::main] async fn main() -> io::Result<()> { + ensure_open_file_limit(NETWORK_PROCESS_OPEN_FILE_LIMIT)?; eprintln!("[capsem-net-proxy] starting (pid {})", process::id()); let attributor = Arc::new(ProcessAttributor::default()); From 9f66cc4440ce615e38ea43efaae324503f146164 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:45:06 +0000 Subject: [PATCH 348/464] test(citadel): bind descriptor budget to client cap --- tests/citadel/test_runtime_limit_authority.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 8722ab9db..3c6026244 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -20,10 +20,11 @@ """ NUMERIC_CONSTANT = re.compile( - r"\b(?:pub(?:\([^)]*\))?\s+)?const\s+([A-Z][A-Z0-9_]*)\s*:\s*usize\s*=\s*(\d+)\s*;" + r"\b(?:pub(?:\([^)]*\))?\s+)?const\s+([A-Z][A-Z0-9_]*)\s*:\s*(?:usize|u64)\s*=\s*(\d+)\s*;" ) CONFIG_OWNED_NAMES = { "NETWORK_CLIENT_CAPACITY", + "NETWORK_PROCESS_OPEN_FILE_LIMIT", "DNS_SESSION_MAX_IN_FLIGHT", "UPSTREAM_GRANT_CAPACITY", "TRAFFIC_GRANT_CAPACITY", @@ -36,6 +37,8 @@ "../../config/data/runtime-limits.toml", 'get("max_concurrent_clients")', "pub const NETWORK_CLIENT_CAPACITY: usize = {capacity}", + 'get("process_open_file_limit")', + "pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit}", ), "crates/capsem-proto/src/lib.rs": ( 'include!(concat!(env!("OUT_DIR"), "/runtime_limits.rs"))', @@ -48,6 +51,10 @@ "crates/capsem-service/src/upstream_broker.rs": ("UPSTREAM_GRANT_CAPACITY",), "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/dns_proxy/session.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), + "crates/capsem-agent/src/net_proxy.rs": ( + "NETWORK_PROCESS_OPEN_FILE_LIMIT", + "ensure_open_file_limit", + ), } @@ -67,6 +74,12 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: RUNTIME_LIMIT_RATIONALE + "\nnetwork.max_concurrent_clients must be an integer from 1 through 4096" ) + open_file_limit = config.get("network", {}).get("process_open_file_limit") + assert type(open_file_limit) is int and 2 * capacity + 64 <= open_file_limit <= 1_048_576, ( + RUNTIME_LIMIT_RATIONALE + + "\nnetwork.process_open_file_limit must cover TCP and VSOCK descriptors " + "for every client plus runtime overhead" + ) missing = [] for relative_path, required_fragments in REQUIRED_WIRING.items(): @@ -93,6 +106,7 @@ def test_numeric_local_ceiling_is_rejected_adversarially() -> None: { "upstream_broker.rs": "const MAX_ACTIVE_GRANTS: usize = 256;", "dns/session.rs": "pub const DNS_SESSION_MAX_IN_FLIGHT: usize = 512;", + "net_proxy.rs": "const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = 4096;", } ) - assert len(violations) == 2, RUNTIME_LIMIT_RATIONALE + assert len(violations) == 3, RUNTIME_LIMIT_RATIONALE From cb51e7b454d066bdbfe527d047004a50b4e3e5ab Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 20:50:11 +0000 Subject: [PATCH 349/464] refactor(proto): derive descriptor budget from client cap --- config/data/runtime-limits.toml | 5 +++-- crates/capsem-proto/build.rs | 14 ++++++++------ tests/citadel/test_runtime_limit_authority.py | 12 +++++++----- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index bd127d150..0b68651f9 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -4,5 +4,6 @@ [network] max_concurrent_clients = 512 # Each intercepted client consumes a guest TCP descriptor and a VSOCK -# descriptor. Leave room for listeners, the Tokio reactor, DNS, and control. -process_open_file_limit = 4096 +# descriptor. The remaining per-client allowance covers listeners, the Tokio +# reactor, DNS, control traffic, and overlap while connections retire. +file_descriptors_per_client = 8 diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index f2ceb036d..d61960b48 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -47,16 +47,18 @@ fn write_runtime_limits() { .unwrap_or_else(|| { panic!("runtime limits: network.max_concurrent_clients must be an integer between 1 and 4096") }); - let open_file_limit = config + let descriptors_per_client = config .get("network") - .and_then(|network| network.get("process_open_file_limit")) + .and_then(|network| network.get("file_descriptors_per_client")) .and_then(toml::Value::as_integer) - .filter(|value| (capacity * 2 + 64..=1_048_576).contains(value)) + .filter(|value| (3..=64).contains(value)) .unwrap_or_else(|| { - panic!( - "runtime limits: network.process_open_file_limit must cover two descriptors per client plus 64, and not exceed 1048576" - ) + panic!("runtime limits: network.file_descriptors_per_client must be an integer between 3 and 64") }); + let open_file_limit = capacity + .checked_mul(descriptors_per_client) + .filter(|value| *value <= 1_048_576) + .expect("runtime limits: derived network descriptor budget exceeds 1048576"); let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set"); std::fs::write( format!("{out_dir}/runtime_limits.rs"), diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 3c6026244..b7172c8f2 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -37,7 +37,8 @@ "../../config/data/runtime-limits.toml", 'get("max_concurrent_clients")', "pub const NETWORK_CLIENT_CAPACITY: usize = {capacity}", - 'get("process_open_file_limit")', + 'get("file_descriptors_per_client")', + ".checked_mul(descriptors_per_client)", "pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit}", ), "crates/capsem-proto/src/lib.rs": ( @@ -74,12 +75,13 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: RUNTIME_LIMIT_RATIONALE + "\nnetwork.max_concurrent_clients must be an integer from 1 through 4096" ) - open_file_limit = config.get("network", {}).get("process_open_file_limit") - assert type(open_file_limit) is int and 2 * capacity + 64 <= open_file_limit <= 1_048_576, ( + descriptors_per_client = config.get("network", {}).get("file_descriptors_per_client") + assert type(descriptors_per_client) is int and 3 <= descriptors_per_client <= 64, ( RUNTIME_LIMIT_RATIONALE - + "\nnetwork.process_open_file_limit must cover TCP and VSOCK descriptors " - "for every client plus runtime overhead" + + "\nnetwork.file_descriptors_per_client must cover TCP and VSOCK " + "descriptors plus runtime overhead" ) + assert capacity * descriptors_per_client <= 1_048_576, RUNTIME_LIMIT_RATIONALE missing = [] for relative_path, required_fragments in REQUIRED_WIRING.items(): From a685dc202d891343f97b65bc0c1feb51617222b7 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:13:58 +0000 Subject: [PATCH 350/464] fix(proto): preserve connection replacement capacity --- config/data/runtime-limits.toml | 2 ++ crates/capsem-proto/build.rs | 17 ++++++++++++ crates/capsem-proto/src/upstream_grant.rs | 4 +-- tests/citadel/test_runtime_limit_authority.py | 27 +++++++++++++++---- 4 files changed, 43 insertions(+), 7 deletions(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index 0b68651f9..4d328a1a8 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -3,6 +3,8 @@ [network] max_concurrent_clients = 512 +# A replacement can be opened before the retiring connection leaves the pool. +traffic_grants_per_client = 2 # Each intercepted client consumes a guest TCP descriptor and a VSOCK # descriptor. The remaining per-client allowance covers listeners, the Tokio # reactor, DNS, control traffic, and overlap while connections retire. diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index d61960b48..f4f61651d 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -47,6 +47,14 @@ fn write_runtime_limits() { .unwrap_or_else(|| { panic!("runtime limits: network.max_concurrent_clients must be an integer between 1 and 4096") }); + let traffic_grants_per_client = config + .get("network") + .and_then(|network| network.get("traffic_grants_per_client")) + .and_then(toml::Value::as_integer) + .filter(|value| (2..=4).contains(value)) + .unwrap_or_else(|| { + panic!("runtime limits: network.traffic_grants_per_client must be an integer between 2 and 4") + }); let descriptors_per_client = config .get("network") .and_then(|network| network.get("file_descriptors_per_client")) @@ -55,6 +63,13 @@ fn write_runtime_limits() { .unwrap_or_else(|| { panic!("runtime limits: network.file_descriptors_per_client must be an integer between 3 and 64") }); + let traffic_grant_capacity = capacity + .checked_mul(traffic_grants_per_client) + .filter(|value| *value <= 16_384) + .expect("runtime limits: derived network traffic-grant capacity exceeds 16384"); + if descriptors_per_client < traffic_grants_per_client * 2 { + panic!("runtime limits: network.file_descriptors_per_client must cover both ends of every traffic grant"); + } let open_file_limit = capacity .checked_mul(descriptors_per_client) .filter(|value| *value <= 1_048_576) @@ -68,6 +83,8 @@ fn write_runtime_limits() { /// TCP descriptor grants and each of the two DNS sessions derive\n\ /// their bounds from this one config-owned value.\n\ pub const NETWORK_CLIENT_CAPACITY: usize = {capacity};\n\ + /// Live and retiring traffic grants allowed during connection replacement.\n\ + pub const NETWORK_TRAFFIC_GRANT_CAPACITY: usize = {traffic_grant_capacity};\n\ /// Minimum descriptor budget for a network-facing process.\n\ pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit};\n" ), diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index 26842e306..b0274e88c 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -28,8 +28,8 @@ pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 4480; pub const UPSTREAM_GRANT_MAX_FDS: usize = 2; /// Per-worker descriptor and pending-selection ceiling. /// -/// The proxy and trusted broker share the public network-client ceiling. -pub const UPSTREAM_GRANT_CAPACITY: usize = crate::NETWORK_CLIENT_CAPACITY; +/// The proxy and trusted broker preserve one replacement slot per client. +pub const UPSTREAM_GRANT_CAPACITY: usize = crate::NETWORK_TRAFFIC_GRANT_CAPACITY; /// DNS names are at most 253 wire-text bytes without a root dot. pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; /// Maximum relative path carried by a guest-share metadata request. diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index b7172c8f2..890f11edc 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -12,10 +12,13 @@ RUNTIME_LIMIT_RATIONALE = """\ The proxy, trusted upstream broker, host DNS session and guest DNS forwarder -must admit the same supported client burst. Local numeric ceilings previously -let one stage accept work that the next stage shed, producing intermittent HTTP -502 responses under load. config/data/runtime-limits.toml is the authority; -capsem-proto generates the shared constant and every consumer derives from it. +must admit the same supported client burst. A traffic connection that is being +replaced overlaps the retiring connection, so descriptor grants must preserve +the config-owned overlap multiple instead of equalling the client count. Local +numeric ceilings previously let one stage accept work that the next stage shed, +producing intermittent HTTP 502 responses under load. +config/data/runtime-limits.toml is the authority; capsem-proto generates the +shared constants and every consumer derives from them. See skills/dev-mitm-proxy/SKILL.md under Cross-process network limits. """ @@ -37,6 +40,9 @@ "../../config/data/runtime-limits.toml", 'get("max_concurrent_clients")', "pub const NETWORK_CLIENT_CAPACITY: usize = {capacity}", + 'get("traffic_grants_per_client")', + ".checked_mul(traffic_grants_per_client)", + "pub const NETWORK_TRAFFIC_GRANT_CAPACITY: usize = {traffic_grant_capacity}", 'get("file_descriptors_per_client")', ".checked_mul(descriptors_per_client)", "pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit}", @@ -46,7 +52,7 @@ "DNS_SESSION_MAX_IN_FLIGHT: usize = NETWORK_CLIENT_CAPACITY", ), "crates/capsem-proto/src/upstream_grant.rs": ( - "UPSTREAM_GRANT_CAPACITY: usize = crate::NETWORK_CLIENT_CAPACITY", + "UPSTREAM_GRANT_CAPACITY: usize = crate::NETWORK_TRAFFIC_GRANT_CAPACITY", ), "crates/capsem-proxy/src/main.rs": ("UPSTREAM_GRANT_CAPACITY",), "crates/capsem-service/src/upstream_broker.rs": ("UPSTREAM_GRANT_CAPACITY",), @@ -75,6 +81,12 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: RUNTIME_LIMIT_RATIONALE + "\nnetwork.max_concurrent_clients must be an integer from 1 through 4096" ) + traffic_grants_per_client = config.get("network", {}).get("traffic_grants_per_client") + assert type(traffic_grants_per_client) is int and 2 <= traffic_grants_per_client <= 4, ( + RUNTIME_LIMIT_RATIONALE + + "\nnetwork.traffic_grants_per_client must cover an active connection " + "and its replacement" + ) descriptors_per_client = config.get("network", {}).get("file_descriptors_per_client") assert type(descriptors_per_client) is int and 3 <= descriptors_per_client <= 64, ( RUNTIME_LIMIT_RATIONALE @@ -82,6 +94,11 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: "descriptors plus runtime overhead" ) assert capacity * descriptors_per_client <= 1_048_576, RUNTIME_LIMIT_RATIONALE + assert descriptors_per_client >= traffic_grants_per_client * 2, ( + RUNTIME_LIMIT_RATIONALE + + "\nnetwork.file_descriptors_per_client must cover both ends of every " + "overlapping traffic grant" + ) missing = [] for relative_path, required_fragments in REQUIRED_WIRING.items(): From eda7f94d65dacc2836bdf4a96efe448a55761669 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:16:10 +0000 Subject: [PATCH 351/464] fix(proxy): raise the host descriptor floor --- CHANGELOG.md | 10 ++++++---- crates/capsem-proxy/src/main.rs | 7 ++++++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f18e1b184..5ec36ee47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -177,10 +177,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 runtime budget. Linux shells commonly supplied only 1,024 descriptors while each relayed connection needs both a TCP and a VSOCK descriptor. -- The confined HTTP proxy admits 512 concurrent clients instead of closing - excess TLS handshakes when its descriptor grant bound is reached. The trusted - upstream broker and DNS sessions derive their ceilings from that same limit - instead of intermittently returning 502 or SERVFAIL during a resolve burst. +- The confined HTTP proxy admits 512 concurrent clients plus one replacement + grant per client instead of closing handoffs while a pooled connection + retires. Network-facing host processes raise their descriptor floor from the + same runtime budget, and the trusted upstream broker and DNS sessions derive + their ceilings from that config instead of intermittently returning 502 or + SERVFAIL during a resolve burst. - Multiplexed DNS sessions admit the dual-stack query burst from 512 concurrent clients instead of shedding valid A or AAAA lookups as SERVFAIL. diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 88b66d899..75acdb32a 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -15,7 +15,7 @@ use capsem_core::net::proxy_engine::{ }; use capsem_foundation::ipc_channel; use capsem_foundation::unix::worker_sandbox::{Policy, Role}; -use capsem_foundation::unix::{fd, router_channel}; +use capsem_foundation::unix::{fd, process::ensure_open_file_limit, router_channel}; use capsem_logger::WriteOp; use capsem_proto::proxy_control::{ decode_proxy_control_request, encode_proxy_control_event, ProxyCapability, ProxyChannelCloseReason, @@ -24,6 +24,7 @@ use capsem_proto::proxy_control::{ }; use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; use capsem_proto::upstream_grant::UPSTREAM_GRANT_CAPACITY; +use capsem_proto::NETWORK_PROCESS_OPEN_FILE_LIMIT; use clap::Parser; use tokio::task::{JoinError, JoinSet}; @@ -60,6 +61,10 @@ fn main() -> Result<()> { capsem_foundation::unix::process::clear_inherited_environment(); fd::close_inherited_descriptors()?; } + ensure_open_file_limit(NETWORK_PROCESS_OPEN_FILE_LIMIT)?; + let args = Args::parse(); + capsem_foundation::telemetry::install_in_band_trace_id(trace_id_from_generation(args.generation)) + .map_err(anyhow::Error::msg)?; let _telemetry = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { service: "capsem-proxy", sink: capsem_foundation::telemetry::LogSink::Stderr, From 6d0ffcd6a0a3acca7ea1b156215006e292880e35 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:16:14 +0000 Subject: [PATCH 352/464] fix(process): raise the relay descriptor floor --- crates/capsem-process/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index f02dedd01..0427b69ea 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -15,8 +15,10 @@ use anyhow::{Context, Result}; use capsem_core::fs_monitor::FsMonitor; use capsem_core::net::upstream_grant::{adopt_inherited, UpstreamGrantClient}; use capsem_core::{prepare_vm, BootOptions, VirtioFsShare, VsockConnection}; +use capsem_foundation::unix::process::ensure_open_file_limit; use capsem_logger::DbWriter; use capsem_proto::ipc::{ProcessToService, ServiceToProcess}; +use capsem_proto::NETWORK_PROCESS_OPEN_FILE_LIMIT; use clap::Parser; use std::os::fd::AsFd as _; use std::os::unix::fs::PermissionsExt as _; @@ -251,6 +253,7 @@ fn main() -> Result<()> { // SAFETY: process entry precedes argument parsing, telemetry, descriptor // owners and runtime threads. Broker grants will be named here explicitly. unsafe { capsem_foundation::unix::fd::close_inherited_descriptors()? }; + ensure_open_file_limit(NETWORK_PROCESS_OPEN_FILE_LIMIT)?; let upstream_socket = adopt_inherited(std::io::stdin().as_fd()).context("adopt inherited upstream grant channel")?; let _telemetry_guard = capsem_foundation::telemetry::init(capsem_foundation::telemetry::TelemetryConfig { From 530cc6b4a4c88bf37cd2b35d085ba588d809579e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:16:20 +0000 Subject: [PATCH 353/464] fix(service): raise the broker descriptor floor --- crates/capsem-service/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index f9ae04fed..b22675f5a 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -13,7 +13,9 @@ use capsem_core::{ net::policy_config::{DetectionLevel, SecurityPluginConfig, SecurityPluginMode, SecurityRuleAction, SettingsFile}, }; use capsem_foundation::ipc_channel::{channel_from_std, Receiver, Sender}; +use capsem_foundation::unix::process::ensure_open_file_limit; use capsem_proto::ipc::{FileBoundaryAction, ProcessToService, ServiceToProcess}; +use capsem_proto::NETWORK_PROCESS_OPEN_FILE_LIMIT; use capsem_service::errors::AppError; use clap::Parser; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -745,6 +747,7 @@ impl ServiceState { #[tokio::main] async fn main() -> Result<()> { + ensure_open_file_limit(NETWORK_PROCESS_OPEN_FILE_LIMIT)?; run_service().await } From 77dc1f0a8c5db24e7270e9abd970be0451e6d4cc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:16:26 +0000 Subject: [PATCH 354/464] test(citadel): require descriptor floors on the relay path --- tests/citadel/test_runtime_limit_authority.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 890f11edc..2ed6c877d 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -54,8 +54,20 @@ "crates/capsem-proto/src/upstream_grant.rs": ( "UPSTREAM_GRANT_CAPACITY: usize = crate::NETWORK_TRAFFIC_GRANT_CAPACITY", ), - "crates/capsem-proxy/src/main.rs": ("UPSTREAM_GRANT_CAPACITY",), + "crates/capsem-proxy/src/main.rs": ( + "UPSTREAM_GRANT_CAPACITY", + "NETWORK_PROCESS_OPEN_FILE_LIMIT", + "ensure_open_file_limit", + ), + "crates/capsem-service/src/main.rs": ( + "NETWORK_PROCESS_OPEN_FILE_LIMIT", + "ensure_open_file_limit", + ), "crates/capsem-service/src/upstream_broker.rs": ("UPSTREAM_GRANT_CAPACITY",), + "crates/capsem-process/src/main.rs": ( + "NETWORK_PROCESS_OPEN_FILE_LIMIT", + "ensure_open_file_limit", + ), "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/dns_proxy/session.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/net_proxy.rs": ( From 45e50f66937ad3af238309b5a8cc38c476e61da2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:18:07 +0000 Subject: [PATCH 355/464] test(proxy): derive replacement overlap from config --- crates/capsem-proxy/src/tests.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index b54ea9375..f6f39449a 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -68,15 +68,20 @@ fn each_traffic_capability_keeps_its_full_bounded_capacity() { } #[test] -fn http_capacity_covers_two_hundred_clients_and_replacement_overlap() { - let active = (1..=400).collect::>(); +fn http_capacity_covers_every_client_and_one_replacement() { + assert!( + UPSTREAM_GRANT_CAPACITY - capsem_proto::NETWORK_CLIENT_CAPACITY >= capsem_proto::NETWORK_CLIENT_CAPACITY, + "traffic capacity must reserve a replacement for every supported client" + ); + let active = (1..UPSTREAM_GRANT_CAPACITY as u64).collect::>(); let grants = BTreeMap::from([(ProxyCapability::HttpTraffic, active.clone())]); let grant_ids = active.into_iter().collect::>(); + let replacement = UPSTREAM_GRANT_CAPACITY as u64; assert_eq!( - grant_rejection(&grants, &grant_ids, ProxyCapability::HttpTraffic, 401), + grant_rejection(&grants, &grant_ids, ProxyCapability::HttpTraffic, replacement), None, - "the supported 200-client load needs one replacement descriptor per client" + "every supported client may overlap its retiring connection" ); } From 4287582ccb02a6dd7b8db30a751bb24308592b3c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:18:30 +0000 Subject: [PATCH 356/464] docs(skills): preserve network limit relationships --- skills/dev-mitm-proxy/SKILL.md | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/skills/dev-mitm-proxy/SKILL.md b/skills/dev-mitm-proxy/SKILL.md index d66f81615..6a1072419 100644 --- a/skills/dev-mitm-proxy/SKILL.md +++ b/skills/dev-mitm-proxy/SKILL.md @@ -60,17 +60,26 @@ Guest curl -> iptables REDIRECT -> capsem-net-proxy (guest, port 10443) ## Cross-process network limits `config/data/runtime-limits.toml` owns the supported concurrent network-client -ceiling. `capsem-proto/build.rs` validates that file and generates -`NETWORK_CLIENT_CAPACITY`; TCP descriptor grants and both DNS sessions derive -their capacities from that shared protocol constant. +ceiling and its required multiples. `capsem-proto/build.rs` validates that file +and generates the shared constants. DNS session capacity equals the client +ceiling. HTTP descriptor-grant capacity multiplies it by +`traffic_grants_per_client` because a pooled connection can be replaced before +the retiring connection closes. The process descriptor floor multiplies it by +`file_descriptors_per_client` and must cover both ends of every overlapping +grant. Never add a numeric connection ceiling to the proxy, service broker, core DNS session, or guest DNS forwarder. A limit duplicated across those processes can make one stage admit work that the next stage sheds, which previously produced -intermittent HTTP 502 responses under load. Change the config-owned value, -exercise every consumer through the shared constant, and run the official -HTTPS and DNS load benchmarks twice before and twice after the change. User -overrides belong in the typed settings contract tracked by issue #349. +intermittent HTTP 502 responses under load. The guest relay, VM owner, service +broker and confined proxy must call +`capsem_foundation::unix::process::ensure_open_file_limit` with the generated +floor before accepting traffic; raising only one process leaves the same cliff +at the next hop. Change the config-owned values, exercise every consumer +through the generated constants, and run the official HTTPS and DNS load +benchmarks twice before and twice after the change. Any traffic error or +capacity refusal fails the run. User overrides belong in the typed settings +contract tracked by issue #349. ## Content-Encoding: the systemic rule From 30caa17311bb9b65d479e56a1ccc92c11fee946d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:29:56 +0000 Subject: [PATCH 357/464] style(proto): retain the source boundary --- crates/capsem-proto/src/lib.rs | 2 -- crates/capsem-proto/src/upstream_grant.rs | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 397c08abd..892570b19 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -201,9 +201,7 @@ pub const BLOCKED_ENV_VARS: &[&str] = &[ "PS4", ]; -// --------------------------------------------------------------------------- // Vsock port constants (shared between host and guest) -// --------------------------------------------------------------------------- /// vsock port for structured control messages (resize, heartbeat, exec, file I/O). pub const VSOCK_PORT_CONTROL: u32 = 5000; diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index b0274e88c..e8bf3d0f7 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -26,9 +26,7 @@ use guest_metadata::{ATIME_SECONDS_RANGE, PATH_RANGE, TIME_FLAGS_OFFSET}; pub const UPSTREAM_GRANT_FRAME_SIZE: usize = 4480; /// A ledger response grants its storage and trusted-checkpoint channels. pub const UPSTREAM_GRANT_MAX_FDS: usize = 2; -/// Per-worker descriptor and pending-selection ceiling. -/// -/// The proxy and trusted broker preserve one replacement slot per client. +/// Per-worker descriptor and pending-selection ceiling with replacement overlap. pub const UPSTREAM_GRANT_CAPACITY: usize = crate::NETWORK_TRAFFIC_GRANT_CAPACITY; /// DNS names are at most 253 wire-text bytes without a root dot. pub const MAX_UPSTREAM_HOST_BYTES: usize = 253; From 57a1fb2b49276cb798bcc8949b1639bd769a4bb4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:30:02 +0000 Subject: [PATCH 358/464] fix(bench): reject retried proxy refusals --- benchmarks/collectors/_guest | 115 ++++++----------------- benchmarks/helpers/guest_measurements.py | 90 ++++++++++++++++++ skills/dev-benchmark/SKILL.md | 4 +- tests/test_benchmark_guest_collector.py | 77 +++++++++++++++ 4 files changed, 197 insertions(+), 89 deletions(-) create mode 100644 benchmarks/helpers/guest_measurements.py diff --git a/benchmarks/collectors/_guest b/benchmarks/collectors/_guest index 2208f57fd..77ce2bca1 100755 --- a/benchmarks/collectors/_guest +++ b/benchmarks/collectors/_guest @@ -34,6 +34,14 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests")) sys.path.insert(0, str(Path(__file__).resolve().parents[2])) import psutil +from benchmarks.helpers.guest_measurements import ( + failed_load_levels, + flatten, + load_quality_metrics, + mitm_runtime_failures, + resource_samples, + unit_for, +) from helpers.constants import ( DEFAULT_CPUS, DEFAULT_RAM_MB, @@ -42,15 +50,6 @@ from helpers.constants import ( from helpers.process_resources import ProcessTreeSampler from helpers.service import ServiceInstance, exec_output_text, wait_exec_ready -# Units the guest reports, by the suffix its key carries. -UNITS = { - "_mbps": "megabits_per_second", - "_ms": "milliseconds", - "_bytes": "bytes", - "iops": "operations", - "rps": "requests_per_second", -} - FORWARDED_ENV = ( "CAPSEM_BENCH_CONCURRENCY", "CAPSEM_BENCH_DURATION_S", @@ -106,28 +105,6 @@ def mitm_fixture_corp_toml(ready: dict) -> str: ) -def failed_load_levels(section: dict) -> list[int]: - """Concurrency levels where any request failed. - - Partial success still means the rail failed to sustain the requested - concurrency. Recording its latency or RPS would ratchet a degraded run. - """ - return [ - int(row.get("concurrency", -1)) - for row in section.get("concurrency_levels", []) - if int(row.get("errors", 0)) > 0 - ] - - -def resource_samples(metrics: dict) -> dict[str, float]: - """Flatten sampled host pressure for diagnostics from rejected load runs.""" - return { - name: measurement["samples"][0] - for name, measurement in sorted(metrics.items()) - if measurement.get("samples") - } - - def guest_command(mode: str) -> str: """One safely quoted command with only documented benchmark controls.""" assignments = [ @@ -138,62 +115,6 @@ def guest_command(mode: str) -> str: return shlex.join(["env", *assignments, "capsem-bench", mode]) -def unit_for(key: str) -> str | None: - """The unit a key declares, or `None` when it declares none. - - A key with no recognised unit is not a measurement. The guest reports the - kernel's block-queue settings alongside its timings -- `rotational`, - `nr_requests`, `read_ahead_kb` -- and walking everything numeric turned - `storage` into 640 "metrics", most of them static configuration that would - then be ratcheted like a duration. - """ - for suffix, unit in UNITS.items(): - if key.endswith(suffix): - return unit - return None - - -def flatten(node, prefix: str = "") -> dict[str, float]: - """Every number the guest reported, under a dotted key. - - The guest sections nest differently per mode -- disk by operation, storage - by mount, startup by command -- so the shape is walked rather than known. - """ - found: dict[str, float] = {} - if isinstance(node, dict): - for key, value in node.items(): - found.update(flatten(value, f"{prefix}.{key}" if prefix else str(key))) - elif isinstance(node, list): - for index, value in enumerate(node): - found.update(flatten(value, f"{prefix}.{index}" if prefix else str(index))) - elif isinstance(node, bool): - pass # a flag is not a measurement - elif isinstance(node, (int, float)): - found[prefix] = float(node) - return found - - -def load_quality_metrics(section: dict) -> dict[str, dict]: - """Store zero-error proof for every load concurrency level.""" - metrics = {} - for index, row in enumerate(section.get("concurrency_levels", [])): - errors = int(row.get("errors", 0)) - total = int(row.get("total_requests", 0)) - metrics[f"concurrency_levels.{index}.errors"] = { - "unit": "count", - "samples": [errors], - } - metrics[f"concurrency_levels.{index}.error_ratio"] = { - "unit": "ratio", - "samples": [errors / total if total else 0.0], - } - metrics[f"concurrency_levels.{index}.requests"] = { - "unit": "count", - "samples": [total], - } - return metrics - - def main() -> int: mode = Path(sys.argv[0]).name parser = argparse.ArgumentParser(description=f"measure {mode} inside a VM") @@ -206,6 +127,7 @@ def main() -> int: name = f"bench-{mode}-{uuid.uuid4().hex[:8]}" service = ServiceInstance() mock_proc = None + service_log = "" client = service.client() try: if mode == "mitm-load": @@ -273,7 +195,11 @@ def main() -> int: finally: with contextlib.suppress(Exception): client.delete(f"/vms/{name}/delete") - service.stop() + if mode == "mitm-load": + service_log = service.stop_and_read_log() + service.stop() + else: + service.stop() if mock_proc is not None: from helpers.mock_server import stop_process @@ -282,6 +208,14 @@ def main() -> int: # The guest writes one document per run keyed by mode; take that section # when present so keys are not prefixed with the mode twice. section = document.get(mode.replace("-", "_"), document) + runtime_failures = mitm_runtime_failures(service_log) if mode == "mitm-load" else [] + if runtime_failures: + raise RuntimeError( + f"capsem-bench {mode}: {len(runtime_failures)} internal traffic " + "descriptor refusal(s) were retried below the client boundary; " + f"first={runtime_failures[:5]!r}; " + f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}" + ) if failed := failed_load_levels(section): raise RuntimeError( f"capsem-bench {mode}: requests failed at concurrency {failed}; " @@ -302,6 +236,11 @@ def main() -> int: if (unit := unit_for(key)) is not None } measured.update(load_quality_metrics(section)) + if mode == "mitm-load": + measured["internal.proxy_traffic_refusals"] = { + "unit": "count", + "samples": [0], + } measured.update(resource_metrics) if not measured: raise RuntimeError( diff --git a/benchmarks/helpers/guest_measurements.py b/benchmarks/helpers/guest_measurements.py new file mode 100644 index 000000000..4d747848a --- /dev/null +++ b/benchmarks/helpers/guest_measurements.py @@ -0,0 +1,90 @@ +"""Pure measurement and quality helpers for the shared guest collector.""" + +from __future__ import annotations + +import json + +UNITS = { + "_mbps": "megabits_per_second", + "_ms": "milliseconds", + "_bytes": "bytes", + "iops": "operations", + "rps": "requests_per_second", +} + + +def failed_load_levels(section: dict) -> list[int]: + """Return concurrency levels where any request failed.""" + return [ + int(row.get("concurrency", -1)) + for row in section.get("concurrency_levels", []) + if int(row.get("errors", 0)) > 0 + ] + + +def mitm_runtime_failures(service_log: str) -> list[str]: + """Return trusted-path traffic refusals that an HTTP client may retry.""" + failures = [] + for line in service_log.splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + fields = record.get("fields", {}) + if fields.get("message") == "proxy worker refused traffic descriptor": + failures.append( + str(fields.get("error", "proxy worker refused traffic descriptor")) + ) + return failures + + +def resource_samples(metrics: dict) -> dict[str, float]: + """Flatten sampled host pressure for diagnostics from rejected load runs.""" + return { + name: measurement["samples"][0] + for name, measurement in sorted(metrics.items()) + if measurement.get("samples") + } + + +def unit_for(key: str) -> str | None: + """Return the unit declared by a measurement-key suffix.""" + for suffix, unit in UNITS.items(): + if key.endswith(suffix): + return unit + return None + + +def flatten(node, prefix: str = "") -> dict[str, float]: + """Return every numeric leaf under its dotted key.""" + found: dict[str, float] = {} + if isinstance(node, dict): + for key, value in node.items(): + found.update(flatten(value, f"{prefix}.{key}" if prefix else str(key))) + elif isinstance(node, list): + for index, value in enumerate(node): + found.update(flatten(value, f"{prefix}.{index}" if prefix else str(index))) + elif not isinstance(node, bool) and isinstance(node, (int, float)): + found[prefix] = float(node) + return found + + +def load_quality_metrics(section: dict) -> dict[str, dict]: + """Store zero-error proof for every load concurrency level.""" + metrics = {} + for index, row in enumerate(section.get("concurrency_levels", [])): + errors = int(row.get("errors", 0)) + total = int(row.get("total_requests", 0)) + metrics[f"concurrency_levels.{index}.errors"] = { + "unit": "count", + "samples": [errors], + } + metrics[f"concurrency_levels.{index}.error_ratio"] = { + "unit": "ratio", + "samples": [errors / total if total else 0.0], + } + metrics[f"concurrency_levels.{index}.requests"] = { + "unit": "count", + "samples": [total], + } + return metrics diff --git a/skills/dev-benchmark/SKILL.md b/skills/dev-benchmark/SKILL.md index a5b995b3d..d9325d1c2 100644 --- a/skills/dev-benchmark/SKILL.md +++ b/skills/dev-benchmark/SKILL.md @@ -94,7 +94,9 @@ fewer requests than its requested concurrency. The guest MITM lane delegates to `capsem-bench-rs http-load`; every client loop must attempt at least one request, and fewer total attempts than clients invalidates the run. Keep cold and warm latency separate so connection admission and TLS setup cannot be -mistaken for steady-state request cost. +mistaken for steady-state request cost. The host collector also reads the +fully flushed service log and rejects proxy traffic-descriptor refusals that +the HTTP client retried internally; zero client errors alone is insufficient. Guest load collectors sample the complete Capsem process tree. Report CPU, file descriptors, RSS, and, where the platform exposes them, PSS and USS for diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index 296837cc0..0ebd6aaaa 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -111,6 +111,81 @@ def test_a_level_where_any_request_failed_is_not_a_measurement() -> None: assert failed({}) == [] +def test_mitm_runtime_refusals_are_fatal_even_when_the_client_retries() -> None: + failures = collector()["mitm_runtime_failures"] + log = "\n".join( + [ + '{"fields":{"message":"service.start"}}', + '{"fields":{"message":"proxy worker refused traffic descriptor",' + '"error":"proxy worker rejected HttpTraffic grant 522: Capacity"}}', + '{"fields":{"message":"proxy worker refused traffic descriptor",' + '"error":"proxy worker rejected HttpTraffic grant 523: Capacity"}}', + ] + ) + + assert failures(log) == [ + "proxy worker rejected HttpTraffic grant 522: Capacity", + "proxy worker rejected HttpTraffic grant 523: Capacity", + ] + assert failures('{"fields":{"message":"service.start"}}') == [] + + +def test_guest_result_rejects_a_hidden_proxy_refusal(monkeypatch, tmp_path) -> None: + document = { + "mitm_load": { + "concurrency_levels": [ + {"concurrency": 512, "total_requests": 600, "errors": 0, "rps": 60.0}, + ] + } + } + client = SimpleNamespace( + post=lambda route, *args, **kwargs: ( + {"id": "bench-test-vm"} + if route == "/vms/create" + else { + "exit_code": 0, + "stdout": {"encoding": "utf8", "data": json.dumps(document)}, + } + ), + delete=lambda *args: None, + ) + log = ( + '{"fields":{"message":"proxy worker refused traffic descriptor",' + '"error":"proxy worker rejected HttpTraffic grant 522: Capacity"}}' + ) + main = collector()["main"] + monkeypatch.setitem( + main.__globals__, + "ServiceInstance", + lambda: SimpleNamespace( + client=lambda: client, + start=lambda: None, + stop=lambda: None, + stop_and_read_log=lambda: log, + tmp_dir=tmp_path, + ), + ) + import helpers.mock_server + + monkeypatch.setattr( + helpers.mock_server, + "start_mock_server", + lambda: ( + object(), + {"dns_udp_addr": "127.0.0.1:41000", "http_addr": "127.0.0.1:3713"}, + ), + ) + monkeypatch.setattr(helpers.mock_server, "stop_process", lambda proc: None) + monkeypatch.setitem(main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True) + monkeypatch.setitem( + main.__globals__, "process_tree_sampler", lambda service: ResourceSamplerStub() + ) + monkeypatch.setattr(sys, "argv", ["mitm-load"]) + + with pytest.raises(RuntimeError, match="1 internal traffic descriptor refusal"): + main() + + def test_load_dimensions_get_more_guest_cpus_bounded_by_the_host() -> None: guest_cpus = collector()["guest_cpus"] assert guest_cpus("disk", host_cpus=16) == 2, "non-load modes keep the default" @@ -197,6 +272,7 @@ def test_guest_result_rejects_one_failed_request(monkeypatch, tmp_path) -> None: client=lambda: client, start=lambda: None, stop=lambda: None, + stop_and_read_log=lambda: "", tmp_dir=tmp_path, ), ) @@ -257,6 +333,7 @@ def test_guest_command_failure_preserves_resource_pressure(monkeypatch, tmp_path client=lambda: client, start=lambda: None, stop=lambda: None, + stop_and_read_log=lambda: "", tmp_dir=tmp_path, ), ) From a9d139fbaa8d43848ec2f5a6c28deca54053442f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:39:51 +0000 Subject: [PATCH 359/464] fix(proto): cover pooled connection lifecycle --- CHANGELOG.md | 6 +++--- config/data/runtime-limits.toml | 4 ++-- crates/capsem-proto/build.rs | 4 ++-- tests/citadel/test_runtime_limit_authority.py | 12 ++++++------ 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ec36ee47..cbe1a071b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -177,9 +177,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 runtime budget. Linux shells commonly supplied only 1,024 descriptors while each relayed connection needs both a TCP and a VSOCK descriptor. -- The confined HTTP proxy admits 512 concurrent clients plus one replacement - grant per client instead of closing handoffs while a pooled connection - retires. Network-facing host processes raise their descriptor floor from the +- The confined HTTP proxy admits 512 concurrent clients while pooled idle and + retiring connections overlap instead of closing valid handoffs during churn. + Network-facing host processes raise their descriptor floor from the same runtime budget, and the trusted upstream broker and DNS sessions derive their ceilings from that config instead of intermittently returning 502 or SERVFAIL during a resolve burst. diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index 4d328a1a8..3318caa65 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -3,8 +3,8 @@ [network] max_concurrent_clients = 512 -# A replacement can be opened before the retiring connection leaves the pool. -traffic_grants_per_client = 2 +# A pool can hold active, idle and retiring connections for one client slot. +traffic_grants_per_client = 3 # Each intercepted client consumes a guest TCP descriptor and a VSOCK # descriptor. The remaining per-client allowance covers listeners, the Tokio # reactor, DNS, control traffic, and overlap while connections retire. diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index f4f61651d..9092b9c0e 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -51,9 +51,9 @@ fn write_runtime_limits() { .get("network") .and_then(|network| network.get("traffic_grants_per_client")) .and_then(toml::Value::as_integer) - .filter(|value| (2..=4).contains(value)) + .filter(|value| (3..=4).contains(value)) .unwrap_or_else(|| { - panic!("runtime limits: network.traffic_grants_per_client must be an integer between 2 and 4") + panic!("runtime limits: network.traffic_grants_per_client must be an integer between 3 and 4") }); let descriptors_per_client = config .get("network") diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 2ed6c877d..8900ecd77 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -12,9 +12,9 @@ RUNTIME_LIMIT_RATIONALE = """\ The proxy, trusted upstream broker, host DNS session and guest DNS forwarder -must admit the same supported client burst. A traffic connection that is being -replaced overlaps the retiring connection, so descriptor grants must preserve -the config-owned overlap multiple instead of equalling the client count. Local +must admit the same supported client burst. A pooled traffic client can have an +active, idle and retiring connection at once, so descriptor grants must +preserve the config-owned overlap multiple instead of equalling the client count. Local numeric ceilings previously let one stage accept work that the next stage shed, producing intermittent HTTP 502 responses under load. config/data/runtime-limits.toml is the authority; capsem-proto generates the @@ -94,10 +94,10 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: + "\nnetwork.max_concurrent_clients must be an integer from 1 through 4096" ) traffic_grants_per_client = config.get("network", {}).get("traffic_grants_per_client") - assert type(traffic_grants_per_client) is int and 2 <= traffic_grants_per_client <= 4, ( + assert type(traffic_grants_per_client) is int and 3 <= traffic_grants_per_client <= 4, ( RUNTIME_LIMIT_RATIONALE - + "\nnetwork.traffic_grants_per_client must cover an active connection " - "and its replacement" + + "\nnetwork.traffic_grants_per_client must cover active, idle and " + "retiring pooled connections" ) descriptors_per_client = config.get("network", {}).get("file_descriptors_per_client") assert type(descriptors_per_client) is int and 3 <= descriptors_per_client <= 64, ( From 71d2a8a289bc786d550c66ed1a86ac10459c12b1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:39:57 +0000 Subject: [PATCH 360/464] test(proxy): cover every pooled connection phase --- crates/capsem-proxy/src/tests.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index f6f39449a..b2d334f91 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -68,10 +68,10 @@ fn each_traffic_capability_keeps_its_full_bounded_capacity() { } #[test] -fn http_capacity_covers_every_client_and_one_replacement() { +fn http_capacity_covers_active_idle_and_retiring_connections() { assert!( - UPSTREAM_GRANT_CAPACITY - capsem_proto::NETWORK_CLIENT_CAPACITY >= capsem_proto::NETWORK_CLIENT_CAPACITY, - "traffic capacity must reserve a replacement for every supported client" + UPSTREAM_GRANT_CAPACITY - capsem_proto::NETWORK_CLIENT_CAPACITY >= capsem_proto::NETWORK_CLIENT_CAPACITY * 2, + "traffic capacity must reserve idle and retiring grants for every active client" ); let active = (1..UPSTREAM_GRANT_CAPACITY as u64).collect::>(); let grants = BTreeMap::from([(ProxyCapability::HttpTraffic, active.clone())]); @@ -81,7 +81,7 @@ fn http_capacity_covers_every_client_and_one_replacement() { assert_eq!( grant_rejection(&grants, &grant_ids, ProxyCapability::HttpTraffic, replacement), None, - "every supported client may overlap its retiring connection" + "the complete pooled lifecycle fits before the configured bound" ); } From dea965f0230c16e7e55002f261afbb2b6173e180 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:40:02 +0000 Subject: [PATCH 361/464] docs(skills): model pooled connection phases --- skills/dev-mitm-proxy/SKILL.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/skills/dev-mitm-proxy/SKILL.md b/skills/dev-mitm-proxy/SKILL.md index 6a1072419..551a3328b 100644 --- a/skills/dev-mitm-proxy/SKILL.md +++ b/skills/dev-mitm-proxy/SKILL.md @@ -63,8 +63,9 @@ Guest curl -> iptables REDIRECT -> capsem-net-proxy (guest, port 10443) ceiling and its required multiples. `capsem-proto/build.rs` validates that file and generates the shared constants. DNS session capacity equals the client ceiling. HTTP descriptor-grant capacity multiplies it by -`traffic_grants_per_client` because a pooled connection can be replaced before -the retiring connection closes. The process descriptor floor multiplies it by +`traffic_grants_per_client` because a pool can hold active and idle connections +while a retiring connection is still closing. The process descriptor floor +multiplies it by `file_descriptors_per_client` and must cover both ends of every overlapping grant. From 707b74193e9479b965296d14de2344ac0c52a2b6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:54:01 +0000 Subject: [PATCH 362/464] fix(bench): bind load clients to keepalive connections --- crates/capsem-bench/src/http_load.rs | 22 +++++---- crates/capsem-bench/src/http_load/tests.rs | 53 ++++++++++++++++++++++ 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/crates/capsem-bench/src/http_load.rs b/crates/capsem-bench/src/http_load.rs index a5b13b9c2..4492f795e 100644 --- a/crates/capsem-bench/src/http_load.rs +++ b/crates/capsem-bench/src/http_load.rs @@ -61,17 +61,14 @@ pub(crate) async fn run(args: Args) -> Result { bail!("--target must start with http:// or https://"); } - let client = Client::builder() - .danger_accept_invalid_certs(true) - .pool_max_idle_per_host(args.concurrency) - .timeout(timeout) - .build() - .context("build pooled HTTP load client")?; + let clients = (0..args.concurrency) + .map(|_| build_client(timeout)) + .collect::>>()?; let started = Instant::now(); let deadline = started + duration; let mut tasks = tokio::task::JoinSet::new(); - for _ in 0..args.concurrency { - tasks.spawn(drive_client(client.clone(), target.clone(), deadline)); + for client in clients { + tasks.spawn(drive_client(client, target.clone(), deadline)); } let mut samples = Vec::new(); @@ -81,6 +78,15 @@ pub(crate) async fn run(args: Args) -> Result { Ok(summarize(samples, args.concurrency, duration, started.elapsed())) } +fn build_client(timeout: Duration) -> Result { + Client::builder() + .danger_accept_invalid_certs(true) + .pool_max_idle_per_host(1) + .timeout(timeout) + .build() + .context("build logical HTTP load client") +} + fn validate(args: &Args) -> Result<()> { if args.concurrency == 0 { bail!("--concurrency must be greater than zero"); diff --git a/crates/capsem-bench/src/http_load/tests.rs b/crates/capsem-bench/src/http_load/tests.rs index 2dc0206e0..381b09c73 100644 --- a/crates/capsem-bench/src/http_load/tests.rs +++ b/crates/capsem-bench/src/http_load/tests.rs @@ -1,4 +1,7 @@ use super::*; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; #[test] fn summary_separates_connection_setup_from_warm_requests() { @@ -49,3 +52,53 @@ fn summary_separates_connection_setup_from_warm_requests() { assert_eq!(report.response_body_bytes, 96); assert_eq!(report.error_distribution["HTTP 502"], 1); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn logical_clients_do_not_multiply_keepalive_connections() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let target = format!("http://{}/tiny", listener.local_addr().unwrap()); + let accepted = Arc::new(AtomicUsize::new(0)); + let accepted_for_server = Arc::clone(&accepted); + let server = tokio::spawn(async move { + loop { + let (mut stream, _) = listener.accept().await.unwrap(); + accepted_for_server.fetch_add(1, Ordering::SeqCst); + tokio::spawn(async move { + let mut request = Vec::new(); + loop { + let mut chunk = [0_u8; 1024]; + let read = stream.read(&mut chunk).await.unwrap(); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\nx") + .await + .unwrap(); + request.clear(); + } + } + }); + } + }); + + let concurrency = 128; + let report = run(Args { + target, + concurrency, + duration_s: 0.5, + timeout_ms: 5_000, + }) + .await + .unwrap(); + server.abort(); + + assert_eq!(report.errors, 0); + assert!(report.total_requests > concurrency); + assert!( + accepted.load(Ordering::SeqCst) <= concurrency, + "one logical client opened more than one live keepalive connection" + ); +} From b81431d7f12463e2d53633747a607e0d625f68a2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 21:54:06 +0000 Subject: [PATCH 363/464] docs(skills): bind load clients to keepalive connections --- skills/dev-benchmark/SKILL.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/skills/dev-benchmark/SKILL.md b/skills/dev-benchmark/SKILL.md index d9325d1c2..db05ee230 100644 --- a/skills/dev-benchmark/SKILL.md +++ b/skills/dev-benchmark/SKILL.md @@ -94,7 +94,10 @@ fewer requests than its requested concurrency. The guest MITM lane delegates to `capsem-bench-rs http-load`; every client loop must attempt at least one request, and fewer total attempts than clients invalidates the run. Keep cold and warm latency separate so connection admission and TLS setup cannot be -mistaken for steady-state request cost. The host collector also reads the +mistaken for steady-state request cost. Each logical client owns one HTTP +client and reuses its keep-alive connection: a shared pool may open multiple +connections per logical client under load and invalidate the requested +concurrency. The host collector also reads the fully flushed service log and rejects proxy traffic-descriptor refusals that the HTTP client retried internally; zero client errors alone is insufficient. From f9439e36159334cdd410d19dcdd65d20b26bc088 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:22:31 +0000 Subject: [PATCH 364/464] fix(bench): reuse logical DNS load sockets --- crates/capsem-bench/src/cli.rs | 2 + crates/capsem-bench/src/dns_load.rs | 193 ++++++++++ crates/capsem-bench/src/dns_load/tests.rs | 43 +++ crates/capsem-bench/src/main.rs | 2 + guest/artifacts/capsem_bench/dns_load.py | 336 ++++-------------- .../test_capsem_bench_mock_server_protocol.py | 80 ++++- 6 files changed, 379 insertions(+), 277 deletions(-) create mode 100644 crates/capsem-bench/src/dns_load.rs create mode 100644 crates/capsem-bench/src/dns_load/tests.rs diff --git a/crates/capsem-bench/src/cli.rs b/crates/capsem-bench/src/cli.rs index 75bc2f5ca..daca46d99 100644 --- a/crates/capsem-bench/src/cli.rs +++ b/crates/capsem-bench/src/cli.rs @@ -22,6 +22,8 @@ pub(crate) enum Command { Ping(crate::ping::Args), /// Drive sustained HTTP load with pooled async clients. HttpLoad(crate::http_load::Args), + /// Drive sustained DNS load with async UDP clients. + DnsLoad(crate::dns_load::Args), /// Run deterministic protocol scenarios against capsem-mock-server. Protocol(ProtocolArgs), /// Run host-direct and guest-through-Capsem protocol lanes, then report delta. diff --git a/crates/capsem-bench/src/dns_load.rs b/crates/capsem-bench/src/dns_load.rs new file mode 100644 index 000000000..15caca141 --- /dev/null +++ b/crates/capsem-bench/src/dns_load.rs @@ -0,0 +1,193 @@ +//! Duration-driven DNS load without a thread per simulated client. + +use crate::protocol::run_one_dns_query; +use crate::scenarios::{build_dns_query, RequestSample}; +use anyhow::{bail, Context, Result}; +use clap::Parser; +use serde::Serialize; +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::time::{Duration, Instant}; +use tokio::net::UdpSocket; + +#[derive(Debug, Parser)] +pub(crate) struct Args { + /// UDP DNS endpoint to query. + #[arg(long, default_value = "127.0.0.1:53")] + pub(crate) target: SocketAddr, + /// DNS name to resolve. + #[arg(long, default_value = "load-test.capsem-bogus")] + pub(crate) qname: String, + /// Numeric DNS query type. + #[arg(long, default_value_t = 1)] + pub(crate) qtype: u16, + /// Number of concurrent query loops. + #[arg(long)] + pub(crate) concurrency: usize, + /// Minimum time to drive queries. Every loop makes at least one query. + #[arg(long)] + pub(crate) duration_s: f64, + /// Per-query timeout. + #[arg(long, default_value_t = 5_000)] + pub(crate) timeout_ms: u64, +} + +#[derive(Debug, Serialize)] +pub(crate) struct Report { + concurrency: usize, + duration_s: f64, + elapsed_s: f64, + total_requests: usize, + errors: usize, + error_ratio: f64, + rps: f64, + p50_ms: f64, + p95_ms: f64, + p99_ms: f64, + p999_ms: f64, + request_bytes: usize, + response_bytes: usize, + decision_distribution: BTreeMap, + error_distribution: BTreeMap, +} + +pub(crate) async fn run(args: Args) -> Result { + validate(&args)?; + let duration = Duration::from_secs_f64(args.duration_s); + let timeout = Duration::from_millis(args.timeout_ms); + let request_size = build_dns_query(&args.qname, args.qtype, 1)?.len(); + let started = Instant::now(); + let deadline = started + duration; + let mut tasks = tokio::task::JoinSet::new(); + for client in 0..args.concurrency { + let qname = args.qname.clone(); + tasks.spawn(drive_client(client, args.target, qname, args.qtype, timeout, deadline)); + } + + let mut samples = Vec::new(); + while let Some(result) = tasks.join_next().await { + samples.extend(result.context("DNS load task panicked")??); + } + Ok(summarize( + samples, + args.concurrency, + duration, + started.elapsed(), + request_size, + )) +} + +fn validate(args: &Args) -> Result<()> { + if args.concurrency == 0 { + bail!("--concurrency must be greater than zero"); + } + if !args.duration_s.is_finite() || args.duration_s <= 0.0 { + bail!("--duration-s must be a finite value greater than zero"); + } + if args.timeout_ms == 0 { + bail!("--timeout-ms must be greater than zero"); + } + Ok(()) +} + +async fn drive_client( + client: usize, + target: SocketAddr, + qname: String, + qtype: u16, + timeout: Duration, + deadline: Instant, +) -> Result> { + let bind: SocketAddr = if target.is_ipv4() { "0.0.0.0:0" } else { "[::]:0" }.parse()?; + let socket = UdpSocket::bind(bind).await.context("bind logical DNS load client")?; + let mut samples = Vec::new(); + let mut sequence = 0_u16; + while samples.is_empty() || Instant::now() < deadline { + let query_id = (client as u16).wrapping_mul(4099).wrapping_add(sequence); + samples.push(run_one_dns_query(&socket, target, &qname, qtype, query_id, timeout).await); + sequence = sequence.wrapping_add(1); + } + Ok(samples) +} + +fn summarize( + samples: Vec, + concurrency: usize, + duration: Duration, + elapsed: Duration, + request_size: usize, +) -> Report { + let total_requests = samples.len(); + let errors = samples.iter().filter(|sample| sample.error.is_some()).count(); + let successful = total_requests - errors; + let mut latencies = samples + .iter() + .filter(|sample| sample.error.is_none()) + .map(|sample| sample.latency_ms) + .collect::>(); + let mut decision_distribution = BTreeMap::new(); + let mut error_distribution = BTreeMap::new(); + for sample in &samples { + if let Some(error) = &sample.error { + *error_distribution.entry(error.clone()).or_insert(0) += 1; + *decision_distribution.entry("transport_error".to_string()).or_insert(0) += 1; + } else { + let decision = match sample.status { + 0 => "allowed".to_string(), + 2 => "error".to_string(), + 3 => "denied".to_string(), + status => format!("rcode_{status}"), + }; + *decision_distribution.entry(decision).or_insert(0) += 1; + } + } + let elapsed_s = elapsed.as_secs_f64(); + Report { + concurrency, + duration_s: duration.as_secs_f64(), + elapsed_s, + total_requests, + errors, + error_ratio: ratio(errors, total_requests), + rps: if elapsed_s > 0.0 { + successful as f64 / elapsed_s + } else { + 0.0 + }, + p50_ms: percentile(&mut latencies, 0.50), + p95_ms: percentile(&mut latencies, 0.95), + p99_ms: percentile(&mut latencies, 0.99), + p999_ms: percentile(&mut latencies, 0.999), + request_bytes: request_size * total_requests, + response_bytes: samples.iter().map(|sample| sample.size).sum(), + decision_distribution, + error_distribution, + } +} + +fn ratio(numerator: usize, denominator: usize) -> f64 { + if denominator == 0 { + 0.0 + } else { + numerator as f64 / denominator as f64 + } +} + +fn percentile(values: &mut [f64], quantile: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + values.sort_by(f64::total_cmp); + let position = quantile * (values.len() - 1) as f64; + let lower = position.floor() as usize; + let upper = position.ceil() as usize; + if lower == upper { + values[lower] + } else { + let weight = position - lower as f64; + values[lower] * (1.0 - weight) + values[upper] * weight + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-bench/src/dns_load/tests.rs b/crates/capsem-bench/src/dns_load/tests.rs new file mode 100644 index 000000000..392a3dcb5 --- /dev/null +++ b/crates/capsem-bench/src/dns_load/tests.rs @@ -0,0 +1,43 @@ +use super::*; +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn logical_clients_reuse_one_udp_socket_each() { + let server = UdpSocket::bind("127.0.0.1:0").await.unwrap(); + let target = server.local_addr().unwrap(); + let peers = Arc::new(Mutex::new(HashSet::new())); + let server_peers = Arc::clone(&peers); + let responder = tokio::spawn(async move { + let mut query = [0_u8; 512]; + loop { + let (length, peer) = server.recv_from(&mut query).await.unwrap(); + server_peers.lock().unwrap().insert(peer); + let mut response = query[..length].to_vec(); + response[2] |= 0x80; + response[3] = (response[3] & 0xf0) | 3; + server.send_to(&response, peer).await.unwrap(); + } + }); + + let concurrency = 16; + let report = run(Args { + target, + qname: "load-test.capsem-bogus".into(), + qtype: 1, + concurrency, + duration_s: 0.25, + timeout_ms: 1_000, + }) + .await + .unwrap(); + responder.abort(); + + assert_eq!(report.errors, 0); + assert!(report.total_requests > concurrency); + assert_eq!(report.decision_distribution["denied"], report.total_requests); + assert!( + peers.lock().unwrap().len() <= concurrency, + "one logical client opened more than one UDP socket" + ); +} diff --git a/crates/capsem-bench/src/main.rs b/crates/capsem-bench/src/main.rs index c420d91c4..9285c19e7 100644 --- a/crates/capsem-bench/src/main.rs +++ b/crates/capsem-bench/src/main.rs @@ -5,6 +5,7 @@ mod collector; mod commands; #[cfg(feature = "host")] mod comparison; +mod dns_load; mod http_load; #[cfg(feature = "host")] mod machine; @@ -64,6 +65,7 @@ async fn main() -> Result<()> { Command::Udp(args) => println!("{}", serde_json::to_string(&udp::run(args).await?)?), Command::Ping(args) => println!("{}", serde_json::to_string(&ping::run(args).await?)?), Command::HttpLoad(args) => println!("{}", serde_json::to_string(&http_load::run(args).await?)?), + Command::DnsLoad(args) => println!("{}", serde_json::to_string(&dns_load::run(args).await?)?), Command::Protocol(args) => { #[cfg(feature = "host")] let destination = args.record.clone(); diff --git a/guest/artifacts/capsem_bench/dns_load.py b/guest/artifacts/capsem_bench/dns_load.py index 0c73400b5..d748eee9f 100644 --- a/guest/artifacts/capsem_bench/dns_load.py +++ b/guest/artifacts/capsem_bench/dns_load.py @@ -1,228 +1,67 @@ -"""dns-load: concurrency-driven load test against the capsem DNS proxy. - -Measures DNS resolution rps + tail latency through the full path: - - guest libc / our wire-format query - -> 127.0.0.1:53 (UDP) - -> iptables nat redirect to 127.0.0.1:1053 - -> capsem-dns-proxy (T3.2) - -> vsock 5007 framed envelope - -> capsem-process serve_dns_session (T3.2 + T3.3) - -> DnsHandler::handle (T3.1 / T3.d) - -> SecurityRuleSet evaluation OR find_dns_redirect OR - UdpSocket forward to 1.1.1.1:53 - -> response wire bytes back over the same path - -Output schema mirrors mitm-load so post-T5 regression checks can -share machinery: - - { - "version": "1.0", - "qname": "api.openai.com", - "qtype": 1, - "concurrency_levels": [ - { - "concurrency": 1, - "duration_s": 10.0, - "total_requests": 1234, - "errors": 0, - "rps": 123.4, - "p50_ms": 1.2, - "p95_ms": 3.4, - "p99_ms": 5.6, - "p999_ms": 12.0, - "request_bytes": 49360, - "response_bytes": 69104, - "decision_distribution": {"denied": 1234} - }, - ... - ] - } - -Default qname is `load-test.capsem-bogus`, a Capsem-local bogus TLD -fixture resolved as local NXDOMAIN in the host DNS handler. That keeps -the benchmark self-contained: no upstream resolver latency, timeouts, -or public DNS variance. Override via `CAPSEM_BENCH_DNS_QNAME` to -benchmark another domain or the upstream-forward path (e.g. `elie.net`). -""" +"""DNS proxy load through the async Rust benchmark driver.""" +import json import os -import random -import socket -import struct -import time -import unittest -from concurrent.futures import ThreadPoolExecutor, as_completed - -from .load_harness import ( - DurationLoadConfig, - render_load_table, - summarize_load_level, -) - -# `rich` and `.helpers` are imported lazily inside `dns_load_bench` -# so the encoder helpers + their unittest module-level tests can run -# host-side via `python3 -m unittest` without needing rich installed -# on the developer machine. The benches themselves only run inside -# the guest where rich + the helpers package are available. - - -def _percentile(sorted_values, q): - """Return the q-th percentile (q in [0, 100]) from a pre-sorted list. - Local copy so this module doesn't need `.helpers` until the bench - entry point itself runs (keeps `python3 -m unittest` host-friendly). - The `.helpers.percentile` shape is identical; we don't import - that one here to avoid a transitive `rich` import. - """ - if not sorted_values: - return 0.0 - k = (len(sorted_values) - 1) * (q / 100.0) - f = int(k) - c = min(f + 1, len(sorted_values) - 1) - return sorted_values[f] + (sorted_values[c] - sorted_values[f]) * (k - f) +import subprocess +from .helpers import console +from .load_harness import DurationLoadConfig, render_load_table DEFAULT_QNAME = "load-test.capsem-bogus" -DEFAULT_QTYPE = 1 # A +DEFAULT_QTYPE = 1 DEFAULT_CONCURRENCY = (1, 10, 50, 200) DEFAULT_DURATION_S = 10.0 DEFAULT_TIMEOUT_S = 5.0 - - -def _encode_qname(name: str) -> bytes: - """Encode a dotted DNS name as length-prefixed labels + root.""" - out = bytearray() - for label in name.split("."): - if not label: - continue - if len(label) > 63: - raise ValueError(f"label too long: {label!r}") - out.append(len(label)) - out.extend(label.encode("ascii")) - out.append(0) # root - return bytes(out) - - -def _build_query(qname: str, qtype: int, qid: int) -> bytes: - """Build a wire-format DNS query for `qname` qtype `qtype`. - - Standard query, RD=1, qdcount=1, no answer/authority/additional. - """ - header = struct.pack( - ">HHHHHH", - qid, - 0x0100, # flags: standard query, RD=1 - 1, # qdcount - 0, # ancount - 0, # nscount - 0, # arcount - ) - qname_bytes = _encode_qname(qname) - question = qname_bytes + struct.pack(">HH", qtype, 1) # qtype + IN - return header + question - - -def _decode_rcode(resp: bytes) -> int: - """RFC 1035 sec 4.1.1: rcode is the low 4 bits of byte 3.""" - if len(resp) < 4: - return 2 # ServFail-shaped on a truncated reply - return resp[3] & 0x0F - - -# Map DNS rcode -> the matching capsem `Decision::as_str` shape so -# the per-level summary distribution lines up with what shows up in -# `dns_events.decision`. -_RCODE_DECISION = { - 0: "allowed", # NoError -- forwarded successfully OR redirect - 2: "error", # ServFail -- upstream unreachable - 3: "denied", # NXDomain -- policy block -} - - -def _do_query(qname: str, qtype: int, timeout_s: float): - """Single UDP DNS query; returns (latency_ms, rcode, error).""" - qid = random.randint(0, 0xFFFF) - pkt = _build_query(qname, qtype, qid) - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.settimeout(timeout_s) - start = time.monotonic() - try: - # iptables nat redirects :53 -> :1053; libc would resolve via - # /etc/resolv.conf -> 127.0.0.1, hitting the same path. We - # bypass libc to keep the timing tight + avoid getaddrinfo - # caching. - sock.sendto(pkt, ("127.0.0.1", 53)) - resp, _peer = sock.recvfrom(4096) - elapsed_ms = (time.monotonic() - start) * 1000 - # Sanity: response id must match query id. - if len(resp) >= 2 and (resp[0] << 8 | resp[1]) != qid: - return (elapsed_ms, 0, f"id mismatch: query=0x{qid:04x}", len(pkt), len(resp)) - return (elapsed_ms, _decode_rcode(resp), None, len(pkt), len(resp)) - except TimeoutError: - elapsed_ms = (time.monotonic() - start) * 1000 - return (elapsed_ms, 0, "timeout", len(pkt), 0) - except OSError as exc: - elapsed_ms = (time.monotonic() - start) * 1000 - return (elapsed_ms, 0, str(exc), len(pkt), 0) - finally: - sock.close() +RUST_BENCH = "/usr/local/bin/capsem-bench-rs" def _drive_at_concurrency(qname, qtype, concurrency, duration_s, timeout_s): - """Spawn `concurrency` workers each looping for `duration_s`. - - Each worker holds its own UDP socket per query (DNS is - connectionless; pooling a socket would be safe but it adds no - real-world signal). Returns the flat list of (latency_ms, rcode, - error) tuples. - """ - deadline = time.monotonic() + duration_s - - def worker(): - out = [] - while time.monotonic() < deadline: - out.append(_do_query(qname, qtype, timeout_s)) - return out - - all_results = [] - with ThreadPoolExecutor(max_workers=concurrency) as pool: - futures = [pool.submit(worker) for _ in range(concurrency)] - for fut in as_completed(futures): - all_results.extend(fut.result()) - return all_results - - -def _summarize(results, concurrency, duration_s): - latencies = [r[0] for r in results if r[2] is None] - errors = sum(1 for r in results if r[2] is not None) - decisions = {} - for _lat, rcode, err, _request_bytes, _response_bytes in results: - if err is not None: - decisions["transport_error"] = decisions.get("transport_error", 0) + 1 - else: - label = _RCODE_DECISION.get(rcode, f"rcode_{rcode}") - decisions[label] = decisions.get(label, 0) + 1 - return summarize_load_level( - latencies, - errors, - concurrency, - duration_s, - extra={ - "decision_distribution": decisions, - "request_bytes": sum(row[3] for row in results), - "response_bytes": sum(row[4] for row in results), - }, + """Run fixed async UDP clients and validate that every client ran.""" + completed = subprocess.run( + [ + RUST_BENCH, + "dns-load", + "--target", + "127.0.0.1:53", + "--qname", + qname, + "--qtype", + str(qtype), + "--concurrency", + str(concurrency), + "--duration-s", + str(duration_s), + "--timeout-ms", + str(round(timeout_s * 1000)), + ], + check=False, + capture_output=True, + text=True, ) + if completed.returncode: + raise RuntimeError( + f"dns-load: Rust load engine exited {completed.returncode}: " + f"{completed.stderr[-2000:].strip()}" + ) + try: + row = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("dns-load: Rust load engine returned invalid JSON") from exc + if row.get("concurrency") != concurrency: + raise RuntimeError("dns-load: Rust load engine reported the wrong concurrency") + if row.get("total_requests", 0) < concurrency: + raise RuntimeError( + f"dns-load: only {row.get('total_requests', 0)} of {concurrency} " + "client loops made a request; the load generator was starved" + ) + return row def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=None): - """Drive the DNS proxy at each concurrency level; return result dict.""" - # Lazy imports -- only the bench entry point needs rich + helpers. - # Keeps `python3 -m unittest dns_load` working host-side. - from .helpers import console - + """Drive the DNS proxy at each concurrency level; return the result dict.""" qname = qname or os.environ.get("CAPSEM_BENCH_DNS_QNAME", DEFAULT_QNAME) qtype = qtype or int(os.environ.get("CAPSEM_BENCH_DNS_QTYPE", DEFAULT_QTYPE)) + timeout_s = float(os.environ.get("CAPSEM_BENCH_DNS_TIMEOUT", DEFAULT_TIMEOUT_S)) config = DurationLoadConfig.from_inputs( "dns-load", default_concurrency=DEFAULT_CONCURRENCY, @@ -230,7 +69,6 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N concurrency_levels=concurrency_levels, duration_s=duration_s, ) - timeout_s = float(os.environ.get("CAPSEM_BENCH_DNS_TIMEOUT", DEFAULT_TIMEOUT_S)) console.print( f"[bold]dns-load[/bold] qname={qname} qtype={qtype} " @@ -239,18 +77,29 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N ) rows = [] - for c in config.concurrency_levels: - console.print(f" concurrency={c} ...") - results = _drive_at_concurrency(qname, qtype, c, config.duration_s, timeout_s) - rows.append(_summarize(results, c, config.duration_s)) + for concurrency in config.concurrency_levels: + console.print(f" concurrency={concurrency} ...") + row = _drive_at_concurrency( + qname, + qtype, + concurrency, + config.duration_s, + timeout_s, + ) + if row["errors"]: + raise RuntimeError( + f"dns-load: {row['errors']} of {row['total_requests']} requests failed " + f"at concurrency {concurrency}: {row['error_distribution']}; " + "the rail did not sustain the requested load" + ) + rows.append(row) out = { - "version": "1.0", + "version": "1.1", "qname": qname, "qtype": qtype, "concurrency_levels": rows, } - render_load_table( f"dns-load (qname={qname}, qtype={qtype}, {config.duration_s}s per level)", rows, @@ -258,63 +107,10 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N ( "decisions", lambda row: ",".join( - f"{k}={v}" for k, v in row["decision_distribution"].items() + f"{key}={value}" + for key, value in row["decision_distribution"].items() ), ), ], ) - return out - - -# ------------------------------------------------------------------- -# Self-tests (host-side; do not require a guest kernel). -# -# Run via: -# python -m unittest guest.artifacts.capsem_bench.dns_load -# ------------------------------------------------------------------- -class DnsLoadEncodingTests(unittest.TestCase): - def test_encode_qname_simple(self): - self.assertEqual( - _encode_qname("anthropic.com"), - b"\x09anthropic\x03com\x00", - ) - - def test_encode_qname_strips_trailing_dot(self): - # Trailing dot produces an empty label; we skip empties so - # "example.com." encodes the same as "example.com". - self.assertEqual( - _encode_qname("example.com."), - _encode_qname("example.com"), - ) - - def test_encode_qname_rejects_oversize_label(self): - with self.assertRaises(ValueError): - _encode_qname("a" * 64 + ".com") - - def test_build_query_header_shape(self): - pkt = _build_query("anthropic.com", 1, 0x1234) - # 12-byte header: id, flags, qdcount=1, an/ns/ar=0 - self.assertEqual(pkt[:2], b"\x12\x34") - self.assertEqual(pkt[2:4], b"\x01\x00") # standard query, RD - self.assertEqual(pkt[4:6], b"\x00\x01") # qdcount=1 - self.assertEqual(pkt[6:8], b"\x00\x00") # ancount=0 - self.assertEqual(pkt[8:10], b"\x00\x00") # nscount=0 - self.assertEqual(pkt[10:12], b"\x00\x00") # arcount=0 - # Question section: qname + qtype=1 + qclass=1 - self.assertEqual(pkt[-4:], b"\x00\x01\x00\x01") - - def test_decode_rcode_nxdomain(self): - # Build a fake response with rcode=3 (NXDomain). - # Byte 3 low 4 bits = rcode. - resp = b"\x12\x34\x81\x83" + b"\x00" * 8 - self.assertEqual(_decode_rcode(resp), 3) - - def test_decode_rcode_truncated_returns_servfail_shape(self): - self.assertEqual(_decode_rcode(b"\x00\x00"), 2) - - def test_rcode_decision_mapping(self): - # Pinned so this stays in lock-step with capsem_logger::Decision. - self.assertEqual(_RCODE_DECISION[0], "allowed") - self.assertEqual(_RCODE_DECISION[2], "error") - self.assertEqual(_RCODE_DECISION[3], "denied") diff --git a/tests/test_capsem_bench_mock_server_protocol.py b/tests/test_capsem_bench_mock_server_protocol.py index bceb101ba..0c33ca11e 100644 --- a/tests/test_capsem_bench_mock_server_protocol.py +++ b/tests/test_capsem_bench_mock_server_protocol.py @@ -79,15 +79,81 @@ def test_mitm_load_rejects_one_failed_request(monkeypatch): ) -def test_dns_load_reports_application_wire_bytes() -> None: - row = dns_load._summarize( - [(1.0, 0, None, 40, 56), (2.0, 3, None, 40, 40)], - concurrency=2, - duration_s=1, +def test_dns_load_delegates_to_async_rust_driver(monkeypatch): + calls = [] + row = { + "concurrency": 2, + "duration_s": 1.0, + "total_requests": 4, + "errors": 0, + "request_bytes": 160, + "response_bytes": 224, + } + + def run(argv, **kwargs): + calls.append((argv, kwargs)) + return types.SimpleNamespace(returncode=0, stdout=json.dumps(row), stderr="") + + monkeypatch.setattr(dns_load.subprocess, "run", run) + + assert dns_load._drive_at_concurrency("fixture.test", 1, 2, 1, 5) == row + assert calls == [ + ( + [ + dns_load.RUST_BENCH, + "dns-load", + "--target", + "127.0.0.1:53", + "--qname", + "fixture.test", + "--qtype", + "1", + "--concurrency", + "2", + "--duration-s", + "1", + "--timeout-ms", + "5000", + ], + {"check": False, "capture_output": True, "text": True}, + ) + ] + + +def test_dns_load_rejects_starved_driver(monkeypatch): + row = {"concurrency": 512, "total_requests": 91, "errors": 0} + monkeypatch.setattr( + dns_load.subprocess, + "run", + lambda *args, **kwargs: types.SimpleNamespace( + returncode=0, stdout=json.dumps(row), stderr="" + ), ) - assert row["request_bytes"] == 80 - assert row["response_bytes"] == 96 + with pytest.raises(RuntimeError, match="91 of 512 client loops"): + dns_load._drive_at_concurrency("fixture.test", 1, 512, 10, 5) + + +def test_dns_load_rejects_one_failed_request(monkeypatch): + monkeypatch.setattr( + dns_load, + "_drive_at_concurrency", + lambda qname, qtype, concurrency, duration_s, timeout_s: { + "concurrency": 1, + "duration_s": 1.0, + "total_requests": 2, + "errors": 1, + "error_distribution": {"timed out": 1}, + }, + ) + + with pytest.raises(RuntimeError, match="1 of 2 requests failed"): + dns_load.dns_load_bench( + qname="fixture.test", + qtype=1, + concurrency_levels=[1], + duration_s=1, + ) class _StubConsole: From cf6b7f3776cb35841afd8277cdee6f0a13605274 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:22:50 +0000 Subject: [PATCH 365/464] docs(skills): require persistent DNS load clients --- skills/dev-benchmark/SKILL.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/skills/dev-benchmark/SKILL.md b/skills/dev-benchmark/SKILL.md index db05ee230..2f6a92a6e 100644 --- a/skills/dev-benchmark/SKILL.md +++ b/skills/dev-benchmark/SKILL.md @@ -36,6 +36,7 @@ this repo took before it existed was taken on an unqualified machine. | `report` | every subject and its trend | | `protocol`, `protocol-delta`, `delta` | the scenario engine: host-direct vs guest-through-Capsem | | `http-load` | duration-driven async HTTP load with pooled keep-alive connections and cold/warm latency | +| `dns-load` | duration-driven async DNS load with one persistent UDP socket per logical client | Modules: `schema.rs` (the record), `stats.rs` (**all** statistics), `store.rs` (SQLite), `collector.rs` (the subprocess protocol), `machine.rs` @@ -97,7 +98,11 @@ and warm latency separate so connection admission and TLS setup cannot be mistaken for steady-state request cost. Each logical client owns one HTTP client and reuses its keep-alive connection: a shared pool may open multiple connections per logical client under load and invalidate the requested -concurrency. The host collector also reads the +concurrency. DNS load follows the same rule: the guest wrapper delegates to +`capsem-bench-rs dns-load`, and each logical client reuses one UDP socket for +the entire level. A Python thread per client or a fresh socket per query +measures scheduler and socket churn rather than the DNS path. The host +collector also reads the fully flushed service log and rejects proxy traffic-descriptor refusals that the HTTP client retried internally; zero client errors alone is insufficient. From 8cb521906552752c73adfe40621d09d2f750eac4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:25:34 +0000 Subject: [PATCH 366/464] refactor(process): extract aggregator lifecycle --- crates/capsem-process/src/aggregator.rs | 82 ++++++++++++++++++ crates/capsem-process/src/main.rs | 106 +----------------------- 2 files changed, 84 insertions(+), 104 deletions(-) create mode 100644 crates/capsem-process/src/aggregator.rs diff --git a/crates/capsem-process/src/aggregator.rs b/crates/capsem-process/src/aggregator.rs new file mode 100644 index 000000000..5922cb183 --- /dev/null +++ b/crates/capsem-process/src/aggregator.rs @@ -0,0 +1,82 @@ +use anyhow::Result; +use capsem_proto::mcp_aggregator::{write_frame, AggregatorClient}; +use std::path::{Path, PathBuf}; +use tracing::{error, info}; + +/// Path to the aggregator's dedicated stderr log within a VM's session +/// directory. Kept out of `process.log` so the parent's JSON tracing +/// stream is not polluted by child text tracing. +pub(crate) fn aggregator_log_path(session_dir: &Path) -> PathBuf { + session_dir.join("mcp-aggregator.stderr.log") +} + +/// Spawn the isolated MCP aggregator subprocess and return its client. +pub(crate) async fn spawn_mcp_aggregator( + servers: &[capsem_proto::mcp_contracts::McpServerDef], + session_dir: &Path, + vm_id: &str, + trace_id: &str, + aggregator_bin: PathBuf, + stderr_file: std::fs::File, +) -> Result { + let (client, rx) = AggregatorClient::channel(64); + let log_path = aggregator_log_path(session_dir); + info!( + bin = %aggregator_bin.display(), + servers = servers.len(), + log = %log_path.display(), + "spawning MCP aggregator" + ); + + let mut cmd = tokio::process::Command::new(&aggregator_bin); + // Preserve the service trace context while keeping the root span's + // explicit trace id authoritative for this child. + for (key, value) in capsem_foundation::telemetry::child_trace_env(vm_id) { + cmd.env(key, value); + } + cmd.env("CAPSEM_TRACE_ID", trace_id); + let mut child = cmd + .arg("--parent-pid") + .arg(std::process::id().to_string()) + .arg("--lock-path") + .arg(session_dir.join("mcp-aggregator.lock")) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::from(stderr_file)) + .spawn()?; + + let mut child_stdin = child.stdin.take().expect("piped aggregator stdin"); + let child_stdout = child.stdout.take().expect("piped aggregator stdout"); + write_frame(&mut child_stdin, &servers.to_vec()).await?; + let inflight = capsem_core::mcp::aggregator_driver::spawn(rx, child_stdin, child_stdout); + + tokio::spawn(async move { + info!("aggregator monitor task started"); + match child.wait().await { + Ok(status) => info!(status = %status, "aggregator subprocess exited"), + Err(error) => error!(%error, "failed to wait on aggregator"), + } + inflight.close(); + }); + Ok(client) +} + +pub(crate) fn resolve_mcp_aggregator_binary(exe_path: &Path) -> Result { + let bin_dir = exe_path.parent().unwrap_or(Path::new(".")); + let mut candidates = vec![bin_dir.join("capsem-mcp-aggregator")]; + if bin_dir.file_name().and_then(|name| name.to_str()) == Some("deps") { + if let Some(target_debug) = bin_dir.parent() { + candidates.push(target_debug.join("capsem-mcp-aggregator")); + } + } + if let Some(candidate) = candidates.iter().find(|candidate| candidate.exists()) { + return Ok(candidate.clone()); + } + + let searched = candidates + .iter() + .map(|path| path.display().to_string()) + .collect::>() + .join(", "); + anyhow::bail!("required MCP aggregator binary capsem-mcp-aggregator is missing; searched: {searched}") +} diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 0427b69ea..f513207df 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -1,3 +1,4 @@ +mod aggregator; mod cables; mod helpers; mod ipc; @@ -27,6 +28,7 @@ use std::sync::Arc; use tokio::sync::{broadcast, mpsc, Mutex}; use tracing::{error, info, warn}; +use aggregator::{aggregator_log_path, resolve_mcp_aggregator_binary, spawn_mcp_aggregator}; use job_store::JobStore; use mcp_runtime::{GuestExposureTools, McpRuntime}; use owner_sandbox::{attest_owner, confine_owner, prepare_owner_sandbox_attestation}; @@ -188,15 +190,6 @@ fn generate_trace_id() -> String { format!("{mixed:016x}") } -/// Path to the aggregator's dedicated stderr log within a VM's session -/// directory. Kept out of `process.log` so the parent's JSON tracing -/// stream isn't polluted by child text tracing (the two used to mix -/// because `Stdio::inherit()` forwarded the aggregator's stderr -/// straight into the parent's log sink). -fn aggregator_log_path(session_dir: &Path) -> PathBuf { - session_dir.join("mcp-aggregator.stderr.log") -} - const OWNER_CHECKPOINT_FILE: &str = "checkpoint.vzsave"; fn owner_checkpoint_path(session_dir: &Path) -> PathBuf { @@ -853,100 +846,5 @@ async fn run_async_main_loop( } } -/// Spawn the isolated MCP aggregator subprocess and return a client handle. -/// -/// The subprocess manages connections to external MCP servers. It communicates -/// via length-prefixed MessagePack frames on stdin/stdout. -/// -/// Frame format: [4 bytes big-endian payload length] [N bytes msgpack] -async fn spawn_mcp_aggregator( - servers: &[capsem_proto::mcp_contracts::McpServerDef], - session_dir: &Path, - vm_id: &str, - trace_id: &str, - aggregator_bin: PathBuf, - stderr_file: std::fs::File, -) -> Result { - use capsem_proto::mcp_aggregator::*; - - let (client, rx) = AggregatorClient::channel(64); - - let log_path = aggregator_log_path(session_dir); - - info!( - bin = %aggregator_bin.display(), - servers = servers.len(), - log = %log_path.display(), - "spawning MCP aggregator" - ); - - let mut cmd = tokio::process::Command::new(&aggregator_bin); - // W4: include CAPSEM_VM_ID, CAPSEM_TRACE_ID, TRACEPARENT, TRACESTATE. - // Caller already has `trace_id` from the root span; we re-derive via - // child_trace_env so the aggregator inherits this process's parent - // traceparent verbatim instead of getting a freshly-synthesized one. - for (k, v) in capsem_foundation::telemetry::child_trace_env(vm_id) { - cmd.env(k, v); - } - // Keep the pre-W4 CAPSEM_TRACE_ID override path so callers that - // pass an explicit trace_id (the root span's value) still win over - // the env-derived id. Belt-and-suspenders for the aggregator's - // structured root span. - cmd.env("CAPSEM_TRACE_ID", trace_id); - let mut child = cmd - .arg("--parent-pid") - .arg(std::process::id().to_string()) - .arg("--lock-path") - .arg(session_dir.join("mcp-aggregator.lock")) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::from(stderr_file)) - .spawn()?; - - let mut child_stdin = child.stdin.take().unwrap(); - let child_stdout = child.stdout.take().unwrap(); - - // Send server definitions as the first frame. - let defs_vec = servers.to_vec(); - write_frame(&mut child_stdin, &defs_vec).await?; - - let inflight = capsem_core::mcp::aggregator_driver::spawn(rx, child_stdin, child_stdout); - - // Monitor child process. - tokio::spawn(async move { - info!("aggregator monitor task started"); - match child.wait().await { - Ok(status) => info!(status = %status, "aggregator subprocess exited"), - Err(e) => error!(error = %e, "failed to wait on aggregator"), - } - inflight.close(); - }); - - Ok(client) -} - -fn resolve_mcp_aggregator_binary(exe_path: &Path) -> Result { - let bin_dir = exe_path.parent().unwrap_or(std::path::Path::new(".")); - let mut candidates = vec![bin_dir.join("capsem-mcp-aggregator")]; - if bin_dir.file_name().and_then(|name| name.to_str()) == Some("deps") { - if let Some(target_debug) = bin_dir.parent() { - candidates.push(target_debug.join("capsem-mcp-aggregator")); - } - } - - for candidate in &candidates { - if candidate.exists() { - return Ok(candidate.clone()); - } - } - - let searched = candidates - .iter() - .map(|path| path.display().to_string()) - .collect::>() - .join(", "); - anyhow::bail!("required MCP aggregator binary capsem-mcp-aggregator is missing; searched: {searched}") -} - #[cfg(test)] mod tests; From 7c11bb3438f94a060ef4497f29e6a50ffc41d688 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:37:42 +0000 Subject: [PATCH 367/464] feat(proto): bound ledger admission batches --- config/data/runtime-limits.toml | 8 ++++ crates/capsem-proto/build.rs | 29 +++++++++++- crates/capsem-proto/src/ledger_commitment.rs | 47 +++++++++++++++++-- .../src/ledger_commitment/tests.rs | 11 +++++ 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index 3318caa65..44d663d9e 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -9,3 +9,11 @@ traffic_grants_per_client = 3 # descriptor. The remaining per-client allowance covers listeners, the Tokio # reactor, DNS, control traffic, and overlap while connections retire. file_descriptors_per_client = 8 + +[ledger] +# Each of the two persistent DNS sessions can have one audit row per client +# waiting for the ledger producer. The queue is derived from the network cap. +producer_events_per_network_client = 2 +# One transport exchange carries this many consecutive producer events. The +# queue must be an exact multiple so a full burst drains without a tail shape. +admission_batch_capacity = 64 diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index 9092b9c0e..ffc19df3d 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -74,6 +74,29 @@ fn write_runtime_limits() { .checked_mul(descriptors_per_client) .filter(|value| *value <= 1_048_576) .expect("runtime limits: derived network descriptor budget exceeds 1048576"); + let producer_events_per_network_client = config + .get("ledger") + .and_then(|ledger| ledger.get("producer_events_per_network_client")) + .and_then(toml::Value::as_integer) + .filter(|value| (1..=16).contains(value)) + .unwrap_or_else(|| { + panic!("runtime limits: ledger.producer_events_per_network_client must be an integer between 1 and 16") + }); + let ledger_queue_capacity = capacity + .checked_mul(producer_events_per_network_client) + .filter(|value| *value <= 65_536) + .expect("runtime limits: derived ledger producer queue exceeds 65536"); + let ledger_batch_capacity = config + .get("ledger") + .and_then(|ledger| ledger.get("admission_batch_capacity")) + .and_then(toml::Value::as_integer) + .filter(|value| (1..=1024).contains(value)) + .unwrap_or_else(|| { + panic!("runtime limits: ledger.admission_batch_capacity must be an integer between 1 and 1024") + }); + if ledger_batch_capacity > ledger_queue_capacity || ledger_queue_capacity % ledger_batch_capacity != 0 { + panic!("runtime limits: ledger producer queue must be an exact multiple of admission batch capacity"); + } let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set"); std::fs::write( format!("{out_dir}/runtime_limits.rs"), @@ -86,7 +109,11 @@ fn write_runtime_limits() { /// Live and retiring traffic grants allowed during connection replacement.\n\ pub const NETWORK_TRAFFIC_GRANT_CAPACITY: usize = {traffic_grant_capacity};\n\ /// Minimum descriptor budget for a network-facing process.\n\ - pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit};\n" + pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit};\n\ + /// Per-producer security-event queue derived from supported clients.\n\ + pub const LEDGER_PRODUCER_QUEUE_CAPACITY: usize = {ledger_queue_capacity};\n\ + /// Consecutive producer events coalesced into one ledger exchange.\n\ + pub const LEDGER_ADMISSION_BATCH_CAPACITY: usize = {ledger_batch_capacity};\n" ), ) .expect("runtime limits: write generated constants"); diff --git a/crates/capsem-proto/src/ledger_commitment.rs b/crates/capsem-proto/src/ledger_commitment.rs index 2691fa041..8af74157b 100644 --- a/crates/capsem-proto/src/ledger_commitment.rs +++ b/crates/capsem-proto/src/ledger_commitment.rs @@ -9,6 +9,35 @@ pub const MAX_COMMITMENT_EVENT_KIND_BYTES: usize = 64; pub const MAX_COMMITMENTS_PER_CHECKPOINT: usize = 4096; pub const ZERO_COMMITMENT_HASH: [u8; 32] = [0; 32]; +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct CommitmentReservation { + event_kind: String, + event_hash: [u8; 32], +} + +impl CommitmentReservation { + pub fn new(event_kind: impl Into, event_hash: [u8; 32]) -> Result { + let reservation = Self { + event_kind: event_kind.into(), + event_hash, + }; + reservation.validate()?; + Ok(reservation) + } + + pub fn validate(&self) -> Result<(), LedgerCommitmentError> { + validate_event_kind(&self.event_kind) + } + + pub fn event_kind(&self) -> &str { + &self.event_kind + } + + pub const fn event_hash(&self) -> [u8; 32] { + self.event_hash + } +} + /// Identity, order and content commitment for one producer record. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct LedgerCommitment { @@ -107,9 +136,7 @@ impl LedgerCommitment { if self.role.producer_name().is_none() { return Err(LedgerCommitmentError::NonProducer); } - if self.event_kind.is_empty() || self.event_kind.len() > MAX_COMMITMENT_EVENT_KIND_BYTES { - return Err(LedgerCommitmentError::InvalidEventKind); - } + validate_event_kind(&self.event_kind)?; Ok(()) } @@ -129,6 +156,14 @@ impl LedgerCommitment { } } +fn validate_event_kind(event_kind: &str) -> Result<(), LedgerCommitmentError> { + if event_kind.is_empty() || event_kind.len() > MAX_COMMITMENT_EVENT_KIND_BYTES { + Err(LedgerCommitmentError::InvalidEventKind) + } else { + Ok(()) + } +} + fn role_tag(role: LedgerClientRole) -> u8 { match role { LedgerClientRole::VmOwner => 1, @@ -148,6 +183,11 @@ pub enum CommitmentCommand { event_hash: [u8; 32], previous_hash: [u8; 32], }, + ReserveBatch { + first_producer_sequence: u64, + previous_hash: [u8; 32], + reservations: Vec, + }, Cancel { global_sequence: u64, }, @@ -159,6 +199,7 @@ pub enum CommitmentCommand { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub enum CommitmentReply { Reserved { global_sequence: u64 }, + ReservedBatch { global_sequences: Vec }, Canceled, Anchored { checkpoint_sequence: u64 }, Failed { message: String }, diff --git a/crates/capsem-proto/src/ledger_commitment/tests.rs b/crates/capsem-proto/src/ledger_commitment/tests.rs index d9261ee23..556a99d2f 100644 --- a/crates/capsem-proto/src/ledger_commitment/tests.rs +++ b/crates/capsem-proto/src/ledger_commitment/tests.rs @@ -40,3 +40,14 @@ fn commitments_reject_reader_and_unbounded_identity() { Err(LedgerCommitmentError::InvalidEventKind) ); } + +#[test] +fn reservation_batches_share_the_commitment_identity_bound() { + let reservation = CommitmentReservation::new("dns_event", [9; 32]).unwrap(); + assert_eq!(reservation.event_kind(), "dns_event"); + assert_eq!(reservation.event_hash(), [9; 32]); + assert_eq!( + CommitmentReservation::new("x".repeat(MAX_COMMITMENT_EVENT_KIND_BYTES + 1), [0; 32]), + Err(LedgerCommitmentError::InvalidEventKind) + ); +} From 3e43356c39361d8c9a164562e50c6de029928446 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:40:03 +0000 Subject: [PATCH 368/464] feat(service): reserve ledger commitments in batches --- .../capsem-service/src/ledger_commitment.rs | 77 ++- .../src/ledger_commitment/tests.rs | 454 ++++++++++++++++++ 2 files changed, 512 insertions(+), 19 deletions(-) diff --git a/crates/capsem-service/src/ledger_commitment.rs b/crates/capsem-service/src/ledger_commitment.rs index b57ef231f..5fa4ce461 100644 --- a/crates/capsem-service/src/ledger_commitment.rs +++ b/crates/capsem-service/src/ledger_commitment.rs @@ -13,8 +13,8 @@ use capsem_proto::ledger::{LedgerChannelGrant, LedgerWelcome}; #[cfg(test)] use capsem_proto::ledger_commitment::ZERO_COMMITMENT_HASH; use capsem_proto::ledger_commitment::{ - CommitmentClientMessage, CommitmentCommand, CommitmentReply, CommitmentServerMessage, LedgerCommitment, - MAX_COMMITMENTS_PER_CHECKPOINT, + CommitmentClientMessage, CommitmentCommand, CommitmentReply, CommitmentReservation, CommitmentServerMessage, + LedgerCommitment, MAX_COMMITMENTS_PER_CHECKPOINT, }; use serde::{Deserialize, Serialize}; @@ -96,6 +96,26 @@ impl CommitmentAuthority { event_hash: [u8; 32], previous_hash: [u8; 32], ) -> Result { + let reservation = CommitmentReservation::new(event_kind, event_hash)?; + let mut sequences = self + .reserve_batch(grant, producer_sequence, previous_hash, vec![reservation]) + .await?; + Ok(sequences.pop().expect("one reservation returns one sequence")) + } + + async fn reserve_batch( + &self, + grant: LedgerChannelGrant, + first_producer_sequence: u64, + previous_hash: [u8; 32], + reservations: Vec, + ) -> Result> { + if reservations.is_empty() || reservations.len() > capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY { + bail!("commitment reservation batch is empty or exceeds its bound"); + } + for reservation in &reservations { + reservation.validate()?; + } let key = ProducerKey::from_grant(grant); let mut state = self.state.lock().await; let tail = state.pending.get(&key).and_then(|pending| pending.last()).map_or_else( @@ -105,28 +125,39 @@ impl CommitmentAuthority { hash: commitment.commitment_hash(), }, ); - if producer_sequence != tail.sequence.checked_add(1).context("producer sequence exhausted")? + if first_producer_sequence != tail.sequence.checked_add(1).context("producer sequence exhausted")? || previous_hash != tail.hash { bail!("producer commitment does not extend its trusted tail"); } - let global_sequence = state - .next_global_sequence - .checked_add(1) - .context("global commitment sequence exhausted")?; - let commitment = LedgerCommitment::new( - grant, - producer_sequence, - global_sequence, - event_kind, - event_hash, - previous_hash, - )?; + let mut producer_sequence = first_producer_sequence; + let mut global_sequence = state.next_global_sequence; + let mut previous_hash = previous_hash; + let mut commitments = Vec::with_capacity(reservations.len()); + let mut global_sequences = Vec::with_capacity(reservations.len()); + for reservation in reservations { + global_sequence = global_sequence + .checked_add(1) + .context("global commitment sequence exhausted")?; + let commitment = LedgerCommitment::new( + grant, + producer_sequence, + global_sequence, + reservation.event_kind(), + reservation.event_hash(), + previous_hash, + )?; + previous_hash = commitment.commitment_hash(); + commitments.push(commitment); + global_sequences.push(global_sequence); + producer_sequence = producer_sequence + .checked_add(1) + .context("producer sequence exhausted")?; + } state.next_global_sequence = global_sequence; - state.global_sequences.insert(global_sequence); - state.pending.entry(key).or_default().push(commitment); - drop(state); - Ok(global_sequence) + state.global_sequences.extend(global_sequences.iter().copied()); + state.pending.entry(key).or_default().extend(commitments); + Ok(global_sequences) } async fn cancel(&self, grant: LedgerChannelGrant, global_sequence: u64) -> Result<()> { @@ -360,6 +391,14 @@ async fn serve_requests( .reserve(grant, producer_sequence, event_kind, event_hash, previous_hash) .await .map(|global_sequence| CommitmentReply::Reserved { global_sequence }), + CommitmentCommand::ReserveBatch { + first_producer_sequence, + previous_hash, + reservations, + } => authority + .reserve_batch(grant, first_producer_sequence, previous_hash, reservations) + .await + .map(|global_sequences| CommitmentReply::ReservedBatch { global_sequences }), CommitmentCommand::Cancel { global_sequence } => authority .cancel(grant, global_sequence) .await diff --git a/crates/capsem-service/src/ledger_commitment/tests.rs b/crates/capsem-service/src/ledger_commitment/tests.rs index cfe652e92..7b853e475 100644 --- a/crates/capsem-service/src/ledger_commitment/tests.rs +++ b/crates/capsem-service/src/ledger_commitment/tests.rs @@ -281,3 +281,457 @@ async fn typed_verification_accepts_exact_rows_and_rejects_anchored_omission_and let client = reader(exact_server, &exact_path).await; exact_authority.verify_ledger(&client).await.unwrap(); } + +fn commitment(grant: LedgerChannelGrant, producer_sequence: u64, global_sequence: u64) -> LedgerCommitment { + LedgerCommitment::new( + grant, + producer_sequence, + global_sequence, + "net_event", + [producer_sequence as u8; 32], + ZERO_COMMITMENT_HASH, + ) + .unwrap() +} + +#[tokio::test] +async fn reservations_can_only_cancel_the_unanchored_tail_and_abandon_revokes_all_pending() { + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "session-cancel").await.unwrap(); + let producer = grant(5, 6); + + assert!(authority.cancel(producer, 1).await.is_err()); + let first = authority + .reserve(producer, 1, "net_event".into(), [1; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(); + let first_commitment = + LedgerCommitment::new(producer, 1, first, "net_event", [1; 32], ZERO_COMMITMENT_HASH).unwrap(); + let second = authority + .reserve( + producer, + 2, + "net_event".into(), + [2; 32], + first_commitment.commitment_hash(), + ) + .await + .unwrap(); + assert!(authority.cancel(producer, first).await.is_err()); + authority.cancel(producer, second).await.unwrap(); + authority.cancel(producer, first).await.unwrap(); + assert!(authority.state.lock().await.pending.is_empty()); + + let third = authority + .reserve(producer, 1, "net_event".into(), [3; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(); + authority.abandon(producer).await; + let state = authority.state.lock().await; + assert!(state.pending.is_empty()); + assert!(!state.global_sequences.contains(&third)); + drop(state); +} + +#[tokio::test] +async fn reservation_and_checkpoint_sequence_exhaustion_fail_closed() { + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "session-overflow") + .await + .unwrap(); + let producer = grant(6, 7); + authority.state.lock().await.next_global_sequence = u64::MAX; + assert!(authority + .reserve(producer, 1, "net_event".into(), [1; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap_err() + .to_string() + .contains("global commitment sequence exhausted")); + + let key = ProducerKey::from_grant(producer); + { + let mut state = authority.state.lock().await; + state.next_global_sequence = 0; + state.anchored.insert( + key, + ProducerTail { + sequence: u64::MAX, + hash: [9; 32], + }, + ); + } + assert!(authority + .reserve(producer, 1, "net_event".into(), [1; 32], [9; 32]) + .await + .unwrap_err() + .to_string() + .contains("producer sequence exhausted")); + + { + let mut state = authority.state.lock().await; + state.anchored.remove(&key); + state.next_checkpoint_sequence = u64::MAX; + } + let global = authority + .reserve(producer, 1, "net_event".into(), [4; 32], ZERO_COMMITMENT_HASH) + .await + .unwrap(); + let reserved = LedgerCommitment::new(producer, 1, global, "net_event", [4; 32], ZERO_COMMITMENT_HASH).unwrap(); + assert!(authority + .anchor(producer, vec![reserved]) + .await + .unwrap_err() + .to_string() + .contains("checkpoint sequence exhausted")); +} + +fn hex(bytes: &[u8]) -> String { + use std::fmt::Write as _; + + bytes + .iter() + .fold(String::with_capacity(bytes.len() * 2), |mut hex, byte| { + write!(hex, "{byte:02x}").unwrap(); + hex + }) +} + +fn commitment_row(commitment: &LedgerCommitment, role: &str) -> Vec { + vec![ + LedgerValue::Integer(commitment.global_sequence() as i64), + LedgerValue::Text(hex(&commitment.generation().as_bytes())), + LedgerValue::Integer(commitment.client_id() as i64), + LedgerValue::Text(role.to_string()), + LedgerValue::Integer(commitment.producer_sequence() as i64), + LedgerValue::Text(commitment.event_kind().to_string()), + LedgerValue::Text(hex(&commitment.event_hash())), + LedgerValue::Text(hex(&commitment.previous_hash())), + LedgerValue::Text(hex(&commitment.commitment_hash())), + ] +} + +#[test] +fn typed_commitment_rows_reject_malformed_or_forged_fields() { + let producer = grant(7, 8); + let expected = commitment(producer, 1, 1); + assert_eq!( + decode_commitment_row(&commitment_row(&expected, "proxy")).unwrap(), + expected + ); + assert!(decode_commitment_row(&[]) + .unwrap_err() + .to_string() + .contains("invalid typed row")); + + for (index, replacement, message) in [ + (1, LedgerValue::Text("00".into()), "wrong length"), + (2, LedgerValue::Integer(-1), "client id is negative"), + (3, LedgerValue::Text("reader".into()), "invalid producer role"), + (4, LedgerValue::Integer(-1), "producer sequence is negative"), + (6, LedgerValue::Text("gg".repeat(32)), "hex field is invalid"), + (8, LedgerValue::Text("00".repeat(32)), "hash does not match"), + ] { + let mut row = commitment_row(&expected, "proxy"); + row[index] = replacement; + assert!(decode_commitment_row(&row).unwrap_err().to_string().contains(message)); + } + let mut row = commitment_row(&expected, "proxy"); + row[0] = LedgerValue::Integer(-1); + assert!(decode_commitment_row(&row) + .unwrap_err() + .to_string() + .contains("global sequence is negative")); + + for (role, role_name) in [ + (LedgerClientRole::VmOwner, "vm_owner"), + (LedgerClientRole::Coordinator, "coordinator"), + ] { + let role_grant = LedgerChannelGrant::new(LedgerGeneration::new([8; 16]), 9, role).unwrap(); + let value = commitment(role_grant, 1, 2); + assert_eq!( + decode_commitment_row(&commitment_row(&value, role_name)).unwrap(), + value + ); + } +} + +fn write_frame(path: &Path, payload: &[u8], checksum: [u8; 32]) { + std::fs::write( + path, + [ + CHECKPOINT_MAGIC.as_slice(), + &(payload.len() as u32).to_be_bytes(), + payload, + &checksum, + ] + .concat(), + ) + .unwrap(); +} + +#[test] +fn checkpoint_loader_rejects_invalid_frames_and_state_transitions() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("checkpoints.log"); + initialize(&path).unwrap(); + + for length in [0_u32, (MAX_CHECKPOINT_FRAME_BYTES as u32) + 1] { + std::fs::write(&path, [CHECKPOINT_MAGIC.as_slice(), &length.to_be_bytes()].concat()).unwrap(); + assert!(read_records(&path) + .unwrap_err() + .to_string() + .contains("frame length is invalid")); + } + write_frame(&path, b"invalid", [0; 32]); + assert!(read_records(&path) + .unwrap_err() + .to_string() + .contains("checksum is corrupt")); + let payload = b"invalid"; + write_frame(&path, payload, *capsem_crypto::blake3::hash(payload).as_bytes()); + assert!(read_records(&path) + .unwrap_err() + .to_string() + .contains("decode commitment checkpoint")); + + let producer = grant(9, 10); + for (record, message) in [ + ( + CheckpointRecord { + checkpoint_sequence: 2, + commitments: vec![commitment(producer, 1, 1)], + }, + "sequence is not contiguous", + ), + ( + CheckpointRecord { + checkpoint_sequence: 1, + commitments: Vec::new(), + }, + "invalid batch size", + ), + ( + CheckpointRecord { + checkpoint_sequence: 1, + commitments: vec![commitment(producer, 2, 1)], + }, + "breaks a producer chain", + ), + ] { + std::fs::write(&path, []).unwrap(); + append(&path, &record).unwrap(); + assert!(load(&path).err().unwrap().to_string().contains(message)); + } + + let other = grant(10, 11); + std::fs::write(&path, []).unwrap(); + append( + &path, + &CheckpointRecord { + checkpoint_sequence: 1, + commitments: vec![commitment(producer, 1, 1), commitment(other, 1, 1)], + }, + ) + .unwrap(); + assert!(load(&path) + .err() + .unwrap() + .to_string() + .contains("repeats a global sequence")); +} + +#[tokio::test] +async fn commitment_channel_enforces_handshake_ids_and_executes_all_commands() { + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "session-channel").await.unwrap(); + let producer = grant(11, 12); + let (client, server) = UnixStream::pair().unwrap(); + let serving = tokio::spawn(serve(Arc::clone(&authority), server, producer)); + let (requests, responses) = + ipc_channel::channel_from_std::(client).unwrap(); + requests + .send(CommitmentClientMessage::Hello { + hello: LedgerHello::for_grant(&producer), + }) + .await + .unwrap(); + assert!(matches!( + responses.recv().await.unwrap(), + CommitmentServerMessage::Welcome { .. } + )); + + requests + .send(CommitmentClientMessage::Request { + request_id: 1, + command: CommitmentCommand::Reserve { + producer_sequence: 1, + event_kind: "net_event".into(), + event_hash: [4; 32], + previous_hash: ZERO_COMMITMENT_HASH, + }, + }) + .await + .unwrap(); + let global_sequence = match responses.recv().await.unwrap() { + CommitmentServerMessage::Response { + request_id: 1, + reply: CommitmentReply::Reserved { global_sequence }, + } => global_sequence, + response => panic!("unexpected reserve response: {response:?}"), + }; + + requests + .send(CommitmentClientMessage::Request { + request_id: 2, + command: CommitmentCommand::Cancel { + global_sequence: global_sequence + 1, + }, + }) + .await + .unwrap(); + assert!(matches!( + responses.recv().await.unwrap(), + CommitmentServerMessage::Response { + request_id: 2, + reply: CommitmentReply::Failed { .. }, + } + )); + + let reserved = + LedgerCommitment::new(producer, 1, global_sequence, "net_event", [4; 32], ZERO_COMMITMENT_HASH).unwrap(); + requests + .send(CommitmentClientMessage::Request { + request_id: 3, + command: CommitmentCommand::Anchor { + commitments: vec![reserved], + }, + }) + .await + .unwrap(); + assert!(matches!( + responses.recv().await.unwrap(), + CommitmentServerMessage::Response { + request_id: 3, + reply: CommitmentReply::Anchored { checkpoint_sequence: 1 }, + } + )); + + let anchored_hash = authority.state.lock().await.anchored[&ProducerKey::from_grant(producer)].hash; + requests + .send(CommitmentClientMessage::Request { + request_id: 4, + command: CommitmentCommand::Reserve { + producer_sequence: 2, + event_kind: "net_event".into(), + event_hash: [5; 32], + previous_hash: anchored_hash, + }, + }) + .await + .unwrap(); + let tail = match responses.recv().await.unwrap() { + CommitmentServerMessage::Response { + request_id: 4, + reply: CommitmentReply::Reserved { global_sequence }, + } => global_sequence, + response => panic!("unexpected reserve response: {response:?}"), + }; + requests + .send(CommitmentClientMessage::Request { + request_id: 5, + command: CommitmentCommand::Cancel { global_sequence: tail }, + }) + .await + .unwrap(); + assert!(matches!( + responses.recv().await.unwrap(), + CommitmentServerMessage::Response { + request_id: 5, + reply: CommitmentReply::Canceled, + } + )); + + requests + .send(CommitmentClientMessage::Hello { + hello: LedgerHello::for_grant(&producer), + }) + .await + .unwrap(); + assert!(serving + .await + .unwrap() + .unwrap_err() + .to_string() + .contains("repeated hello")); + + let (client, server) = UnixStream::pair().unwrap(); + let serving = tokio::spawn(serve(authority, server, producer)); + let (requests, responses) = + ipc_channel::channel_from_std::(client).unwrap(); + requests + .send(CommitmentClientMessage::Request { + request_id: 0, + command: CommitmentCommand::Cancel { global_sequence: 1 }, + }) + .await + .unwrap(); + drop(responses); + assert!(serving.await.unwrap().unwrap_err().to_string().contains("before hello")); + + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "session-zero-id").await.unwrap(); + let (client, server) = UnixStream::pair().unwrap(); + let serving = tokio::spawn(serve(authority, server, producer)); + let (requests, responses) = + ipc_channel::channel_from_std::(client).unwrap(); + requests + .send(CommitmentClientMessage::Hello { + hello: LedgerHello::for_grant(&producer), + }) + .await + .unwrap(); + assert!(matches!( + responses.recv().await.unwrap(), + CommitmentServerMessage::Welcome { .. } + )); + requests + .send(CommitmentClientMessage::Request { + request_id: 0, + command: CommitmentCommand::Cancel { global_sequence: 1 }, + }) + .await + .unwrap(); + assert!(serving + .await + .unwrap() + .unwrap_err() + .to_string() + .contains("request id must not be zero")); +} + +#[tokio::test] +async fn reservation_batch_extends_one_trusted_tail_atomically() { + let root = tempfile::tempdir().unwrap(); + let authority = CommitmentAuthority::open(root.path(), "batch").await.unwrap(); + let producer = grant(21, 22); + let sequences = authority + .reserve_batch( + producer, + 1, + ZERO_COMMITMENT_HASH, + vec![ + CommitmentReservation::new("dns_event", [1; 32]).unwrap(), + CommitmentReservation::new("security_rule_event", [2; 32]).unwrap(), + ], + ) + .await + .unwrap(); + + assert_eq!(sequences, vec![1, 2]); + let state = authority.state.lock().await; + let pending = &state.pending[&ProducerKey::from_grant(producer)]; + assert_eq!(pending.len(), 2); + assert_eq!(pending[0].producer_sequence(), 1); + assert_eq!(pending[1].producer_sequence(), 2); + assert_eq!(pending[1].previous_hash(), pending[0].commitment_hash()); +} From 666c01e3240f8887cf962c54102e8af8ddf65358 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:49:50 +0000 Subject: [PATCH 369/464] perf(proto): bound ledger coalescing delay --- config/data/runtime-limits.toml | 3 +++ crates/capsem-proto/build.rs | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index 44d663d9e..d2c1cfc7d 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -17,3 +17,6 @@ producer_events_per_network_client = 2 # One transport exchange carries this many consecutive producer events. The # queue must be an exact multiple so a full burst drains without a tail shape. admission_batch_capacity = 64 +# A lone event waits briefly for its immediate followers. This is far below a +# network round trip while avoiding singleton IPC under concurrent bursts. +admission_coalesce_wait_microseconds = 100 diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index ffc19df3d..4210edd5a 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -97,6 +97,14 @@ fn write_runtime_limits() { if ledger_batch_capacity > ledger_queue_capacity || ledger_queue_capacity % ledger_batch_capacity != 0 { panic!("runtime limits: ledger producer queue must be an exact multiple of admission batch capacity"); } + let ledger_coalesce_wait_microseconds = config + .get("ledger") + .and_then(|ledger| ledger.get("admission_coalesce_wait_microseconds")) + .and_then(toml::Value::as_integer) + .filter(|value| (1..=10_000).contains(value)) + .unwrap_or_else(|| { + panic!("runtime limits: ledger.admission_coalesce_wait_microseconds must be an integer between 1 and 10000") + }); let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set"); std::fs::write( format!("{out_dir}/runtime_limits.rs"), @@ -113,7 +121,9 @@ fn write_runtime_limits() { /// Per-producer security-event queue derived from supported clients.\n\ pub const LEDGER_PRODUCER_QUEUE_CAPACITY: usize = {ledger_queue_capacity};\n\ /// Consecutive producer events coalesced into one ledger exchange.\n\ - pub const LEDGER_ADMISSION_BATCH_CAPACITY: usize = {ledger_batch_capacity};\n" + pub const LEDGER_ADMISSION_BATCH_CAPACITY: usize = {ledger_batch_capacity};\n\ + /// Maximum wait for an immediate follower before admitting a singleton.\n\ + pub const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = {ledger_coalesce_wait_microseconds};\n" ), ) .expect("runtime limits: write generated constants"); From ffb64e0506e9420d6f65008e79d6f2afdb1aacdd Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:49:53 +0000 Subject: [PATCH 370/464] perf(logger): coalesce remote ledger admissions --- CHANGELOG.md | 4 + crates/capsem-logger/src/ledger_protocol.rs | 66 ++++- .../src/ledger_protocol/tests.rs | 31 ++ crates/capsem-logger/src/ledger_server.rs | 54 ++++ .../capsem-logger/src/ledger_server/tests.rs | 71 ++++- crates/capsem-logger/src/writer/remote.rs | 277 +++++++++++++----- .../capsem-logger/src/writer/remote/tests.rs | 98 +++---- 7 files changed, 472 insertions(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbe1a071b..3364f1ded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -272,6 +272,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Ledger producers coalesce bounded consecutive security events into one + commitment reservation and admission exchange, avoiding audit backpressure + during concurrent DNS and gateway traffic. + - Runtime memory caches use one bounded, concurrent implementation with uniform hit, miss, mutation, load-time and entry metrics. DNS retains its TTL and policy checks, while concurrent TLS handshakes share one leaf mint. diff --git a/crates/capsem-logger/src/ledger_protocol.rs b/crates/capsem-logger/src/ledger_protocol.rs index 5c8840489..58e07562a 100644 --- a/crates/capsem-logger/src/ledger_protocol.rs +++ b/crates/capsem-logger/src/ledger_protocol.rs @@ -31,6 +31,44 @@ const MAX_EVENT_ID_BYTES: usize = 12; const MAX_TIMESTAMP_BYTES: usize = 64; const SNAPSHOT_ID_BYTES: usize = 16; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LedgerAdmission { + event: Box, + commitment: LedgerCommitment, +} + +impl LedgerAdmission { + pub fn new(event: WriteOp, commitment: LedgerCommitment) -> Self { + Self { + event: Box::new(event), + commitment, + } + } + + pub fn validate(&self) -> Result<(), LedgerProtocolError> { + self.commitment + .validate() + .map_err(|_| LedgerProtocolError::InvalidOperation)?; + let (event_hash, event_bytes) = crate::writer::commitment_event_fingerprint(&self.event) + .map_err(|_| LedgerProtocolError::InvalidOperation)?; + if self.commitment.event_kind() != self.event.kind() || self.commitment.event_hash() != event_hash { + return Err(LedgerProtocolError::InvalidOperation); + } + if event_bytes > MAX_LEDGER_OPERATION_BYTES - MAX_ADMIT_METADATA_BYTES { + return Err(LedgerProtocolError::OperationTooLarge); + } + Ok(()) + } + + pub fn into_parts(self) -> (WriteOp, LedgerCommitment) { + (*self.event, self.commitment) + } + + pub fn commitment(&self) -> &LedgerCommitment { + &self.commitment + } +} + pub type LedgerResponse = capsem_proto::ledger::LedgerResponse; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -56,6 +94,9 @@ pub enum LedgerCommand { event: Box, commitment: LedgerCommitment, }, + AdmitBatch { + admissions: Vec, + }, /// Make every accepted event through the returned admission id durable. Flush, Query { @@ -80,7 +121,7 @@ pub enum LedgerCommand { impl LedgerOperation for LedgerCommand { fn capability(&self) -> LedgerCapability { match self { - Self::Admit { .. } => LedgerCapability::Admit, + Self::Admit { .. } | Self::AdmitBatch { .. } => LedgerCapability::Admit, Self::Flush => LedgerCapability::Flush, Self::Query { .. } | Self::ReadBodies { .. } | Self::Counters => LedgerCapability::Read, Self::Retain { .. } => LedgerCapability::Retain, @@ -106,6 +147,14 @@ impl LedgerOperation for LedgerCommand { } return Ok(()); } + Self::AdmitBatch { admissions } => { + if admissions.is_empty() || admissions.len() > capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY { + return Err(LedgerProtocolError::InvalidOperation); + } + for admission in admissions { + admission.validate()?; + } + } Self::Query { query } => query.validate()?, Self::ReadBodies { event_id } if !valid_event_id(event_id) => { return Err(LedgerProtocolError::InvalidOperation) @@ -289,6 +338,11 @@ pub enum LedgerReply { Accepted { admission_id: u64, }, + AcceptedBatch { + first_admission_id: u64, + through_admission_id: u64, + accepted: u16, + }, Durable { through_admission_id: u64, }, @@ -328,6 +382,16 @@ impl LedgerReply { pub fn validate(&self) -> Result<(), LedgerProtocolError> { let valid = match self { Self::Accepted { admission_id } => *admission_id != 0, + Self::AcceptedBatch { + first_admission_id, + through_admission_id, + accepted, + } => { + *first_admission_id != 0 + && usize::from(*accepted) <= capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY + && *accepted != 0 + && first_admission_id.checked_add(u64::from(*accepted) - 1) == Some(*through_admission_id) + } Self::Durable { .. } | Self::Counters { .. } | Self::BodiesComplete | Self::Retained { .. } => true, Self::Query { sets } => { sets.len() <= MAX_LEDGER_RESULT_SETS && sets.iter().all(|rows| rows.validate().is_ok()) diff --git a/crates/capsem-logger/src/ledger_protocol/tests.rs b/crates/capsem-logger/src/ledger_protocol/tests.rs index a83ca026c..0da46b53c 100644 --- a/crates/capsem-logger/src/ledger_protocol/tests.rs +++ b/crates/capsem-logger/src/ledger_protocol/tests.rs @@ -59,6 +59,37 @@ async fn request_round_trips_over_the_landed_bounded_channel() { )); } +#[test] +fn admission_batch_is_bounded_and_correlated_as_one_reply() { + let LedgerCommand::Admit { event, commitment } = file_write() else { + unreachable!() + }; + let admission = LedgerAdmission::new(*event, commitment); + LedgerCommand::AdmitBatch { + admissions: vec![admission.clone(), admission], + } + .validate() + .unwrap(); + assert_eq!( + LedgerCommand::AdmitBatch { admissions: vec![] }.validate(), + Err(LedgerProtocolError::InvalidOperation) + ); + LedgerReply::AcceptedBatch { + first_admission_id: 41, + through_admission_id: 42, + accepted: 2, + } + .validate() + .unwrap(); + assert!(LedgerReply::AcceptedBatch { + first_admission_id: 41, + through_admission_id: 43, + accepted: 2, + } + .validate() + .is_err()); +} + #[test] fn roles_admit_only_their_operation_classes() { for (role, operation) in [ diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index a88e6422d..9a8d8a137 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -140,6 +140,60 @@ impl LedgerServer { Err(error) => Err(error), } } + LedgerCommand::AdmitBatch { admissions } => { + let accepted = { + let mut admission_state = self.admission.lock().await; + let first_admission_id = admission_state + .last_accepted + .checked_add(1) + .ok_or_else(|| "ledger admission id exhausted".to_string()); + let validated = admissions.iter().try_for_each(|admission| { + admission.commitment().validate_grant(grant).map_err(|e| e.to_string()) + }); + match (first_admission_id, validated) { + (Ok(first_admission_id), Ok(())) => { + let count = u16::try_from(admissions.len()) + .map_err(|_| "ledger admission batch count exceeds u16".to_string()); + match count { + Ok(count) => { + let mut failure = None; + for admission in admissions { + let (event, commitment) = admission.into_parts(); + if let Err(error) = self.db.write_committed(event, commitment).await { + failure = Some(storage(error)); + break; + } + admission_state.last_accepted = admission_state + .last_accepted + .checked_add(1) + .ok_or_else(|| "ledger admission id exhausted".to_string())?; + } + let through_admission_id = admission_state.last_accepted; + drop(admission_state); + failure.map_or_else(|| Ok((first_admission_id, through_admission_id, count)), Err) + } + Err(error) => Err(error), + } + } + (Err(error), _) | (_, Err(error)) => Err(error), + } + }; + match accepted { + Ok((first_admission_id, through_admission_id, accepted)) => { + send_success( + sender, + request_id, + LedgerReply::AcceptedBatch { + first_admission_id, + through_admission_id, + accepted, + }, + ) + .await + } + Err(error) => Err(error), + } + } LedgerCommand::Flush => { // The lock orders the writer enqueue and this barrier against // every client, so the returned id cannot name a later write. diff --git a/crates/capsem-logger/src/ledger_server/tests.rs b/crates/capsem-logger/src/ledger_server/tests.rs index ea7ffb700..ec64a2a2f 100644 --- a/crates/capsem-logger/src/ledger_server/tests.rs +++ b/crates/capsem-logger/src/ledger_server/tests.rs @@ -9,7 +9,8 @@ use capsem_proto::ledger::{ use super::*; use crate::ledger_protocol::{ - LedgerClientMessage, LedgerCommand, LedgerQuery, LedgerReply, LedgerServerMessage, LedgerTimelineLayer, + LedgerAdmission, LedgerClientMessage, LedgerCommand, LedgerQuery, LedgerReply, LedgerServerMessage, + LedgerTimelineLayer, }; use crate::{Decision, NetEvent, WriteOp}; @@ -103,6 +104,74 @@ fn prepared_server_holds_writer_authority_until_workers_start() { drop(LedgerServer::prepare_with_codecs(&database, capsem_archive::ArchiveCodecs::default()).unwrap()); } +#[tokio::test] +async fn admission_batch_gets_one_contiguous_acceptance_reply() { + let dir = tempfile::tempdir().unwrap(); + let database = dir.path().join("session.db"); + let server = Arc::new(LedgerServer::open(&database).unwrap()); + let producer_grant = grant(LedgerClientRole::VmOwner); + let (producer, replies, task) = connect(Arc::clone(&server), producer_grant).await; + let mut first = body_event(Vec::new()); + let WriteOp::NetEvent(first_event) = &mut first else { + unreachable!() + }; + first_event.event_id = Some("0123456789ac".into()); + first_event.domain = "first.test".into(); + let first_commitment = capsem_proto::ledger_commitment::LedgerCommitment::new( + producer_grant, + 1, + 1, + first.kind(), + crate::writer::commitment_event_hash(&first).unwrap(), + capsem_proto::ledger_commitment::ZERO_COMMITMENT_HASH, + ) + .unwrap(); + let mut second = body_event(Vec::new()); + let WriteOp::NetEvent(second_event) = &mut second else { + unreachable!() + }; + second_event.event_id = Some("0123456789ad".into()); + second_event.domain = "second.test".into(); + let second_commitment = capsem_proto::ledger_commitment::LedgerCommitment::new( + producer_grant, + 2, + 2, + second.kind(), + crate::writer::commitment_event_hash(&second).unwrap(), + first_commitment.commitment_hash(), + ) + .unwrap(); + + assert!(matches!( + request( + &producer, + &replies, + 1, + LedgerCommand::AdmitBatch { + admissions: vec![ + LedgerAdmission::new(first, first_commitment), + LedgerAdmission::new(second, second_commitment), + ], + }, + ) + .await, + LedgerReply::AcceptedBatch { + first_admission_id: 1, + through_admission_id: 2, + accepted: 2, + } + )); + assert!(matches!( + request(&producer, &replies, 2, LedgerCommand::Flush).await, + LedgerReply::Durable { + through_admission_id: 2 + } + )); + drop(producer); + drop(replies); + assert_eq!(task.await.unwrap().unwrap(), LedgerClientExit::Disconnected); +} + #[tokio::test] async fn accepted_durable_query_body_retention_and_warc_run_through_one_owner() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/capsem-logger/src/writer/remote.rs b/crates/capsem-logger/src/writer/remote.rs index 17af2dc59..b0a2b290a 100644 --- a/crates/capsem-logger/src/writer/remote.rs +++ b/crates/capsem-logger/src/writer/remote.rs @@ -8,12 +8,12 @@ use std::time::Duration; use capsem_foundation::ipc_channel; use capsem_proto::ledger::{LedgerChannelGrant, LedgerHello, LedgerOutcome, LedgerRequest}; use capsem_proto::ledger_commitment::{ - CommitmentClientMessage, CommitmentCommand, CommitmentReply, CommitmentServerMessage, LedgerCommitment, - ZERO_COMMITMENT_HASH, + CommitmentClientMessage, CommitmentCommand, CommitmentReply, CommitmentReservation, CommitmentServerMessage, + LedgerCommitment, MAX_COMMITMENTS_PER_CHECKPOINT, ZERO_COMMITMENT_HASH, }; use tokio::sync::oneshot; -use crate::ledger_protocol::{LedgerClientMessage, LedgerCommand, LedgerReply, LedgerServerMessage}; +use crate::ledger_protocol::{LedgerAdmission, LedgerClientMessage, LedgerCommand, LedgerReply, LedgerServerMessage}; use crate::{RetainOutcome, WriteOp}; const STARTUP_TIMEOUT: Duration = Duration::from_secs(10); @@ -65,17 +65,17 @@ enum RemoteMessage { } enum AdmissionMessage { - Operation { - message: RemoteMessage, - commitment: Option, + Operations { + messages: Vec, + commitments: Vec, }, Shutdown, } enum AdmissionResult { - Operation { - message: RemoteMessage, - commitment: Option, + Operations { + messages: Vec, + commitments: Vec, result: Result, }, Shutdown(Result), @@ -304,6 +304,7 @@ fn run( let mut commitments = CommitmentState::new(commitment_sender, commitment_receiver, grant); let mut failure = None; + let mut deferred = None; loop { while let Ok(result) = result_rx.try_recv() { if let Err(error) = apply_admission_result(&runtime, &mut commitments, result) { @@ -314,7 +315,10 @@ fn run( if failure.is_some() { break; } - let message = match messages.recv_timeout(Duration::from_millis(1)) { + let message = match deferred + .take() + .map_or_else(|| messages.recv_timeout(Duration::from_millis(1)), Ok) + { Ok(message) => message, Err(mpsc::RecvTimeoutError::Timeout) => continue, Err(mpsc::RecvTimeoutError::Disconnected) => { @@ -347,28 +351,63 @@ fn run( return; } }; - let commitment = if let RemoteMessage::Admit { event, .. } = &message { + let mut batch = vec![message]; + if matches!(batch.first(), Some(RemoteMessage::Admit { .. })) { + while batch.len() < capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY { + match messages.try_recv() { + Ok(message @ RemoteMessage::Admit { .. }) => batch.push(message), + Ok(message) => { + deferred = Some(message); + break; + } + Err(mpsc::TryRecvError::Empty) if batch.len() == 1 => { + match messages.recv_timeout(Duration::from_micros( + capsem_proto::LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS, + )) { + Ok(message @ RemoteMessage::Admit { .. }) => batch.push(message), + Ok(message) => { + deferred = Some(message); + break; + } + Err(_) => break, + } + } + Err(_) => break, + } + } + } + let reserved = if matches!(batch.first(), Some(RemoteMessage::Admit { .. })) { + let events = batch + .iter() + .map(|message| match message { + RemoteMessage::Admit { event, .. } => event.as_ref(), + _ => unreachable!("admission batch contains only events"), + }) + .collect::>(); match runtime.block_on(async { - tokio::time::timeout(OPERATION_TIMEOUT, commitments.reserve(event)) + tokio::time::timeout(OPERATION_TIMEOUT, commitments.reserve_batch(&events)) .await .map_err(|_| "ledger commitment reservation timed out".to_string())? }) { - Ok(commitment) => Some(commitment), + Ok(commitments) => commitments, Err(error) => { - finish(message, Err(error.clone())); + finish_all(batch, Err(error.clone())); failure = Some(error); break; } } } else { - None + Vec::new() }; - if let Err(error) = admission_tx.send(AdmissionMessage::Operation { message, commitment }) { - let AdmissionMessage::Operation { message, .. } = error.0 else { + if let Err(error) = admission_tx.send(AdmissionMessage::Operations { + messages: batch, + commitments: reserved, + }) { + let AdmissionMessage::Operations { messages, .. } = error.0 else { unreachable!("only operation messages are sent here") }; let reason = "ledger admission stage stopped".to_string(); - finish(message, Err(reason.clone())); + finish_all(messages, Err(reason.clone())); failure = Some(reason); break; } @@ -425,11 +464,11 @@ fn admission_run( let mut request_id = 1_u64; while let Ok(message) = messages.recv() { let (result, response) = match message { - AdmissionMessage::Operation { message, commitment } => { + AdmissionMessage::Operations { messages, commitments } => { let result = runtime.block_on(async { tokio::time::timeout( OPERATION_TIMEOUT, - dispatch_admission(&sender, &receiver, request_id, &message, commitment.as_ref()), + dispatch_admissions(&sender, &receiver, request_id, &messages, &commitments), ) .await .map_err(|_| "ledger client operation timed out".to_string())? @@ -437,9 +476,9 @@ fn admission_run( let failed = result.is_err(); ( failed, - AdmissionResult::Operation { - message, - commitment, + AdmissionResult::Operations { + messages, + commitments, result, }, ) @@ -567,37 +606,58 @@ impl CommitmentState { } } - async fn reserve(&mut self, event: &WriteOp) -> Result { - let producer_sequence = self + async fn reserve_batch(&mut self, events: &[&WriteOp]) -> Result, String> { + if events.is_empty() || events.len() > capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY { + return Err("ledger commitment reservation batch is empty or exceeds its bound".into()); + } + let first_producer_sequence = self .producer_sequence .checked_add(1) .ok_or_else(|| "ledger producer sequence exhausted".to_string())?; - let event_hash = super::operation::commitment_event_hash(event)?; + let reservations = events + .iter() + .map(|event| { + CommitmentReservation::new(event.kind(), super::operation::commitment_event_hash(event)?) + .map_err(|error| error.to_string()) + }) + .collect::, String>>()?; let reply = self - .request(CommitmentCommand::Reserve { - producer_sequence, - event_kind: event.kind().to_string(), - event_hash, + .request(CommitmentCommand::ReserveBatch { + first_producer_sequence, previous_hash: self.previous_hash, + reservations: reservations.clone(), }) .await?; - let CommitmentReply::Reserved { global_sequence } = reply else { + let CommitmentReply::ReservedBatch { global_sequences } = reply else { return Err(format!( - "ledger commitment coordinator returned {reply:?} for reservation" + "ledger commitment coordinator returned {reply:?} for reservation batch" )); }; - let commitment = LedgerCommitment::new( - self.grant, - producer_sequence, - global_sequence, - event.kind(), - event_hash, - self.previous_hash, - ) - .map_err(|error| error.to_string())?; - self.producer_sequence = producer_sequence; - self.previous_hash = commitment.commitment_hash(); - Ok(commitment) + if global_sequences.len() != reservations.len() { + return Err("ledger commitment coordinator returned the wrong batch length".into()); + } + let mut producer_sequence = first_producer_sequence; + let mut previous_hash = self.previous_hash; + let mut commitments = Vec::with_capacity(events.len()); + for (reservation, global_sequence) in reservations.into_iter().zip(global_sequences) { + let commitment = LedgerCommitment::new( + self.grant, + producer_sequence, + global_sequence, + reservation.event_kind(), + reservation.event_hash(), + previous_hash, + ) + .map_err(|error| error.to_string())?; + previous_hash = commitment.commitment_hash(); + commitments.push(commitment); + producer_sequence = producer_sequence + .checked_add(1) + .ok_or_else(|| "ledger producer sequence exhausted".to_string())?; + } + self.producer_sequence = producer_sequence - 1; + self.previous_hash = previous_hash; + Ok(commitments) } async fn anchor_through(&mut self, through_admission_id: u64) -> Result<(), String> { @@ -624,20 +684,32 @@ impl CommitmentState { } } -async fn dispatch_admission( +async fn dispatch_admissions( sender: &ClientSender, receiver: &ClientReceiver, request_id: u64, - message: &RemoteMessage, - commitment: Option<&LedgerCommitment>, + messages: &[RemoteMessage], + commitments: &[LedgerCommitment], ) -> Result { - let command = match message { - RemoteMessage::Admit { event, .. } => LedgerCommand::Admit { - event: event.clone(), - commitment: commitment.cloned().expect("admission was reserved before dispatch"), - }, - RemoteMessage::Flush { .. } => LedgerCommand::Flush, - RemoteMessage::Retain { cutoff, .. } => LedgerCommand::Retain { cutoff: cutoff.clone() }, + let command = match messages { + [RemoteMessage::Flush { .. }] if commitments.is_empty() => LedgerCommand::Flush, + [RemoteMessage::Retain { cutoff, .. }] if commitments.is_empty() => { + LedgerCommand::Retain { cutoff: cutoff.clone() } + } + _ if messages.len() == commitments.len() && !messages.is_empty() => { + let admissions = messages + .iter() + .zip(commitments) + .map(|(message, commitment)| { + let RemoteMessage::Admit { event, .. } = message else { + return Err("ledger admission batch contains a barrier".to_string()); + }; + Ok(LedgerAdmission::new(event.as_ref().clone(), commitment.clone())) + }) + .collect::, String>>()?; + LedgerCommand::AdmitBatch { admissions } + } + _ => return Err("ledger operation batch has inconsistent commitments".into()), }; sender .send(LedgerClientMessage::Request { @@ -672,14 +744,14 @@ async fn flush_admissions( receiver: &ClientReceiver, request_id: u64, ) -> Result { - dispatch_admission( + dispatch_admissions( sender, receiver, request_id, - &RemoteMessage::Flush { + &[RemoteMessage::Flush { reply: oneshot::channel().0, - }, - None, + }], + &[], ) .await } @@ -689,20 +761,33 @@ fn apply_admission_result( commitments: &mut CommitmentState, result: AdmissionResult, ) -> Result<(), String> { - let AdmissionResult::Operation { - message, - commitment, + let AdmissionResult::Operations { + messages, + commitments: reserved, result, } = result else { return Err("ledger admission stage sent an unexpected shutdown result".into()); }; - let result = match (result, commitment) { - (Ok(LedgerReply::Accepted { admission_id }), Some(commitment)) => { - commitments.pending.push((admission_id, commitment)); - Ok(LedgerReply::Accepted { admission_id }) + let result = match (result, reserved.as_slice()) { + ( + Ok(LedgerReply::AcceptedBatch { + first_admission_id, + through_admission_id, + accepted, + }), + reserved, + ) if usize::from(accepted) == reserved.len() => { + commitments + .pending + .extend((first_admission_id..=through_admission_id).zip(reserved.iter().cloned())); + Ok(LedgerReply::AcceptedBatch { + first_admission_id, + through_admission_id, + accepted, + }) } - (Ok(LedgerReply::Durable { through_admission_id }), None) => { + (Ok(LedgerReply::Durable { through_admission_id }), []) => { runtime.block_on(async { tokio::time::timeout(OPERATION_TIMEOUT, commitments.anchor_through(through_admission_id)) .await @@ -710,12 +795,12 @@ fn apply_admission_result( })?; Ok(LedgerReply::Durable { through_admission_id }) } - (Ok(reply @ LedgerReply::Retained { .. }), None) => Ok(reply), + (Ok(reply @ LedgerReply::Retained { .. }), []) => Ok(reply), (Ok(reply), _) => Err(format!("ledger returned {reply:?} for the staged operation")), (Err(error), _) => Err(error), }; let failed = result.as_ref().err().cloned(); - finish(message, result); + finish_all(messages, result); failed.map_or(Ok(()), Err) } @@ -736,10 +821,10 @@ fn apply_shutdown_result( fn fail_admission_message(message: AdmissionMessage, results: &mpsc::Sender, reason: &str) { match message { - AdmissionMessage::Operation { message, commitment } => { - let _ = results.send(AdmissionResult::Operation { - message, - commitment, + AdmissionMessage::Operations { messages, commitments } => { + let _ = results.send(AdmissionResult::Operations { + messages, + commitments, result: Err(reason.into()), }); } @@ -750,8 +835,14 @@ fn fail_admission_message(message: AdmissionMessage, results: &mpsc::Sender, result: Result) { + for message in messages { + finish(message, result.clone()); } } @@ -759,7 +850,7 @@ fn finish(message: RemoteMessage, result: Result) { match message { RemoteMessage::Admit { reply, .. } => { let result = result.and_then(|reply| match reply { - LedgerReply::Accepted { .. } => Ok(()), + LedgerReply::Accepted { .. } | LedgerReply::AcceptedBatch { .. } => Ok(()), reply => Err(format!("ledger returned {reply:?} for admission")), }); if let Some(reply) = reply { @@ -841,6 +932,42 @@ pub(crate) fn test_commitment_channel( }, } } + CommitmentCommand::ReserveBatch { + first_producer_sequence, + previous_hash, + reservations, + } => { + let mut previous_hash = previous_hash; + let mut sequences = Vec::with_capacity(reservations.len()); + let mut failed = None; + for (producer_sequence, reservation) in (first_producer_sequence..).zip(reservations) { + global_sequence += 1; + match LedgerCommitment::new( + grant, + producer_sequence, + global_sequence, + reservation.event_kind(), + reservation.event_hash(), + previous_hash, + ) { + Ok(commitment) => { + previous_hash = commitment.commitment_hash(); + pending.push(commitment); + sequences.push(global_sequence); + } + Err(error) => { + failed = Some(error.to_string()); + break; + } + } + } + failed.map_or( + CommitmentReply::ReservedBatch { + global_sequences: sequences, + }, + |message| CommitmentReply::Failed { message }, + ) + } CommitmentCommand::Cancel { global_sequence } => { if pending.last().map(LedgerCommitment::global_sequence) == Some(global_sequence) { pending.pop(); diff --git a/crates/capsem-logger/src/writer/remote/tests.rs b/crates/capsem-logger/src/writer/remote/tests.rs index 44ac72f20..ce4f8db14 100644 --- a/crates/capsem-logger/src/writer/remote/tests.rs +++ b/crates/capsem-logger/src/writer/remote/tests.rs @@ -144,12 +144,10 @@ async fn descriptor_writer_preserves_blocking_and_try_admission_order() { } #[tokio::test] -async fn the_next_commitment_reservation_overlaps_the_previous_admission() { +async fn consecutive_admissions_share_one_commitment_and_ledger_exchange() { let grant = LedgerChannelGrant::new(LedgerGeneration::new([8; 16]), 12, LedgerClientRole::VmOwner).unwrap(); let (ledger_client, ledger_server) = UnixStream::pair().unwrap(); let (commitment_client, commitment_server) = UnixStream::pair().unwrap(); - let second_reserved = Arc::new(tokio::sync::Notify::new()); - let commitment_signal = Arc::clone(&second_reserved); let commitment_task = tokio::spawn(async move { let (responses, requests) = capsem_foundation::ipc_channel::channel_from_std::< CommitmentServerMessage, @@ -166,35 +164,36 @@ async fn the_next_commitment_reservation_overlaps_the_previous_admission() { }) .await .unwrap(); - let mut sequence = 0_u64; let mut pending = Vec::new(); loop { let CommitmentClientMessage::Request { request_id, command } = requests.recv().await.unwrap() else { panic!("unexpected repeated commitment hello") }; let reply = match command { - capsem_proto::ledger_commitment::CommitmentCommand::Reserve { - producer_sequence, - event_kind, - event_hash, + capsem_proto::ledger_commitment::CommitmentCommand::ReserveBatch { + first_producer_sequence, previous_hash, + reservations, } => { - sequence += 1; - let commitment = LedgerCommitment::new( - grant, - producer_sequence, - sequence, - event_kind, - event_hash, - previous_hash, - ) - .unwrap(); - pending.push(commitment); - if sequence == 2 { - commitment_signal.notify_one(); + assert_eq!(first_producer_sequence, 1); + assert_eq!(reservations.len(), 2); + let mut previous_hash = previous_hash; + for (index, reservation) in reservations.into_iter().enumerate() { + let sequence = u64::try_from(index + 1).unwrap(); + let commitment = LedgerCommitment::new( + grant, + sequence, + sequence, + reservation.event_kind(), + reservation.event_hash(), + previous_hash, + ) + .unwrap(); + previous_hash = commitment.commitment_hash(); + pending.push(commitment); } - CommitmentReply::Reserved { - global_sequence: sequence, + CommitmentReply::ReservedBatch { + global_sequences: vec![1, 2], } } capsem_proto::ledger_commitment::CommitmentCommand::Anchor { commitments } => { @@ -208,7 +207,7 @@ async fn the_next_commitment_reservation_overlaps_the_previous_admission() { .send(CommitmentServerMessage::Response { request_id, reply }) .await .unwrap(); - if sequence == 2 && pending.is_empty() { + if pending.is_empty() { return; } } @@ -227,28 +226,27 @@ async fn the_next_commitment_reservation_overlaps_the_previous_admission() { }) .await .unwrap(); - for admission_id in 1..=2 { - let LedgerClientMessage::Request { request } = requests.recv().await.unwrap() else { - panic!("unexpected repeated ledger hello") - }; - assert!(matches!( - request.operation(), - crate::ledger_protocol::LedgerCommand::Admit { .. } - )); - if admission_id == 1 { - second_reserved.notified().await; - } - responses - .send(LedgerServerMessage::Response { - response: LedgerResponse::success( - request.request_id(), - crate::ledger_protocol::LedgerReply::Accepted { admission_id }, - ) - .unwrap(), - }) - .await - .unwrap(); - } + let LedgerClientMessage::Request { request } = requests.recv().await.unwrap() else { + panic!("unexpected repeated ledger hello") + }; + let crate::ledger_protocol::LedgerCommand::AdmitBatch { admissions } = request.operation() else { + panic!("expected one admission batch") + }; + assert_eq!(admissions.len(), 2); + responses + .send(LedgerServerMessage::Response { + response: LedgerResponse::success( + request.request_id(), + crate::ledger_protocol::LedgerReply::AcceptedBatch { + first_admission_id: 1, + through_admission_id: 2, + accepted: 2, + }, + ) + .unwrap(), + }) + .await + .unwrap(); let LedgerClientMessage::Request { request } = requests.recv().await.unwrap() else { panic!("unexpected repeated ledger hello") }; @@ -282,12 +280,8 @@ async fn the_next_commitment_reservation_overlaps_the_previous_admission() { .unwrap() .unwrap(); - let (first, second) = tokio::join!( - writer.write_checked(event_with_id("one.example", "0123456789ae")), - writer.write_checked(event_with_id("two.example", "0123456789af")), - ); - first.unwrap(); - second.unwrap(); + assert!(writer.try_write(event_with_id("one.example", "0123456789ae"))); + assert!(writer.try_write(event_with_id("two.example", "0123456789af"))); writer.flush_checked().await.unwrap(); commitment_task.await.unwrap(); ledger_task.await.unwrap(); From 596b0b17d25b14777117fb00f556d873d7143d5d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:51:04 +0000 Subject: [PATCH 371/464] refactor(process): use shared ledger queue limit --- crates/capsem-process/src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index f513207df..9cda5b130 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -577,9 +577,9 @@ async fn run_async_main_loop( .await .context("acquire supervised session ledger")?; let ledger_path = session_dir.join("session.db"); - // 1024 queued events: a guest resolving and fetching in parallel enqueues - // several rows per request, and a full queue makes every producer sleep - // in 5 ms steps on its reply path (see `DbWriter::send_with_backpressure`). + // The config-owned queue covers the supported concurrent network burst. + // A full queue makes every producer sleep in 5 ms steps on its reply path + // (see `DbWriter::send_with_backpressure`). // The logical path is diagnostic identity only; the descriptor selects // the worker that exclusively owns and opens the database. let db = Arc::new( @@ -589,7 +589,7 @@ async fn run_async_main_loop( commitment_stream, ledger_grant, &ledger_path, - 1024, + capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, ) }) .await From 5d5f53163c265ff14b1d28a73d4c8468a40f9a5d Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:51:49 +0000 Subject: [PATCH 372/464] refactor(proxy): use shared ledger queue limit --- crates/capsem-proxy/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index 75acdb32a..afbb6abcb 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -223,7 +223,7 @@ async fn run_control( commitment, ledger_grant, Path::new("capability:session-ledger"), - 1024, + capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, ) }) .await From 358676d5897dfc62df27f5c7d4a0cb56ff7730c6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:52:22 +0000 Subject: [PATCH 373/464] test(citadel): guard ledger runtime limits --- tests/citadel/test_runtime_limit_authority.py | 74 ++++++++++++++++--- 1 file changed, 63 insertions(+), 11 deletions(-) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 8900ecd77..2ce6fe207 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -11,12 +11,13 @@ CONFIG = ROOT / "config/data/runtime-limits.toml" RUNTIME_LIMIT_RATIONALE = """\ -The proxy, trusted upstream broker, host DNS session and guest DNS forwarder -must admit the same supported client burst. A pooled traffic client can have an -active, idle and retiring connection at once, so descriptor grants must -preserve the config-owned overlap multiple instead of equalling the client count. Local -numeric ceilings previously let one stage accept work that the next stage shed, -producing intermittent HTTP 502 responses under load. +The proxy, trusted upstream broker, host DNS session, guest DNS forwarder and +security ledger must admit the same supported client burst. A pooled traffic +client can have an active, idle and retiring connection at once, and each +client can produce several audit rows, so downstream capacities must preserve +their config-owned multiples. Local numeric ceilings previously let one stage +accept work that the next stage shed, producing intermittent HTTP 502 and DNS +SERVFAIL responses under load. config/data/runtime-limits.toml is the authority; capsem-proto generates the shared constants and every consumer derives from them. See skills/dev-mitm-proxy/SKILL.md under Cross-process network limits. @@ -33,6 +34,9 @@ "TRAFFIC_GRANT_CAPACITY", "MAX_SELECTIONS", "MAX_ACTIVE_GRANTS", + "LEDGER_PRODUCER_QUEUE_CAPACITY", + "LEDGER_ADMISSION_BATCH_CAPACITY", + "LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS", } REQUIRED_WIRING = { @@ -46,6 +50,14 @@ 'get("file_descriptors_per_client")', ".checked_mul(descriptors_per_client)", "pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit}", + 'get("producer_events_per_network_client")', + ".checked_mul(producer_events_per_network_client)", + 'get("admission_batch_capacity")', + "ledger_queue_capacity % ledger_batch_capacity", + "pub const LEDGER_PRODUCER_QUEUE_CAPACITY: usize = {ledger_queue_capacity}", + "pub const LEDGER_ADMISSION_BATCH_CAPACITY: usize = {ledger_batch_capacity}", + 'get("admission_coalesce_wait_microseconds")', + "pub const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = {ledger_coalesce_wait_microseconds}", ), "crates/capsem-proto/src/lib.rs": ( 'include!(concat!(env!("OUT_DIR"), "/runtime_limits.rs"))', @@ -57,6 +69,7 @@ "crates/capsem-proxy/src/main.rs": ( "UPSTREAM_GRANT_CAPACITY", "NETWORK_PROCESS_OPEN_FILE_LIMIT", + "LEDGER_PRODUCER_QUEUE_CAPACITY", "ensure_open_file_limit", ), "crates/capsem-service/src/main.rs": ( @@ -66,8 +79,15 @@ "crates/capsem-service/src/upstream_broker.rs": ("UPSTREAM_GRANT_CAPACITY",), "crates/capsem-process/src/main.rs": ( "NETWORK_PROCESS_OPEN_FILE_LIMIT", + "LEDGER_PRODUCER_QUEUE_CAPACITY", "ensure_open_file_limit", ), + "crates/capsem-logger/src/writer/remote.rs": ( + "LEDGER_ADMISSION_BATCH_CAPACITY", + "LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS", + "ReserveBatch", + "AdmitBatch", + ), "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/dns_proxy/session.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/net_proxy.rs": ( @@ -93,13 +113,19 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: RUNTIME_LIMIT_RATIONALE + "\nnetwork.max_concurrent_clients must be an integer from 1 through 4096" ) - traffic_grants_per_client = config.get("network", {}).get("traffic_grants_per_client") - assert type(traffic_grants_per_client) is int and 3 <= traffic_grants_per_client <= 4, ( + traffic_grants_per_client = config.get("network", {}).get( + "traffic_grants_per_client" + ) + assert ( + type(traffic_grants_per_client) is int and 3 <= traffic_grants_per_client <= 4 + ), ( RUNTIME_LIMIT_RATIONALE + "\nnetwork.traffic_grants_per_client must cover active, idle and " "retiring pooled connections" ) - descriptors_per_client = config.get("network", {}).get("file_descriptors_per_client") + descriptors_per_client = config.get("network", {}).get( + "file_descriptors_per_client" + ) assert type(descriptors_per_client) is int and 3 <= descriptors_per_client <= 64, ( RUNTIME_LIMIT_RATIONALE + "\nnetwork.file_descriptors_per_client must cover TCP and VSOCK " @@ -111,6 +137,29 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: + "\nnetwork.file_descriptors_per_client must cover both ends of every " "overlapping traffic grant" ) + producer_events_per_network_client = config.get("ledger", {}).get( + "producer_events_per_network_client" + ) + assert ( + type(producer_events_per_network_client) is int + and 1 <= producer_events_per_network_client <= 16 + ), RUNTIME_LIMIT_RATIONALE + ledger_queue_capacity = capacity * producer_events_per_network_client + assert ledger_queue_capacity <= 65_536, RUNTIME_LIMIT_RATIONALE + admission_batch_capacity = config.get("ledger", {}).get("admission_batch_capacity") + assert ( + type(admission_batch_capacity) is int + and 1 <= admission_batch_capacity <= 1024 + and admission_batch_capacity <= ledger_queue_capacity + and ledger_queue_capacity % admission_batch_capacity == 0 + ), RUNTIME_LIMIT_RATIONALE + coalesce_wait_microseconds = config.get("ledger", {}).get( + "admission_coalesce_wait_microseconds" + ) + assert ( + type(coalesce_wait_microseconds) is int + and 1 <= coalesce_wait_microseconds <= 10_000 + ), RUNTIME_LIMIT_RATIONALE missing = [] for relative_path, required_fragments in REQUIRED_WIRING.items(): @@ -123,7 +172,7 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: assert not missing, RUNTIME_LIMIT_RATIONALE + "\n" + "\n".join(missing) -def test_network_limit_names_have_no_numeric_rust_authority() -> None: +def test_runtime_limit_names_have_no_numeric_rust_authority() -> None: sources = { str(path.relative_to(ROOT)): path.read_text() for path in (ROOT / "crates").glob("*/src/**/*.rs") @@ -138,6 +187,9 @@ def test_numeric_local_ceiling_is_rejected_adversarially() -> None: "upstream_broker.rs": "const MAX_ACTIVE_GRANTS: usize = 256;", "dns/session.rs": "pub const DNS_SESSION_MAX_IN_FLIGHT: usize = 512;", "net_proxy.rs": "const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = 4096;", + "process.rs": "const LEDGER_PRODUCER_QUEUE_CAPACITY: usize = 1024;", + "logger.rs": "const LEDGER_ADMISSION_BATCH_CAPACITY: usize = 64;", + "remote.rs": "const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = 100;", } ) - assert len(violations) == 3, RUNTIME_LIMIT_RATIONALE + assert len(violations) == 6, RUNTIME_LIMIT_RATIONALE From 27cd01cd9131ce5bd90cf121bce3985978aed5f3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 22:53:03 +0000 Subject: [PATCH 374/464] docs(skills): centralize ledger load limits --- skills/dev-benchmark/SKILL.md | 6 ++++++ skills/dev-mitm-proxy/SKILL.md | 21 ++++++++++++++------- skills/dev-rust-patterns/SKILL.md | 7 +++++-- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/skills/dev-benchmark/SKILL.md b/skills/dev-benchmark/SKILL.md index 2f6a92a6e..eaa224df5 100644 --- a/skills/dev-benchmark/SKILL.md +++ b/skills/dev-benchmark/SKILL.md @@ -105,6 +105,12 @@ measures scheduler and socket churn rather than the DNS path. The host collector also reads the fully flushed service log and rejects proxy traffic-descriptor refusals that the HTTP client retried internally; zero client errors alone is insufficient. +The DNS lane likewise rejects audit refusals and SERVFAIL responses. When DNS +RPS flattens or errors rise with concurrency, correlate the failed level with +the proxy log before raising a socket cap: synchronous ledger commitment and +admission exchanges can make the audit queue the real bottleneck. Changes to +that path belong in the shared benchmark lane and must keep its strict error +gate; do not replace it with a one-off load script. Guest load collectors sample the complete Capsem process tree. Report CPU, file descriptors, RSS, and, where the platform exposes them, PSS and USS for diff --git a/skills/dev-mitm-proxy/SKILL.md b/skills/dev-mitm-proxy/SKILL.md index 551a3328b..1c39e00b1 100644 --- a/skills/dev-mitm-proxy/SKILL.md +++ b/skills/dev-mitm-proxy/SKILL.md @@ -67,12 +67,17 @@ ceiling. HTTP descriptor-grant capacity multiplies it by while a retiring connection is still closing. The process descriptor floor multiplies it by `file_descriptors_per_client` and must cover both ends of every overlapping -grant. - -Never add a numeric connection ceiling to the proxy, service broker, core DNS -session, or guest DNS forwarder. A limit duplicated across those processes can -make one stage admit work that the next stage sheds, which previously produced -intermittent HTTP 502 responses under load. The guest relay, VM owner, service +grant. The ledger producer queue multiplies the client ceiling by +`producer_events_per_network_client`; its admission batch must divide that +queue exactly. `admission_coalesce_wait_microseconds` gives a lone audit event +a bounded chance to join its immediate followers before the logger performs +the commitment and admission IPC exchanges. + +Never add a numeric connection, ledger queue, batch or coalescing ceiling to +the proxy, service broker, VM owner, logger, core DNS session, or guest DNS +forwarder. A limit duplicated across those processes can make one stage admit +work that the next stage sheds, which previously produced intermittent HTTP +502 and DNS SERVFAIL responses under load. The guest relay, VM owner, service broker and confined proxy must call `capsem_foundation::unix::process::ensure_open_file_limit` with the generated floor before accepting traffic; raising only one process leaves the same cliff @@ -80,7 +85,9 @@ at the next hop. Change the config-owned values, exercise every consumer through the generated constants, and run the official HTTPS and DNS load benchmarks twice before and twice after the change. Any traffic error or capacity refusal fails the run. User overrides belong in the typed settings -contract tracked by issue #349. +contract tracked by issue #349. Extend and run +`tests/citadel/test_runtime_limit_authority.py` whenever a new consumer or +derived limit is added. ## Content-Encoding: the systemic rule diff --git a/skills/dev-rust-patterns/SKILL.md b/skills/dev-rust-patterns/SKILL.md index c1b162448..a798be04e 100644 --- a/skills/dev-rust-patterns/SKILL.md +++ b/skills/dev-rust-patterns/SKILL.md @@ -201,8 +201,11 @@ struct FunctionCall { - **Config-owned cross-process ceilings**: Network concurrency is owned by `config/data/runtime-limits.toml` and generated into `capsem-proto`. Import the shared protocol constant in host and guest consumers; never duplicate a - numeric capacity in individual crates. See `/dev-mitm-proxy` for the load - proof required when it changes. + numeric capacity in individual crates. Ledger queue and batch capacities + derive from the same supported-client ceiling, and bounded coalescing waits + use the generated duration rather than a local sleep literal. Extend the + runtime-limit Citadel guard for every new consumer. See `/dev-mitm-proxy` for + the load proof required when a limit changes. ### Host-serialization locks for per-host critical sections From f9e2fcc4892167582910bfec6429f5a8f695b8cf Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:04:40 +0000 Subject: [PATCH 375/464] fix(bench): expose configured DNS load ceiling --- Cargo.lock | 1 + crates/capsem-bench/Cargo.toml | 1 + crates/capsem-bench/src/cli.rs | 2 ++ crates/capsem-bench/src/dns_load.rs | 11 ++++-- crates/capsem-bench/src/dns_load/tests.rs | 36 +++++++++++++++++++ crates/capsem-bench/src/main.rs | 2 ++ crates/capsem-bench/src/runtime_limits.rs | 23 ++++++++++++ .../capsem-bench/src/runtime_limits/tests.rs | 19 ++++++++++ 8 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 crates/capsem-bench/src/runtime_limits.rs create mode 100644 crates/capsem-bench/src/runtime_limits/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 68889c777..9f3d24ec6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -839,6 +839,7 @@ version = "0.7.0" dependencies = [ "anyhow", "capsem-foundation", + "capsem-proto", "clap", "futures", "reqwest 0.12.28", diff --git a/crates/capsem-bench/Cargo.toml b/crates/capsem-bench/Cargo.toml index 7f79889d9..23549c455 100644 --- a/crates/capsem-bench/Cargo.toml +++ b/crates/capsem-bench/Cargo.toml @@ -21,6 +21,7 @@ path = "src/main.rs" [dependencies] anyhow.workspace = true capsem-foundation = { path = "../capsem-foundation", default-features = false, optional = true } +capsem-proto = { path = "../capsem-proto" } clap.workspace = true futures.workspace = true reqwest.workspace = true diff --git a/crates/capsem-bench/src/cli.rs b/crates/capsem-bench/src/cli.rs index daca46d99..f19aed772 100644 --- a/crates/capsem-bench/src/cli.rs +++ b/crates/capsem-bench/src/cli.rs @@ -24,6 +24,8 @@ pub(crate) enum Command { HttpLoad(crate::http_load::Args), /// Drive sustained DNS load with async UDP clients. DnsLoad(crate::dns_load::Args), + /// Print the config-owned capacities used to shape load curves. + RuntimeLimits, /// Run deterministic protocol scenarios against capsem-mock-server. Protocol(ProtocolArgs), /// Run host-direct and guest-through-Capsem protocol lanes, then report delta. diff --git a/crates/capsem-bench/src/dns_load.rs b/crates/capsem-bench/src/dns_load.rs index 15caca141..fc7cc27b9 100644 --- a/crates/capsem-bench/src/dns_load.rs +++ b/crates/capsem-bench/src/dns_load.rs @@ -118,11 +118,14 @@ fn summarize( request_size: usize, ) -> Report { let total_requests = samples.len(); - let errors = samples.iter().filter(|sample| sample.error.is_some()).count(); + let errors = samples + .iter() + .filter(|sample| sample.error.is_some() || sample.status == 2) + .count(); let successful = total_requests - errors; let mut latencies = samples .iter() - .filter(|sample| sample.error.is_none()) + .filter(|sample| sample.error.is_none() && sample.status != 2) .map(|sample| sample.latency_ms) .collect::>(); let mut decision_distribution = BTreeMap::new(); @@ -131,10 +134,12 @@ fn summarize( if let Some(error) = &sample.error { *error_distribution.entry(error.clone()).or_insert(0) += 1; *decision_distribution.entry("transport_error".to_string()).or_insert(0) += 1; + } else if sample.status == 2 { + *error_distribution.entry("SERVFAIL".to_string()).or_insert(0) += 1; + *decision_distribution.entry("error".to_string()).or_insert(0) += 1; } else { let decision = match sample.status { 0 => "allowed".to_string(), - 2 => "error".to_string(), 3 => "denied".to_string(), status => format!("rcode_{status}"), }; diff --git a/crates/capsem-bench/src/dns_load/tests.rs b/crates/capsem-bench/src/dns_load/tests.rs index 392a3dcb5..abe52bfa5 100644 --- a/crates/capsem-bench/src/dns_load/tests.rs +++ b/crates/capsem-bench/src/dns_load/tests.rs @@ -41,3 +41,39 @@ async fn logical_clients_reuse_one_udp_socket_each() { "one logical client opened more than one UDP socket" ); } + +#[test] +fn servfail_is_a_failed_request_not_a_successful_decision() { + let report = summarize( + vec![ + RequestSample { + status: 2, + size: 40, + latency_ms: 2.0, + attempts: 1, + error: None, + required_text_present: false, + secret_shaped_fixture_seen: false, + }, + RequestSample { + status: 3, + size: 40, + latency_ms: 3.0, + attempts: 1, + error: None, + required_text_present: false, + secret_shaped_fixture_seen: false, + }, + ], + 2, + Duration::from_secs(1), + Duration::from_secs(1), + 40, + ); + + assert_eq!(report.errors, 1); + assert_eq!(report.error_distribution["SERVFAIL"], 1); + assert_eq!(report.decision_distribution["error"], 1); + assert_eq!(report.decision_distribution["denied"], 1); + assert_eq!(report.rps, 1.0); +} diff --git a/crates/capsem-bench/src/main.rs b/crates/capsem-bench/src/main.rs index 9285c19e7..50394ada9 100644 --- a/crates/capsem-bench/src/main.rs +++ b/crates/capsem-bench/src/main.rs @@ -16,6 +16,7 @@ mod protocol_record; #[path = "stats/rates.rs"] mod rates; mod redis; +mod runtime_limits; mod scenarios; #[cfg(feature = "host")] mod schema; @@ -66,6 +67,7 @@ async fn main() -> Result<()> { Command::Ping(args) => println!("{}", serde_json::to_string(&ping::run(args).await?)?), Command::HttpLoad(args) => println!("{}", serde_json::to_string(&http_load::run(args).await?)?), Command::DnsLoad(args) => println!("{}", serde_json::to_string(&dns_load::run(args).await?)?), + Command::RuntimeLimits => println!("{}", serde_json::to_string(&runtime_limits::report())?), Command::Protocol(args) => { #[cfg(feature = "host")] let destination = args.record.clone(); diff --git a/crates/capsem-bench/src/runtime_limits.rs b/crates/capsem-bench/src/runtime_limits.rs new file mode 100644 index 000000000..1fef752e6 --- /dev/null +++ b/crates/capsem-bench/src/runtime_limits.rs @@ -0,0 +1,23 @@ +//! Config-owned capacities exposed to the benchmark curve planner. + +use serde::Serialize; + +#[derive(Debug, Serialize)] +pub(crate) struct Report { + network_client_capacity: usize, + ledger_producer_queue_capacity: usize, + ledger_admission_batch_capacity: usize, + ledger_admission_coalesce_wait_microseconds: u64, +} + +pub(crate) const fn report() -> Report { + Report { + network_client_capacity: capsem_proto::NETWORK_CLIENT_CAPACITY, + ledger_producer_queue_capacity: capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, + ledger_admission_batch_capacity: capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY, + ledger_admission_coalesce_wait_microseconds: capsem_proto::LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS, + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-bench/src/runtime_limits/tests.rs b/crates/capsem-bench/src/runtime_limits/tests.rs new file mode 100644 index 000000000..dcb9645be --- /dev/null +++ b/crates/capsem-bench/src/runtime_limits/tests.rs @@ -0,0 +1,19 @@ +use super::*; + +#[test] +fn benchmark_limits_are_the_protocol_authority() { + let limits = report(); + assert_eq!(limits.network_client_capacity, capsem_proto::NETWORK_CLIENT_CAPACITY); + assert_eq!( + limits.ledger_producer_queue_capacity, + capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY + ); + assert_eq!( + limits.ledger_admission_batch_capacity, + capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY + ); + assert_eq!( + limits.ledger_admission_coalesce_wait_microseconds, + capsem_proto::LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS + ); +} From 892d3149cfd4704c7e3e6c8388dc16c46bc7848a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:05:11 +0000 Subject: [PATCH 376/464] fix(bench): derive DNS curve from runtime limit --- guest/artifacts/capsem_bench/dns_load.py | 34 +++++++++++++++++-- .../test_capsem_bench_mock_server_protocol.py | 32 +++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/guest/artifacts/capsem_bench/dns_load.py b/guest/artifacts/capsem_bench/dns_load.py index d748eee9f..be81d97fc 100644 --- a/guest/artifacts/capsem_bench/dns_load.py +++ b/guest/artifacts/capsem_bench/dns_load.py @@ -9,12 +9,39 @@ DEFAULT_QNAME = "load-test.capsem-bogus" DEFAULT_QTYPE = 1 -DEFAULT_CONCURRENCY = (1, 10, 50, 200) DEFAULT_DURATION_S = 10.0 DEFAULT_TIMEOUT_S = 5.0 RUST_BENCH = "/usr/local/bin/capsem-bench-rs" +def _runtime_limits(): + completed = subprocess.run( + [RUST_BENCH, "runtime-limits"], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode: + raise RuntimeError( + f"dns-load: read runtime limits failed with {completed.returncode}: " + f"{completed.stderr[-2000:].strip()}" + ) + try: + limits = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("dns-load: runtime limits returned invalid JSON") from exc + capacity = limits.get("network_client_capacity") + if type(capacity) is not int or capacity < 1: + raise RuntimeError("dns-load: runtime limits omitted network client capacity") + return limits + + +def _default_concurrency(): + capacity = _runtime_limits()["network_client_capacity"] + levels = (1, capacity // 32, capacity // 8, capacity // 4, capacity // 2, capacity) + return tuple(dict.fromkeys(max(1, level) for level in levels)) + + def _drive_at_concurrency(qname, qtype, concurrency, duration_s, timeout_s): """Run fixed async UDP clients and validate that every client ran.""" completed = subprocess.run( @@ -62,9 +89,10 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N qname = qname or os.environ.get("CAPSEM_BENCH_DNS_QNAME", DEFAULT_QNAME) qtype = qtype or int(os.environ.get("CAPSEM_BENCH_DNS_QTYPE", DEFAULT_QTYPE)) timeout_s = float(os.environ.get("CAPSEM_BENCH_DNS_TIMEOUT", DEFAULT_TIMEOUT_S)) + default_concurrency = _default_concurrency() if concurrency_levels is None else () config = DurationLoadConfig.from_inputs( "dns-load", - default_concurrency=DEFAULT_CONCURRENCY, + default_concurrency=default_concurrency, default_duration_s=DEFAULT_DURATION_S, concurrency_levels=concurrency_levels, duration_s=duration_s, @@ -95,7 +123,7 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N rows.append(row) out = { - "version": "1.1", + "version": "1.2", "qname": qname, "qtype": qtype, "concurrency_levels": rows, diff --git a/tests/test_capsem_bench_mock_server_protocol.py b/tests/test_capsem_bench_mock_server_protocol.py index 0c33ca11e..d93107dce 100644 --- a/tests/test_capsem_bench_mock_server_protocol.py +++ b/tests/test_capsem_bench_mock_server_protocol.py @@ -120,6 +120,38 @@ def run(argv, **kwargs): ] +def test_dns_load_curve_reaches_the_config_owned_client_ceiling(monkeypatch): + monkeypatch.setattr( + dns_load, + "_runtime_limits", + lambda: {"network_client_capacity": 512}, + ) + + assert dns_load._default_concurrency() == (1, 16, 64, 128, 256, 512) + + +def test_dns_load_reads_runtime_limits_from_the_rust_harness(monkeypatch): + calls = [] + + def run(argv, **kwargs): + calls.append((argv, kwargs)) + return types.SimpleNamespace( + returncode=0, + stdout=json.dumps({"network_client_capacity": 512}), + stderr="", + ) + + monkeypatch.setattr(dns_load.subprocess, "run", run) + + assert dns_load._runtime_limits()["network_client_capacity"] == 512 + assert calls == [ + ( + [dns_load.RUST_BENCH, "runtime-limits"], + {"check": False, "capture_output": True, "text": True}, + ) + ] + + def test_dns_load_rejects_starved_driver(monkeypatch): row = {"concurrency": 512, "total_requests": 91, "errors": 0} monkeypatch.setattr( From 1cb9cf00e15ff7a83cdd8b60ad3cefd4bafa61e9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:05:29 +0000 Subject: [PATCH 377/464] test(citadel): guard DNS load capacity source --- tests/citadel/test_runtime_limit_authority.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 2ce6fe207..f1b17d50a 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -94,6 +94,17 @@ "NETWORK_PROCESS_OPEN_FILE_LIMIT", "ensure_open_file_limit", ), + "crates/capsem-bench/src/runtime_limits.rs": ( + "NETWORK_CLIENT_CAPACITY", + "LEDGER_PRODUCER_QUEUE_CAPACITY", + "LEDGER_ADMISSION_BATCH_CAPACITY", + "LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS", + ), + "guest/artifacts/capsem_bench/dns_load.py": ( + '"runtime-limits"', + '"network_client_capacity"', + "capacity // 2", + ), } From 46ced55ddcb1bf7ff6e0eb80fcb6ad69efee7495 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:15:52 +0000 Subject: [PATCH 378/464] perf(proto): absorb ledger flush overlap --- config/data/runtime-limits.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index d2c1cfc7d..6c382dbae 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -11,9 +11,10 @@ traffic_grants_per_client = 3 file_descriptors_per_client = 8 [ledger] -# Each of the two persistent DNS sessions can have one audit row per client -# waiting for the ledger producer. The queue is derived from the network cap. -producer_events_per_network_client = 2 +# A DNS query contributes its primary audit row plus its default-rule row. +# Two persistent DNS sessions can overlap, and the same burst must remain +# admissible while the ledger owner flushes the preceding rows to disk. +producer_events_per_network_client = 8 # One transport exchange carries this many consecutive producer events. The # queue must be an exact multiple so a full burst drains without a tail shape. admission_batch_capacity = 64 From db842e912e3fc7a54ecb8c8d0e333af08b6c4914 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:15:52 +0000 Subject: [PATCH 379/464] perf(logger): use shared producer queue capacity --- CHANGELOG.md | 3 ++- crates/capsem-logger/src/db.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3364f1ded..8eafd8563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -274,7 +274,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ledger producers coalesce bounded consecutive security events into one commitment reservation and admission exchange, avoiding audit backpressure - during concurrent DNS and gateway traffic. + during concurrent DNS and gateway traffic. Their config-derived queues also + absorb supported traffic while the ledger owner flushes earlier rows. - Runtime memory caches use one bounded, concurrent implementation with uniform hit, miss, mutation, load-time and entry metrics. DNS retains its diff --git a/crates/capsem-logger/src/db.rs b/crates/capsem-logger/src/db.rs index 472d60334..da4b66854 100644 --- a/crates/capsem-logger/src/db.rs +++ b/crates/capsem-logger/src/db.rs @@ -276,7 +276,7 @@ impl DbHandle { pub(crate) fn prepare_with_codecs(path: &Path, codecs: ArchiveCodecs) -> rusqlite::Result { let started = Instant::now(); - let writer = DbWriter::prepare_with_codecs(path, 1024, codecs.clone())?; + let writer = DbWriter::prepare_with_codecs(path, capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, codecs.clone())?; // Reads go to the file, like every other handle's: the writer's // memory holds only rows it has not flushed, and SQLite's // `data_version` tells the reader when a flush landed. From d9bf155ae4d7ac37ed1f24b3f4f0f9cd85ba4340 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:15:52 +0000 Subject: [PATCH 380/464] test(citadel): guard ledger owner queue capacity --- tests/citadel/test_runtime_limit_authority.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index f1b17d50a..1d07350b3 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -88,6 +88,7 @@ "ReserveBatch", "AdmitBatch", ), + "crates/capsem-logger/src/db.rs": ("LEDGER_PRODUCER_QUEUE_CAPACITY",), "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/dns_proxy/session.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/net_proxy.rs": ( From b4823bad55a7033d82ed68017404b236f51974dc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:26:41 +0000 Subject: [PATCH 381/464] perf(logger): preserve admission batches through SQLite --- CHANGELOG.md | 6 +-- crates/capsem-logger/src/db.rs | 24 ++++++++++- crates/capsem-logger/src/ledger_server.rs | 22 ++++------ crates/capsem-logger/src/writer.rs | 45 ++++++++++++++++++--- crates/capsem-logger/src/writer/barriers.rs | 4 ++ crates/capsem-logger/src/writer/producer.rs | 18 +++++++++ crates/capsem-logger/src/writer/tests.rs | 15 +++++++ 7 files changed, 110 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8eafd8563..b85838132 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -273,9 +273,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Ledger producers coalesce bounded consecutive security events into one - commitment reservation and admission exchange, avoiding audit backpressure - during concurrent DNS and gateway traffic. Their config-derived queues also - absorb supported traffic while the ledger owner flushes earlier rows. + commitment reservation, admission exchange and SQLite transaction, avoiding + per-row scheduling and audit backpressure during concurrent DNS and gateway + traffic. Their config-derived queues absorb bounded handoff bursts. - Runtime memory caches use one bounded, concurrent implementation with uniform hit, miss, mutation, load-time and entry metrics. DNS retains its diff --git a/crates/capsem-logger/src/db.rs b/crates/capsem-logger/src/db.rs index da4b66854..2daa1b227 100644 --- a/crates/capsem-logger/src/db.rs +++ b/crates/capsem-logger/src/db.rs @@ -276,7 +276,12 @@ impl DbHandle { pub(crate) fn prepare_with_codecs(path: &Path, codecs: ArchiveCodecs) -> rusqlite::Result { let started = Instant::now(); - let writer = DbWriter::prepare_with_codecs(path, capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, codecs.clone())?; + let writer = DbWriter::prepare_with_capacities( + path, + capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY / capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY, + capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY, + codecs.clone(), + )?; // Reads go to the file, like every other handle's: the writer's // memory holds only rows it has not flushed, and SQLite's // `data_version` tells the reader when a flush landed. @@ -784,6 +789,23 @@ impl DbHandle { Ok(()) } + pub(crate) async fn write_committed_batch( + &self, + writes: Vec<(WriteOp, capsem_proto::ledger_commitment::LedgerCommitment)>, + ) -> DbResult<()> { + let affects_session_summary = writes + .iter() + .any(|(op, _)| !matches!(op, WriteOp::PolicyMutationEvent(_) | WriteOp::HostEvent(_))); + let writer = self + .inner + .writer + .as_ref() + .ok_or_else(|| "db handle is read-only; no writer is available".to_string())?; + writer.write_committed_batch(writes).await?; + self.invalidate_after_write(affects_session_summary); + Ok(()) + } + /// Flush accepted writes through the DB-owned writer path. /// /// Tests and read-after-write callers use this as the visibility barrier. diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index 9a8d8a137..e3f3540fc 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -156,21 +156,15 @@ impl LedgerServer { .map_err(|_| "ledger admission batch count exceeds u16".to_string()); match count { Ok(count) => { - let mut failure = None; - for admission in admissions { - let (event, commitment) = admission.into_parts(); - if let Err(error) = self.db.write_committed(event, commitment).await { - failure = Some(storage(error)); - break; - } - admission_state.last_accepted = admission_state - .last_accepted - .checked_add(1) - .ok_or_else(|| "ledger admission id exhausted".to_string())?; - } - let through_admission_id = admission_state.last_accepted; + let through_admission_id = first_admission_id + .checked_add(u64::from(count) - 1) + .ok_or_else(|| "ledger admission id exhausted".to_string())?; + let writes = + admissions.into_iter().map(|admission| admission.into_parts()).collect(); + self.db.write_committed_batch(writes).await.map_err(storage)?; + admission_state.last_accepted = through_admission_id; drop(admission_state); - failure.map_or_else(|| Ok((first_admission_id, through_admission_id, count)), Err) + Ok((first_admission_id, through_admission_id, count)) } Err(error) => Err(error), } diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index bf87151af..0441e5077 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -256,6 +256,10 @@ type RetainReply = tokio::sync::oneshot::Sender>; #[derive(Debug)] enum WriterMessage { Write(WriteEnvelope), + /// A producer batch that must reach the SQLite transaction as one unit. + /// Keeping the vector intact avoids rebuilding a protocol batch through + /// one channel send and one scheduler wakeup per row. + WriteBatch(Vec), /// Commit everything queued before this and report whether the disk flush /// happened. Flush(tokio::sync::oneshot::Sender), @@ -279,6 +283,18 @@ impl WriterMessage { }) } + fn committed_batch(writes: Vec<(WriteOp, capsem_proto::ledger_commitment::LedgerCommitment)>) -> Self { + Self::WriteBatch( + writes + .into_iter() + .map(|(op, commitment)| WriteEnvelope { + op, + commitment: Some(commitment), + }) + .collect(), + ) + } + fn flush(reply: tokio::sync::oneshot::Sender) -> Self { Self::Flush(reply) } @@ -387,12 +403,13 @@ impl DbWriter { Self::prepare_with_clock_and_codecs(path, capacity, now, ArchiveCodecs::default()).map(PreparedDbWriter::start) } - pub(crate) fn prepare_with_codecs( + pub(crate) fn prepare_with_capacities( path: &Path, - capacity: usize, + channel_capacity: usize, + batch_capacity: usize, codecs: ArchiveCodecs, ) -> rusqlite::Result { - Self::prepare_with_clock_and_codecs(path, capacity, SystemTime::now, codecs) + Self::prepare_with_clock_and_capacities(path, channel_capacity, batch_capacity, SystemTime::now, codecs) } fn prepare_with_clock_and_codecs( @@ -400,6 +417,16 @@ impl DbWriter { capacity: usize, now: LedgerClock, codecs: ArchiveCodecs, + ) -> rusqlite::Result { + Self::prepare_with_clock_and_capacities(path, capacity, capacity, now, codecs) + } + + fn prepare_with_clock_and_capacities( + path: &Path, + channel_capacity: usize, + batch_capacity: usize, + now: LedgerClock, + codecs: ArchiveCodecs, ) -> rusqlite::Result { if let Some(parent) = path.parent() { let _ = std::fs::create_dir_all(parent); @@ -407,7 +434,7 @@ impl DbWriter { let mut last_busy = None; for _ in 0..50 { - match Self::prepare_once(path, capacity, now, codecs.clone()) { + match Self::prepare_once(path, channel_capacity, batch_capacity, now, codecs.clone()) { Ok(prepared) => return Ok(prepared), Err(error) if is_sqlite_busy(&error) => { last_busy = Some(error); @@ -424,7 +451,8 @@ impl DbWriter { fn prepare_once( path: &Path, - capacity: usize, + channel_capacity: usize, + batch_capacity: usize, now: LedgerClock, codecs: ArchiveCodecs, ) -> rusqlite::Result { @@ -467,7 +495,12 @@ impl DbWriter { // scan of its rows. let tally = LedgerTally::restored(counters::load(&conn)?); - let (tx, rx) = writer_channel(schedule.capacity); + let batch_capacity = if batch_capacity == 0 { + DEFAULT_BATCH_CAPACITY + } else { + batch_capacity + }; + let (tx, rx) = writer_channel(channel_capacity); let db_path = path.to_path_buf(); let pending_body_bytes = Arc::new(AtomicU64::new(0)); Ok(PreparedDbWriter { diff --git a/crates/capsem-logger/src/writer/barriers.rs b/crates/capsem-logger/src/writer/barriers.rs index 65165f07b..7d4c1e26b 100644 --- a/crates/capsem-logger/src/writer/barriers.rs +++ b/crates/capsem-logger/src/writer/barriers.rs @@ -34,6 +34,10 @@ impl Barriers { batch.push(op); false } + WriterMessage::WriteBatch(mut writes) => { + batch.append(&mut writes); + false + } WriterMessage::Flush(reply) => { self.flushes.push(reply); true diff --git a/crates/capsem-logger/src/writer/producer.rs b/crates/capsem-logger/src/writer/producer.rs index aad8c701d..4bb2bea61 100644 --- a/crates/capsem-logger/src/writer/producer.rs +++ b/crates/capsem-logger/src/writer/producer.rs @@ -58,6 +58,24 @@ impl DbWriter { .map_err(|error| format!("db writer channel closed: {error}")) } + pub(crate) async fn write_committed_batch( + &self, + writes: Vec<(WriteOp, capsem_proto::ledger_commitment::LedgerCommitment)>, + ) -> Result<(), String> { + if self.remote.is_some() { + return Err("a remote ledger writer cannot accept prebuilt commitments".into()); + } + if writes.is_empty() || writes.len() > capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY { + return Err("committed write batch is empty or exceeds its bound".into()); + } + let Some(tx) = self.clone_sender() else { + return Err("db writer sender missing".to_string()); + }; + send_with_backpressure(&tx, WriterMessage::committed_batch(writes)) + .await + .map_err(|error| format!("db writer channel closed: {error}")) + } + /// Try to enqueue without blocking. Returns false when the queue is full or closed. pub fn try_write(&self, op: WriteOp) -> bool { let span = tracing::debug_span!( diff --git a/crates/capsem-logger/src/writer/tests.rs b/crates/capsem-logger/src/writer/tests.rs index 515ae66ae..3bc859b8f 100644 --- a/crates/capsem-logger/src/writer/tests.rs +++ b/crates/capsem-logger/src/writer/tests.rs @@ -684,6 +684,21 @@ fn writer_channel_capacity_applies_backpressure() { )); } +#[test] +fn queued_write_batch_reaches_one_writer_transaction_batch() { + let mut barriers = Barriers::default(); + let mut batch = Vec::new(); + let writes = vec![ + WriteEnvelope::plain(file_event_with_credential("/batch/one", None)), + WriteEnvelope::plain(file_event_with_credential("/batch/two", None)), + ]; + + assert!(!barriers.accept(WriterMessage::WriteBatch(writes), &mut batch)); + assert_eq!(batch.len(), 2); + assert!(matches!(&batch[0].op, WriteOp::FileEvent(event) if event.path == "/batch/one")); + assert!(matches!(&batch[1].op, WriteOp::FileEvent(event) if event.path == "/batch/two")); +} + #[test] fn brokered_substitution_persists_reference_and_not_secret() { let dir = tempfile::tempdir().unwrap(); From d0b41c28f7a9b9e6d52c6f81c5fc486ef14691d6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:26:41 +0000 Subject: [PATCH 382/464] docs(config): explain ledger event budget --- config/data/runtime-limits.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index 6c382dbae..daa01bde9 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -11,9 +11,9 @@ traffic_grants_per_client = 3 file_descriptors_per_client = 8 [ledger] -# A DNS query contributes its primary audit row plus its default-rule row. -# Two persistent DNS sessions can overlap, and the same burst must remain -# admissible while the ledger owner flushes the preceding rows to disk. +# A DNS query contributes a primary row, a decision transition and a rule row. +# Two persistent DNS sessions can overlap; two additional event slots per +# client absorb the bounded handoff between protocol and SQLite batches. producer_events_per_network_client = 8 # One transport exchange carries this many consecutive producer events. The # queue must be an exact multiple so a full burst drains without a tail shape. From a9648876d7d8bfd01854b5aa9bf3790f0d1bbbb3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:26:41 +0000 Subject: [PATCH 383/464] test(citadel): guard ledger batch authority --- tests/citadel/test_runtime_limit_authority.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 1d07350b3..b7812de33 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -88,7 +88,10 @@ "ReserveBatch", "AdmitBatch", ), - "crates/capsem-logger/src/db.rs": ("LEDGER_PRODUCER_QUEUE_CAPACITY",), + "crates/capsem-logger/src/db.rs": ( + "LEDGER_PRODUCER_QUEUE_CAPACITY", + "LEDGER_ADMISSION_BATCH_CAPACITY", + ), "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/dns_proxy/session.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/net_proxy.rs": ( From 2431bf16887bead32662752c8e5ef9529b5568cf Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:34:52 +0000 Subject: [PATCH 384/464] perf(proto): bound ledger disk flush interval --- config/data/runtime-limits.toml | 5 +++++ crates/capsem-proto/build.rs | 23 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index daa01bde9..3ca77e97d 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -21,3 +21,8 @@ admission_batch_capacity = 64 # A lone event waits briefly for its immediate followers. This is far below a # network round trip while avoiding singleton IPC under concurrent bursts. admission_coalesce_wait_microseconds = 100 +# A fail-closed network action waits at most one second for admission. Flush +# often enough that copying preceding memory rows cannot consume that budget. +disk_flush_interval_milliseconds = 1000 +# A burst also flushes before the timer at this exact batch multiple. +disk_flush_max_pending_events = 1048576 diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index 4210edd5a..b3015b904 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -105,6 +105,23 @@ fn write_runtime_limits() { .unwrap_or_else(|| { panic!("runtime limits: ledger.admission_coalesce_wait_microseconds must be an integer between 1 and 10000") }); + let ledger_disk_flush_interval_milliseconds = config + .get("ledger") + .and_then(|ledger| ledger.get("disk_flush_interval_milliseconds")) + .and_then(toml::Value::as_integer) + .filter(|value| (100..=5_000).contains(value)) + .unwrap_or_else(|| { + panic!("runtime limits: ledger.disk_flush_interval_milliseconds must be an integer between 100 and 5000") + }); + let ledger_disk_flush_max_pending_events = config + .get("ledger") + .and_then(|ledger| ledger.get("disk_flush_max_pending_events")) + .and_then(toml::Value::as_integer) + .filter(|value| (ledger_batch_capacity..=10_000_000).contains(value)) + .filter(|value| value % ledger_batch_capacity == 0) + .unwrap_or_else(|| { + panic!("runtime limits: ledger.disk_flush_max_pending_events must be a batch multiple between the batch capacity and 10000000") + }); let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set"); std::fs::write( format!("{out_dir}/runtime_limits.rs"), @@ -123,7 +140,11 @@ fn write_runtime_limits() { /// Consecutive producer events coalesced into one ledger exchange.\n\ pub const LEDGER_ADMISSION_BATCH_CAPACITY: usize = {ledger_batch_capacity};\n\ /// Maximum wait for an immediate follower before admitting a singleton.\n\ - pub const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = {ledger_coalesce_wait_microseconds};\n" + pub const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = {ledger_coalesce_wait_microseconds};\n\ + /// Maximum time accepted rows stay only in the memory schema.\n\ + pub const LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS: u64 = {ledger_disk_flush_interval_milliseconds};\n\ + /// Pending events that force a disk flush before the timer.\n\ + pub const LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS: usize = {ledger_disk_flush_max_pending_events};\n" ), ) .expect("runtime limits: write generated constants"); From 81632044e04f0bfb1ccee441f0a27215ce82f9e3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:34:59 +0000 Subject: [PATCH 385/464] perf(logger): keep periodic flush below admission deadline --- CHANGELOG.md | 4 +++- crates/capsem-logger/benches/db_read_write_micro.rs | 2 +- crates/capsem-logger/src/writer.rs | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b85838132..cc7dfcee8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -275,7 +275,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ledger producers coalesce bounded consecutive security events into one commitment reservation, admission exchange and SQLite transaction, avoiding per-row scheduling and audit backpressure during concurrent DNS and gateway - traffic. Their config-derived queues absorb bounded handoff bursts. + traffic. Their config-derived queues absorb bounded handoff bursts, and a + one-second disk flush interval keeps each durable copy inside the fail-closed + network admission budget. - Runtime memory caches use one bounded, concurrent implementation with uniform hit, miss, mutation, load-time and entry metrics. DNS retains its diff --git a/crates/capsem-logger/benches/db_read_write_micro.rs b/crates/capsem-logger/benches/db_read_write_micro.rs index 8771559b6..ccc3f4beb 100644 --- a/crates/capsem-logger/benches/db_read_write_micro.rs +++ b/crates/capsem-logger/benches/db_read_write_micro.rs @@ -19,7 +19,7 @@ const READ_ROWS: usize = 1_000_000; /// of the block that is still being written. const ARCHIVE_EXCHANGES: usize = 140; /// A disk flush every this many exchanges, standing in for the writer's -/// five-second timer on a live session. +/// periodic timer on a live session. const ARCHIVE_FLUSH_EVERY: usize = 10; /// Timings of the archive path a polled route sees. diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 0441e5077..08ea5c42f 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -78,8 +78,8 @@ const DEFAULT_BATCH_CAPACITY: usize = 10_000; /// spent its accept path waiting on ten disk flushes and was accepted 3-4x /// slower (see `benches/db_read_write_micro.rs`), where the whole point of /// the memory schema is that accepting a row never waits on the disk. -const DISK_FLUSH_THRESHOLD_OPS: usize = 1_000_000; -const DISK_FLUSH_INTERVAL: Duration = Duration::from_secs(5); +const DISK_FLUSH_THRESHOLD_OPS: usize = capsem_proto::LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS; +const DISK_FLUSH_INTERVAL: Duration = Duration::from_millis(capsem_proto::LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS); #[derive(Clone, Copy)] struct WriterSchedule { From f626708829b6e79c710662a57fd4eac387b7f9c3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:35:05 +0000 Subject: [PATCH 386/464] feat(bench): expose ledger flush limits --- crates/capsem-bench/src/runtime_limits.rs | 4 ++++ crates/capsem-bench/src/runtime_limits/tests.rs | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/crates/capsem-bench/src/runtime_limits.rs b/crates/capsem-bench/src/runtime_limits.rs index 1fef752e6..3199dd3f5 100644 --- a/crates/capsem-bench/src/runtime_limits.rs +++ b/crates/capsem-bench/src/runtime_limits.rs @@ -8,6 +8,8 @@ pub(crate) struct Report { ledger_producer_queue_capacity: usize, ledger_admission_batch_capacity: usize, ledger_admission_coalesce_wait_microseconds: u64, + ledger_disk_flush_interval_milliseconds: u64, + ledger_disk_flush_max_pending_events: usize, } pub(crate) const fn report() -> Report { @@ -16,6 +18,8 @@ pub(crate) const fn report() -> Report { ledger_producer_queue_capacity: capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, ledger_admission_batch_capacity: capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY, ledger_admission_coalesce_wait_microseconds: capsem_proto::LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS, + ledger_disk_flush_interval_milliseconds: capsem_proto::LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS, + ledger_disk_flush_max_pending_events: capsem_proto::LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS, } } diff --git a/crates/capsem-bench/src/runtime_limits/tests.rs b/crates/capsem-bench/src/runtime_limits/tests.rs index dcb9645be..335064ed6 100644 --- a/crates/capsem-bench/src/runtime_limits/tests.rs +++ b/crates/capsem-bench/src/runtime_limits/tests.rs @@ -16,4 +16,12 @@ fn benchmark_limits_are_the_protocol_authority() { limits.ledger_admission_coalesce_wait_microseconds, capsem_proto::LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS ); + assert_eq!( + limits.ledger_disk_flush_interval_milliseconds, + capsem_proto::LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS + ); + assert_eq!( + limits.ledger_disk_flush_max_pending_events, + capsem_proto::LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS + ); } From 2a1f07ab4117d378ba7db5fc2412e6c00adcab58 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:35:13 +0000 Subject: [PATCH 387/464] test(citadel): guard ledger flush limits --- tests/citadel/test_runtime_limit_authority.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index b7812de33..ef31af1a4 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -58,6 +58,10 @@ "pub const LEDGER_ADMISSION_BATCH_CAPACITY: usize = {ledger_batch_capacity}", 'get("admission_coalesce_wait_microseconds")', "pub const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = {ledger_coalesce_wait_microseconds}", + 'get("disk_flush_interval_milliseconds")', + "pub const LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS: u64 = {ledger_disk_flush_interval_milliseconds}", + 'get("disk_flush_max_pending_events")', + "pub const LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS: usize = {ledger_disk_flush_max_pending_events}", ), "crates/capsem-proto/src/lib.rs": ( 'include!(concat!(env!("OUT_DIR"), "/runtime_limits.rs"))', @@ -92,6 +96,10 @@ "LEDGER_PRODUCER_QUEUE_CAPACITY", "LEDGER_ADMISSION_BATCH_CAPACITY", ), + "crates/capsem-logger/src/writer.rs": ( + "LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS", + "LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS", + ), "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/dns_proxy/session.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/net_proxy.rs": ( @@ -175,6 +183,21 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: type(coalesce_wait_microseconds) is int and 1 <= coalesce_wait_microseconds <= 10_000 ), RUNTIME_LIMIT_RATIONALE + disk_flush_interval_milliseconds = config.get("ledger", {}).get( + "disk_flush_interval_milliseconds" + ) + assert ( + type(disk_flush_interval_milliseconds) is int + and 100 <= disk_flush_interval_milliseconds <= 5_000 + ), RUNTIME_LIMIT_RATIONALE + disk_flush_max_pending_events = config.get("ledger", {}).get( + "disk_flush_max_pending_events" + ) + assert ( + type(disk_flush_max_pending_events) is int + and admission_batch_capacity <= disk_flush_max_pending_events <= 10_000_000 + and disk_flush_max_pending_events % admission_batch_capacity == 0 + ), RUNTIME_LIMIT_RATIONALE missing = [] for relative_path, required_fragments in REQUIRED_WIRING.items(): From 1ef432c78a167df532510cf818162b21d326f089 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:47:19 +0000 Subject: [PATCH 388/464] perf(logger): expose slow disk flushes --- crates/capsem-logger/src/writer.rs | 44 +++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 08ea5c42f..2d06ad15f 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -815,16 +815,14 @@ fn writer_loop( match rx.recv_timeout(interval.saturating_sub(last_disk_flush.elapsed())) { Ok(message) => Some(message), Err(mpsc::RecvTimeoutError::Timeout) => { - // A failed attempt retains its dirty rows but must not - // keep receiving against an already-expired deadline. - last_disk_flush = Instant::now(); - if let Err(error) = flush_dirty_tables_to_disk( + if let Err(error) = flush_dirty_tables_to_disk_recorded( &conn, &mut dirty_tables, &mut flush_watermarks, db_path.as_deref(), &mut bodies, tally.counters(), + dirty_ops, ) { warn!(error = %error, "db interval flush failed"); } else { @@ -902,14 +900,14 @@ fn writer_loop( Err(format!("db rejected {rejected_ops} accepted write operations")) }; if disk_flush_due { - last_disk_flush = Instant::now(); - match flush_dirty_tables_to_disk( + match flush_dirty_tables_to_disk_recorded( &conn, &mut dirty_tables, &mut flush_watermarks, db_path.as_deref(), &mut bodies, tally.counters(), + dirty_ops, ) { Ok(()) => { dirty_ops = 0; @@ -942,13 +940,14 @@ fn writer_loop( // The open block ends with its FINAL segment, committed by this flush. bodies.close_block(); - if let Err(error) = flush_dirty_tables_to_disk( + if let Err(error) = flush_dirty_tables_to_disk_recorded( &conn, &mut dirty_tables, &mut flush_watermarks, db_path.as_deref(), &mut bodies, tally.counters(), + dirty_ops, ) { warn!(error = %error, "db shutdown dirty table flush failed"); // There is no next flush to retry into, so the bodies still waiting @@ -1046,6 +1045,37 @@ fn flush_dirty_tables_to_disk( Ok(()) } +fn flush_dirty_tables_to_disk_recorded( + conn: &Connection, + dirty_tables: &mut BTreeSet<&'static str>, + flush_watermarks: &mut schema::MemoryFlushWatermarks, + db_path: Option<&Path>, + bodies: &mut BodyArchive, + counters: &LedgerCounters, + pending_ops: usize, +) -> rusqlite::Result<()> { + let table_count = dirty_tables.len(); + let started = Instant::now(); + let result = flush_dirty_tables_to_disk(conn, dirty_tables, flush_watermarks, db_path, bodies, counters); + let elapsed = started.elapsed(); + let status = if result.is_ok() { "ok" } else { "error" }; + tracing::debug!( + target: "capsem.db", + duration_ms = elapsed.as_secs_f64() * 1000.0, + pending_ops, + table_count, + status, + "db disk flush completed" + ); + if elapsed >= DISK_FLUSH_INTERVAL { + warn!( + duration_ms = elapsed.as_secs_f64() * 1000.0, + pending_ops, table_count, status, "db disk flush exceeded its configured interval" + ); + } + result +} + /// Execute through the connection's prepared-statement cache. /// /// Every insert here is one of a small fixed set of statements (one per From fd20fbeac30bbe356dc99ccd917efc106307e0ad Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:52:57 +0000 Subject: [PATCH 389/464] perf(logger): split slow flush phase timing --- crates/capsem-logger/src/writer.rs | 60 +++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 2d06ad15f..0533a49f1 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -994,16 +994,17 @@ fn flush_dirty_tables_to_disk( db_path: Option<&Path>, bodies: &mut BodyArchive, counters: &LedgerCounters, -) -> rusqlite::Result<()> { +) -> rusqlite::Result { if dirty_tables.is_empty() && !bodies.has_work() { - return Ok(()); + return Ok(FlushTimings::default()); } let started = Instant::now(); // Bytes before index: the segment reaches the archive file here, and the // rows that name it are inserted in the transaction below. A crash in // between costs unreferenced bytes, never a row pointing past EOF. + let started = Instant::now(); bodies.flush_segment(); - let archive_finished = Instant::now(); + let archive_ms = started.elapsed().as_secs_f64() * 1000.0; let tables: Vec<&'static str> = dirty_tables.iter().copied().collect(); let tx = conn.unchecked_transaction()?; bodies.commit_index_rows(&tx)?; @@ -1016,14 +1017,18 @@ fn flush_dirty_tables_to_disk( "injected disk flush failure before copy".to_string(), )); } + let started = Instant::now(); let advanced_watermarks = schema::with_memory_schema_lock(|| { schema::flush_memory_tables_to_disk(&tx, tables.iter().copied(), flush_watermarks) })?; - let tables_finished = Instant::now(); + let table_copy_ms = started.elapsed().as_secs_f64() * 1000.0; // The snapshot lands with the rows it counts, or neither does. + let started = Instant::now(); counters::store(&tx, counters)?; + let counter_snapshot_ms = started.elapsed().as_secs_f64() * 1000.0; + let started = Instant::now(); tx.commit()?; - let commit_finished = Instant::now(); + let commit_ms = started.elapsed().as_secs_f64() * 1000.0; // Only now are the written segments indexed: anything above this line rolls // the transaction back, and the next flush retries both halves together. bodies.index_rows_committed(); @@ -1032,17 +1037,20 @@ fn flush_dirty_tables_to_disk( schema::record_sqlite_mmap_telemetry(conn, path, "writer", "flush"); } dirty_tables.clear(); - tracing::debug!( - target: "capsem.db", - duration_ms = started.elapsed().as_secs_f64() * 1000.0, - archive_ms = archive_finished.duration_since(started).as_secs_f64() * 1000.0, - index_ms = index_finished.duration_since(archive_finished).as_secs_f64() * 1000.0, - tables_ms = tables_finished.duration_since(index_finished).as_secs_f64() * 1000.0, - commit_ms = commit_finished.duration_since(tables_finished).as_secs_f64() * 1000.0, - tables = tables.len(), - "db disk flush completed" - ); - Ok(()) + Ok(FlushTimings { + archive_ms, + table_copy_ms, + counter_snapshot_ms, + commit_ms, + }) +} + +#[derive(Default)] +struct FlushTimings { + archive_ms: f64, + table_copy_ms: f64, + counter_snapshot_ms: f64, + commit_ms: f64, } fn flush_dirty_tables_to_disk_recorded( @@ -1059,9 +1067,18 @@ fn flush_dirty_tables_to_disk_recorded( let result = flush_dirty_tables_to_disk(conn, dirty_tables, flush_watermarks, db_path, bodies, counters); let elapsed = started.elapsed(); let status = if result.is_ok() { "ok" } else { "error" }; + let timings = result.as_ref().ok(); + let archive_ms = timings.map_or(0.0, |timings| timings.archive_ms); + let table_copy_ms = timings.map_or(0.0, |timings| timings.table_copy_ms); + let counter_snapshot_ms = timings.map_or(0.0, |timings| timings.counter_snapshot_ms); + let commit_ms = timings.map_or(0.0, |timings| timings.commit_ms); tracing::debug!( target: "capsem.db", duration_ms = elapsed.as_secs_f64() * 1000.0, + archive_ms, + table_copy_ms, + counter_snapshot_ms, + commit_ms, pending_ops, table_count, status, @@ -1070,10 +1087,17 @@ fn flush_dirty_tables_to_disk_recorded( if elapsed >= DISK_FLUSH_INTERVAL { warn!( duration_ms = elapsed.as_secs_f64() * 1000.0, - pending_ops, table_count, status, "db disk flush exceeded its configured interval" + archive_ms, + table_copy_ms, + counter_snapshot_ms, + commit_ms, + pending_ops, + table_count, + status, + "db disk flush exceeded its configured interval" ); } - result + result.map(|_| ()) } /// Execute through the connection's prepared-statement cache. From 8252f3e50d32d7317db7b6e46579148087f5c9f1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Tue, 13 Oct 2026 23:58:11 +0000 Subject: [PATCH 390/464] perf(logger): keep telemetry off live flushes --- CHANGELOG.md | 3 ++- crates/capsem-logger/src/writer.rs | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc7dfcee8..ffce38614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -277,7 +277,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 per-row scheduling and audit backpressure during concurrent DNS and gateway traffic. Their config-derived queues absorb bounded handoff bursts, and a one-second disk flush interval keeps each durable copy inside the fail-closed - network admission budget. + network admission budget. SQLite mmap and file-size diagnostics run at open + and shutdown instead of pausing live admissions after every flush. - Runtime memory caches use one bounded, concurrent implementation with uniform hit, miss, mutation, load-time and entry metrics. DNS retains its diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 0533a49f1..58169dcc4 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -971,6 +971,13 @@ fn writer_loop( let status = if result.is_ok() { "ok" } else { "error" }; ::metrics::histogram!(DB_SHUTDOWN_FLUSH_MS, "status" => status).record(elapsed_ms); span.record("status", status); + // mmap coverage and file-size diagnostics query SQLite and the filesystem. + // Sampling them on every periodic flush once held this admission thread + // for more than a second under DNS load. Shutdown is outside the producer + // path and still records the final ledger size and mmap budget. + if let Some(path) = db_path.as_deref() { + schema::record_sqlite_mmap_telemetry(&conn, path, "writer", "shutdown"); + } } #[derive(Clone, Copy)] @@ -1033,9 +1040,6 @@ fn flush_dirty_tables_to_disk( // the transaction back, and the next flush retries both halves together. bodies.index_rows_committed(); flush_watermarks.extend(advanced_watermarks); - if let Some(path) = db_path { - schema::record_sqlite_mmap_telemetry(conn, path, "writer", "flush"); - } dirty_tables.clear(); Ok(FlushTimings { archive_ms, From 7738e0f777657017f9e6d9f4558dcd3caa98ae65 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 00:05:56 +0000 Subject: [PATCH 391/464] feat(archive): expose independent durability handle --- crates/capsem-archive/src/lib.rs | 2 +- crates/capsem-archive/src/writer.rs | 22 ++++++++++++++++++++++ crates/capsem-archive/src/writer/tests.rs | 16 ++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/crates/capsem-archive/src/lib.rs b/crates/capsem-archive/src/lib.rs index 8dc1cd9f6..0760d4b77 100644 --- a/crates/capsem-archive/src/lib.rs +++ b/crates/capsem-archive/src/lib.rs @@ -35,7 +35,7 @@ pub use format::{ pub use reader::{BlockExtent, BodyLogReader}; pub use retain::{commit_retained, stage_retained_blocks, validate_block_extent, RetainedStaging}; pub use warc::{write_record, WarcRecord}; -pub use writer::{BodyLogWriter, SegmentWritten}; +pub use writer::{BodyLogSync, BodyLogWriter, SegmentWritten}; #[cfg(test)] mod tests; diff --git a/crates/capsem-archive/src/writer.rs b/crates/capsem-archive/src/writer.rs index 92f5daa0c..7a56ce19e 100644 --- a/crates/capsem-archive/src/writer.rs +++ b/crates/capsem-archive/src/writer.rs @@ -23,6 +23,7 @@ use std::ffi::OsStr; use std::fs::File; use std::io::{self, Read, Seek, SeekFrom, Write}; +use std::os::fd::AsFd as _; use std::os::unix::fs::PermissionsExt; use std::path::Path; use std::time::Instant; @@ -89,6 +90,20 @@ pub struct BodyLogWriter { fail_write_after: Option, } +/// A duplicated archive descriptor that can run the durability barrier on a +/// helper thread while the writer continues appending later segments. +pub struct BodyLogSync { + file: File, +} + +impl BodyLogSync { + /// Make every archive write completed before this call durable. + pub fn sync(&self) -> Result<()> { + unix_fs::durable_sync_file(&self.file)?; + Ok(()) + } +} + impl Drop for BodyLogWriter { fn drop(&mut self) { // Not while unwinding, and not when poisoned: a poisoned writer can @@ -103,6 +118,13 @@ impl Drop for BodyLogWriter { } impl BodyLogWriter { + /// Duplicate the archive descriptor for an independent sync worker. + pub fn sync_handle(&self) -> Result { + Ok(BodyLogSync { + file: File::from(capsem_foundation::unix::fd::duplicate(self.file.as_fd())?), + }) + } + /// Create or reopen `session.bodies`. /// /// Creates the file with mode 0o600, refuses a symlink, and opens without diff --git a/crates/capsem-archive/src/writer/tests.rs b/crates/capsem-archive/src/writer/tests.rs index 31b94f243..fde78809d 100644 --- a/crates/capsem-archive/src/writer/tests.rs +++ b/crates/capsem-archive/src/writer/tests.rs @@ -60,6 +60,22 @@ fn a_flush_with_nothing_staged_writes_nothing() { assert_eq!(file_len(&path), end); } +#[test] +fn a_duplicated_sync_handle_can_flush_while_the_writer_stays_open() { + let dir = tempfile::tempdir().unwrap(); + let path = archive(&dir); + let mut writer = BodyLogWriter::open(&path).unwrap(); + writer.stage(b"first body").unwrap(); + writer.flush_segment().unwrap().expect("one segment"); + + let sync = writer.sync_handle().unwrap(); + sync.sync().unwrap(); + + writer.stage(b"later body").unwrap(); + assert!(writer.flush_segment().unwrap().is_some()); + writer.sync().unwrap(); +} + /// The whole point of the format: one block across many flushes, its raw /// and on-disk lengths growing under one offset, only the first segment /// carrying the block header. From 27c8db7f0dcfcba063a7cb5ba6cbc7dd249eb372 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 00:14:00 +0000 Subject: [PATCH 392/464] perf(logger): move archive sync off admission path --- CHANGELOG.md | 4 +- crates/capsem-logger/src/writer.rs | 44 ++- crates/capsem-logger/src/writer/bodies.rs | 305 ++++++++++++++---- .../capsem-logger/src/writer/tests/bodies.rs | 32 ++ .../src/writer/tests/counters.rs | 1 + 5 files changed, 313 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffce38614..4ca46687c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -278,7 +278,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 traffic. Their config-derived queues absorb bounded handoff bursts, and a one-second disk flush interval keeps each durable copy inside the fail-closed network admission budget. SQLite mmap and file-size diagnostics run at open - and shutdown instead of pausing live admissions after every flush. + and shutdown instead of pausing live admissions after every flush. Body + archive durability runs on a small ledger-owned helper, and SQLite publishes + an archive index row only after the matching bytes reach durable storage. - Runtime memory caches use one bounded, concurrent implementation with uniform hit, miss, mutation, load-time and entry metrics. DNS retains its diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 58169dcc4..26b51e521 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -809,7 +809,7 @@ fn writer_loop( // 1. Block until at least one op arrives. Returns None when all // Senders are dropped (clean shutdown) and ends the loop. loop { - let first_message = if dirty_ops == 0 { + let first_message = if dirty_ops == 0 && !bodies.has_work() { rx.recv().ok() } else { match rx.recv_timeout(interval.saturating_sub(last_disk_flush.elapsed())) { @@ -822,7 +822,10 @@ fn writer_loop( db_path.as_deref(), &mut bodies, tally.counters(), - dirty_ops, + FlushRequest { + pending_ops: dirty_ops, + wait_for_body_sync: false, + }, ) { warn!(error = %error, "db interval flush failed"); } else { @@ -907,7 +910,10 @@ fn writer_loop( db_path.as_deref(), &mut bodies, tally.counters(), - dirty_ops, + FlushRequest { + pending_ops: dirty_ops, + wait_for_body_sync: barriers.waiting(), + }, ) { Ok(()) => { dirty_ops = 0; @@ -947,7 +953,10 @@ fn writer_loop( db_path.as_deref(), &mut bodies, tally.counters(), - dirty_ops, + FlushRequest { + pending_ops: dirty_ops, + wait_for_body_sync: true, + }, ) { warn!(error = %error, "db shutdown dirty table flush failed"); // There is no next flush to retry into, so the bodies still waiting @@ -1001,6 +1010,7 @@ fn flush_dirty_tables_to_disk( db_path: Option<&Path>, bodies: &mut BodyArchive, counters: &LedgerCounters, + wait_for_body_sync: bool, ) -> rusqlite::Result { if dirty_tables.is_empty() && !bodies.has_work() { return Ok(FlushTimings::default()); @@ -1011,11 +1021,11 @@ fn flush_dirty_tables_to_disk( // between costs unreferenced bytes, never a row pointing past EOF. let started = Instant::now(); bodies.flush_segment(); + bodies.prepare_index_rows(wait_for_body_sync); let archive_ms = started.elapsed().as_secs_f64() * 1000.0; let tables: Vec<&'static str> = dirty_tables.iter().copied().collect(); let tx = conn.unchecked_transaction()?; - bodies.commit_index_rows(&tx)?; - let index_finished = Instant::now(); + bodies.commit_prepared_index_rows(&tx)?; // Injected where a real flush failure lands: inside the transaction, with // the archive's index rows already written into it. Everything here rolls // back together, and the retry has to bring the bodies back with the rows. @@ -1057,6 +1067,12 @@ struct FlushTimings { commit_ms: f64, } +#[derive(Clone, Copy)] +struct FlushRequest { + pending_ops: usize, + wait_for_body_sync: bool, +} + fn flush_dirty_tables_to_disk_recorded( conn: &Connection, dirty_tables: &mut BTreeSet<&'static str>, @@ -1064,11 +1080,19 @@ fn flush_dirty_tables_to_disk_recorded( db_path: Option<&Path>, bodies: &mut BodyArchive, counters: &LedgerCounters, - pending_ops: usize, + request: FlushRequest, ) -> rusqlite::Result<()> { let table_count = dirty_tables.len(); let started = Instant::now(); - let result = flush_dirty_tables_to_disk(conn, dirty_tables, flush_watermarks, db_path, bodies, counters); + let result = flush_dirty_tables_to_disk( + conn, + dirty_tables, + flush_watermarks, + db_path, + bodies, + counters, + request.wait_for_body_sync, + ); let elapsed = started.elapsed(); let status = if result.is_ok() { "ok" } else { "error" }; let timings = result.as_ref().ok(); @@ -1083,7 +1107,7 @@ fn flush_dirty_tables_to_disk_recorded( table_copy_ms, counter_snapshot_ms, commit_ms, - pending_ops, + pending_ops = request.pending_ops, table_count, status, "db disk flush completed" @@ -1095,7 +1119,7 @@ fn flush_dirty_tables_to_disk_recorded( table_copy_ms, counter_snapshot_ms, commit_ms, - pending_ops, + pending_ops = request.pending_ops, table_count, status, "db disk flush exceeded its configured interval" diff --git a/crates/capsem-logger/src/writer/bodies.rs b/crates/capsem-logger/src/writer/bodies.rs index 50cd0d566..e01541a7f 100644 --- a/crates/capsem-logger/src/writer/bodies.rs +++ b/crates/capsem-logger/src/writer/bodies.rs @@ -5,15 +5,17 @@ //! on disk. The order is the whole point: every disk flush writes one segment //! of the open block -- everything staged since the last one -- and syncs it //! *before* the transaction that inserts its rows and updates its -//! `body_blocks` row commits. A crash between the two leaves unreferenced +//! `body_blocks` row commits. Periodic flushes perform that device sync on a +//! small dedicated thread so the ledger can keep accepting events, while +//! explicit barriers wait for it. A crash between the two leaves unreferenced //! bytes at the end of the file, which cost their bytes and nothing else; the //! reverse order would leave index rows pointing past EOF, which is a ledger //! that lies. //! //! A sync-flushed segment leaves the block open, so later bodies reuse its //! dictionary. A block closes at its target size, `MAX_BLOCK_AGE`, shutdown, -//! or retention. Compression runs synchronously on this writer thread; each -//! flush costs only the sync point. +//! or retention. Compression runs synchronously on this writer thread; only +//! the durability syscall leaves it. //! //! Identical bytes within the open block are stored once. An event that //! matches three rules archives its payload three times over, and the @@ -27,11 +29,13 @@ use std::collections::HashMap; use std::ffi::OsString; use std::path::{Path, PathBuf}; +use std::sync::mpsc; use std::time::{Duration, Instant, SystemTime}; use super::{blake3_bytes_ref, execute_cached, format_timestamp, LedgerClock, MAX_BODY_BLOB_BYTES}; use capsem_archive::{ - ArchiveCodecs, ArchiveError, ArchiveId, BodyLogWriter, BodyRef, FileHeader, GenerationId, SegmentWritten, + ArchiveCodecs, ArchiveError, ArchiveId, BodyLogSync, BodyLogWriter, BodyRef, FileHeader, GenerationId, + SegmentWritten, }; use capsem_foundation::unix::contained::{ContainedDir, EntryKind}; use capsem_foundation::unix::fs::{durable_sync_directory, ensure_private_dir}; @@ -103,6 +107,68 @@ struct BodyIndexRow { created_at: String, } +type AppendedSegment = (SegmentWritten, Vec); + +struct ArchiveSyncWorker { + requests: Option>, + completions: mpsc::Receiver>, + join: Option>, +} + +impl ArchiveSyncWorker { + fn start(sync: BodyLogSync) -> std::io::Result { + let (requests, work) = mpsc::sync_channel(1); + let (completed, completions) = mpsc::sync_channel(1); + let join = std::thread::Builder::new() + .name("capsem-archive-sync".into()) + .stack_size(128 * 1024) + .spawn(move || { + while work.recv().is_ok() { + let result = sync.sync().map_err(|error| error.to_string()); + if completed.send(result).is_err() { + break; + } + } + })?; + Ok(Self { + requests: Some(requests), + completions, + join: Some(join), + }) + } + + fn request(&self) -> Result<(), String> { + self.requests + .as_ref() + .ok_or_else(|| "archive sync worker stopped".to_string())? + .send(()) + .map_err(|_| "archive sync worker stopped".to_string()) + } + + fn try_complete(&self) -> Option> { + match self.completions.try_recv() { + Ok(result) => Some(result), + Err(mpsc::TryRecvError::Empty) => None, + Err(mpsc::TryRecvError::Disconnected) => Some(Err("archive sync worker stopped".into())), + } + } + + fn complete(&self) -> Result<(), String> { + self.completions + .recv() + .map_err(|_| "archive sync worker stopped".to_string())? + } +} + +impl Drop for ArchiveSyncWorker { + fn drop(&mut self) { + self.requests.take(); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} + /// The writer thread's archive: one `session.bodies` beside `session.db`. pub(super) struct BodyArchive { db_path: Option, @@ -133,9 +199,15 @@ pub(super) struct BodyArchive { block_spans: HashMap, /// Sequence number the next staged row takes. next_seq: u64, - /// Segments on disk, each with the rows it made placeable, waiting for - /// those rows to commit. - appended: Vec<(SegmentWritten, Vec)>, + /// Segments written through the page cache and waiting to start a device + /// sync. + appended: Vec, + /// Segments whose device sync completed and whose rows may enter SQLite. + synced: Vec, + /// Segments covered by the durability barrier currently running. + syncing: Option>, + /// Lazily-created helper that keeps fdatasync off the admission thread. + sync_worker: Option, /// Test builds only: make the next segment write fail the way a full /// disk does. Injected at this seam rather than inside the archive /// writer because what is under test is what *this* type does once the @@ -457,6 +529,9 @@ impl BodyArchive { block_spans: HashMap::new(), next_seq: 0, appended: Vec::new(), + synced: Vec::new(), + syncing: None, + sync_worker: None, #[cfg(test)] fail_next_append: false, #[cfg(test)] @@ -628,6 +703,7 @@ impl BodyArchive { /// written goes with it. Every caller has already counted the rows that /// named those bytes. fn retire_writer(&mut self) { + self.sync_worker = None; if let Some(writer) = self.writer.take() { writer.abandon(); } @@ -648,6 +724,7 @@ impl BodyArchive { pub(super) fn adopt_generation(&mut self, writer: BodyLogWriter) { debug_assert!(!self.has_work()); + self.sync_worker = None; self.writer = Some(writer); self.block_spans.clear(); self.block_opened_at = None; @@ -669,6 +746,14 @@ impl BodyArchive { for (_, rows) in &mut self.appended { rows.retain(|row| row.seq < mark); } + for (_, rows) in &mut self.synced { + rows.retain(|row| row.seq < mark); + } + if let Some(syncing) = &mut self.syncing { + for (_, rows) in syncing { + rows.retain(|row| row.seq < mark); + } + } } /// Close the open block if it is full or too old. Called between the @@ -690,7 +775,12 @@ impl BodyArchive { /// were dropped by a rollback -- they still have to reach the file, or the /// writer closes holding them. pub(super) fn has_work(&self) -> bool { - !self.appended.is_empty() || !self.staged.is_empty() || !self.reused.is_empty() || self.pending_bytes() > 0 + !self.appended.is_empty() + || !self.synced.is_empty() + || self.syncing.is_some() + || !self.staged.is_empty() + || !self.reused.is_empty() + || self.pending_bytes() > 0 } /// Write everything staged since the last flush as one segment of the @@ -758,7 +848,102 @@ impl BodyArchive { } } - /// Record every written segment's block and the index rows it holds. + /// Start or collect the durability work that makes index rows safe. + /// + /// Periodic flushes poll without waiting so archive storage latency never + /// blocks new ledger admissions. Explicit barriers and shutdown wait for + /// every segment, preserving their read-after-write and durability + /// contracts. + pub(super) fn prepare_index_rows(&mut self, wait: bool) { + if !self.finish_archive_sync(wait) { + return; + } + while self.syncing.is_none() && !self.appended.is_empty() { + if !self.start_archive_sync() || !wait || !self.finish_archive_sync(true) { + return; + } + } + } + + fn start_archive_sync(&mut self) -> bool { + if self.appended.is_empty() { + return true; + } + if self.writer.is_none() { + self.abandon_uncommitted("poisoned"); + return false; + } + if self.sync_worker.is_none() { + let started = self + .writer + .as_ref() + .expect("writer checked above") + .sync_handle() + .map_err(|error| error.to_string()) + .and_then(|sync| ArchiveSyncWorker::start(sync).map_err(|error| error.to_string())); + match started { + Ok(worker) => self.sync_worker = Some(worker), + Err(error) => { + warn!(error, "session body archive sync worker could not start"); + self.abandon_uncommitted("sync"); + self.retire_writer(); + return false; + } + } + } + if let Err(error) = self.sync_worker.as_ref().expect("worker created above").request() { + warn!(error, "session body archive sync worker stopped"); + self.abandon_uncommitted("sync"); + self.retire_writer(); + return false; + } + self.syncing = Some(std::mem::take(&mut self.appended)); + true + } + + fn finish_archive_sync(&mut self, wait: bool) -> bool { + if self.syncing.is_none() { + return true; + } + let result = if wait { + Some( + self.sync_worker + .as_ref() + .expect("an in-flight sync has a worker") + .complete(), + ) + } else { + self.sync_worker + .as_ref() + .expect("an in-flight sync has a worker") + .try_complete() + }; + let Some(result) = result else { + return true; + }; + let syncing = self.syncing.take().expect("the completed sync was in flight"); + match result { + Ok(()) => { + self.synced.extend(syncing); + #[cfg(test)] + self.steps.push("sync"); + true + } + Err(error) => { + let dropped_segments = syncing.len(); + self.appended.extend(syncing); + warn!( + error, + dropped_segments, "session body archive could not be flushed; no further bodies will be stored" + ); + self.abandon_uncommitted("sync"); + self.retire_writer(); + false + } + } + } + + /// Record every durably-written segment's block and the index rows it holds. /// /// Runs inside the caller's transaction, after `flush_segment` put the /// bytes in the file. Nothing here can make a row visible before its @@ -786,11 +971,12 @@ impl BodyArchive { /// beside it and the commit after it; all three roll back every row, and /// the next flush retries these with them. pub(super) fn commit_index_rows(&mut self, conn: &Connection) -> rusqlite::Result<()> { - if self.appended.is_empty() && self.reused.is_empty() { - return Ok(()); - } - let started = Instant::now(); - if !self.appended.is_empty() && !self.sync_appended_segments() { + self.prepare_index_rows(true); + self.commit_prepared_index_rows(conn) + } + + pub(super) fn commit_prepared_index_rows(&mut self, conn: &Connection) -> rusqlite::Result<()> { + if self.synced.is_empty() && self.reused.is_empty() { return Ok(()); } let synced = Instant::now(); @@ -800,11 +986,11 @@ impl BodyArchive { #[cfg(test)] self.steps.push("commit"); let sealed_at = format_timestamp((self.now)()); - for (segment, _) in &self.appended { + for (segment, _) in &self.synced { upsert_block(conn, segment, &sealed_at)?; } let mut committed_end = None; - for (segment, _) in &self.appended { + for (segment, _) in &self.synced { let end = segment.block_offset.checked_add(segment.disk_len).ok_or_else(|| { rusqlite::Error::InvalidParameterName("archive committed extent overflows u64".into()) })?; @@ -821,58 +1007,45 @@ impl BodyArchive { [committed_end], )?; } - let blocks_finished = Instant::now(); - index_batch::insert( - conn, - self.appended.iter().flat_map(|(_, rows)| rows).chain(&self.reused), - )?; - let rows_finished = Instant::now(); - tracing::debug!( - target: "capsem.db", - archive_sync_ms = synced.duration_since(started).as_secs_f64() * 1000.0, - index_sql_ms = synced.elapsed().as_secs_f64() * 1000.0, - block_sql_ms = blocks_finished.duration_since(synced).as_secs_f64() * 1000.0, - row_sql_ms = rows_finished.duration_since(blocks_finished).as_secs_f64() * 1000.0, - segments = self.appended.len(), - index_rows = self.appended.iter().map(|(_, rows)| rows.len()).sum::() + self.reused.len(), - "db archive index flushed" - ); + for row in self.synced.iter().flat_map(|(_, rows)| rows).chain(&self.reused) { + execute_cached( + conn, + "INSERT OR REPLACE INTO event_body_blobs ( + event_id, event_type, source_table, direction, content_type, + original_bytes, stored_bytes, truncated, body_hash, + block_offset, body_offset, body_len, + trace_id, turn_id, created_at + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)", + params![ + &row.event_id, + row.event_type, + row.source_table, + row.direction, + &row.content_type, + row.original_bytes, + row.stored_bytes, + i64::from(row.truncated), + &row.body_hash, + row.block_offset as i64, + row.body_offset, + row.stored_bytes, + &row.trace_id, + &row.turn_id, + &row.created_at, + ], + )?; + } Ok(()) } /// The transaction that `commit_index_rows` wrote into has committed, so /// those segments are indexed and this archive is done with them. pub(super) fn index_rows_committed(&mut self) { - self.appended.clear(); + self.synced.clear(); self.reused.clear(); } - /// Flush the written segments to the device before their rows commit. - /// `false` when the flush failed and the rows must not be written. - fn sync_appended_segments(&mut self) -> bool { - let Some(writer) = self.writer.as_mut() else { - // The writer is already gone -- an earlier write or flush gave - // up -- so these segments will never be vouched for. Dropping them - // is the same fail-closed answer, and keeping them would leave - // every later flush opening a transaction to do nothing with. - self.abandon_uncommitted("poisoned"); - return false; - }; - if let Err(error) = writer.sync() { - warn!( - error = %error, - dropped_segments = self.appended.len(), - "session body archive could not be flushed; no further bodies will be stored" - ); - self.abandon_uncommitted("sync"); - self.retire_writer(); - return false; - } - #[cfg(test)] - self.steps.push("sync"); - true - } - /// Durability barrier for session close. The block was closed and its /// rows committed by the final flush; this makes sure the device has it. pub(super) fn sync(&mut self) { @@ -888,9 +1061,17 @@ impl BodyArchive { /// and the rows of bytes not yet written are equally unreachable once the /// archive stops vouching for its own file. pub(super) fn abandon_uncommitted(&mut self, reason: &'static str) { - let dropped: usize = - self.appended.iter().map(|(_, rows)| rows.len()).sum::() + self.staged.len() + self.reused.len(); + let dropped: usize = self.appended.iter().map(|(_, rows)| rows.len()).sum::() + + self.synced.iter().map(|(_, rows)| rows.len()).sum::() + + self + .syncing + .as_ref() + .map_or(0, |segments| segments.iter().map(|(_, rows)| rows.len()).sum()) + + self.staged.len() + + self.reused.len(); self.appended.clear(); + self.synced.clear(); + self.syncing = None; self.staged.clear(); self.reused.clear(); self.drop_bodies(dropped, reason); @@ -935,7 +1116,7 @@ impl BodyArchive { /// Segments written but not yet indexed. #[cfg(test)] pub(super) fn appended_len_for_tests(&self) -> usize { - self.appended.len() + self.appended.len() + self.synced.len() + self.syncing.as_ref().map_or(0, Vec::len) } /// The flush/commit steps this archive has taken, in order. diff --git a/crates/capsem-logger/src/writer/tests/bodies.rs b/crates/capsem-logger/src/writer/tests/bodies.rs index a1479776a..f38b86e22 100644 --- a/crates/capsem-logger/src/writer/tests/bodies.rs +++ b/crates/capsem-logger/src/writer/tests/bodies.rs @@ -334,6 +334,38 @@ fn appended_blocks_are_flushed_before_their_index_rows_commit() { archive.sync(); } +/// A periodic flush may leave archive durability running in parallel with +/// later admissions, but it must not publish the corresponding index rows +/// until that durability work has completed. +#[test] +fn nonblocking_archive_sync_defers_index_rows_until_completion() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("nonblocking-sync.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + crate::schema::create_tables(&conn).unwrap(); + + let mut archive = BodyArchive::open_for_tests(Some(&db_path), SystemTime::now, &conn); + archive.stage( + &conn, + body_blob("0f1f2f3f4f60", "a body synchronized away from admissions"), + ); + archive.flush_segment(); + archive.prepare_index_rows(false); + archive.commit_prepared_index_rows(&conn).unwrap(); + archive.index_rows_committed(); + + let rows = || { + conn.query_row("SELECT COUNT(*) FROM event_body_blobs", [], |row| row.get::<_, i64>(0)) + .unwrap() + }; + assert_eq!(rows(), 0, "an in-flight sync cannot publish its rows"); + + archive.commit_index_rows(&conn).unwrap(); + archive.index_rows_committed(); + assert_eq!(rows(), 1, "the row becomes eligible after the sync finishes"); + assert_eq!(archive.steps_for_tests(), ["sync", "commit"]); +} + /// A body that cannot fit beside what the open block holds closes the block /// and retries, rather than failing or panicking. Two 10 MiB bodies cannot /// share a 16 MiB block, so they land in two. diff --git a/crates/capsem-logger/src/writer/tests/counters.rs b/crates/capsem-logger/src/writer/tests/counters.rs index e928ae97d..5991f390f 100644 --- a/crates/capsem-logger/src/writer/tests/counters.rs +++ b/crates/capsem-logger/src/writer/tests/counters.rs @@ -37,6 +37,7 @@ fn completing_a_flushed_exec_commits_its_count_with_the_row() { None, &mut bodies, tally.counters(), + true, ) .unwrap(); assert_eq!(on_disk(&path).0.exec.started, 1); From 13167b7c65615e9344592d776fed455d3eec6280 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 00:56:06 +0000 Subject: [PATCH 393/464] perf(build): optimize ledger sqlite in dev --- Cargo.toml | 13 +++++++++++++ tests/citadel/test_hot_build_contract.py | 16 ++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 150cacb62..b86fcbe2a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,6 +66,19 @@ opt-level = 3 [profile.dev.package.capsem-cache] opt-level = 3 +[profile.dev.package.capsem-logger] +opt-level = 3 + +# The ledger's SQLite transaction is part of live DNS/HTTP admission. Package +# overrides do not flow into dependencies, so optimize both the Rust wrapper +# and bundled C engine explicitly in the debug binaries used by installs and +# gateway benchmarks. +[profile.dev.package.rusqlite] +opt-level = 3 + +[profile.dev.package.libsqlite3-sys] +opt-level = 3 + [profile.dev.package.quick_cache] opt-level = 3 diff --git a/tests/citadel/test_hot_build_contract.py b/tests/citadel/test_hot_build_contract.py index ce5300f38..76c38db59 100644 --- a/tests/citadel/test_hot_build_contract.py +++ b/tests/citadel/test_hot_build_contract.py @@ -18,6 +18,8 @@ "sha2", "capsem-cache", "capsem-crypto", + "capsem-logger", + "libsqlite3-sys", "blake2b_simd", "blake2s_simd", "serde", @@ -30,19 +32,17 @@ "itoa", "ryu", "quick_cache", + "rusqlite", } RATIONALE = """\ -Hot codec package lost dev optimization. +Hot dev package lost optimization. Capsem's hands-on local install path uses debug binaries, so JSON, MessagePack, -DNS wire parsing, and BLAKE3 must stay optimized in the dev profile. If this -guard fails, route/gateway latency can regress from compiler configuration -before the DB or network code even runs. SQLite, its adapter and the ledger -writer must also stay optimized: a native concurrency 64 reproduction took 2.07 seconds to flush its -archive index and ledger tables, filling the bounded producer queue and holding -the final HTTP frame for 2.18 seconds (#350). Preserve audit admission and FULL -durability; do not hide compiler cost by raising queue limits or dropping rows. +DNS wire parsing, BLAKE3, and the admission-path ledger writer must stay +optimized in the dev profile. If this guard fails, route/gateway latency can +regress from compiler configuration before the network code can drain its +admission queue. """ From 1d6ca6c0c9839c3019a69e58916c059066425fef Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 00:57:43 +0000 Subject: [PATCH 394/464] test(bench): preserve failed load measurements --- guest/artifacts/capsem_bench/dns_load.py | 13 ++-- guest/artifacts/capsem_bench/mitm_load.py | 13 ++-- .../test_capsem_bench_mock_server_protocol.py | 70 +++++++++++++++---- 3 files changed, 67 insertions(+), 29 deletions(-) diff --git a/guest/artifacts/capsem_bench/dns_load.py b/guest/artifacts/capsem_bench/dns_load.py index be81d97fc..e2b90b7dc 100644 --- a/guest/artifacts/capsem_bench/dns_load.py +++ b/guest/artifacts/capsem_bench/dns_load.py @@ -114,16 +114,15 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N config.duration_s, timeout_s, ) - if row["errors"]: - raise RuntimeError( - f"dns-load: {row['errors']} of {row['total_requests']} requests failed " - f"at concurrency {concurrency}: {row['error_distribution']}; " - "the rail did not sustain the requested load" - ) rows.append(row) + if row["errors"]: + # Preserve the failed row and every lower successful level in the + # JSON artifact. The host collector rejects non-zero errors after + # recording the structured RPS, latency and pressure evidence. + break out = { - "version": "1.2", + "version": "1.3", "qname": qname, "qtype": qtype, "concurrency_levels": rows, diff --git a/guest/artifacts/capsem_bench/mitm_load.py b/guest/artifacts/capsem_bench/mitm_load.py index 0cab1566f..5d5a75d96 100644 --- a/guest/artifacts/capsem_bench/mitm_load.py +++ b/guest/artifacts/capsem_bench/mitm_load.py @@ -114,16 +114,15 @@ def mitm_load_bench(target=None, concurrency_levels=None, duration_s=None): for c in config.concurrency_levels: console.print(f" concurrency={c} ...") row = _drive_at_concurrency(target, c, config.duration_s) - if row["errors"]: - raise RuntimeError( - f"mitm-load: {row['errors']} of {row['total_requests']} requests failed " - f"at concurrency {c}: {row['error_distribution']}; " - "the rail did not sustain the requested load" - ) rows.append(row) + if row["errors"]: + # The collector is the quality gate. Return the failed row so it + # can reject the run without discarding the measurements that + # explain the failure. + break out = { - "version": "1.1", + "version": "1.2", "target": target, "concurrency_levels": rows, } diff --git a/tests/test_capsem_bench_mock_server_protocol.py b/tests/test_capsem_bench_mock_server_protocol.py index d93107dce..ffe5ba3aa 100644 --- a/tests/test_capsem_bench_mock_server_protocol.py +++ b/tests/test_capsem_bench_mock_server_protocol.py @@ -58,7 +58,7 @@ def test_mitm_load_rejects_starved_driver(monkeypatch): mitm_load._drive_at_concurrency("https://fixture.test/tiny", 512, 10) -def test_mitm_load_rejects_one_failed_request(monkeypatch): +def test_mitm_load_preserves_failed_level_for_collector_rejection(monkeypatch): monkeypatch.setattr( mitm_load, "_drive_at_concurrency", @@ -68,15 +68,34 @@ def test_mitm_load_rejects_one_failed_request(monkeypatch): "total_requests": 2, "errors": 1, "error_distribution": {"timed out": 1}, + "rps": 2.0, + "p50_ms": 10.0, + "p95_ms": 20.0, + "p99_ms": 20.0, + "p999_ms": 20.0, }, ) - with pytest.raises(RuntimeError, match="1 of 2 requests failed"): - mitm_load.mitm_load_bench( - target="https://fixture.capsem.test/tiny", - concurrency_levels=[1], - duration_s=1, - ) + result = mitm_load.mitm_load_bench( + target="https://fixture.capsem.test/tiny", + concurrency_levels=[1], + duration_s=1, + ) + + assert result["concurrency_levels"] == [ + { + "concurrency": 1, + "duration_s": 1.0, + "total_requests": 2, + "errors": 1, + "error_distribution": {"timed out": 1}, + "rps": 2.0, + "p50_ms": 10.0, + "p95_ms": 20.0, + "p99_ms": 20.0, + "p999_ms": 20.0, + } + ] def test_dns_load_delegates_to_async_rust_driver(monkeypatch): @@ -166,7 +185,7 @@ def test_dns_load_rejects_starved_driver(monkeypatch): dns_load._drive_at_concurrency("fixture.test", 1, 512, 10, 5) -def test_dns_load_rejects_one_failed_request(monkeypatch): +def test_dns_load_preserves_failed_level_for_collector_rejection(monkeypatch): monkeypatch.setattr( dns_load, "_drive_at_concurrency", @@ -176,16 +195,37 @@ def test_dns_load_rejects_one_failed_request(monkeypatch): "total_requests": 2, "errors": 1, "error_distribution": {"timed out": 1}, + "decision_distribution": {"allow": 1}, + "rps": 2.0, + "p50_ms": 10.0, + "p95_ms": 20.0, + "p99_ms": 20.0, + "p999_ms": 20.0, }, ) - with pytest.raises(RuntimeError, match="1 of 2 requests failed"): - dns_load.dns_load_bench( - qname="fixture.test", - qtype=1, - concurrency_levels=[1], - duration_s=1, - ) + result = dns_load.dns_load_bench( + qname="fixture.test", + qtype=1, + concurrency_levels=[1], + duration_s=1, + ) + + assert result["concurrency_levels"] == [ + { + "concurrency": 1, + "duration_s": 1.0, + "total_requests": 2, + "errors": 1, + "error_distribution": {"timed out": 1}, + "decision_distribution": {"allow": 1}, + "rps": 2.0, + "p50_ms": 10.0, + "p95_ms": 20.0, + "p99_ms": 20.0, + "p999_ms": 20.0, + } + ] class _StubConsole: From b9d1029cabcb64814569acba1e7a9239a1fa4b64 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 01:05:31 +0000 Subject: [PATCH 395/464] test(bench): retain rejected load evidence --- benchmarks/collectors/_guest | 18 +++++++++++------- tests/test_benchmark_guest_collector.py | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/benchmarks/collectors/_guest b/benchmarks/collectors/_guest index 77ce2bca1..cd56ef194 100755 --- a/benchmarks/collectors/_guest +++ b/benchmarks/collectors/_guest @@ -192,6 +192,17 @@ def main() -> int: f"response={json.dumps(artifact_result)[:1000]}" ) document = json.loads(exec_output_text(artifact_result)) + # Reject while the fixture is still live. ServiceInstance sees the + # active exception during teardown and preserves the ledger/proxy + # evidence instead of treating this as a clean benchmark shutdown. + section = document.get(mode.replace("-", "_"), document) + if failed := failed_load_levels(section): + raise RuntimeError( + f"capsem-bench {mode}: requests failed at concurrency {failed}; " + "the rail did not sustain the requested load, so the run is rejected; " + f"artifact={json.dumps(document, sort_keys=True)}; " + f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}" + ) finally: with contextlib.suppress(Exception): client.delete(f"/vms/{name}/delete") @@ -207,7 +218,6 @@ def main() -> int: # The guest writes one document per run keyed by mode; take that section # when present so keys are not prefixed with the mode twice. - section = document.get(mode.replace("-", "_"), document) runtime_failures = mitm_runtime_failures(service_log) if mode == "mitm-load" else [] if runtime_failures: raise RuntimeError( @@ -216,12 +226,6 @@ def main() -> int: f"first={runtime_failures[:5]!r}; " f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}" ) - if failed := failed_load_levels(section): - raise RuntimeError( - f"capsem-bench {mode}: requests failed at concurrency {failed}; " - "the rail did not sustain the requested load, so the run is rejected; " - f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}" - ) numbers = flatten(section) if not numbers: raise RuntimeError( diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index 0ebd6aaaa..addbfe834 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -312,6 +312,7 @@ def stop(self) -> dict: main() message = str(failure.value) assert "requests failed at concurrency [200]" in message + assert f'artifact={json.dumps(document, sort_keys=True)}' in message assert '"resources.total.cpu_seconds": 12.5' in message assert '"resources.total.peak_rss_bytes": 1048576' in message From 1b2e0d1fc63fc07b4c85cff4a0784e35de74bd39 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 01:21:12 +0000 Subject: [PATCH 396/464] perf(logger): batch archive index writes --- crates/capsem-logger/src/writer.rs | 14 ++++ crates/capsem-logger/src/writer/bodies.rs | 93 +++++++++++++++-------- 2 files changed, 75 insertions(+), 32 deletions(-) diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 26b51e521..2befd99be 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -1024,8 +1024,12 @@ fn flush_dirty_tables_to_disk( bodies.prepare_index_rows(wait_for_body_sync); let archive_ms = started.elapsed().as_secs_f64() * 1000.0; let tables: Vec<&'static str> = dirty_tables.iter().copied().collect(); + let started = Instant::now(); let tx = conn.unchecked_transaction()?; + let transaction_begin_ms = started.elapsed().as_secs_f64() * 1000.0; + let started = Instant::now(); bodies.commit_prepared_index_rows(&tx)?; + let body_index_ms = started.elapsed().as_secs_f64() * 1000.0; // Injected where a real flush failure lands: inside the transaction, with // the archive's index rows already written into it. Everything here rolls // back together, and the retry has to bring the bodies back with the rows. @@ -1053,6 +1057,8 @@ fn flush_dirty_tables_to_disk( dirty_tables.clear(); Ok(FlushTimings { archive_ms, + transaction_begin_ms, + body_index_ms, table_copy_ms, counter_snapshot_ms, commit_ms, @@ -1062,6 +1068,8 @@ fn flush_dirty_tables_to_disk( #[derive(Default)] struct FlushTimings { archive_ms: f64, + transaction_begin_ms: f64, + body_index_ms: f64, table_copy_ms: f64, counter_snapshot_ms: f64, commit_ms: f64, @@ -1097,6 +1105,8 @@ fn flush_dirty_tables_to_disk_recorded( let status = if result.is_ok() { "ok" } else { "error" }; let timings = result.as_ref().ok(); let archive_ms = timings.map_or(0.0, |timings| timings.archive_ms); + let transaction_begin_ms = timings.map_or(0.0, |timings| timings.transaction_begin_ms); + let body_index_ms = timings.map_or(0.0, |timings| timings.body_index_ms); let table_copy_ms = timings.map_or(0.0, |timings| timings.table_copy_ms); let counter_snapshot_ms = timings.map_or(0.0, |timings| timings.counter_snapshot_ms); let commit_ms = timings.map_or(0.0, |timings| timings.commit_ms); @@ -1104,6 +1114,8 @@ fn flush_dirty_tables_to_disk_recorded( target: "capsem.db", duration_ms = elapsed.as_secs_f64() * 1000.0, archive_ms, + transaction_begin_ms, + body_index_ms, table_copy_ms, counter_snapshot_ms, commit_ms, @@ -1116,6 +1128,8 @@ fn flush_dirty_tables_to_disk_recorded( warn!( duration_ms = elapsed.as_secs_f64() * 1000.0, archive_ms, + transaction_begin_ms, + body_index_ms, table_copy_ms, counter_snapshot_ms, commit_ms, diff --git a/crates/capsem-logger/src/writer/bodies.rs b/crates/capsem-logger/src/writer/bodies.rs index e01541a7f..129dca6c1 100644 --- a/crates/capsem-logger/src/writer/bodies.rs +++ b/crates/capsem-logger/src/writer/bodies.rs @@ -41,7 +41,7 @@ use capsem_foundation::unix::contained::{ContainedDir, EntryKind}; use capsem_foundation::unix::fs::{durable_sync_directory, ensure_private_dir}; use capsem_foundation::unix::lock::{self, FileLock, LockAttempt, LockMode}; use capsem_telemetry::db::{DB_ARCHIVE_BODIES_DEDUPLICATED_TOTAL, DB_ARCHIVE_BODIES_DROPPED_TOTAL}; -use rusqlite::{params, Connection, OptionalExtension}; +use rusqlite::{params, params_from_iter, Connection, OptionalExtension, ToSql}; use tracing::warn; /// How long a block may stay open before the next body starts a new one. @@ -100,7 +100,7 @@ struct BodyIndexRow { stored_bytes: i64, truncated: bool, body_hash: String, - block_offset: u64, + block_offset: i64, body_offset: i64, trace_id: Option, turn_id: Option, @@ -225,6 +225,13 @@ pub(super) struct BodyArchive { /// would cost more than the bytes it saves. const MIN_ARCHIVE_DEDUP_BYTES: usize = 512; +/// Fifteen bound values per body row fit 64 rows below SQLite's portable +/// 999-variable ceiling. The statement and transient reference vector stay +/// bounded while a busy flush pays statement dispatch once per 64 bodies +/// instead of once per body. +const BODY_INDEX_ROWS_PER_INSERT: usize = 64; +const BODY_INDEX_VALUE_LIST: &str = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + /// Where identical stored bytes already sit in a committed segment, if they /// do. Committed spans are immutable and synced, so pointing a new row at one /// is as safe as pointing at fresh bytes; retention keeps a block while any @@ -587,7 +594,7 @@ impl BodyArchive { // same stored bytes from bodies of different sizes. truncated: original_bytes > stored_len as u64, body_hash, - block_offset: reference.block_offset, + block_offset: reference.block_offset as i64, body_offset: i64::from(reference.offset), trace_id: blob.trace_id.map(str::to_string), turn_id: blob.turn_id.map(str::to_string), @@ -1007,35 +1014,13 @@ impl BodyArchive { [committed_end], )?; } - for row in self.synced.iter().flat_map(|(_, rows)| rows).chain(&self.reused) { - execute_cached( - conn, - "INSERT OR REPLACE INTO event_body_blobs ( - event_id, event_type, source_table, direction, content_type, - original_bytes, stored_bytes, truncated, body_hash, - block_offset, body_offset, body_len, - trace_id, turn_id, created_at - ) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15)", - params![ - &row.event_id, - row.event_type, - row.source_table, - row.direction, - &row.content_type, - row.original_bytes, - row.stored_bytes, - i64::from(row.truncated), - &row.body_hash, - row.block_offset as i64, - row.body_offset, - row.stored_bytes, - &row.trace_id, - &row.turn_id, - &row.created_at, - ], - )?; - } + let rows = self + .synced + .iter() + .flat_map(|(_, rows)| rows) + .chain(&self.reused) + .collect::>(); + insert_body_index_rows(conn, &rows)?; Ok(()) } @@ -1126,6 +1111,50 @@ impl BodyArchive { } } +fn insert_body_index_rows(conn: &Connection, rows: &[&BodyIndexRow]) -> rusqlite::Result<()> { + for chunk in rows.chunks(BODY_INDEX_ROWS_PER_INSERT) { + let mut sql = String::from( + "INSERT OR REPLACE INTO event_body_blobs ( + event_id, event_type, source_table, direction, content_type, + original_bytes, stored_bytes, truncated, body_hash, + block_offset, body_offset, body_len, + trace_id, turn_id, created_at + ) VALUES ", + ); + for index in 0..chunk.len() { + if index > 0 { + sql.push_str(", "); + } + sql.push_str(BODY_INDEX_VALUE_LIST); + } + + // References only: the rows keep ownership until the surrounding + // transaction commits, so batching does not clone any body metadata. + let mut values: Vec<&dyn ToSql> = Vec::with_capacity(chunk.len() * 15); + for row in chunk { + values.extend([ + &row.event_id as &dyn ToSql, + &row.event_type as &dyn ToSql, + &row.source_table as &dyn ToSql, + &row.direction as &dyn ToSql, + &row.content_type as &dyn ToSql, + &row.original_bytes as &dyn ToSql, + &row.stored_bytes as &dyn ToSql, + &row.truncated as &dyn ToSql, + &row.body_hash as &dyn ToSql, + &row.block_offset as &dyn ToSql, + &row.body_offset as &dyn ToSql, + &row.stored_bytes as &dyn ToSql, + &row.trace_id as &dyn ToSql, + &row.turn_id as &dyn ToSql, + &row.created_at as &dyn ToSql, + ]); + } + execute_cached(conn, &sql, params_from_iter(values))?; + } + Ok(()) +} + /// Record one written segment against its block: insert the block's row on /// its first segment, grow it on every later one. /// From 2d9a80d9fbdd030f26384efef3b2085bb47454f6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 01:28:18 +0000 Subject: [PATCH 397/464] test(bench): attribute ledger resource pressure --- tests/helpers/process_resources.py | 1 + tests/test_process_resource_sampler.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/tests/helpers/process_resources.py b/tests/helpers/process_resources.py index 64a1edf57..7ad342f90 100644 --- a/tests/helpers/process_resources.py +++ b/tests/helpers/process_resources.py @@ -16,6 +16,7 @@ TRACKED_PROCESS_NAMES = frozenset( { "capsem_network", + "capsem_ledger", "capsem_process", "capsem_proxy", "capsem_router", diff --git a/tests/test_process_resource_sampler.py b/tests/test_process_resource_sampler.py index 2f4f8e874..32db817de 100644 --- a/tests/test_process_resource_sampler.py +++ b/tests/test_process_resource_sampler.py @@ -131,6 +131,17 @@ def test_process_that_exits_keeps_its_last_cpu_sample() -> None: assert sampler.metrics()["resources.capsem_proxy.cpu_s"]["samples"] == [1.0] +def test_sampler_attributes_dedicated_ledger_pressure() -> None: + sampler = ProcessTreeSampler(None) + sampler.observe(snapshot(usage(8, "capsem-ledger", cpu="2", rss=100, pss=80, uss=70, fds=5))) + sampler.observe(snapshot(usage(8, "capsem-ledger", cpu="3", rss=250, pss=220, uss=200, fds=7))) + + metrics = sampler.metrics() + assert metrics["resources.capsem_ledger.cpu_s"]["samples"] == [1.0] + assert metrics["resources.capsem_ledger.peak_pss_bytes"]["samples"] == [220] + assert metrics["resources.capsem_ledger.peak_pss_delta_bytes"]["samples"] == [140] + + def test_platform_without_proportional_memory_omits_those_metrics() -> None: sampler = ProcessTreeSampler(None) sampler.observe( From 0c3a57b198ff33d004712489676cd73232b03378 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 01:46:57 +0000 Subject: [PATCH 398/464] fix(logger): bound producer queues by events --- crates/capsem-logger/src/db.rs | 2 +- crates/capsem-logger/src/writer.rs | 143 ++++++++++++++++++-- crates/capsem-logger/src/writer/producer.rs | 6 +- crates/capsem-logger/src/writer/tests.rs | 32 ++++- 4 files changed, 163 insertions(+), 20 deletions(-) diff --git a/crates/capsem-logger/src/db.rs b/crates/capsem-logger/src/db.rs index 2daa1b227..852d0c503 100644 --- a/crates/capsem-logger/src/db.rs +++ b/crates/capsem-logger/src/db.rs @@ -278,7 +278,7 @@ impl DbHandle { let started = Instant::now(); let writer = DbWriter::prepare_with_capacities( path, - capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY / capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY, + capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY, codecs.clone(), )?; diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 2befd99be..c82961672 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -1,6 +1,6 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{mpsc, Arc}; use std::time::{Duration, Instant, SystemTime}; @@ -298,6 +298,14 @@ impl WriterMessage { fn flush(reply: tokio::sync::oneshot::Sender) -> Self { Self::Flush(reply) } + + fn event_count(&self) -> usize { + match self { + Self::Write(_) => 1, + Self::WriteBatch(writes) => writes.len(), + Self::Flush(_) | Self::Retain { .. } => 0, + } + } } #[derive(Debug)] @@ -318,11 +326,122 @@ impl WriteEnvelope { /// message that outgrows it by a lot deserves the question asked again. const _: () = assert!(std::mem::size_of::() <= 1024); -type WriterSender = mpsc::SyncSender; -type WriterReceiver = mpsc::Receiver; +#[derive(Clone)] +struct WriterSender { + tx: mpsc::SyncSender, + budget: Arc, +} + +type WriterReceiver = mpsc::Receiver; + +struct QueuedWriterMessage { + message: WriterMessage, + _permit: WriterQueuePermit, +} + +impl QueuedWriterMessage { + fn into_message(self) -> WriterMessage { + self.message + } +} + +struct WriterQueueBudget { + capacity: usize, + used: AtomicUsize, +} + +struct WriterQueuePermit { + budget: Arc, + weight: usize, +} + +impl Drop for WriterQueuePermit { + fn drop(&mut self) { + self.budget.used.fetch_sub(self.weight, Ordering::Release); + } +} + +impl WriterQueueBudget { + fn try_reserve(self: &Arc, weight: usize) -> Option { + let mut used = self.used.load(Ordering::Acquire); + loop { + let next = used.checked_add(weight)?; + if next > self.capacity { + return None; + } + match self + .used + .compare_exchange_weak(used, next, Ordering::AcqRel, Ordering::Acquire) + { + Ok(_) => { + return Some(WriterQueuePermit { + budget: Arc::clone(self), + weight, + }) + } + Err(current) => used = current, + } + } + } +} + +#[derive(Debug, PartialEq, Eq)] +enum WriterTrySendError { + Full, + Disconnected, +} + +impl WriterSender { + fn try_send(&self, message: &mut Option) -> Result<(), WriterTrySendError> { + let weight = message.as_ref().expect("writer message is present").event_count(); + let Some(permit) = self.budget.try_reserve(weight) else { + return Err(WriterTrySendError::Full); + }; + match self.tx.try_send(QueuedWriterMessage { + message: message.take().expect("writer message is present"), + _permit: permit, + }) { + Ok(()) => Ok(()), + Err(mpsc::TrySendError::Full(queued)) => { + *message = Some(queued.into_message()); + Err(WriterTrySendError::Full) + } + Err(mpsc::TrySendError::Disconnected(queued)) => { + *message = Some(queued.into_message()); + Err(WriterTrySendError::Disconnected) + } + } + } + + fn send(&self, message: WriterMessage) -> Result<(), &'static str> { + let mut message = Some(message); + let mut wait = Duration::from_micros(50); + loop { + match self.try_send(&mut message) { + Ok(()) => return Ok(()), + Err(WriterTrySendError::Full) => { + std::thread::sleep(wait); + wait = (wait * 2).min(Duration::from_millis(5)); + } + Err(WriterTrySendError::Disconnected) => return Err("db writer channel closed"), + } + } + } +} fn writer_channel(capacity: usize) -> (WriterSender, WriterReceiver) { - mpsc::sync_channel(capacity.max(1)) + let capacity = capacity.max(1); + let (tx, rx) = mpsc::sync_channel(capacity); + ( + WriterSender { + tx, + budget: Arc::new(WriterQueueBudget { + capacity, + used: AtomicUsize::new(0), + }), + }, + rx, + ) } mod barriers; @@ -752,14 +871,14 @@ const BACKPRESSURE_MAX_WAIT: Duration = Duration::from_millis(5); /// backoff. Retrying immediately after `yield_now` pinned every producer task /// at full CPU for as long as the writer thread was inside a disk flush, which /// can be seconds with a large dirty set or a busy-timeout on the file. -async fn send_with_backpressure(tx: &WriterSender, mut message: WriterMessage) -> Result<(), String> { +async fn send_with_backpressure(tx: &WriterSender, message: WriterMessage) -> Result<(), String> { + let mut message = Some(message); let mut wait = Duration::from_micros(50); let mut attempts = 0u32; loop { - match tx.try_send(message) { + match tx.try_send(&mut message) { Ok(()) => return Ok(()), - Err(mpsc::TrySendError::Full(returned)) => { - message = returned; + Err(WriterTrySendError::Full) => { if attempts == 0 { tokio::task::yield_now().await; } else { @@ -768,7 +887,7 @@ async fn send_with_backpressure(tx: &WriterSender, mut message: WriterMessage) - } attempts += 1; } - Err(mpsc::TrySendError::Disconnected(_)) => { + Err(WriterTrySendError::Disconnected) => { return Err("db writer channel closed".to_string()); } } @@ -778,7 +897,7 @@ async fn send_with_backpressure(tx: &WriterSender, mut message: WriterMessage) - /// The writer thread loop: block-then-drain batching. fn writer_loop( conn: Connection, - rx: mpsc::Receiver, + rx: WriterReceiver, db_path: Option, schedule: WriterSchedule, pending_body_bytes: &AtomicU64, @@ -845,12 +964,12 @@ fn writer_loop( let mut batch = Vec::with_capacity(batch_capacity); let mut barriers = Barriers::default(); - let mut at_barrier = barriers.accept(first_message, &mut batch); + let mut at_barrier = barriers.accept(first_message.into_message(), &mut batch); // 2. Drain any ops already queued (non-blocking). while !at_barrier && batch.len() < batch_capacity { match rx.try_recv() { - Ok(message) => at_barrier = barriers.accept(message, &mut batch), + Ok(message) => at_barrier = barriers.accept(message.into_message(), &mut batch), Err(_) => break, } } diff --git a/crates/capsem-logger/src/writer/producer.rs b/crates/capsem-logger/src/writer/producer.rs index 4bb2bea61..3f0846d84 100644 --- a/crates/capsem-logger/src/writer/producer.rs +++ b/crates/capsem-logger/src/writer/producer.rs @@ -91,10 +91,10 @@ impl DbWriter { return accepted; } let queue_result = match self.clone_sender() { - Some(tx) => match tx.try_send(WriterMessage::write(op)) { + Some(tx) => match tx.try_send(&mut Some(WriterMessage::write(op))) { Ok(()) => "queued", - Err(mpsc::TrySendError::Full(_)) => "full", - Err(mpsc::TrySendError::Disconnected(_)) => "closed", + Err(WriterTrySendError::Full) => "full", + Err(WriterTrySendError::Disconnected) => "closed", }, None => "missing_sender", }; diff --git a/crates/capsem-logger/src/writer/tests.rs b/crates/capsem-logger/src/writer/tests.rs index 3bc859b8f..d00aea2ea 100644 --- a/crates/capsem-logger/src/writer/tests.rs +++ b/crates/capsem-logger/src/writer/tests.rs @@ -675,15 +675,39 @@ fn try_write_on_open_writer_succeeds() { #[test] fn writer_channel_capacity_applies_backpressure() { let (tx, _rx) = writer_channel(1); - tx.try_send(WriterMessage::write(file_event_with_credential("/queued", None))) - .unwrap(); + tx.try_send(&mut Some(WriterMessage::write(file_event_with_credential( + "/queued", None, + )))) + .unwrap(); assert!(matches!( - tx.try_send(WriterMessage::write(file_event_with_credential("/full", None,))), - Err(std::sync::mpsc::TrySendError::Full(_)) + tx.try_send(&mut Some(WriterMessage::write(file_event_with_credential( + "/full", None, + )))), + Err(WriterTrySendError::Full) )); } +#[test] +fn writer_channel_capacity_counts_events_inside_batches() { + let (tx, rx) = writer_channel(3); + let batch = || { + WriterMessage::WriteBatch(vec![ + WriteEnvelope::plain(file_event_with_credential("/batch/one", None)), + WriteEnvelope::plain(file_event_with_credential("/batch/two", None)), + ]) + }; + + tx.try_send(&mut Some(batch())).unwrap(); + assert!(matches!(tx.try_send(&mut Some(batch())), Err(WriterTrySendError::Full))); + tx.try_send(&mut Some(WriterMessage::write(file_event_with_credential( + "/third", None, + )))) + .unwrap(); + drop(rx.recv().unwrap()); + tx.try_send(&mut Some(batch())).unwrap(); +} + #[test] fn queued_write_batch_reaches_one_writer_transaction_batch() { let mut barriers = Barriers::default(); From aa9c723ae8802c6c32a28d856fae89306a3e4bcc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 01:58:47 +0000 Subject: [PATCH 399/464] security(core): preserve supplemental ledger admission --- CHANGELOG.md | 35 +++++++++++++++++ crates/capsem-core/src/net/dns/session/mod.rs | 1 + crates/capsem-core/src/security_engine/mod.rs | 25 ++++++++---- .../src/security_engine/rule_emission.rs | 26 ++++++++----- .../security_engine/tests/decision_ledger.rs | 39 +++++++++++++++++++ 5 files changed, 109 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ca46687c..7fd8f879f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Fail DNS closed when any decision, rule or ask row is refused, and coalesce + related supplemental admissions so concurrent queries do not serialize one + ledger round trip per row. + +- VM boot admission retains an exclusive generation lifetime fence. Service + restart cleanup confirms exit in the original kernel boot and PID namespace + before retiring grants and session bytes; delayed children cannot replay boot. + +- Startup retirement verifies loss of the original service parent and uses + stable native identity on Linux and Mac. Hosts without guarded signals await + the owner's parent-loss exit and refuse unconfirmed retirement. + +- Fresh and resumed VM launches carry the service's original PID and exact + spawn generation into process lifetime and boot admission. + +- VM processes verify their original service parent and trusted spawn + generation before boot, and terminate when that service disappears. + +- Linux startup pins original orphan processes and verifies fresh native + arguments before signalling. TERM/KILL and exit waits retain those handles; + uncertain identity or exit refuses startup before stale socket removal. + +- VM teardown signals and awaits its original owned child rather than a + cached PID. Shutdown, suspend and failed restore refuse uncertain exit; + service shutdown retires the child before deleting ephemeral session bytes. + +- Service startup orphan selection preserves native argument boundaries and + requires one session path directly under its own run roots. Neighbouring + root prefixes, incidental command text, duplicate options and traversal + targets cannot select unrelated workloads. + +- Managed VM expiry and authenticated close retire the original trusted + effects even when ownership storage is missing or corrupt. Failed persistence + cannot acknowledge a replacement generation as closed. + - Replace the archived `serde_yaml`/`unsafe-libyaml` Sigma-rule parser with a maintained safe-Rust parser, rejecting duplicate detection keys instead of silently accepting the last value. diff --git a/crates/capsem-core/src/net/dns/session/mod.rs b/crates/capsem-core/src/net/dns/session/mod.rs index aba7620ec..1c88fa871 100644 --- a/crates/capsem-core/src/net/dns/session/mod.rs +++ b/crates/capsem-core/src/net/dns/session/mod.rs @@ -179,6 +179,7 @@ pub async fn emit_dns_security_write_and_rules( .await { warn!(error = %error, "failed to emit DNS security rule ledger rows"); + return None; } Some(event_id) } diff --git a/crates/capsem-core/src/security_engine/mod.rs b/crates/capsem-core/src/security_engine/mod.rs index cd139999c..2ff331388 100644 --- a/crates/capsem-core/src/security_engine/mod.rs +++ b/crates/capsem-core/src/security_engine/mod.rs @@ -679,7 +679,9 @@ pub async fn emit_security_decision_transition( timestamp_unix_ms: i64, ) -> Result<(), String> { let decision_event = security_decision_event(event_id, event_type, rule, event, decision_state, timestamp_unix_ms)?; - emit_security_write(db, WriteOp::SecurityDecisionEvent(decision_event)).await; + emit_security_write(db, WriteOp::SecurityDecisionEvent(decision_event)) + .await + .ok_or_else(|| "ledger refused security decision row".to_string())?; Ok(()) } @@ -693,7 +695,8 @@ pub fn emit_security_decision_transition_blocking( timestamp_unix_ms: i64, ) -> Result<(), String> { let decision_event = security_decision_event(event_id, event_type, rule, event, decision_state, timestamp_unix_ms)?; - emit_security_write_blocking(db, WriteOp::SecurityDecisionEvent(decision_event)); + emit_security_write_blocking(db, WriteOp::SecurityDecisionEvent(decision_event)) + .ok_or_else(|| "ledger refused security decision row".to_string())?; Ok(()) } @@ -763,7 +766,9 @@ pub async fn emit_security_rule_match( ) -> Result<(), String> { let rule_event = security_rule_event(event_id, event_type, rule, event, timestamp_unix_ms)?; trace_security_rule_match(&rule_event, rule); - emit_security_write(db, WriteOp::SecurityRuleEvent(rule_event)).await; + emit_security_write(db, WriteOp::SecurityRuleEvent(rule_event)) + .await + .ok_or_else(|| "ledger refused security rule row".to_string())?; Ok(()) } @@ -801,7 +806,9 @@ pub async fn emit_security_ask_pending( ) -> Result { let ask_id = SecurityEventId::new_uuid4(); let ask_event = security_ask_pending_event(ask_id.clone(), event_id, event_type, rule, event, timestamp_unix_ms)?; - emit_security_write(db, WriteOp::SecurityAskEvent(ask_event)).await; + emit_security_write(db, WriteOp::SecurityAskEvent(ask_event)) + .await + .ok_or_else(|| "ledger refused pending security ask row".to_string())?; Ok(ask_id) } @@ -815,7 +822,8 @@ pub fn emit_security_ask_pending_blocking( ) -> Result { let ask_id = SecurityEventId::new_uuid4(); let ask_event = security_ask_pending_event(ask_id.clone(), event_id, event_type, rule, event, timestamp_unix_ms)?; - emit_security_write_blocking(db, WriteOp::SecurityAskEvent(ask_event)); + emit_security_write_blocking(db, WriteOp::SecurityAskEvent(ask_event)) + .ok_or_else(|| "ledger refused pending security ask row".to_string())?; Ok(ask_id) } @@ -828,7 +836,8 @@ pub fn emit_security_ask_resolution_blocking( timestamp_unix_ms: i64, ) -> Result<(), String> { let event = security_ask_resolution_event(pending, status, resolver, reason, timestamp_unix_ms)?; - emit_security_write_blocking(db, WriteOp::SecurityAskEvent(event)); + emit_security_write_blocking(db, WriteOp::SecurityAskEvent(event)) + .ok_or_else(|| "ledger refused resolved security ask row".to_string())?; Ok(()) } @@ -841,7 +850,9 @@ pub async fn emit_security_ask_resolution( timestamp_unix_ms: i64, ) -> Result<(), String> { let event = security_ask_resolution_event(pending, status, resolver, reason, timestamp_unix_ms)?; - emit_security_write(db, WriteOp::SecurityAskEvent(event)).await; + emit_security_write(db, WriteOp::SecurityAskEvent(event)) + .await + .ok_or_else(|| "ledger refused resolved security ask row".to_string())?; Ok(()) } diff --git a/crates/capsem-core/src/security_engine/rule_emission.rs b/crates/capsem-core/src/security_engine/rule_emission.rs index eb511ff9f..a6f2f69e2 100644 --- a/crates/capsem-core/src/security_engine/rule_emission.rs +++ b/crates/capsem-core/src/security_engine/rule_emission.rs @@ -120,17 +120,17 @@ async fn emit_rule_rows( let mut enforcement = security_enforcement_decision(selected_rule, &mut enriched_event); let mut decision_state = enriched_event.decision.clone(); let mut rule_events = Vec::new(); + let mut writes = Vec::new(); if let Some(rule) = selected_rule { - emit_security_decision_transition( - db, + let decision_event = security_decision_event( event_id.clone(), event_type, rule, &enriched_event, &mut decision_state, timestamp_unix_ms, - ) - .await?; + )?; + writes.push(WriteOp::SecurityDecisionEvent(decision_event)); if recorded == RuleRowDecision::Enforced { enriched_event.decision = decision_state.clone(); } @@ -138,7 +138,7 @@ async fn emit_rule_rows( for rule in evaluation.matched_rules() { let rule_event = security_rule_event(event_id.clone(), event_type, rule, &enriched_event, timestamp_unix_ms)?; trace_security_rule_match(&rule_event, rule); - emit_security_write(db, WriteOp::SecurityRuleEvent(rule_event.clone())).await; + writes.push(WriteOp::SecurityRuleEvent(rule_event.clone())); rule_events.push(rule_event); emitted += 1; } @@ -146,17 +146,22 @@ async fn emit_rule_rows( let Some(rule) = selected_rule else { return Err("ask enforcement decision did not carry a rule".to_string()); }; - let ask_id = emit_security_ask_pending( - db, + let ask_id = SecurityEventId::new_uuid4(); + let ask_event = security_ask_pending_event( + ask_id.clone(), event_id.clone(), event_type, rule, &enriched_event, timestamp_unix_ms, - ) - .await?; + )?; + writes.push(WriteOp::SecurityAskEvent(ask_event)); enforcement.ask_id = Some(ask_id); } + let admitted = futures::future::join_all(writes.into_iter().map(|write| emit_security_write(db, write))).await; + if admitted.iter().any(Option::is_none) { + return Err("ledger refused one or more supplemental security rows".to_string()); + } Ok(SecurityRuleEmission { event_id, emitted, @@ -200,7 +205,8 @@ fn emit_rule_rows_blocking( for rule in evaluation.matched_rules() { let rule_event = security_rule_event(event_id.clone(), event_type, rule, &enriched_event, timestamp_unix_ms)?; trace_security_rule_match(&rule_event, rule); - emit_security_write_blocking(db, WriteOp::SecurityRuleEvent(rule_event.clone())); + emit_security_write_blocking(db, WriteOp::SecurityRuleEvent(rule_event.clone())) + .ok_or_else(|| "ledger refused security rule row".to_string())?; rule_events.push(rule_event); emitted += 1; } diff --git a/crates/capsem-core/src/security_engine/tests/decision_ledger.rs b/crates/capsem-core/src/security_engine/tests/decision_ledger.rs index 1471c3a9a..6caaa7971 100644 --- a/crates/capsem-core/src/security_engine/tests/decision_ledger.rs +++ b/crates/capsem-core/src/security_engine/tests/decision_ledger.rs @@ -435,6 +435,45 @@ match = 'http.path.startsWith("/v1/")' ); } +#[tokio::test] +async fn emit_matching_security_rules_reports_supplemental_admission_failure() { + let writer = capsem_logger::DbWriter::open_in_memory(16).unwrap(); + writer.shutdown_blocking(); + let profile = SecurityRuleProfile::parse_toml( + r#" +[profiles.rules.http_block] +name = "http_block" +action = "block" +detection_level = "critical" +match = 'http.path.startsWith("/v1/")' +"#, + ) + .unwrap(); + let rules = + crate::net::policy_config::SecurityRuleSet::compile_profile(&profile, SecurityRuleSource::User).unwrap(); + let event = SecurityEvent::new(RuntimeSecurityEventType::HttpRequest).with_http(HttpSecurityEvent { + host: Some("api.openai.com".into()), + method: Some("POST".into()), + path: Some("/v1/responses".into()), + query: None, + status: None, + body: None, + }); + + let error = emit_matching_security_rules( + &writer, + SecurityEventId::new_uuid4(), + RuntimeSecurityEventType::HttpRequest, + &rules, + &event, + 1_789_000_000_200, + ) + .await + .unwrap_err(); + + assert!(error.contains("supplemental security rows"), "{error}"); +} + #[tokio::test] async fn emit_matching_security_rules_with_decision_uses_same_evaluation_as_ledger() { let tmp = tempfile::tempdir().unwrap(); From 9403ff6a222ce3f57a58b14f15b9ceaf90c4769f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 04:29:37 +0000 Subject: [PATCH 400/464] feat(proto): centralize DNS and ledger capacity limits --- config/data/runtime-limits.toml | 18 +++++- crates/capsem-proto/build.rs | 47 +++++++++++++++ tests/citadel/test_runtime_limit_authority.py | 58 ++++++++++++++++++- 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index 3ca77e97d..4edba4daa 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -9,6 +9,12 @@ traffic_grants_per_client = 3 # descriptor. The remaining per-client allowance covers listeners, the Tokio # reactor, DNS, control traffic, and overlap while connections retire. file_descriptors_per_client = 8 +# Two independent guest-to-host sessions cover the usual A+AAAA pair and keep +# resolution available while one session reconnects. +dns_sessions_per_vm = 2 +# RFC 6891's practical EDNS UDP payload ceiling. The listener buffer derives +# from this, the session count and the supported client count below. +dns_max_udp_datagram_bytes = 4096 [ledger] # A DNS query contributes a primary row, a decision transition and a rule row. @@ -17,10 +23,20 @@ file_descriptors_per_client = 8 producer_events_per_network_client = 8 # One transport exchange carries this many consecutive producer events. The # queue must be an exact multiple so a full burst drains without a tail shape. -admission_batch_capacity = 64 +admission_batch_capacity = 256 # A lone event waits briefly for its immediate followers. This is far below a # network round trip while avoiding singleton IPC under concurrent bursts. admission_coalesce_wait_microseconds = 100 +# Body indexes are published only after their archive segment reaches the +# device. A slow sync can make several seconds of rows eligible at once; cap +# each following SQLite transaction so admission never waits on that backlog. +# This is an exact admission-batch multiple so the drain has no tail shape. +archive_index_rows_per_flush = 8192 +# The ledger is append-heavy and its polled queries use narrow indexes. A +# 256 MiB window made each busy VM retain almost the full ledger as resident +# file-backed memory; keep a 32 MiB query working set and let the kernel cache +# colder pages globally across many concurrent VMs. +sqlite_mmap_bytes = 33554432 # A fail-closed network action waits at most one second for admission. Flush # often enough that copying preceding memory rows cannot consume that budget. disk_flush_interval_milliseconds = 1000 diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index b3015b904..0bb6b339c 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -74,6 +74,25 @@ fn write_runtime_limits() { .checked_mul(descriptors_per_client) .filter(|value| *value <= 1_048_576) .expect("runtime limits: derived network descriptor budget exceeds 1048576"); + let dns_sessions_per_vm = config + .get("network") + .and_then(|network| network.get("dns_sessions_per_vm")) + .and_then(toml::Value::as_integer) + .filter(|value| (2..=8).contains(value)) + .unwrap_or_else(|| panic!("runtime limits: network.dns_sessions_per_vm must be an integer between 2 and 8")); + let dns_max_udp_datagram_bytes = config + .get("network") + .and_then(|network| network.get("dns_max_udp_datagram_bytes")) + .and_then(toml::Value::as_integer) + .filter(|value| (4096..=65_507).contains(value)) + .unwrap_or_else(|| { + panic!("runtime limits: network.dns_max_udp_datagram_bytes must be an integer between 4096 and 65507") + }); + let dns_udp_socket_buffer_bytes = capacity + .checked_mul(dns_sessions_per_vm) + .and_then(|value| value.checked_mul(dns_max_udp_datagram_bytes)) + .filter(|value| *value <= 64 * 1024 * 1024) + .expect("runtime limits: derived DNS UDP socket buffer exceeds 64 MiB"); let producer_events_per_network_client = config .get("ledger") .and_then(|ledger| ledger.get("producer_events_per_network_client")) @@ -105,6 +124,24 @@ fn write_runtime_limits() { .unwrap_or_else(|| { panic!("runtime limits: ledger.admission_coalesce_wait_microseconds must be an integer between 1 and 10000") }); + let ledger_archive_index_rows_per_flush = config + .get("ledger") + .and_then(|ledger| ledger.get("archive_index_rows_per_flush")) + .and_then(toml::Value::as_integer) + .filter(|value| (ledger_batch_capacity..=65_536).contains(value)) + .filter(|value| value % ledger_batch_capacity == 0) + .unwrap_or_else(|| { + panic!("runtime limits: ledger.archive_index_rows_per_flush must be a batch multiple between the batch capacity and 65536") + }); + let ledger_sqlite_mmap_bytes = config + .get("ledger") + .and_then(|ledger| ledger.get("sqlite_mmap_bytes")) + .and_then(toml::Value::as_integer) + .filter(|value| (4 * 1024 * 1024..=256 * 1024 * 1024).contains(value)) + .filter(|value| value % (1024 * 1024) == 0) + .unwrap_or_else(|| { + panic!("runtime limits: ledger.sqlite_mmap_bytes must be a whole MiB between 4 MiB and 256 MiB") + }); let ledger_disk_flush_interval_milliseconds = config .get("ledger") .and_then(|ledger| ledger.get("disk_flush_interval_milliseconds")) @@ -135,12 +172,22 @@ fn write_runtime_limits() { pub const NETWORK_TRAFFIC_GRANT_CAPACITY: usize = {traffic_grant_capacity};\n\ /// Minimum descriptor budget for a network-facing process.\n\ pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit};\n\ + /// Independent guest-to-host DNS sessions per VM.\n\ + pub const NETWORK_DNS_SESSIONS_PER_VM: usize = {dns_sessions_per_vm};\n\ + /// Maximum DNS datagram accepted by the guest proxy.\n\ + pub const NETWORK_DNS_MAX_UDP_DATAGRAM_BYTES: usize = {dns_max_udp_datagram_bytes};\n\ + /// Receive buffer for one full supported DNS client burst.\n\ + pub const NETWORK_DNS_UDP_SOCKET_BUFFER_BYTES: usize = {dns_udp_socket_buffer_bytes};\n\ /// Per-producer security-event queue derived from supported clients.\n\ pub const LEDGER_PRODUCER_QUEUE_CAPACITY: usize = {ledger_queue_capacity};\n\ /// Consecutive producer events coalesced into one ledger exchange.\n\ pub const LEDGER_ADMISSION_BATCH_CAPACITY: usize = {ledger_batch_capacity};\n\ /// Maximum wait for an immediate follower before admitting a singleton.\n\ pub const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = {ledger_coalesce_wait_microseconds};\n\ + /// Archive body-index rows published in one disk flush transaction.\n\ + pub const LEDGER_ARCHIVE_INDEX_ROWS_PER_FLUSH: usize = {ledger_archive_index_rows_per_flush};\n\ + /// Maximum file-backed ledger window mapped into one process.\n\ + pub const LEDGER_SQLITE_MMAP_BYTES: i64 = {ledger_sqlite_mmap_bytes};\n\ /// Maximum time accepted rows stay only in the memory schema.\n\ pub const LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS: u64 = {ledger_disk_flush_interval_milliseconds};\n\ /// Pending events that force a disk flush before the timer.\n\ diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index ef31af1a4..08b486dce 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -29,6 +29,9 @@ CONFIG_OWNED_NAMES = { "NETWORK_CLIENT_CAPACITY", "NETWORK_PROCESS_OPEN_FILE_LIMIT", + "NETWORK_DNS_SESSIONS_PER_VM", + "NETWORK_DNS_MAX_UDP_DATAGRAM_BYTES", + "NETWORK_DNS_UDP_SOCKET_BUFFER_BYTES", "DNS_SESSION_MAX_IN_FLIGHT", "UPSTREAM_GRANT_CAPACITY", "TRAFFIC_GRANT_CAPACITY", @@ -37,6 +40,8 @@ "LEDGER_PRODUCER_QUEUE_CAPACITY", "LEDGER_ADMISSION_BATCH_CAPACITY", "LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS", + "LEDGER_ARCHIVE_INDEX_ROWS_PER_FLUSH", + "LEDGER_SQLITE_MMAP_BYTES", } REQUIRED_WIRING = { @@ -50,6 +55,11 @@ 'get("file_descriptors_per_client")', ".checked_mul(descriptors_per_client)", "pub const NETWORK_PROCESS_OPEN_FILE_LIMIT: u64 = {open_file_limit}", + 'get("dns_sessions_per_vm")', + 'get("dns_max_udp_datagram_bytes")', + "pub const NETWORK_DNS_SESSIONS_PER_VM: usize = {dns_sessions_per_vm}", + "pub const NETWORK_DNS_MAX_UDP_DATAGRAM_BYTES: usize = {dns_max_udp_datagram_bytes}", + "pub const NETWORK_DNS_UDP_SOCKET_BUFFER_BYTES: usize = {dns_udp_socket_buffer_bytes}", 'get("producer_events_per_network_client")', ".checked_mul(producer_events_per_network_client)", 'get("admission_batch_capacity")', @@ -58,6 +68,10 @@ "pub const LEDGER_ADMISSION_BATCH_CAPACITY: usize = {ledger_batch_capacity}", 'get("admission_coalesce_wait_microseconds")', "pub const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = {ledger_coalesce_wait_microseconds}", + 'get("archive_index_rows_per_flush")', + "pub const LEDGER_ARCHIVE_INDEX_ROWS_PER_FLUSH: usize = {ledger_archive_index_rows_per_flush}", + 'get("sqlite_mmap_bytes")', + "pub const LEDGER_SQLITE_MMAP_BYTES: i64 = {ledger_sqlite_mmap_bytes}", 'get("disk_flush_interval_milliseconds")', "pub const LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS: u64 = {ledger_disk_flush_interval_milliseconds}", 'get("disk_flush_max_pending_events")', @@ -100,17 +114,32 @@ "LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS", "LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS", ), + "crates/capsem-logger/src/schema/pragmas.rs": ( + "capsem_proto::LEDGER_SQLITE_MMAP_BYTES", + ), "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), - "crates/capsem-agent/src/dns_proxy/session.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), + "crates/capsem-agent/src/dns_proxy/session.rs": ( + "DNS_SESSION_MAX_IN_FLIGHT", + "NETWORK_DNS_SESSIONS_PER_VM", + ), + "crates/capsem-agent/src/dns_proxy.rs": ( + "NETWORK_DNS_MAX_UDP_DATAGRAM_BYTES", + "NETWORK_DNS_UDP_SOCKET_BUFFER_BYTES", + "set_receive_buffer", + ), "crates/capsem-agent/src/net_proxy.rs": ( "NETWORK_PROCESS_OPEN_FILE_LIMIT", "ensure_open_file_limit", ), "crates/capsem-bench/src/runtime_limits.rs": ( "NETWORK_CLIENT_CAPACITY", + "NETWORK_DNS_SESSIONS_PER_VM", + "NETWORK_DNS_MAX_UDP_DATAGRAM_BYTES", + "NETWORK_DNS_UDP_SOCKET_BUFFER_BYTES", "LEDGER_PRODUCER_QUEUE_CAPACITY", "LEDGER_ADMISSION_BATCH_CAPACITY", "LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS", + "LEDGER_SQLITE_MMAP_BYTES", ), "guest/artifacts/capsem_bench/dns_load.py": ( '"runtime-limits"', @@ -160,6 +189,19 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: + "\nnetwork.file_descriptors_per_client must cover both ends of every " "overlapping traffic grant" ) + dns_sessions_per_vm = config.get("network", {}).get("dns_sessions_per_vm") + assert type(dns_sessions_per_vm) is int and 2 <= dns_sessions_per_vm <= 8, ( + RUNTIME_LIMIT_RATIONALE + ) + dns_max_udp_datagram_bytes = config.get("network", {}).get( + "dns_max_udp_datagram_bytes" + ) + assert ( + type(dns_max_udp_datagram_bytes) is int + and 4096 <= dns_max_udp_datagram_bytes <= 65_507 + and capacity * dns_sessions_per_vm * dns_max_udp_datagram_bytes + <= 64 * 1024 * 1024 + ), RUNTIME_LIMIT_RATIONALE producer_events_per_network_client = config.get("ledger", {}).get( "producer_events_per_network_client" ) @@ -183,6 +225,20 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: type(coalesce_wait_microseconds) is int and 1 <= coalesce_wait_microseconds <= 10_000 ), RUNTIME_LIMIT_RATIONALE + archive_index_rows_per_flush = config.get("ledger", {}).get( + "archive_index_rows_per_flush" + ) + assert ( + type(archive_index_rows_per_flush) is int + and admission_batch_capacity <= archive_index_rows_per_flush <= 65_536 + and archive_index_rows_per_flush % admission_batch_capacity == 0 + ), RUNTIME_LIMIT_RATIONALE + sqlite_mmap_bytes = config.get("ledger", {}).get("sqlite_mmap_bytes") + assert ( + type(sqlite_mmap_bytes) is int + and 4 * 1024 * 1024 <= sqlite_mmap_bytes <= 256 * 1024 * 1024 + and sqlite_mmap_bytes % (1024 * 1024) == 0 + ), RUNTIME_LIMIT_RATIONALE disk_flush_interval_milliseconds = config.get("ledger", {}).get( "disk_flush_interval_milliseconds" ) From 948b97937d1229e31674374879c61029ba1d26ad Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 04:29:44 +0000 Subject: [PATCH 401/464] feat(foundation): size datagram receive buffers --- crates/capsem-foundation/src/unix/fd.rs | 23 ++++++++++++++----- crates/capsem-foundation/src/unix/fd/tests.rs | 8 +++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/crates/capsem-foundation/src/unix/fd.rs b/crates/capsem-foundation/src/unix/fd.rs index 583faa2ac..e19f43a43 100644 --- a/crates/capsem-foundation/src/unix/fd.rs +++ b/crates/capsem-foundation/src/unix/fd.rs @@ -268,12 +268,7 @@ pub fn tcp_clear_reset_on_close(fd: BorrowedFd<'_>) -> io::Result { /// Linux accounts up to twice the requested bytes for TCP bookkeeping. VSOCK /// uses separate credit-buffer options, so those are fixed as well on Linux. pub fn set_stream_buffers(fd: BorrowedFd<'_>, bytes: usize) -> io::Result<()> { - if bytes == 0 || bytes > i32::MAX as usize { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "invalid socket buffer size", - )); - } + validate_socket_buffer_size(bytes)?; socket::setsockopt(&fd, socket::sockopt::SndBuf, &bytes).map_err(errno::io)?; socket::setsockopt(&fd, socket::sockopt::RcvBuf, &bytes).map_err(errno::io)?; #[cfg(target_os = "linux")] @@ -309,6 +304,22 @@ pub fn set_stream_buffers(fd: BorrowedFd<'_>, bytes: usize) -> io::Result<()> { Ok(()) } +/// Size a datagram receive queue for a known bounded burst. +pub fn set_receive_buffer(fd: BorrowedFd<'_>, bytes: usize) -> io::Result<()> { + validate_socket_buffer_size(bytes)?; + socket::setsockopt(&fd, socket::sockopt::RcvBuf, &bytes).map_err(errno::io) +} + +fn validate_socket_buffer_size(bytes: usize) -> io::Result<()> { + if bytes == 0 || bytes > i32::MAX as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid socket buffer size", + )); + } + Ok(()) +} + /// What the kernel holds for one connected stream socket, for saying which /// end of a stalled copy stopped moving. Each count is read on its own and is /// `None` where the platform or socket family cannot answer. diff --git a/crates/capsem-foundation/src/unix/fd/tests.rs b/crates/capsem-foundation/src/unix/fd/tests.rs index 5facbe187..2538e19df 100644 --- a/crates/capsem-foundation/src/unix/fd/tests.rs +++ b/crates/capsem-foundation/src/unix/fd/tests.rs @@ -168,6 +168,14 @@ fn stream_buffer_limits_replace_large_kernel_queues() { assert!(getsockopt(&client, sockopt::RcvBuf).unwrap() <= 64 * 1024); } +#[test] +fn datagram_receive_buffer_can_hold_a_concurrent_burst() { + use nix::sys::socket::{getsockopt, sockopt}; + let socket = std::net::UdpSocket::bind("127.0.0.1:0").unwrap(); + super::set_receive_buffer(socket.as_fd(), 512 * 1024).unwrap(); + assert!(getsockopt(&socket, sockopt::RcvBuf).unwrap() >= 512 * 1024); +} + #[test] fn zero_buffer_limit_is_refused_without_changing_the_socket() { use nix::sys::socket::{getsockopt, sockopt}; From 496a48cbaa7593d2298166f55712f71685792733 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 04:29:55 +0000 Subject: [PATCH 402/464] fix(agent): absorb supported DNS bursts --- CHANGELOG.md | 4 ++++ crates/capsem-agent/src/dns_proxy.rs | 4 +++- crates/capsem-agent/src/dns_proxy/session.rs | 2 +- crates/capsem-agent/src/dns_proxy/tests.rs | 10 ++++++++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fd8f879f..b2b825383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -207,6 +207,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- The guest DNS listener sizes its UDP receive queue from the supported client, + session and EDNS datagram limits, preventing kernel receive drops during a + 512-client dual-stack burst. + - Sustain the configured 512 concurrent intercepted connections in the guest network relay by raising its inherited descriptor limit to the centralized runtime budget. Linux shells commonly supplied only 1,024 descriptors while diff --git a/crates/capsem-agent/src/dns_proxy.rs b/crates/capsem-agent/src/dns_proxy.rs index 4b11ade6f..b3f52e906 100644 --- a/crates/capsem-agent/src/dns_proxy.rs +++ b/crates/capsem-agent/src/dns_proxy.rs @@ -40,6 +40,7 @@ mod vsock_io; mod wire; use std::io; +use std::os::fd::AsFd as _; use std::process; use std::sync::Arc; @@ -62,10 +63,11 @@ const LISTEN_PORT: u16 = capsem_proto::GUEST_DNS_PROXY_PORT; /// Maximum bytes for one DNS UDP datagram. RFC 6891 caps practical /// EDNS responses at ~4096; standard queries fit in 512. 4096 is what /// hickory uses internally. -const MAX_UDP_DNS_BYTES: usize = 4096; +const MAX_UDP_DNS_BYTES: usize = capsem_proto::NETWORK_DNS_MAX_UDP_DATAGRAM_BYTES; /// UDP listener: read one datagram, forward, send the response back. async fn run_udp_listener(forwarder: Arc) -> io::Result<()> { let sock = UdpSocket::bind((LISTEN_BIND, LISTEN_PORT)).await?; + capsem_foundation::unix::fd::set_receive_buffer(sock.as_fd(), capsem_proto::NETWORK_DNS_UDP_SOCKET_BUFFER_BYTES)?; eprintln!("[capsem-dns-proxy] udp listening on {LISTEN_BIND}:{LISTEN_PORT}"); let sock = std::sync::Arc::new(sock); loop { diff --git a/crates/capsem-agent/src/dns_proxy/session.rs b/crates/capsem-agent/src/dns_proxy/session.rs index fa432ce62..9db01fb17 100644 --- a/crates/capsem-agent/src/dns_proxy/session.rs +++ b/crates/capsem-agent/src/dns_proxy/session.rs @@ -29,7 +29,7 @@ use capsem_proto::{decode_dns_response, encode_dns_request, DnsRequest, DnsRespo /// Sessions per forwarder. Two keep the guest resolving while one /// reconnects, and give the host two independent frame streams. -pub(crate) const DNS_SESSIONS: usize = 2; +pub(crate) const DNS_SESSIONS: usize = capsem_proto::NETWORK_DNS_SESSIONS_PER_VM; /// Queries one session holds in flight before it sheds. Past the shared /// guest/host bound the client gets SERVFAIL rather than an unbounded queue. pub(crate) use capsem_proto::DNS_SESSION_MAX_IN_FLIGHT; diff --git a/crates/capsem-agent/src/dns_proxy/tests.rs b/crates/capsem-agent/src/dns_proxy/tests.rs index 7b1753e00..a85e7d1b8 100644 --- a/crates/capsem-agent/src/dns_proxy/tests.rs +++ b/crates/capsem-agent/src/dns_proxy/tests.rs @@ -35,6 +35,16 @@ fn max_udp_dns_bytes_supports_edns() { const _: () = assert!(MAX_UDP_DNS_BYTES >= 4096); } +#[test] +fn udp_burst_buffer_is_derived_from_the_shared_client_capacity() { + assert_eq!(DNS_SESSIONS, capsem_proto::NETWORK_DNS_SESSIONS_PER_VM); + assert_eq!(MAX_UDP_DNS_BYTES, capsem_proto::NETWORK_DNS_MAX_UDP_DATAGRAM_BYTES); + assert_eq!( + capsem_proto::NETWORK_DNS_UDP_SOCKET_BUFFER_BYTES, + capsem_proto::NETWORK_CLIENT_CAPACITY * DNS_SESSIONS * MAX_UDP_DNS_BYTES + ); +} + #[test] fn dns_proxy_keeps_more_than_one_persistent_session() { const _: () = assert!( From 7fa232537fae2449b86150ffa92754ee8c2d5c98 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 04:30:33 +0000 Subject: [PATCH 403/464] refactor(core): split security event engine --- config/gate.toml | 5 +- .../capsem-core/src/security_engine/engine.rs | 80 ++++++++ crates/capsem-core/src/security_engine/mod.rs | 82 +------- .../security_engine/tests/decision_ledger.rs | 189 +---------------- .../src/security_engine/tests/story.rs | 190 ++++++++++++++++++ 5 files changed, 276 insertions(+), 270 deletions(-) create mode 100644 crates/capsem-core/src/security_engine/engine.rs create mode 100644 crates/capsem-core/src/security_engine/tests/story.rs diff --git a/config/gate.toml b/config/gate.toml index 1e02f26da..f7513266d 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1104,8 +1104,9 @@ must_stay_below_lines = 3000 "crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs" = 2041 "crates/capsem-core/src/net/mitm_proxy/mod.rs" = 2528 "crates/capsem-core/src/net/policy_config/security_rule_profile/tests.rs" = 1549 -"crates/capsem-core/src/security_engine/mod.rs" = 2073 -"crates/capsem-core/tests/mitm_integration.rs" = 2014 +"crates/capsem-core/src/security_engine/mod.rs" = 2048 +"crates/capsem-core/tests/mitm_integration.rs" = 2087 +"crates/capsem-logger/src/schema/tests.rs" = 1003 "crates/capsem-logger/src/writer/tests.rs" = 1395 "crates/capsem-mock-server/src/main.rs" = 1167 "crates/capsem-process/src/vsock.rs" = 1239 diff --git a/crates/capsem-core/src/security_engine/engine.rs b/crates/capsem-core/src/security_engine/engine.rs new file mode 100644 index 000000000..7884e6a99 --- /dev/null +++ b/crates/capsem-core/src/security_engine/engine.rs @@ -0,0 +1,80 @@ +use super::*; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SecurityEmitError { + message: String, +} + +impl SecurityEmitError { + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl fmt::Display for SecurityEmitError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for SecurityEmitError {} + +/// Single auditable event emission boundary. +pub trait SecurityEventEmitter: Send + Sync { + fn emit(&self, event: SecurityEvent) -> Result<(), SecurityEmitError>; +} + +/// Security-event execution boundary for matched rule actions. +/// +/// Runtime/parser paths hand this engine a canonical `SecurityEvent` plus the +/// matched action-bearing rules. The engine applies actions in deterministic +/// order, then emits exactly the final post-action event. +pub struct SecurityEventEngine { + action_registry: SecurityActionRegistry, + emitter: Arc, +} + +impl SecurityEventEngine { + pub fn new(action_registry: SecurityActionRegistry, emitter: Arc) -> Self { + Self { + action_registry, + emitter, + } + } + + pub fn with_builtin_actions(emitter: Arc) -> Self { + Self::new(SecurityActionRegistry::with_builtin_actions(), emitter) + } + + pub fn apply_matching_rules_and_emit( + &self, + rules: &SecurityRuleSet, + mut event: SecurityEvent, + ) -> Result { + event.validate_network(event.event_type)?; + event = self + .action_registry + .apply_security_plugins(SecurityPluginStage::Preprocess, event)?; + + let evaluation = rules.evaluate(&event).map_err(SecurityActionError::new)?; + for rule in evaluation.matched_rules() { + record_rule_detection(&mut event, rule); + } + event.request_decision(requested_boundary_decision( + selected_enforcement_rule(&evaluation), + event.event_type, + )); + event = self + .action_registry + .apply_security_plugins(SecurityPluginStage::Postprocess, event)?; + event = self + .action_registry + .apply_security_plugins(SecurityPluginStage::Logging, event)?; + self.emitter + .emit(event.clone()) + .map_err(|error| SecurityActionError::new(error.to_string()))?; + Ok(event) + } +} diff --git a/crates/capsem-core/src/security_engine/mod.rs b/crates/capsem-core/src/security_engine/mod.rs index 2ff331388..c8e680314 100644 --- a/crates/capsem-core/src/security_engine/mod.rs +++ b/crates/capsem-core/src/security_engine/mod.rs @@ -22,7 +22,8 @@ mod builtin_actions; mod container; mod credential_grants; pub use container::ContainerSecurityEvent; -pub use credential_grants::CredentialGrantAdmission; +mod engine; +pub use engine::*; mod event_id; pub use event_id::SecurityEventId; mod forensics; @@ -2040,84 +2041,5 @@ fn model_contains_text(model: &ModelSecurityEvent, needle: &str) -> bool { .any(|content| content.contains(needle)) } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SecurityEmitError { - message: String, -} - -impl SecurityEmitError { - pub fn new(message: impl Into) -> Self { - Self { - message: message.into(), - } - } -} - -impl fmt::Display for SecurityEmitError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&self.message) - } -} - -impl std::error::Error for SecurityEmitError {} - -/// Single auditable event emission boundary. -pub trait SecurityEventEmitter: Send + Sync { - fn emit(&self, event: SecurityEvent) -> Result<(), SecurityEmitError>; -} - -/// Security-event execution boundary for matched rule actions. -/// -/// Runtime/parser paths hand this engine a canonical `SecurityEvent` plus the -/// matched action-bearing rules. The engine applies actions in deterministic -/// order, then emits exactly the final post-action event. -pub struct SecurityEventEngine { - action_registry: SecurityActionRegistry, - emitter: Arc, -} - -impl SecurityEventEngine { - pub fn new(action_registry: SecurityActionRegistry, emitter: Arc) -> Self { - Self { - action_registry, - emitter, - } - } - - pub fn with_builtin_actions(emitter: Arc) -> Self { - Self::new(SecurityActionRegistry::with_builtin_actions(), emitter) - } - - pub fn apply_matching_rules_and_emit( - &self, - rules: &SecurityRuleSet, - mut event: SecurityEvent, - ) -> Result { - event.validate_network(event.event_type)?; - event = self - .action_registry - .apply_security_plugins(SecurityPluginStage::Preprocess, event)?; - - let evaluation = rules.evaluate(&event).map_err(SecurityActionError::new)?; - for rule in evaluation.matched_rules() { - record_rule_detection(&mut event, rule); - } - event.request_decision(requested_boundary_decision( - selected_enforcement_rule(&evaluation), - event.event_type, - )); - event = self - .action_registry - .apply_security_plugins(SecurityPluginStage::Postprocess, event)?; - event = self - .action_registry - .apply_security_plugins(SecurityPluginStage::Logging, event)?; - self.emitter - .emit(event.clone()) - .map_err(|error| SecurityActionError::new(error.to_string()))?; - Ok(event) - } -} - #[cfg(test)] mod tests; diff --git a/crates/capsem-core/src/security_engine/tests/decision_ledger.rs b/crates/capsem-core/src/security_engine/tests/decision_ledger.rs index 6caaa7971..84b9c86d6 100644 --- a/crates/capsem-core/src/security_engine/tests/decision_ledger.rs +++ b/crates/capsem-core/src/security_engine/tests/decision_ledger.rs @@ -821,191 +821,4 @@ match = 'ip.value == "127.0.0.1" || http.host == "127.0.0.1"' ); } -#[tokio::test] -async fn session_db_regenerates_rule_enforcement_detection_and_ask_story() { - let tmp = tempfile::tempdir().unwrap(); - let db_path = tmp.path().join("session.db"); - let writer = capsem_logger::DbWriter::open(&db_path, 16).unwrap(); - let github_rules = security_rule_set( - r#" -[corp.rules.github_block] -name = "github_block" -action = "block" -detection_level = "critical" -priority = -10 -reason = "corp block" -match = 'http.host == "github.com"' - -[profiles.rules.github_detect] -name = "github_detect" -action = "allow" -detection_level = "high" -match = 'http.host == "github.com"' - -[profiles.rules.github_postprocess] -name = "github_postprocess" -action = "postprocess" -detection_level = "informational" -match = 'http.host == "github.com"' -"#, - ); - let github_event_id = emit_security_write(&writer, net_write(None)) - .await - .expect("primary HTTP event must receive an id"); - let github_event = SecurityEvent::new(RuntimeSecurityEventType::HttpRequest) - .with_trace_id("trace_github") - .with_http(HttpSecurityEvent { - host: Some("github.com".into()), - method: Some("GET".into()), - path: Some("/settings/tokens".into()), - ..Default::default() - }); - - let github_emission = emit_matching_security_rules_with_decision( - &writer, - github_event_id.clone(), - RuntimeSecurityEventType::HttpRequest, - &github_rules, - &github_event, - 1_789_000_000_310, - ) - .await - .unwrap(); - assert_eq!(github_emission.emitted, 3); - assert_eq!(github_emission.enforcement.action, SecurityEnforcementAction::Block); - assert_eq!( - github_emission.enforcement.rule_id.as_deref(), - Some("corp.rules.github_block") - ); - - let ask_rules = security_rule_set( - r#" -[profiles.rules.ask_openai] -name = "ask_openai" -action = "ask" -reason = "manual approval required" -match = 'http.host == "api.openai.com"' -"#, - ); - let ask_event_id = emit_security_write(&writer, net_write(None)) - .await - .expect("primary HTTP event must receive an id"); - let ask_event = SecurityEvent::new(RuntimeSecurityEventType::HttpRequest) - .with_trace_id("trace_openai_ask") - .with_http(HttpSecurityEvent { - host: Some("api.openai.com".into()), - method: Some("POST".into()), - path: Some("/v1/responses".into()), - ..Default::default() - }); - - let ask_emission = emit_matching_security_rules_with_decision( - &writer, - ask_event_id.clone(), - RuntimeSecurityEventType::HttpRequest, - &ask_rules, - &ask_event, - 1_789_000_000_320, - ) - .await - .unwrap(); - let ask_id = ask_emission - .enforcement - .ask_id - .clone() - .expect("ask decision must return ask_id"); - let ask_rule = ask_rules - .rules() - .iter() - .find(|rule| rule.rule_id == "profiles.rules.ask_openai") - .expect("ask rule must compile"); - let pending = security_ask_pending_event( - ask_id.clone(), - ask_event_id.clone(), - RuntimeSecurityEventType::HttpRequest, - ask_rule, - &ask_event, - 1_789_000_000_320, - ) - .unwrap(); - emit_security_ask_resolution( - &writer, - &pending, - capsem_logger::SecurityAskStatus::Denied, - "tester", - Some("denied for test".to_string()), - 1_789_000_000_330, - ) - .await - .unwrap(); - writer.shutdown_blocking(); - - let reader = capsem_logger::DbReader::open(&db_path).unwrap(); - let rows = reader.recent_security_rule_events(10).unwrap(); - assert_eq!(rows.len(), 4); - - let postprocess_row = rows - .iter() - .find(|row| row.rule_id == "profiles.rules.github_postprocess") - .expect("postprocess detection rule row must be present"); - assert_eq!(postprocess_row.event_id, github_event_id.as_str()); - assert_eq!(postprocess_row.event_type, "http.request"); - assert_eq!( - postprocess_row.rule_action, - capsem_logger::SecurityRuleAction::Postprocess - ); - assert_eq!( - postprocess_row.detection_level, - capsem_logger::SecurityDetectionLevel::Informational - ); - let postprocess_rule: serde_json::Value = serde_json::from_str(&postprocess_row.rule_json).unwrap(); - assert_eq!(postprocess_rule["provider"], "profiles"); - assert_eq!(postprocess_rule["rule_action"], "postprocess"); - assert_eq!(postprocess_rule["detection_level"], "informational"); - assert!(postprocess_rule.get("plugin").is_none()); - let postprocess_payload = archived_payload(&db_path, github_event_id.as_str()).await; - let postprocess_event: serde_json::Value = serde_json::from_str(&postprocess_payload).unwrap(); - assert_eq!(postprocess_event["event_type"], "http.request"); - assert_eq!(postprocess_event["http"]["host"], "github.com"); - - let block_row = rows - .iter() - .find(|row| row.rule_id == "corp.rules.github_block") - .expect("enforcement block row must be present"); - assert_eq!(block_row.rule_action, capsem_logger::SecurityRuleAction::Block); - assert_eq!( - block_row.detection_level, - capsem_logger::SecurityDetectionLevel::Critical - ); - let block_rule: serde_json::Value = serde_json::from_str(&block_row.rule_json).unwrap(); - assert_eq!(block_rule["reason"], "corp block"); - assert_eq!(block_rule["priority"], -10); - - let detect_row = rows - .iter() - .find(|row| row.rule_id == "profiles.rules.github_detect") - .expect("detection row must be present"); - assert_eq!(detect_row.detection_level, capsem_logger::SecurityDetectionLevel::High); - - let ask_rows = reader.recent_security_ask_events(10).unwrap(); - assert_eq!(ask_rows.len(), 2); - assert_eq!(ask_rows[0].status, capsem_logger::SecurityAskStatus::Denied); - assert_eq!(ask_rows[0].ask_id, ask_id.as_str()); - assert_eq!(ask_rows[0].event_id, ask_event_id.as_str()); - assert_eq!(ask_rows[0].rule_id, "profiles.rules.ask_openai"); - assert_eq!(ask_rows[0].resolver.as_deref(), Some("tester")); - assert_eq!(ask_rows[1].status, capsem_logger::SecurityAskStatus::Pending); - - let security = capsem_logger::DbHandle::open_external_reader(&db_path) - .unwrap() - .ledger_counters() - .await - .unwrap() - .security - .clone(); - assert_eq!(security.matches, 4); - assert_eq!(security.by_action.get("block"), Some(&1)); - assert_eq!(security.by_action.get("postprocess"), Some(&1)); - let postprocess = &security.by_rule["profiles.rules.github_postprocess"]["postprocess"]["informational"]; - assert_eq!(postprocess.latest_event_id, github_event_id.as_str()); -} +mod story; diff --git a/crates/capsem-core/src/security_engine/tests/story.rs b/crates/capsem-core/src/security_engine/tests/story.rs new file mode 100644 index 000000000..b234bda5e --- /dev/null +++ b/crates/capsem-core/src/security_engine/tests/story.rs @@ -0,0 +1,190 @@ +use super::*; + +#[tokio::test] +async fn session_db_regenerates_rule_enforcement_detection_and_ask_story() { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("session.db"); + let writer = capsem_logger::DbWriter::open(&db_path, 16).unwrap(); + let github_rules = security_rule_set( + r#" +[corp.rules.github_block] +name = "github_block" +action = "block" +detection_level = "critical" +priority = -10 +reason = "corp block" +match = 'http.host == "github.com"' + +[profiles.rules.github_detect] +name = "github_detect" +action = "allow" +detection_level = "high" +match = 'http.host == "github.com"' + +[profiles.rules.github_postprocess] +name = "github_postprocess" +action = "postprocess" +detection_level = "informational" +match = 'http.host == "github.com"' +"#, + ); + let github_event_id = emit_security_write(&writer, net_write(None)) + .await + .expect("primary HTTP event must receive an id"); + let github_event = SecurityEvent::new(RuntimeSecurityEventType::HttpRequest) + .with_trace_id("trace_github") + .with_http(HttpSecurityEvent { + host: Some("github.com".into()), + method: Some("GET".into()), + path: Some("/settings/tokens".into()), + ..Default::default() + }); + + let github_emission = emit_matching_security_rules_with_decision( + &writer, + github_event_id.clone(), + RuntimeSecurityEventType::HttpRequest, + &github_rules, + &github_event, + 1_789_000_000_310, + ) + .await + .unwrap(); + assert_eq!(github_emission.emitted, 3); + assert_eq!(github_emission.enforcement.action, SecurityEnforcementAction::Block); + assert_eq!( + github_emission.enforcement.rule_id.as_deref(), + Some("corp.rules.github_block") + ); + + let ask_rules = security_rule_set( + r#" +[profiles.rules.ask_openai] +name = "ask_openai" +action = "ask" +reason = "manual approval required" +match = 'http.host == "api.openai.com"' +"#, + ); + let ask_event_id = emit_security_write(&writer, net_write(None)) + .await + .expect("primary HTTP event must receive an id"); + let ask_event = SecurityEvent::new(RuntimeSecurityEventType::HttpRequest) + .with_trace_id("trace_openai_ask") + .with_http(HttpSecurityEvent { + host: Some("api.openai.com".into()), + method: Some("POST".into()), + path: Some("/v1/responses".into()), + ..Default::default() + }); + + let ask_emission = emit_matching_security_rules_with_decision( + &writer, + ask_event_id.clone(), + RuntimeSecurityEventType::HttpRequest, + &ask_rules, + &ask_event, + 1_789_000_000_320, + ) + .await + .unwrap(); + let ask_id = ask_emission + .enforcement + .ask_id + .clone() + .expect("ask decision must return ask_id"); + let ask_rule = ask_rules + .rules() + .iter() + .find(|rule| rule.rule_id == "profiles.rules.ask_openai") + .expect("ask rule must compile"); + let pending = security_ask_pending_event( + ask_id.clone(), + ask_event_id.clone(), + RuntimeSecurityEventType::HttpRequest, + ask_rule, + &ask_event, + 1_789_000_000_320, + ) + .unwrap(); + emit_security_ask_resolution( + &writer, + &pending, + capsem_logger::SecurityAskStatus::Denied, + "tester", + Some("denied for test".to_string()), + 1_789_000_000_330, + ) + .await + .unwrap(); + writer.shutdown_blocking(); + + let reader = capsem_logger::DbReader::open(&db_path).unwrap(); + let rows = reader.recent_security_rule_events(10).unwrap(); + assert_eq!(rows.len(), 4); + + let postprocess_row = rows + .iter() + .find(|row| row.rule_id == "profiles.rules.github_postprocess") + .expect("postprocess detection rule row must be present"); + assert_eq!(postprocess_row.event_id, github_event_id.as_str()); + assert_eq!(postprocess_row.event_type, "http.request"); + assert_eq!( + postprocess_row.rule_action, + capsem_logger::SecurityRuleAction::Postprocess + ); + assert_eq!( + postprocess_row.detection_level, + capsem_logger::SecurityDetectionLevel::Informational + ); + let postprocess_rule: serde_json::Value = serde_json::from_str(&postprocess_row.rule_json).unwrap(); + assert_eq!(postprocess_rule["provider"], "profiles"); + assert_eq!(postprocess_rule["rule_action"], "postprocess"); + assert_eq!(postprocess_rule["detection_level"], "informational"); + assert!(postprocess_rule.get("plugin").is_none()); + let postprocess_payload = archived_payload(&db_path, github_event_id.as_str()).await; + let postprocess_event: serde_json::Value = serde_json::from_str(&postprocess_payload).unwrap(); + assert_eq!(postprocess_event["event_type"], "http.request"); + assert_eq!(postprocess_event["http"]["host"], "github.com"); + + let block_row = rows + .iter() + .find(|row| row.rule_id == "corp.rules.github_block") + .expect("enforcement block row must be present"); + assert_eq!(block_row.rule_action, capsem_logger::SecurityRuleAction::Block); + assert_eq!( + block_row.detection_level, + capsem_logger::SecurityDetectionLevel::Critical + ); + let block_rule: serde_json::Value = serde_json::from_str(&block_row.rule_json).unwrap(); + assert_eq!(block_rule["reason"], "corp block"); + assert_eq!(block_rule["priority"], -10); + + let detect_row = rows + .iter() + .find(|row| row.rule_id == "profiles.rules.github_detect") + .expect("detection row must be present"); + assert_eq!(detect_row.detection_level, capsem_logger::SecurityDetectionLevel::High); + + let ask_rows = reader.recent_security_ask_events(10).unwrap(); + assert_eq!(ask_rows.len(), 2); + assert_eq!(ask_rows[0].status, capsem_logger::SecurityAskStatus::Denied); + assert_eq!(ask_rows[0].ask_id, ask_id.as_str()); + assert_eq!(ask_rows[0].event_id, ask_event_id.as_str()); + assert_eq!(ask_rows[0].rule_id, "profiles.rules.ask_openai"); + assert_eq!(ask_rows[0].resolver.as_deref(), Some("tester")); + assert_eq!(ask_rows[1].status, capsem_logger::SecurityAskStatus::Pending); + + let security = capsem_logger::DbHandle::open_external_reader(&db_path) + .unwrap() + .ledger_counters() + .await + .unwrap() + .security + .clone(); + assert_eq!(security.matches, 4); + assert_eq!(security.by_action.get("block"), Some(&1)); + assert_eq!(security.by_action.get("postprocess"), Some(&1)); + let postprocess = &security.by_rule["profiles.rules.github_postprocess"]["postprocess"]["informational"]; + assert_eq!(postprocess.latest_event_id, github_event_id.as_str()); +} From 866179ffe44756dda33d51cc47f747688ed7b59e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 04:30:56 +0000 Subject: [PATCH 404/464] perf(logger): bound archive publication memory --- CHANGELOG.md | 5 + config/gate.toml | 3 +- crates/capsem-logger/src/schema.rs | 5 + crates/capsem-logger/src/schema/pragmas.rs | 2 +- crates/capsem-logger/src/schema/tests.rs | 110 ++---- .../capsem-logger/src/schema/tests/pragmas.rs | 84 ++++ crates/capsem-logger/src/writer.rs | 358 +----------------- crates/capsem-logger/src/writer/bodies.rs | 287 ++------------ .../capsem-logger/src/writer/bodies/index.rs | 224 +++++++++++ .../src/writer/bodies/sync_worker.rs | 65 ++++ crates/capsem-logger/src/writer/flush.rs | 152 ++++++++ crates/capsem-logger/src/writer/remote.rs | 5 +- crates/capsem-logger/src/writer/tests.rs | 201 +--------- .../capsem-logger/src/writer/tests/bodies.rs | 123 +++--- .../src/writer/tests/bodies/aging.rs | 54 +++ .../capsem-logger/src/writer/tests/salvage.rs | 201 ++++++++++ crates/capsem-logger/src/writer/worker.rs | 199 ++++++++++ 17 files changed, 1140 insertions(+), 938 deletions(-) create mode 100644 crates/capsem-logger/src/schema/tests/pragmas.rs create mode 100644 crates/capsem-logger/src/writer/bodies/index.rs create mode 100644 crates/capsem-logger/src/writer/bodies/sync_worker.rs create mode 100644 crates/capsem-logger/src/writer/flush.rs create mode 100644 crates/capsem-logger/src/writer/tests/bodies/aging.rs create mode 100644 crates/capsem-logger/src/writer/tests/salvage.rs create mode 100644 crates/capsem-logger/src/writer/worker.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index b2b825383..655b3531f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -311,6 +311,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Ledger staging returns SQLite pages after each flush, caps each process's + file-backed map at 32 MiB, and publishes sync-completed body indexes in + bounded transactions without replaying block metadata. Busy sessions no + longer retain their largest burst or stall admission on an index backlog. + - Ledger producers coalesce bounded consecutive security events into one commitment reservation, admission exchange and SQLite transaction, avoiding per-row scheduling and audit backpressure during concurrent DNS and gateway diff --git a/config/gate.toml b/config/gate.toml index f7513266d..275a64b3e 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1106,8 +1106,7 @@ must_stay_below_lines = 3000 "crates/capsem-core/src/net/policy_config/security_rule_profile/tests.rs" = 1549 "crates/capsem-core/src/security_engine/mod.rs" = 2048 "crates/capsem-core/tests/mitm_integration.rs" = 2087 -"crates/capsem-logger/src/schema/tests.rs" = 1003 -"crates/capsem-logger/src/writer/tests.rs" = 1395 +"crates/capsem-logger/src/writer/tests.rs" = 1237 "crates/capsem-mock-server/src/main.rs" = 1167 "crates/capsem-process/src/vsock.rs" = 1239 "crates/capsem-proto/src/lib.rs" = 1008 diff --git a/crates/capsem-logger/src/schema.rs b/crates/capsem-logger/src/schema.rs index a114bec28..a4724195b 100644 --- a/crates/capsem-logger/src/schema.rs +++ b/crates/capsem-logger/src/schema.rs @@ -259,6 +259,11 @@ pub(crate) fn with_memory_schema_lock(operation: impl FnOnce() -> rusqlite::R pub fn create_memory_tables(conn: &Connection, memory_uri: &str) -> rusqlite::Result<()> { integrity::validate(conn)?; attach_memory_schema(conn, memory_uri)?; + // DELETE alone leaves an in-memory SQLite database's freed pages attached + // to the process forever. The staging schema is emptied every flush, so + // return those pages at the same commit instead of retaining the largest + // burst for the rest of the VM's lifetime. + conn.execute_batch("PRAGMA mem.auto_vacuum = FULL")?; reconcile_memory_tables_from_disk(conn) } diff --git a/crates/capsem-logger/src/schema/pragmas.rs b/crates/capsem-logger/src/schema/pragmas.rs index 107fd269b..facfdc79d 100644 --- a/crates/capsem-logger/src/schema/pragmas.rs +++ b/crates/capsem-logger/src/schema/pragmas.rs @@ -21,7 +21,7 @@ const READER_BUSY_TIMEOUT: Duration = Duration::from_secs(5); /// Keep this in the DB layer: routes and security components should not know /// whether a query reads through SQLite's page cache, mmap, or DB-owned memory /// tables. -pub const SQLITE_MMAP_SIZE_BYTES: i64 = 256 * 1024 * 1024; +pub const SQLITE_MMAP_SIZE_BYTES: i64 = capsem_proto::LEDGER_SQLITE_MMAP_BYTES; fn apply_mmap_pragma(conn: &Connection) -> rusqlite::Result<()> { conn.pragma_update(None, "mmap_size", SQLITE_MMAP_SIZE_BYTES) diff --git a/crates/capsem-logger/src/schema/tests.rs b/crates/capsem-logger/src/schema/tests.rs index 5d4efb5ed..3bd9c0e34 100644 --- a/crates/capsem-logger/src/schema/tests.rs +++ b/crates/capsem-logger/src/schema/tests.rs @@ -1,8 +1,4 @@ use super::*; -use capsem_telemetry::db::{ - DB_SQLITE_FILE_SIZE_BYTES, DB_SQLITE_MMAP_BUDGET_CHECKS_TOTAL, DB_SQLITE_MMAP_CONFIG_BYTES, - DB_SQLITE_MMAP_COVERAGE_RATIO, DB_SQLITE_MMAP_EFFECTIVE_BYTES, DB_SQLITE_WAL_SIZE_BYTES, -}; use rusqlite::OpenFlags; mod archive; @@ -161,6 +157,33 @@ fn db_mem_flush_copies_each_row_once_and_empties_memory() { ); } +#[test] +fn db_mem_flush_releases_pages_from_completed_bursts() { + let conn = Connection::open_in_memory().unwrap(); + create_tables(&conn).unwrap(); + create_memory_tables(&conn, &memory_uri_for_name("db_mem_flush_releases_pages")).unwrap(); + let initial_pages: i64 = conn.query_row("PRAGMA mem.page_count", [], |row| row.get(0)).unwrap(); + let payload = "x".repeat(4096); + for index in 0..2048 { + conn.execute( + "INSERT INTO mem.net_events (timestamp, domain, decision) + VALUES ('2026-10-14T00:00:00Z', ?1, 'denied')", + [format!("{index}-{payload}")], + ) + .unwrap(); + } + let peak_pages: i64 = conn.query_row("PRAGMA mem.page_count", [], |row| row.get(0)).unwrap(); + assert!(peak_pages > initial_pages + 1000, "fixture must create memory pressure"); + + let watermarks = initial_memory_flush_watermarks(&conn, ["net_events"]).unwrap(); + flush_memory_tables_to_disk(&conn, ["net_events"], &watermarks).unwrap(); + let retained_pages: i64 = conn.query_row("PRAGMA mem.page_count", [], |row| row.get(0)).unwrap(); + assert!( + retained_pages <= initial_pages + 16, + "empty staging tables retained {retained_pages} pages after a {peak_pages}-page burst" + ); +} + #[test] fn memory_sequences_start_above_every_id_the_disk_handed_out() { let conn = Connection::open_in_memory().unwrap(); @@ -885,84 +908,7 @@ fn security_rule_events_reject_non_json_forensic_payloads() { ); } -/// Writer pragmas (WAL + synchronous) must only be applied to read-write -/// connections. Read-only connections must use apply_reader_pragmas instead. -#[test] -fn reader_pragmas_work_on_readonly_connection() { - // Create a file-backed DB first (writer sets WAL). - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("test.db"); - { - let conn = Connection::open(&path).unwrap(); - apply_pragmas(&conn).unwrap(); - create_tables(&conn).unwrap(); - } - - // Open read-only -- apply_reader_pragmas must not fail. - let flags = rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX; - let conn = Connection::open_with_flags(&path, flags).unwrap(); - apply_reader_pragmas(&conn).unwrap(); -} - -#[test] -fn reader_pragmas_enable_mmap_before_query_only() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("test.db"); - { - let conn = Connection::open(&path).unwrap(); - apply_pragmas(&conn).unwrap(); - create_tables(&conn).unwrap(); - } - - let flags = rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX; - let conn = Connection::open_with_flags(&path, flags).unwrap(); - apply_reader_pragmas(&conn).unwrap(); - - let mmap_size: i64 = conn.query_row("PRAGMA mmap_size", [], |row| row.get(0)).unwrap(); - assert!( - mmap_size >= SQLITE_MMAP_SIZE_BYTES, - "reader worker connections must enable SQLite mmap before query_only; got {mmap_size}" - ); - - let query_only: i64 = conn.query_row("PRAGMA query_only", [], |row| row.get(0)).unwrap(); - assert_eq!(query_only, 1, "reader worker must still be query-only"); -} - -#[test] -fn mmap_telemetry_records_budget_and_size_metrics() { - use metrics_util::debugging::{DebugValue, DebuggingRecorder}; - - let recorder = DebuggingRecorder::new(); - let snapshotter = recorder.snapshotter(); - let _guard = metrics::set_default_local_recorder(&recorder); - - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("test.db"); - let conn = Connection::open(&path).unwrap(); - apply_pragmas(&conn).unwrap(); - create_tables(&conn).unwrap(); - record_sqlite_mmap_telemetry(&conn, &path, "writer", "test"); - - let snapshot = snapshotter.snapshot().into_vec(); - assert!(snapshot.iter().any(|(key, _, _, value)| { - key.key().name() == DB_SQLITE_MMAP_CONFIG_BYTES && matches!(value, DebugValue::Gauge(_)) - })); - assert!(snapshot.iter().any(|(key, _, _, value)| { - key.key().name() == DB_SQLITE_MMAP_EFFECTIVE_BYTES && matches!(value, DebugValue::Gauge(_)) - })); - assert!(snapshot.iter().any(|(key, _, _, value)| { - key.key().name() == DB_SQLITE_FILE_SIZE_BYTES && matches!(value, DebugValue::Gauge(_)) - })); - assert!(snapshot.iter().any(|(key, _, _, value)| { - key.key().name() == DB_SQLITE_WAL_SIZE_BYTES && matches!(value, DebugValue::Gauge(_)) - })); - assert!(snapshot.iter().any(|(key, _, _, value)| { - key.key().name() == DB_SQLITE_MMAP_COVERAGE_RATIO && matches!(value, DebugValue::Gauge(_)) - })); - assert!(snapshot.iter().any(|(key, _, _, value)| { - key.key().name() == DB_SQLITE_MMAP_BUDGET_CHECKS_TOTAL && matches!(value, DebugValue::Counter(1)) - })); -} +mod pragmas; #[test] fn create_tables_keeps_snapshots_out_of_session_db() { diff --git a/crates/capsem-logger/src/schema/tests/pragmas.rs b/crates/capsem-logger/src/schema/tests/pragmas.rs new file mode 100644 index 000000000..bbd62bfb1 --- /dev/null +++ b/crates/capsem-logger/src/schema/tests/pragmas.rs @@ -0,0 +1,84 @@ +use super::*; +use capsem_telemetry::db::{ + DB_SQLITE_FILE_SIZE_BYTES, DB_SQLITE_MMAP_BUDGET_CHECKS_TOTAL, DB_SQLITE_MMAP_CONFIG_BYTES, + DB_SQLITE_MMAP_COVERAGE_RATIO, DB_SQLITE_MMAP_EFFECTIVE_BYTES, DB_SQLITE_WAL_SIZE_BYTES, +}; + +/// Writer pragmas (WAL + synchronous) must only be applied to read-write +/// connections. Read-only connections must use apply_reader_pragmas instead. +#[test] +fn reader_pragmas_work_on_readonly_connection() { + // Create a file-backed DB first (writer sets WAL). + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + { + let conn = Connection::open(&path).unwrap(); + apply_pragmas(&conn).unwrap(); + create_tables(&conn).unwrap(); + } + + // Open read-only -- apply_reader_pragmas must not fail. + let flags = rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX; + let conn = Connection::open_with_flags(&path, flags).unwrap(); + apply_reader_pragmas(&conn).unwrap(); +} + +#[test] +fn reader_pragmas_enable_mmap_before_query_only() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + { + let conn = Connection::open(&path).unwrap(); + apply_pragmas(&conn).unwrap(); + create_tables(&conn).unwrap(); + } + + let flags = rusqlite::OpenFlags::SQLITE_OPEN_READ_WRITE | rusqlite::OpenFlags::SQLITE_OPEN_NO_MUTEX; + let conn = Connection::open_with_flags(&path, flags).unwrap(); + apply_reader_pragmas(&conn).unwrap(); + + let mmap_size: i64 = conn.query_row("PRAGMA mmap_size", [], |row| row.get(0)).unwrap(); + assert!( + mmap_size >= SQLITE_MMAP_SIZE_BYTES, + "reader worker connections must enable SQLite mmap before query_only; got {mmap_size}" + ); + + let query_only: i64 = conn.query_row("PRAGMA query_only", [], |row| row.get(0)).unwrap(); + assert_eq!(query_only, 1, "reader worker must still be query-only"); +} + +#[test] +fn mmap_telemetry_records_budget_and_size_metrics() { + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + + let recorder = DebuggingRecorder::new(); + let snapshotter = recorder.snapshotter(); + let _guard = metrics::set_default_local_recorder(&recorder); + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("test.db"); + let conn = Connection::open(&path).unwrap(); + apply_pragmas(&conn).unwrap(); + create_tables(&conn).unwrap(); + record_sqlite_mmap_telemetry(&conn, &path, "writer", "test"); + + let snapshot = snapshotter.snapshot().into_vec(); + assert!(snapshot.iter().any(|(key, _, _, value)| { + key.key().name() == DB_SQLITE_MMAP_CONFIG_BYTES && matches!(value, DebugValue::Gauge(_)) + })); + assert!(snapshot.iter().any(|(key, _, _, value)| { + key.key().name() == DB_SQLITE_MMAP_EFFECTIVE_BYTES && matches!(value, DebugValue::Gauge(_)) + })); + assert!(snapshot.iter().any(|(key, _, _, value)| { + key.key().name() == DB_SQLITE_FILE_SIZE_BYTES && matches!(value, DebugValue::Gauge(_)) + })); + assert!(snapshot.iter().any(|(key, _, _, value)| { + key.key().name() == DB_SQLITE_WAL_SIZE_BYTES && matches!(value, DebugValue::Gauge(_)) + })); + assert!(snapshot.iter().any(|(key, _, _, value)| { + key.key().name() == DB_SQLITE_MMAP_COVERAGE_RATIO && matches!(value, DebugValue::Gauge(_)) + })); + assert!(snapshot.iter().any(|(key, _, _, value)| { + key.key().name() == DB_SQLITE_MMAP_BUDGET_CHECKS_TOTAL && matches!(value, DebugValue::Counter(1)) + })); +} diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index c82961672..473598c07 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -19,7 +19,11 @@ use crate::events::{ use crate::schema; mod bodies; +mod flush; mod flush_faults; +#[cfg(test)] +use flush::flush_dirty_tables_to_disk; +use flush::{flush_dirty_tables_to_disk_recorded, FlushRequest}; mod legacy; mod model_rows; mod producer; @@ -455,9 +459,11 @@ mod remote; pub(crate) use remote::test_commitment_channel; mod retention; mod retention_faults; +mod worker; mod writer_lock; #[cfg(test)] pub(crate) use retention_faults::{fail_retention_for_path_for_tests, RetentionFault}; +use worker::writer_loop; use barriers::Barriers; use batch::{execute_memory_batch, retry_batch_ops_individually}; @@ -895,219 +901,6 @@ async fn send_with_backpressure(tx: &WriterSender, message: WriterMessage) -> Re } /// The writer thread loop: block-then-drain batching. -fn writer_loop( - conn: Connection, - rx: WriterReceiver, - db_path: Option, - schedule: WriterSchedule, - pending_body_bytes: &AtomicU64, - mut bodies: BodyArchive, - mut tally: LedgerTally, -) { - let WriterSchedule { - capacity: batch_capacity, - interval, - } = schedule; - let mut flush_watermarks = - schema::with_memory_schema_lock(|| schema::initial_memory_flush_watermarks(&conn, schema::hot_ledger_tables())) - .unwrap_or_else(|error| { - warn!(error = %error, "db initial memory flush watermark load failed"); - schema::MemoryFlushWatermarks::new() - }); - // Rows at or below the open-time watermark were written by an earlier - // writer; exec ids restart with the process, so a completion must not - // reach one of those. - let exec_floor = flush_watermarks.get("exec_events").copied().unwrap_or(0); - let mut dirty_tables = BTreeSet::new(); - let mut dirty_ops = 0_usize; - // Producer acceptance is not SQL success. Preserve rejected-operation - // failures across ordinary batches and interval flushes until the next - // explicit durability boundary reports that its queued prefix failed. - let mut rejected_ops = 0_usize; - let mut last_disk_flush = Instant::now(); - // 1. Block until at least one op arrives. Returns None when all - // Senders are dropped (clean shutdown) and ends the loop. - loop { - let first_message = if dirty_ops == 0 && !bodies.has_work() { - rx.recv().ok() - } else { - match rx.recv_timeout(interval.saturating_sub(last_disk_flush.elapsed())) { - Ok(message) => Some(message), - Err(mpsc::RecvTimeoutError::Timeout) => { - if let Err(error) = flush_dirty_tables_to_disk_recorded( - &conn, - &mut dirty_tables, - &mut flush_watermarks, - db_path.as_deref(), - &mut bodies, - tally.counters(), - FlushRequest { - pending_ops: dirty_ops, - wait_for_body_sync: false, - }, - ) { - warn!(error = %error, "db interval flush failed"); - } else { - dirty_ops = 0; - last_disk_flush = Instant::now(); - pending_body_bytes.store(bodies.pending_bytes() as u64, Ordering::Release); - } - ::metrics::gauge!(DB_MEMORY_UNFLUSHED_OPS).set(dirty_ops as f64); - continue; - } - Err(mpsc::RecvTimeoutError::Disconnected) => None, - } - }; - let Some(first_message) = first_message else { - break; - }; - - let mut batch = Vec::with_capacity(batch_capacity); - let mut barriers = Barriers::default(); - let mut at_barrier = barriers.accept(first_message.into_message(), &mut batch); - - // 2. Drain any ops already queued (non-blocking). - while !at_barrier && batch.len() < batch_capacity { - match rx.try_recv() { - Ok(message) => at_barrier = barriers.accept(message.into_message(), &mut batch), - Err(_) => break, - } - } - - // 3. Execute entire batch in a single transaction. - let batch_size = batch.len(); - let batch_bucket = batch_size_bucket(batch_size); - let span = tracing::debug_span!( - target: "capsem.db", - DB_WRITE_BATCH_SPAN, - batch_size_bucket = batch_bucket, - status = tracing::field::Empty, - ); - let started = Instant::now(); - let batch_capacity = batch.capacity(); - if batch.is_empty() { - record_batch(started, batch_size, batch_capacity, batch_bucket, "ok", &span); - } else { - match span.in_scope(|| execute_memory_batch(&conn, &batch, &mut bodies, exec_floor, &mut tally)) { - Ok(outcome) => { - dirty_tables.extend(outcome.tables); - dirty_ops += outcome.written; - record_batch(started, batch_size, batch_capacity, batch_bucket, "ok", &span); - } - Err(e) => { - record_batch(started, batch_size, batch_capacity, batch_bucket, "error", &span); - warn!( - error = %e, - count = batch.len(), - "db memory write batch failed; retrying its ops individually" - ); - let salvaged = span - .in_scope(|| retry_batch_ops_individually(&conn, &batch, &mut bodies, exec_floor, &mut tally)); - dirty_tables.extend(salvaged.tables); - dirty_ops += salvaged.written; - rejected_ops = rejected_ops.saturating_add(salvaged.rejected); - } - } - } - // A burst of large bodies closes its block as soon as the batch ends - // rather than growing it until the interval expires. Its index rows - // wait for the next transaction; its bytes are already on disk. - bodies.close_if_due(); - pending_body_bytes.store(bodies.pending_bytes() as u64, Ordering::Release); - let disk_flush_due = - dirty_ops >= DISK_FLUSH_THRESHOLD_OPS || last_disk_flush.elapsed() >= interval || barriers.waiting(); - let mut barrier_outcome: FlushOutcome = if rejected_ops == 0 { - Ok(()) - } else { - Err(format!("db rejected {rejected_ops} accepted write operations")) - }; - if disk_flush_due { - match flush_dirty_tables_to_disk_recorded( - &conn, - &mut dirty_tables, - &mut flush_watermarks, - db_path.as_deref(), - &mut bodies, - tally.counters(), - FlushRequest { - pending_ops: dirty_ops, - wait_for_body_sync: barriers.waiting(), - }, - ) { - Ok(()) => { - dirty_ops = 0; - last_disk_flush = Instant::now(); - pending_body_bytes.store(bodies.pending_bytes() as u64, Ordering::Release); - } - Err(error) => { - warn!(error = %error, "db dirty table flush failed"); - barrier_outcome = Err(format!("db dirty table flush failed: {error}")); - } - } - } - ::metrics::gauge!(DB_MEMORY_UNFLUSHED_OPS).set(dirty_ops as f64); - let explicit_boundary = barriers.waiting(); - barriers.answer(&conn, &mut bodies, &barrier_outcome); - if explicit_boundary { - rejected_ops = 0; - } - } - - // Test hook: lets `test_wal_absent_after_clean_shutdown`-style tests - // simulate a slow checkpoint so the explicit-cleanup path can be - // distinguished from implicit tokio-runtime-drop ordering. Gated on - // an env var so it's a no-op in production. - if let Ok(ms) = std::env::var("CAPSEM_TEST_SLOW_CHECKPOINT_MS") { - if let Ok(ms) = ms.parse::() { - std::thread::sleep(std::time::Duration::from_millis(ms)); - } - } - - // The open block ends with its FINAL segment, committed by this flush. - bodies.close_block(); - if let Err(error) = flush_dirty_tables_to_disk_recorded( - &conn, - &mut dirty_tables, - &mut flush_watermarks, - db_path.as_deref(), - &mut bodies, - tally.counters(), - FlushRequest { - pending_ops: dirty_ops, - wait_for_body_sync: true, - }, - ) { - warn!(error = %error, "db shutdown dirty table flush failed"); - // There is no next flush to retry into, so the bodies still waiting - // for an index row are lost here. Counted, so the drop counter closes - // over the session rather than ending on a number that is short by - // however much the last flush was carrying. - bodies.abandon_uncommitted("shutdown"); - } - bodies.sync(); - pending_body_bytes.store(bodies.pending_bytes() as u64, Ordering::Release); - - // All senders dropped -- checkpoint WAL before closing connection. - let span = tracing::debug_span!( - target: "capsem.db", - DB_SHUTDOWN_FLUSH_SPAN, - status = tracing::field::Empty, - ); - let started = Instant::now(); - let result = span.in_scope(|| conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")); - let elapsed_ms = started.elapsed().as_secs_f64() * 1000.0; - let status = if result.is_ok() { "ok" } else { "error" }; - ::metrics::histogram!(DB_SHUTDOWN_FLUSH_MS, "status" => status).record(elapsed_ms); - span.record("status", status); - // mmap coverage and file-size diagnostics query SQLite and the filesystem. - // Sampling them on every periodic flush once held this admission thread - // for more than a second under DNS load. Shutdown is outside the producer - // path and still records the final ledger size and mmap budget. - if let Some(path) = db_path.as_deref() { - schema::record_sqlite_mmap_telemetry(&conn, path, "writer", "shutdown"); - } -} - #[derive(Clone, Copy)] enum WriteTarget { Memory, @@ -1122,145 +915,6 @@ impl WriteTarget { } } -fn flush_dirty_tables_to_disk( - conn: &Connection, - dirty_tables: &mut BTreeSet<&'static str>, - flush_watermarks: &mut schema::MemoryFlushWatermarks, - db_path: Option<&Path>, - bodies: &mut BodyArchive, - counters: &LedgerCounters, - wait_for_body_sync: bool, -) -> rusqlite::Result { - if dirty_tables.is_empty() && !bodies.has_work() { - return Ok(FlushTimings::default()); - } - let started = Instant::now(); - // Bytes before index: the segment reaches the archive file here, and the - // rows that name it are inserted in the transaction below. A crash in - // between costs unreferenced bytes, never a row pointing past EOF. - let started = Instant::now(); - bodies.flush_segment(); - bodies.prepare_index_rows(wait_for_body_sync); - let archive_ms = started.elapsed().as_secs_f64() * 1000.0; - let tables: Vec<&'static str> = dirty_tables.iter().copied().collect(); - let started = Instant::now(); - let tx = conn.unchecked_transaction()?; - let transaction_begin_ms = started.elapsed().as_secs_f64() * 1000.0; - let started = Instant::now(); - bodies.commit_prepared_index_rows(&tx)?; - let body_index_ms = started.elapsed().as_secs_f64() * 1000.0; - // Injected where a real flush failure lands: inside the transaction, with - // the archive's index rows already written into it. Everything here rolls - // back together, and the retry has to bring the bodies back with the rows. - if take_disk_flush_failure_for_tests(db_path) { - return Err(rusqlite::Error::InvalidParameterName( - "injected disk flush failure before copy".to_string(), - )); - } - let started = Instant::now(); - let advanced_watermarks = schema::with_memory_schema_lock(|| { - schema::flush_memory_tables_to_disk(&tx, tables.iter().copied(), flush_watermarks) - })?; - let table_copy_ms = started.elapsed().as_secs_f64() * 1000.0; - // The snapshot lands with the rows it counts, or neither does. - let started = Instant::now(); - counters::store(&tx, counters)?; - let counter_snapshot_ms = started.elapsed().as_secs_f64() * 1000.0; - let started = Instant::now(); - tx.commit()?; - let commit_ms = started.elapsed().as_secs_f64() * 1000.0; - // Only now are the written segments indexed: anything above this line rolls - // the transaction back, and the next flush retries both halves together. - bodies.index_rows_committed(); - flush_watermarks.extend(advanced_watermarks); - dirty_tables.clear(); - Ok(FlushTimings { - archive_ms, - transaction_begin_ms, - body_index_ms, - table_copy_ms, - counter_snapshot_ms, - commit_ms, - }) -} - -#[derive(Default)] -struct FlushTimings { - archive_ms: f64, - transaction_begin_ms: f64, - body_index_ms: f64, - table_copy_ms: f64, - counter_snapshot_ms: f64, - commit_ms: f64, -} - -#[derive(Clone, Copy)] -struct FlushRequest { - pending_ops: usize, - wait_for_body_sync: bool, -} - -fn flush_dirty_tables_to_disk_recorded( - conn: &Connection, - dirty_tables: &mut BTreeSet<&'static str>, - flush_watermarks: &mut schema::MemoryFlushWatermarks, - db_path: Option<&Path>, - bodies: &mut BodyArchive, - counters: &LedgerCounters, - request: FlushRequest, -) -> rusqlite::Result<()> { - let table_count = dirty_tables.len(); - let started = Instant::now(); - let result = flush_dirty_tables_to_disk( - conn, - dirty_tables, - flush_watermarks, - db_path, - bodies, - counters, - request.wait_for_body_sync, - ); - let elapsed = started.elapsed(); - let status = if result.is_ok() { "ok" } else { "error" }; - let timings = result.as_ref().ok(); - let archive_ms = timings.map_or(0.0, |timings| timings.archive_ms); - let transaction_begin_ms = timings.map_or(0.0, |timings| timings.transaction_begin_ms); - let body_index_ms = timings.map_or(0.0, |timings| timings.body_index_ms); - let table_copy_ms = timings.map_or(0.0, |timings| timings.table_copy_ms); - let counter_snapshot_ms = timings.map_or(0.0, |timings| timings.counter_snapshot_ms); - let commit_ms = timings.map_or(0.0, |timings| timings.commit_ms); - tracing::debug!( - target: "capsem.db", - duration_ms = elapsed.as_secs_f64() * 1000.0, - archive_ms, - transaction_begin_ms, - body_index_ms, - table_copy_ms, - counter_snapshot_ms, - commit_ms, - pending_ops = request.pending_ops, - table_count, - status, - "db disk flush completed" - ); - if elapsed >= DISK_FLUSH_INTERVAL { - warn!( - duration_ms = elapsed.as_secs_f64() * 1000.0, - archive_ms, - transaction_begin_ms, - body_index_ms, - table_copy_ms, - counter_snapshot_ms, - commit_ms, - pending_ops = request.pending_ops, - table_count, - status, - "db disk flush exceeded its configured interval" - ); - } - result.map(|_| ()) -} - /// Execute through the connection's prepared-statement cache. /// /// Every insert here is one of a small fixed set of statements (one per diff --git a/crates/capsem-logger/src/writer/bodies.rs b/crates/capsem-logger/src/writer/bodies.rs index 129dca6c1..be3580b5e 100644 --- a/crates/capsem-logger/src/writer/bodies.rs +++ b/crates/capsem-logger/src/writer/bodies.rs @@ -29,21 +29,23 @@ use std::collections::HashMap; use std::ffi::OsString; use std::path::{Path, PathBuf}; -use std::sync::mpsc; use std::time::{Duration, Instant, SystemTime}; use super::{blake3_bytes_ref, execute_cached, format_timestamp, LedgerClock, MAX_BODY_BLOB_BYTES}; use capsem_archive::{ - ArchiveCodecs, ArchiveError, ArchiveId, BodyLogSync, BodyLogWriter, BodyRef, FileHeader, GenerationId, - SegmentWritten, + ArchiveCodecs, ArchiveError, ArchiveId, BodyLogWriter, BodyRef, FileHeader, GenerationId, SegmentWritten, }; use capsem_foundation::unix::contained::{ContainedDir, EntryKind}; use capsem_foundation::unix::fs::{durable_sync_directory, ensure_private_dir}; use capsem_foundation::unix::lock::{self, FileLock, LockAttempt, LockMode}; use capsem_telemetry::db::{DB_ARCHIVE_BODIES_DEDUPLICATED_TOTAL, DB_ARCHIVE_BODIES_DROPPED_TOTAL}; -use rusqlite::{params, params_from_iter, Connection, OptionalExtension, ToSql}; +use rusqlite::{params, Connection, OptionalExtension}; use tracing::warn; +mod index; +mod sync_worker; +use sync_worker::ArchiveSyncWorker; + /// How long a block may stay open before the next body starts a new one. /// /// Retention drops whole blocks and keeps a block while its newest segment @@ -107,66 +109,10 @@ struct BodyIndexRow { created_at: String, } -type AppendedSegment = (SegmentWritten, Vec); - -struct ArchiveSyncWorker { - requests: Option>, - completions: mpsc::Receiver>, - join: Option>, -} - -impl ArchiveSyncWorker { - fn start(sync: BodyLogSync) -> std::io::Result { - let (requests, work) = mpsc::sync_channel(1); - let (completed, completions) = mpsc::sync_channel(1); - let join = std::thread::Builder::new() - .name("capsem-archive-sync".into()) - .stack_size(128 * 1024) - .spawn(move || { - while work.recv().is_ok() { - let result = sync.sync().map_err(|error| error.to_string()); - if completed.send(result).is_err() { - break; - } - } - })?; - Ok(Self { - requests: Some(requests), - completions, - join: Some(join), - }) - } - - fn request(&self) -> Result<(), String> { - self.requests - .as_ref() - .ok_or_else(|| "archive sync worker stopped".to_string())? - .send(()) - .map_err(|_| "archive sync worker stopped".to_string()) - } - - fn try_complete(&self) -> Option> { - match self.completions.try_recv() { - Ok(result) => Some(result), - Err(mpsc::TryRecvError::Empty) => None, - Err(mpsc::TryRecvError::Disconnected) => Some(Err("archive sync worker stopped".into())), - } - } - - fn complete(&self) -> Result<(), String> { - self.completions - .recv() - .map_err(|_| "archive sync worker stopped".to_string())? - } -} - -impl Drop for ArchiveSyncWorker { - fn drop(&mut self) { - self.requests.take(); - if let Some(join) = self.join.take() { - let _ = join.join(); - } - } +struct AppendedSegment { + segment: SegmentWritten, + rows: Vec, + block_committed: bool, } /// The writer thread's archive: one `session.bodies` beside `session.db`. @@ -204,6 +150,10 @@ pub(super) struct BodyArchive { appended: Vec, /// Segments whose device sync completed and whose rows may enter SQLite. synced: Vec, + /// Rows written by the current SQLite transaction. They leave `synced` + /// and `reused` only after that transaction commits; a rollback retries + /// the same prefix on the next flush. + prepared_index_rows: Option, /// Segments covered by the durability barrier currently running. syncing: Option>, /// Lazily-created helper that keeps fdatasync off the admission thread. @@ -220,6 +170,19 @@ pub(super) struct BodyArchive { steps: Vec<&'static str>, } +#[derive(Default)] +struct PreparedIndexRows { + synced: usize, + reused: usize, + blocks: Vec, +} + +impl PreparedIndexRows { + const fn total(&self) -> usize { + self.synced + self.reused + } +} + /// Bodies shorter than this are not looked up in the archive index. Deflate's /// window already finds a repeat that small inside the block, and the lookup /// would cost more than the bytes it saves. @@ -229,9 +192,6 @@ const MIN_ARCHIVE_DEDUP_BYTES: usize = 512; /// 999-variable ceiling. The statement and transient reference vector stay /// bounded while a busy flush pays statement dispatch once per 64 bodies /// instead of once per body. -const BODY_INDEX_ROWS_PER_INSERT: usize = 64; -const BODY_INDEX_VALUE_LIST: &str = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - /// Where identical stored bytes already sit in a committed segment, if they /// do. Committed spans are immutable and synced, so pointing a new row at one /// is as safe as pointing at fresh bytes; retention keeps a block while any @@ -537,6 +497,7 @@ impl BodyArchive { next_seq: 0, appended: Vec::new(), synced: Vec::new(), + prepared_index_rows: None, syncing: None, sync_worker: None, #[cfg(test)] @@ -750,15 +711,15 @@ impl BodyArchive { pub(super) fn rollback_staged(&mut self, mark: u64) { self.staged.retain(|row| row.seq < mark); self.reused.retain(|row| row.seq < mark); - for (_, rows) in &mut self.appended { - rows.retain(|row| row.seq < mark); + for segment in &mut self.appended { + segment.rows.retain(|row| row.seq < mark); } - for (_, rows) in &mut self.synced { - rows.retain(|row| row.seq < mark); + for segment in &mut self.synced { + segment.rows.retain(|row| row.seq < mark); } if let Some(syncing) = &mut self.syncing { - for (_, rows) in syncing { - rows.retain(|row| row.seq < mark); + for segment in syncing { + segment.rows.retain(|row| row.seq < mark); } } } @@ -836,7 +797,11 @@ impl BodyArchive { match written { Ok(Some(segment)) => { let rows = std::mem::take(&mut self.staged); - self.appended.push((segment, rows)); + self.appended.push(AppendedSegment { + segment, + rows, + block_committed: false, + }); } Ok(None) => {} Err(error) => { @@ -950,87 +915,6 @@ impl BodyArchive { } } - /// Record every durably-written segment's block and the index rows it holds. - /// - /// Runs inside the caller's transaction, after `flush_segment` put the - /// bytes in the file. Nothing here can make a row visible before its - /// bytes. - /// - /// The archive is flushed to the device first. The segment write only - /// went through the page cache, so without this a power loss could leave - /// SQLite's durably-committed index rows naming segments that never - /// reached the disk. The cost is one `fdatasync` per flush that has - /// segments to index -- at most once per flush interval, never once per - /// body. A failed flush means those bytes may not be there, so their rows - /// are dropped and archiving stops. Losing bodies is recoverable; an index - /// that lies is not. - /// - /// `body_blocks` holds one row per block, and each segment grows it: the - /// upsert records the block's committed raw length and on-disk extent, - /// and never shrinks them, so a flush retried after a rollback is the - /// same as the first. Not `REPLACE`: that is a delete and an insert, which - /// under enforced foreign keys would take every index row naming the block - /// with it. The blocks are all written before any row, so a row always - /// names a block the transaction already holds. - /// - /// The segments stay in hand until the caller says the transaction - /// committed. An insert here can fail, and so can the memory-table copy - /// beside it and the commit after it; all three roll back every row, and - /// the next flush retries these with them. - pub(super) fn commit_index_rows(&mut self, conn: &Connection) -> rusqlite::Result<()> { - self.prepare_index_rows(true); - self.commit_prepared_index_rows(conn) - } - - pub(super) fn commit_prepared_index_rows(&mut self, conn: &Connection) -> rusqlite::Result<()> { - if self.synced.is_empty() && self.reused.is_empty() { - return Ok(()); - } - let synced = Instant::now(); - // Recorded here, where the first row is actually written, and not - // beside the flush that precedes it: a trace whose two steps are - // pushed from one place proves their order only to itself. - #[cfg(test)] - self.steps.push("commit"); - let sealed_at = format_timestamp((self.now)()); - for (segment, _) in &self.synced { - upsert_block(conn, segment, &sealed_at)?; - } - let mut committed_end = None; - for (segment, _) in &self.synced { - let end = segment.block_offset.checked_add(segment.disk_len).ok_or_else(|| { - rusqlite::Error::InvalidParameterName("archive committed extent overflows u64".into()) - })?; - committed_end = Some(committed_end.map_or(end, |current: u64| current.max(end))); - } - if let Some(committed_end) = committed_end { - let committed_end = i64::try_from(committed_end).map_err(|_| { - rusqlite::Error::InvalidParameterName("archive committed extent exceeds SQLite INTEGER".into()) - })?; - conn.execute( - "UPDATE archive_state - SET committed_end = MAX(committed_end, ?1) - WHERE singleton = 1", - [committed_end], - )?; - } - let rows = self - .synced - .iter() - .flat_map(|(_, rows)| rows) - .chain(&self.reused) - .collect::>(); - insert_body_index_rows(conn, &rows)?; - Ok(()) - } - - /// The transaction that `commit_index_rows` wrote into has committed, so - /// those segments are indexed and this archive is done with them. - pub(super) fn index_rows_committed(&mut self) { - self.synced.clear(); - self.reused.clear(); - } - /// Durability barrier for session close. The block was closed and its /// rows committed by the final flush; this makes sure the device has it. pub(super) fn sync(&mut self) { @@ -1046,12 +930,12 @@ impl BodyArchive { /// and the rows of bytes not yet written are equally unreachable once the /// archive stops vouching for its own file. pub(super) fn abandon_uncommitted(&mut self, reason: &'static str) { - let dropped: usize = self.appended.iter().map(|(_, rows)| rows.len()).sum::() - + self.synced.iter().map(|(_, rows)| rows.len()).sum::() + let dropped: usize = self.appended.iter().map(|segment| segment.rows.len()).sum::() + + self.synced.iter().map(|segment| segment.rows.len()).sum::() + self .syncing .as_ref() - .map_or(0, |segments| segments.iter().map(|(_, rows)| rows.len()).sum()) + .map_or(0, |segments| segments.iter().map(|segment| segment.rows.len()).sum()) + self.staged.len() + self.reused.len(); self.appended.clear(); @@ -1110,90 +994,3 @@ impl BodyArchive { &self.steps } } - -fn insert_body_index_rows(conn: &Connection, rows: &[&BodyIndexRow]) -> rusqlite::Result<()> { - for chunk in rows.chunks(BODY_INDEX_ROWS_PER_INSERT) { - let mut sql = String::from( - "INSERT OR REPLACE INTO event_body_blobs ( - event_id, event_type, source_table, direction, content_type, - original_bytes, stored_bytes, truncated, body_hash, - block_offset, body_offset, body_len, - trace_id, turn_id, created_at - ) VALUES ", - ); - for index in 0..chunk.len() { - if index > 0 { - sql.push_str(", "); - } - sql.push_str(BODY_INDEX_VALUE_LIST); - } - - // References only: the rows keep ownership until the surrounding - // transaction commits, so batching does not clone any body metadata. - let mut values: Vec<&dyn ToSql> = Vec::with_capacity(chunk.len() * 15); - for row in chunk { - values.extend([ - &row.event_id as &dyn ToSql, - &row.event_type as &dyn ToSql, - &row.source_table as &dyn ToSql, - &row.direction as &dyn ToSql, - &row.content_type as &dyn ToSql, - &row.original_bytes as &dyn ToSql, - &row.stored_bytes as &dyn ToSql, - &row.truncated as &dyn ToSql, - &row.body_hash as &dyn ToSql, - &row.block_offset as &dyn ToSql, - &row.body_offset as &dyn ToSql, - &row.stored_bytes as &dyn ToSql, - &row.trace_id as &dyn ToSql, - &row.turn_id as &dyn ToSql, - &row.created_at as &dyn ToSql, - ]); - } - execute_cached(conn, &sql, params_from_iter(values))?; - } - Ok(()) -} - -/// Record one written segment against its block: insert the block's row on -/// its first segment, grow it on every later one. -/// -/// `WHERE excluded.disk_len >= disk_len` keeps it monotone, so a retried -/// flush that replays an older segment after a newer one committed cannot -/// shrink the extent a reader or retention relies on. A retry that finds a -/// *larger* extent recorded is the one case that should never happen, and -/// is said out loud. -/// -/// `sealed_at` moves only when the segment brought bodies. Retention cuts by -/// it, and it closes the open block before it looks: an empty FINAL segment -/// stamped "now" would make every idle block the newest thing in the archive -/// at the very moment retention asked how old it was, and nothing would ever -/// age out of a block that was open when retention ran. -fn upsert_block(conn: &Connection, segment: &SegmentWritten, sealed_at: &str) -> rusqlite::Result<()> { - let recorded = execute_cached( - conn, - "INSERT INTO body_blocks (block_offset, raw_len, disk_len, sealed_at) - VALUES (?1, ?2, ?3, ?4) - ON CONFLICT(block_offset) DO UPDATE SET - raw_len = excluded.raw_len, - disk_len = excluded.disk_len, - sealed_at = CASE WHEN excluded.raw_len > body_blocks.raw_len - THEN excluded.sealed_at ELSE body_blocks.sealed_at END - WHERE excluded.disk_len >= body_blocks.disk_len", - params![ - segment.block_offset as i64, - i64::from(segment.raw_len), - segment.disk_len as i64, - sealed_at, - ], - )?; - if recorded == 0 { - warn!( - block_offset = segment.block_offset, - raw_len = segment.raw_len, - disk_len = segment.disk_len, - "session body index already records a longer extent for this block; kept the longer one" - ); - } - Ok(()) -} diff --git a/crates/capsem-logger/src/writer/bodies/index.rs b/crates/capsem-logger/src/writer/bodies/index.rs new file mode 100644 index 000000000..35b8e5ebd --- /dev/null +++ b/crates/capsem-logger/src/writer/bodies/index.rs @@ -0,0 +1,224 @@ +//! SQLite publication of durable archive segments and body index rows. + +use super::*; +use rusqlite::{params_from_iter, ToSql}; + +const BODY_INDEX_ROWS_PER_INSERT: usize = 64; +const BODY_INDEX_VALUE_LIST: &str = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; + +impl BodyArchive { + /// Record every durably-written segment's block and the index rows it holds. + /// + /// Runs inside the caller's transaction, after `flush_segment` put the + /// bytes in the file. Nothing here can make a row visible before its + /// bytes. + /// + /// The archive is flushed to the device first. The segment write only + /// went through the page cache, so without this a power loss could leave + /// SQLite's durably-committed index rows naming segments that never + /// reached the disk. The cost is one `fdatasync` per flush that has + /// segments to index -- at most once per flush interval, never once per + /// body. A failed flush means those bytes may not be there, so their rows + /// are dropped and archiving stops. Losing bodies is recoverable; an index + /// that lies is not. + /// + /// `body_blocks` holds one row per block, and each segment grows it: the + /// upsert records the block's committed raw length and on-disk extent, + /// and never shrinks them, so a flush retried after a rollback is the + /// same as the first. Not `REPLACE`: that is a delete and an insert, which + /// under enforced foreign keys would take every index row naming the block + /// with it. The blocks are all written before any row, so a row always + /// names a block the transaction already holds. + /// + /// The segments stay in hand until the caller says the transaction + /// committed. An insert here can fail, and so can the memory-table copy + /// beside it and the commit after it; all three roll back every row, and + /// the next flush retries these with them. + pub(in crate::writer) fn commit_index_rows(&mut self, conn: &Connection) -> rusqlite::Result<()> { + self.prepare_index_rows(true); + self.commit_prepared_index_rows_with_limit(conn, usize::MAX) + } + + pub(in crate::writer) fn commit_prepared_index_rows_with_limit( + &mut self, + conn: &Connection, + max_rows: usize, + ) -> rusqlite::Result<()> { + self.prepared_index_rows = None; + if self.synced.is_empty() && self.reused.is_empty() { + return Ok(()); + } + // Recorded here, where the first row is actually written, and not + // beside the flush that precedes it: a trace whose two steps are + // pushed from one place proves their order only to itself. + #[cfg(test)] + self.steps.push("commit"); + let synced_rows = self.synced.iter().map(|segment| segment.rows.len()).sum::(); + let mut prepared = PreparedIndexRows { + synced: synced_rows.min(max_rows), + reused: self.reused.len().min(max_rows.saturating_sub(synced_rows)), + blocks: Vec::new(), + }; + let sealed_at = format_timestamp((self.now)()); + let mut committed_end = None; + for (index, queued) in self.synced.iter().enumerate() { + // A zero-row FINAL segment still advances the durable extent. All + // newly synced block metadata is therefore published once even + // when this transaction takes only a bounded prefix of body rows. + if queued.block_committed { + continue; + } + upsert_block(conn, &queued.segment, &sealed_at)?; + prepared.blocks.push(index); + let end = queued + .segment + .block_offset + .checked_add(queued.segment.disk_len) + .ok_or_else(|| { + rusqlite::Error::InvalidParameterName("archive committed extent overflows u64".into()) + })?; + committed_end = Some(committed_end.map_or(end, |current: u64| current.max(end))); + } + if let Some(committed_end) = committed_end { + let committed_end = i64::try_from(committed_end).map_err(|_| { + rusqlite::Error::InvalidParameterName("archive committed extent exceeds SQLite INTEGER".into()) + })?; + conn.execute( + "UPDATE archive_state + SET committed_end = MAX(committed_end, ?1) + WHERE singleton = 1", + [committed_end], + )?; + } + let rows = self + .synced + .iter() + .flat_map(|segment| &segment.rows) + .chain(&self.reused) + .take(prepared.total()) + .collect::>(); + insert_body_index_rows(conn, &rows)?; + self.prepared_index_rows = Some(prepared); + Ok(()) + } + + pub(in crate::writer) fn prepared_index_row_count(&self) -> usize { + self.prepared_index_rows.as_ref().map_or(0, |prepared| prepared.total()) + } + + /// The transaction that `commit_index_rows` wrote into has committed, so + /// those segments are indexed and this archive is done with them. + pub(in crate::writer) fn index_rows_committed(&mut self) { + let Some(prepared) = self.prepared_index_rows.take() else { + return; + }; + for index in prepared.blocks { + self.synced[index].block_committed = true; + } + let mut synced = prepared.synced; + let mut committed_segments = 0; + for segment in &mut self.synced { + let row_count = segment.rows.len(); + if row_count == 0 { + committed_segments += 1; + } else if row_count <= synced { + synced -= row_count; + committed_segments += 1; + } else if synced > 0 { + segment.rows.drain(..synced); + break; + } else { + break; + } + } + self.synced.drain(..committed_segments); + self.reused.drain(..prepared.reused); + } +} + +fn insert_body_index_rows(conn: &Connection, rows: &[&BodyIndexRow]) -> rusqlite::Result<()> { + for chunk in rows.chunks(BODY_INDEX_ROWS_PER_INSERT) { + let mut sql = String::from( + "INSERT OR REPLACE INTO event_body_blobs ( + event_id, event_type, source_table, direction, content_type, + original_bytes, stored_bytes, truncated, body_hash, + block_offset, body_offset, body_len, + trace_id, turn_id, created_at + ) VALUES ", + ); + for index in 0..chunk.len() { + if index > 0 { + sql.push_str(", "); + } + sql.push_str(BODY_INDEX_VALUE_LIST); + } + + // References only: the rows keep ownership until the surrounding + // transaction commits, so batching does not clone any body metadata. + let mut values: Vec<&dyn ToSql> = Vec::with_capacity(chunk.len() * 15); + for row in chunk { + values.extend([ + &row.event_id as &dyn ToSql, + &row.event_type as &dyn ToSql, + &row.source_table as &dyn ToSql, + &row.direction as &dyn ToSql, + &row.content_type as &dyn ToSql, + &row.original_bytes as &dyn ToSql, + &row.stored_bytes as &dyn ToSql, + &row.truncated as &dyn ToSql, + &row.body_hash as &dyn ToSql, + &row.block_offset as &dyn ToSql, + &row.body_offset as &dyn ToSql, + &row.stored_bytes as &dyn ToSql, + &row.trace_id as &dyn ToSql, + &row.turn_id as &dyn ToSql, + &row.created_at as &dyn ToSql, + ]); + } + execute_cached(conn, &sql, params_from_iter(values))?; + } + Ok(()) +} + +/// Record one written segment against its block: insert the block's row on +/// its first segment, grow it on every later one. +/// +/// `WHERE excluded.disk_len >= disk_len` keeps it monotone, so a retried +/// flush that replays an older segment after a newer one committed cannot +/// shrink the extent a reader or retention relies on. A retry that finds a +/// *larger* extent recorded is the one case that should never happen, and +/// is said out loud. +/// +/// `sealed_at` moves only when the segment brought bodies. Retention cuts by +/// it, and it closes the open block before it looks: an empty FINAL segment +/// stamped "now" would make every idle block the newest thing in the archive +/// at the very moment retention asked how old it was, and nothing would ever +/// age out of a block that was open when retention ran. +fn upsert_block(conn: &Connection, segment: &SegmentWritten, sealed_at: &str) -> rusqlite::Result<()> { + let recorded = execute_cached( + conn, + "INSERT INTO body_blocks (block_offset, raw_len, disk_len, sealed_at) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT(block_offset) DO UPDATE SET + raw_len = excluded.raw_len, + disk_len = excluded.disk_len, + sealed_at = CASE WHEN excluded.raw_len > body_blocks.raw_len + THEN excluded.sealed_at ELSE body_blocks.sealed_at END + WHERE excluded.disk_len >= body_blocks.disk_len", + params![ + segment.block_offset as i64, + i64::from(segment.raw_len), + segment.disk_len as i64, + sealed_at, + ], + )?; + if recorded == 0 { + warn!( + block_offset = segment.block_offset, + raw_len = segment.raw_len, + disk_len = segment.disk_len, + "session body index already records a longer extent for this block; kept the longer one" + ); + } + Ok(()) +} diff --git a/crates/capsem-logger/src/writer/bodies/sync_worker.rs b/crates/capsem-logger/src/writer/bodies/sync_worker.rs new file mode 100644 index 000000000..f071ea379 --- /dev/null +++ b/crates/capsem-logger/src/writer/bodies/sync_worker.rs @@ -0,0 +1,65 @@ +//! Dedicated archive durability worker. + +use std::sync::mpsc; + +use capsem_archive::BodyLogSync; + +pub(super) struct ArchiveSyncWorker { + requests: Option>, + completions: mpsc::Receiver>, + join: Option>, +} + +impl ArchiveSyncWorker { + pub(super) fn start(sync: BodyLogSync) -> std::io::Result { + let (requests, work) = mpsc::sync_channel(1); + let (completed, completions) = mpsc::sync_channel(1); + let join = std::thread::Builder::new() + .name("capsem-archive-sync".into()) + .stack_size(128 * 1024) + .spawn(move || { + while work.recv().is_ok() { + let result = sync.sync().map_err(|error| error.to_string()); + if completed.send(result).is_err() { + break; + } + } + })?; + Ok(Self { + requests: Some(requests), + completions, + join: Some(join), + }) + } + + pub(super) fn request(&self) -> Result<(), String> { + self.requests + .as_ref() + .ok_or_else(|| "archive sync worker stopped".to_string())? + .send(()) + .map_err(|_| "archive sync worker stopped".to_string()) + } + + pub(super) fn try_complete(&self) -> Option> { + match self.completions.try_recv() { + Ok(result) => Some(result), + Err(mpsc::TryRecvError::Empty) => None, + Err(mpsc::TryRecvError::Disconnected) => Some(Err("archive sync worker stopped".into())), + } + } + + pub(super) fn complete(&self) -> Result<(), String> { + self.completions + .recv() + .map_err(|_| "archive sync worker stopped".to_string())? + } +} + +impl Drop for ArchiveSyncWorker { + fn drop(&mut self) { + self.requests.take(); + if let Some(join) = self.join.take() { + let _ = join.join(); + } + } +} diff --git a/crates/capsem-logger/src/writer/flush.rs b/crates/capsem-logger/src/writer/flush.rs new file mode 100644 index 000000000..fc2d63eea --- /dev/null +++ b/crates/capsem-logger/src/writer/flush.rs @@ -0,0 +1,152 @@ +//! Bounded publication of in-memory ledger rows and archive indexes. + +use super::*; + +pub(in crate::writer) fn flush_dirty_tables_to_disk( + conn: &Connection, + dirty_tables: &mut BTreeSet<&'static str>, + flush_watermarks: &mut schema::MemoryFlushWatermarks, + db_path: Option<&Path>, + bodies: &mut BodyArchive, + counters: &LedgerCounters, + wait_for_body_sync: bool, +) -> rusqlite::Result { + if dirty_tables.is_empty() && !bodies.has_work() { + return Ok(FlushTimings::default()); + } + // Bytes before index: the segment reaches the archive file here, and the + // rows that name it are inserted in the transaction below. A crash in + // between costs unreferenced bytes, never a row pointing past EOF. + let started = Instant::now(); + bodies.flush_segment(); + bodies.prepare_index_rows(wait_for_body_sync); + let archive_ms = started.elapsed().as_secs_f64() * 1000.0; + let tables: Vec<&'static str> = dirty_tables.iter().copied().collect(); + let started = Instant::now(); + let tx = conn.unchecked_transaction()?; + let transaction_begin_ms = started.elapsed().as_secs_f64() * 1000.0; + let started = Instant::now(); + let body_index_limit = if wait_for_body_sync { + usize::MAX + } else { + capsem_proto::LEDGER_ARCHIVE_INDEX_ROWS_PER_FLUSH + }; + bodies.commit_prepared_index_rows_with_limit(&tx, body_index_limit)?; + let body_index_rows = bodies.prepared_index_row_count(); + let body_index_ms = started.elapsed().as_secs_f64() * 1000.0; + // Injected where a real flush failure lands: inside the transaction, with + // the archive's index rows already written into it. Everything here rolls + // back together, and the retry has to bring the bodies back with the rows. + if take_disk_flush_failure_for_tests(db_path) { + return Err(rusqlite::Error::InvalidParameterName( + "injected disk flush failure before copy".to_string(), + )); + } + let started = Instant::now(); + let advanced_watermarks = schema::with_memory_schema_lock(|| { + schema::flush_memory_tables_to_disk(&tx, tables.iter().copied(), flush_watermarks) + })?; + let table_copy_ms = started.elapsed().as_secs_f64() * 1000.0; + // The snapshot lands with the rows it counts, or neither does. + let started = Instant::now(); + counters::store(&tx, counters)?; + let counter_snapshot_ms = started.elapsed().as_secs_f64() * 1000.0; + let started = Instant::now(); + tx.commit()?; + let commit_ms = started.elapsed().as_secs_f64() * 1000.0; + // Only now are the written segments indexed: anything above this line rolls + // the transaction back, and the next flush retries both halves together. + bodies.index_rows_committed(); + flush_watermarks.extend(advanced_watermarks); + dirty_tables.clear(); + Ok(FlushTimings { + archive_ms, + transaction_begin_ms, + body_index_rows, + body_index_ms, + table_copy_ms, + counter_snapshot_ms, + commit_ms, + }) +} + +#[derive(Default)] +pub(in crate::writer) struct FlushTimings { + archive_ms: f64, + transaction_begin_ms: f64, + body_index_rows: usize, + body_index_ms: f64, + table_copy_ms: f64, + counter_snapshot_ms: f64, + commit_ms: f64, +} + +#[derive(Clone, Copy)] +pub(super) struct FlushRequest { + pub(super) pending_ops: usize, + pub(super) wait_for_body_sync: bool, +} + +pub(super) fn flush_dirty_tables_to_disk_recorded( + conn: &Connection, + dirty_tables: &mut BTreeSet<&'static str>, + flush_watermarks: &mut schema::MemoryFlushWatermarks, + db_path: Option<&Path>, + bodies: &mut BodyArchive, + counters: &LedgerCounters, + request: FlushRequest, +) -> rusqlite::Result<()> { + let table_count = dirty_tables.len(); + let started = Instant::now(); + let result = flush_dirty_tables_to_disk( + conn, + dirty_tables, + flush_watermarks, + db_path, + bodies, + counters, + request.wait_for_body_sync, + ); + let elapsed = started.elapsed(); + let status = if result.is_ok() { "ok" } else { "error" }; + let timings = result.as_ref().ok(); + let archive_ms = timings.map_or(0.0, |timings| timings.archive_ms); + let transaction_begin_ms = timings.map_or(0.0, |timings| timings.transaction_begin_ms); + let body_index_rows = timings.map_or(0, |timings| timings.body_index_rows); + let body_index_ms = timings.map_or(0.0, |timings| timings.body_index_ms); + let table_copy_ms = timings.map_or(0.0, |timings| timings.table_copy_ms); + let counter_snapshot_ms = timings.map_or(0.0, |timings| timings.counter_snapshot_ms); + let commit_ms = timings.map_or(0.0, |timings| timings.commit_ms); + tracing::debug!( + target: "capsem.db", + duration_ms = elapsed.as_secs_f64() * 1000.0, + archive_ms, + transaction_begin_ms, + body_index_rows, + body_index_ms, + table_copy_ms, + counter_snapshot_ms, + commit_ms, + pending_ops = request.pending_ops, + table_count, + status, + "db disk flush completed" + ); + if elapsed >= DISK_FLUSH_INTERVAL { + warn!( + duration_ms = elapsed.as_secs_f64() * 1000.0, + archive_ms, + transaction_begin_ms, + body_index_rows, + body_index_ms, + table_copy_ms, + counter_snapshot_ms, + commit_ms, + pending_ops = request.pending_ops, + table_count, + status, + "db disk flush exceeded its configured interval" + ); + } + result.map(|_| ()) +} diff --git a/crates/capsem-logger/src/writer/remote.rs b/crates/capsem-logger/src/writer/remote.rs index b0a2b290a..15cb27f92 100644 --- a/crates/capsem-logger/src/writer/remote.rs +++ b/crates/capsem-logger/src/writer/remote.rs @@ -20,10 +20,7 @@ const STARTUP_TIMEOUT: Duration = Duration::from_secs(10); #[cfg(not(test))] const OPERATION_TIMEOUT: Duration = Duration::from_secs(5); #[cfg(test)] -// Large body admission includes framing, archive compression, SQLite commit, -// and its producer commitment. Two hundred milliseconds made the WARC -// backpressure regression fail during fixture setup on otherwise healthy -// macOS and loaded Linux builders. +// Covers archive compression and SQLite commit on loaded builders. const OPERATION_TIMEOUT: Duration = Duration::from_secs(2); const BACKPRESSURE_MAX_WAIT: Duration = Duration::from_millis(5); diff --git a/crates/capsem-logger/src/writer/tests.rs b/crates/capsem-logger/src/writer/tests.rs index d00aea2ea..bf39fbb48 100644 --- a/crates/capsem-logger/src/writer/tests.rs +++ b/crates/capsem-logger/src/writer/tests.rs @@ -1233,202 +1233,5 @@ fn dns_events_indexed_by_trace_id_for_join() { assert_eq!(count, 1, "missing idx_dns_events_trace_id"); } -fn file_event_with_credential(path: &str, credential_ref: Option<&str>) -> WriteOp { - WriteOp::FileEvent(crate::events::FileEvent { - event_id: None, - timestamp: std::time::SystemTime::now(), - action: crate::events::FileAction::Created, - path: path.to_string(), - size: Some(1), - kind: crate::events::FileKind::File, - trace_id: None, - credential_ref: credential_ref.map(str::to_string), - }) -} - -/// One row the schema rejects must not take its neighbours down with it. -/// -/// The write batch is a single transaction for throughput, so a CHECK violation -/// on one op used to roll back every op batched alongside it -- valid security -/// telemetry discarded because an unrelated producer built one bad row, and only -/// a warn line to show for it. -#[test] -fn one_rejected_op_does_not_discard_the_rest_of_its_batch() { - let tmp = tempfile::tempdir().unwrap(); - let db_path = tmp.path().join("session.db"); - let writer = DbWriter::open(&db_path, 64).unwrap(); - let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); - - rt.block_on(async { - // Enqueued together so they land in one batch. The middle op carries a - // credential_ref the schema CHECK rejects. - writer.write(file_event_with_credential("/good/before", None)).await; - writer - .write(file_event_with_credential( - "/bad/malformed", - Some("credential:blake3:not-a-real-digest"), - )) - .await; - writer.write(file_event_with_credential("/good/after", None)).await; - writer.flush().await; - }); - writer.shutdown_blocking(); - - let conn = rusqlite::Connection::open(&db_path).unwrap(); - let paths = conn - .prepare("SELECT path FROM fs_events ORDER BY id") - .unwrap() - .query_map([], |row| row.get::<_, String>(0)) - .unwrap() - .collect::, _>>() - .unwrap(); - - assert_eq!( - paths, - vec!["/good/before".to_string(), "/good/after".to_string()], - "the two valid rows must survive the one the schema refused" - ); -} - -fn minimal_model_call(trace_id: &str) -> WriteOp { - WriteOp::ModelCall(crate::events::ModelCall { - event_id: None, - timestamp: std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000), - provider: "anthropic".to_string(), - protocol: Some("anthropic".to_string()), - model: Some("claude-sonnet-4-20250514".to_string()), - process_name: None, - pid: None, - method: "POST".to_string(), - path: "/v1/messages".to_string(), - stream: false, - system_prompt_preview: None, - messages_count: 1, - tools_count: 0, - request_bytes: 16, - request_body: None, - message_id: Some("msg_salvage".to_string()), - status_code: Some(200), - text_content: Some("hello".to_string()), - thinking_content: None, - response_body: None, - stop_reason: Some("end_turn".to_string()), - input_tokens: Some(1), - output_tokens: Some(1), - usage_details: std::collections::BTreeMap::new(), - duration_ms: 1, - response_bytes: 16, - estimated_cost_usd: 0.0, - trace_id: Some(trace_id.to_string()), - credential_ref: None, - tool_calls: vec![], - tool_responses: vec![], - }) -} - -#[test] -fn salvaging_a_failed_batch_preserves_model_items() { - let tmp = tempfile::tempdir().unwrap(); - let db_path = tmp.path().join("session.db"); - let writer = DbWriter::open(&db_path, 64).unwrap(); - let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); - - rt.block_on(async { - writer.write(minimal_model_call("trace_salvage")).await; - writer - .write(file_event_with_credential( - "/bad/malformed", - Some("credential:blake3:not-a-real-digest"), - )) - .await; - writer.flush().await; - }); - writer.shutdown_blocking(); - - let conn = rusqlite::Connection::open(&db_path).unwrap(); - let calls: i64 = conn - .query_row("SELECT count(*) FROM model_calls", [], |row| row.get(0)) - .unwrap(); - let items: i64 = conn - .query_row("SELECT count(*) FROM model_items", [], |row| row.get(0)) - .unwrap(); - - assert_eq!(calls, 1, "the model call survives the rejected row beside it"); - assert!( - items > 0, - "the model items must survive retrying the rejected batch one row at a time" - ); -} - -#[test] -fn a_rejected_op_is_dropped_wherever_it_sits_in_the_batch() { - for (label, position) in [("first", 0usize), ("middle", 1), ("last", 2)] { - let tmp = tempfile::tempdir().unwrap(); - let db_path = tmp.path().join("session.db"); - let writer = DbWriter::open(&db_path, 64).unwrap(); - let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); - - rt.block_on(async { - for index in 0..3 { - if index == position { - writer - .write(file_event_with_credential( - "/bad/malformed", - Some("credential:blake3:not-a-real-digest"), - )) - .await; - } else { - writer - .write(file_event_with_credential(&format!("/good/{index}"), None)) - .await; - } - } - writer.flush().await; - }); - writer.shutdown_blocking(); - - let conn = rusqlite::Connection::open(&db_path).unwrap(); - let surviving: i64 = conn - .query_row("SELECT count(*) FROM fs_events", [], |row| row.get(0)) - .unwrap(); - assert_eq!( - surviving, 2, - "a rejected op in the {label} position must cost only itself" - ); - } -} - -#[test] -fn a_batch_of_only_rejected_ops_writes_nothing_and_does_not_wedge_the_writer() { - let tmp = tempfile::tempdir().unwrap(); - let db_path = tmp.path().join("session.db"); - let writer = DbWriter::open(&db_path, 64).unwrap(); - let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); - - rt.block_on(async { - for _ in 0..3 { - writer - .write(file_event_with_credential( - "/bad/malformed", - Some("credential:blake3:not-a-real-digest"), - )) - .await; - } - writer.flush().await; - // The writer must still accept work after salvaging an all-bad batch. - writer.write(file_event_with_credential("/good/after", None)).await; - writer.flush().await; - }); - writer.shutdown_blocking(); - - let conn = rusqlite::Connection::open(&db_path).unwrap(); - let paths = conn - .prepare("SELECT path FROM fs_events ORDER BY id") - .unwrap() - .query_map([], |row| row.get::<_, String>(0)) - .unwrap() - .collect::, _>>() - .unwrap(); - - assert_eq!(paths, vec!["/good/after".to_string()]); -} +mod salvage; +use salvage::{file_event_with_credential, minimal_model_call}; diff --git a/crates/capsem-logger/src/writer/tests/bodies.rs b/crates/capsem-logger/src/writer/tests/bodies.rs index f38b86e22..fe1a2389b 100644 --- a/crates/capsem-logger/src/writer/tests/bodies.rs +++ b/crates/capsem-logger/src/writer/tests/bodies.rs @@ -351,7 +351,9 @@ fn nonblocking_archive_sync_defers_index_rows_until_completion() { ); archive.flush_segment(); archive.prepare_index_rows(false); - archive.commit_prepared_index_rows(&conn).unwrap(); + archive + .commit_prepared_index_rows_with_limit(&conn, capsem_proto::LEDGER_ARCHIVE_INDEX_ROWS_PER_FLUSH) + .unwrap(); archive.index_rows_committed(); let rows = || { @@ -366,6 +368,72 @@ fn nonblocking_archive_sync_defers_index_rows_until_completion() { assert_eq!(archive.steps_for_tests(), ["sync", "commit"]); } +/// A completed device sync may make several seconds of body rows eligible at +/// once. Each SQLite transaction publishes only its bounded prefix, and a +/// committed prefix is removed while the remainder stays available to the +/// next flush. +#[test] +fn synchronized_index_backlog_drains_across_bounded_transactions() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("bounded-index-publish.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + crate::schema::create_tables(&conn).unwrap(); + + let mut archive = BodyArchive::open_for_tests(Some(&db_path), SystemTime::now, &conn); + for index in 0..5 { + archive.stage( + &conn, + body_blob(&format!("{index:012x}"), &format!("unique body {index}")), + ); + } + archive.flush_segment(); + archive.prepare_index_rows(true); + + for expected in [2, 4, 5] { + let tx = conn.unchecked_transaction().unwrap(); + archive.commit_prepared_index_rows_with_limit(&tx, 2).unwrap(); + assert!(archive.prepared_index_row_count() <= 2); + tx.commit().unwrap(); + archive.index_rows_committed(); + let rows: i64 = conn + .query_row("SELECT COUNT(*) FROM event_body_blobs", [], |row| row.get(0)) + .unwrap(); + assert_eq!(rows, expected); + if expected == 2 { + conn.execute_batch( + "CREATE TRIGGER body_block_metadata_is_written_once + BEFORE UPDATE ON body_blocks + BEGIN + SELECT RAISE(FAIL, 'body block metadata rewritten'); + END;", + ) + .unwrap(); + } + } + assert_eq!(archive.prepared_index_row_count(), 0); +} + +#[test] +fn committed_empty_segment_leaves_no_archive_work() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("empty-segment.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + crate::schema::create_tables(&conn).unwrap(); + let mut archive = BodyArchive::open_for_tests(Some(&db_path), SystemTime::now, &conn); + + let mark = archive.staged_mark(); + archive.stage(&conn, body_blob("empty-segment", "rolled back body")); + archive.flush_segment(); + archive.rollback_staged(mark); + archive.prepare_index_rows(true); + let tx = conn.unchecked_transaction().unwrap(); + archive.commit_prepared_index_rows_with_limit(&tx, 2).unwrap(); + tx.commit().unwrap(); + archive.index_rows_committed(); + + assert!(!archive.has_work(), "the committed empty segment has no backlog"); +} + /// A body that cannot fit beside what the open block holds closes the block /// and retries, rather than failing or panicking. Two 10 MiB bodies cannot /// share a 16 MiB block, so they land in two. @@ -903,55 +971,4 @@ fn a_crash_between_a_segment_and_its_commit_strands_bytes_and_nothing_else() { assert_eq!(read_indexed(&conn, &db_path, "0a0a0a0a0a03"), b"after the restart"); } -/// Seconds since the epoch the test clock reads, so a test can move time. -static TEST_CLOCK_SECS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1_800_000_000); - -fn test_clock() -> SystemTime { - SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(TEST_CLOCK_SECS.load(std::sync::atomic::Ordering::SeqCst)) -} - -fn advance_test_clock(by: std::time::Duration) { - TEST_CLOCK_SECS.fetch_add(by.as_secs(), std::sync::atomic::Ordering::SeqCst); -} - -/// A quiet session must not keep one block open for days: retention drops -/// whole blocks by their last write, so a block that never closed would keep -/// every body in it for as long as anything new joined it. A block older -/// than `MAX_BLOCK_AGE` closes, and the next body starts a new one. -#[test] -fn a_block_open_longer_than_its_age_limit_closes_before_the_next_body() { - use crate::writer::bodies::MAX_BLOCK_AGE; - - let dir = tempfile::tempdir().unwrap(); - let db_path = dir.path().join("block-age.db"); - let conn = rusqlite::Connection::open(&db_path).unwrap(); - crate::schema::create_tables(&conn).unwrap(); - - let mut archive = BodyArchive::open_for_tests(Some(&db_path), test_clock, &conn); - archive.stage(&conn, body_blob("0a0a0a0a0b01", "opens the block")); - archive.flush_segment(); - advance_test_clock(MAX_BLOCK_AGE.saturating_sub(std::time::Duration::from_secs(60))); - archive.stage(&conn, body_blob("0a0a0a0a0b02", "still inside the age limit")); - archive.flush_segment(); - advance_test_clock(std::time::Duration::from_secs(120)); - archive.stage(&conn, body_blob("0a0a0a0a0b03", "past it")); - archive.flush_segment(); - commit(&mut archive, &conn); - - let blocks: Vec = conn - .prepare("SELECT block_offset FROM event_body_blobs ORDER BY event_id") - .unwrap() - .query_map([], |row| row.get(0)) - .unwrap() - .collect::>() - .unwrap(); - assert_eq!(blocks[0], blocks[1], "inside the limit, one block"); - assert!(blocks[2] > blocks[1], "past it, a new block: {blocks:?}"); - assert_eq!(read_indexed(&conn, &db_path, "0a0a0a0a0b03"), b"past it"); - - // And an idle block closes at the next flush, not only at the next body. - advance_test_clock(MAX_BLOCK_AGE); - archive.flush_segment(); - assert_eq!(archive.appended_len_for_tests(), 1, "the stale block closed"); - commit(&mut archive, &conn); -} +mod aging; diff --git a/crates/capsem-logger/src/writer/tests/bodies/aging.rs b/crates/capsem-logger/src/writer/tests/bodies/aging.rs new file mode 100644 index 000000000..1ee5c5e9b --- /dev/null +++ b/crates/capsem-logger/src/writer/tests/bodies/aging.rs @@ -0,0 +1,54 @@ +use super::*; + +/// Seconds since the epoch the test clock reads, so a test can move time. +static TEST_CLOCK_SECS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1_800_000_000); + +fn test_clock() -> SystemTime { + SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(TEST_CLOCK_SECS.load(std::sync::atomic::Ordering::SeqCst)) +} + +fn advance_test_clock(by: std::time::Duration) { + TEST_CLOCK_SECS.fetch_add(by.as_secs(), std::sync::atomic::Ordering::SeqCst); +} + +/// A quiet session must not keep one block open for days: retention drops +/// whole blocks by their last write, so a block that never closed would keep +/// every body in it for as long as anything new joined it. A block older +/// than `MAX_BLOCK_AGE` closes, and the next body starts a new one. +#[test] +fn a_block_open_longer_than_its_age_limit_closes_before_the_next_body() { + use crate::writer::bodies::MAX_BLOCK_AGE; + + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("block-age.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + crate::schema::create_tables(&conn).unwrap(); + + let mut archive = BodyArchive::open_for_tests(Some(&db_path), test_clock, &conn); + archive.stage(&conn, body_blob("0a0a0a0a0b01", "opens the block")); + archive.flush_segment(); + advance_test_clock(MAX_BLOCK_AGE.saturating_sub(std::time::Duration::from_secs(60))); + archive.stage(&conn, body_blob("0a0a0a0a0b02", "still inside the age limit")); + archive.flush_segment(); + advance_test_clock(std::time::Duration::from_secs(120)); + archive.stage(&conn, body_blob("0a0a0a0a0b03", "past it")); + archive.flush_segment(); + commit(&mut archive, &conn); + + let blocks: Vec = conn + .prepare("SELECT block_offset FROM event_body_blobs ORDER BY event_id") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(blocks[0], blocks[1], "inside the limit, one block"); + assert!(blocks[2] > blocks[1], "past it, a new block: {blocks:?}"); + assert_eq!(read_indexed(&conn, &db_path, "0a0a0a0a0b03"), b"past it"); + + // And an idle block closes at the next flush, not only at the next body. + advance_test_clock(MAX_BLOCK_AGE); + archive.flush_segment(); + assert_eq!(archive.appended_len_for_tests(), 1, "the stale block closed"); + commit(&mut archive, &conn); +} diff --git a/crates/capsem-logger/src/writer/tests/salvage.rs b/crates/capsem-logger/src/writer/tests/salvage.rs new file mode 100644 index 000000000..a05f438b0 --- /dev/null +++ b/crates/capsem-logger/src/writer/tests/salvage.rs @@ -0,0 +1,201 @@ +use super::*; + +pub(super) fn file_event_with_credential(path: &str, credential_ref: Option<&str>) -> WriteOp { + WriteOp::FileEvent(crate::events::FileEvent { + event_id: None, + timestamp: std::time::SystemTime::now(), + action: crate::events::FileAction::Created, + path: path.to_string(), + size: Some(1), + kind: crate::events::FileKind::File, + trace_id: None, + credential_ref: credential_ref.map(str::to_string), + }) +} + +/// One row the schema rejects must not take its neighbours down with it. +/// +/// The write batch is a single transaction for throughput, so a CHECK violation +/// on one op used to roll back every op batched alongside it -- valid security +/// telemetry discarded because an unrelated producer built one bad row, and only +/// a warn line to show for it. +#[test] +fn one_rejected_op_does_not_discard_the_rest_of_its_batch() { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("session.db"); + let writer = DbWriter::open(&db_path, 64).unwrap(); + let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); + + rt.block_on(async { + // Enqueued together so they land in one batch. The middle op carries a + // credential_ref the schema CHECK rejects. + writer.write(file_event_with_credential("/good/before", None)).await; + writer + .write(file_event_with_credential( + "/bad/malformed", + Some("credential:blake3:not-a-real-digest"), + )) + .await; + writer.write(file_event_with_credential("/good/after", None)).await; + writer.flush().await; + }); + writer.shutdown_blocking(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let paths = conn + .prepare("SELECT path FROM fs_events ORDER BY id") + .unwrap() + .query_map([], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + + assert_eq!( + paths, + vec!["/good/before".to_string(), "/good/after".to_string()], + "the two valid rows must survive the one the schema refused" + ); +} + +pub(super) fn minimal_model_call(trace_id: &str) -> WriteOp { + WriteOp::ModelCall(crate::events::ModelCall { + event_id: None, + timestamp: std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(1_700_000_000), + provider: "anthropic".to_string(), + protocol: Some("anthropic".to_string()), + model: Some("claude-sonnet-4-20250514".to_string()), + process_name: None, + pid: None, + method: "POST".to_string(), + path: "/v1/messages".to_string(), + stream: false, + system_prompt_preview: None, + messages_count: 1, + tools_count: 0, + request_bytes: 16, + request_body: None, + message_id: Some("msg_salvage".to_string()), + status_code: Some(200), + text_content: Some("hello".to_string()), + thinking_content: None, + response_body: None, + stop_reason: Some("end_turn".to_string()), + input_tokens: Some(1), + output_tokens: Some(1), + usage_details: std::collections::BTreeMap::new(), + duration_ms: 1, + response_bytes: 16, + estimated_cost_usd: 0.0, + trace_id: Some(trace_id.to_string()), + credential_ref: None, + tool_calls: vec![], + tool_responses: vec![], + }) +} + +#[test] +fn salvaging_a_failed_batch_preserves_model_items() { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("session.db"); + let writer = DbWriter::open(&db_path, 64).unwrap(); + let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); + + rt.block_on(async { + writer.write(minimal_model_call("trace_salvage")).await; + writer + .write(file_event_with_credential( + "/bad/malformed", + Some("credential:blake3:not-a-real-digest"), + )) + .await; + writer.flush().await; + }); + writer.shutdown_blocking(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let calls: i64 = conn + .query_row("SELECT count(*) FROM model_calls", [], |row| row.get(0)) + .unwrap(); + let items: i64 = conn + .query_row("SELECT count(*) FROM model_items", [], |row| row.get(0)) + .unwrap(); + + assert_eq!(calls, 1, "the model call survives the rejected row beside it"); + assert!( + items > 0, + "the model items must survive retrying the rejected batch one row at a time" + ); +} + +#[test] +fn a_rejected_op_is_dropped_wherever_it_sits_in_the_batch() { + for (label, position) in [("first", 0usize), ("middle", 1), ("last", 2)] { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("session.db"); + let writer = DbWriter::open(&db_path, 64).unwrap(); + let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); + + rt.block_on(async { + for index in 0..3 { + if index == position { + writer + .write(file_event_with_credential( + "/bad/malformed", + Some("credential:blake3:not-a-real-digest"), + )) + .await; + } else { + writer + .write(file_event_with_credential(&format!("/good/{index}"), None)) + .await; + } + } + writer.flush().await; + }); + writer.shutdown_blocking(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let surviving: i64 = conn + .query_row("SELECT count(*) FROM fs_events", [], |row| row.get(0)) + .unwrap(); + assert_eq!( + surviving, 2, + "a rejected op in the {label} position must cost only itself" + ); + } +} + +#[test] +fn a_batch_of_only_rejected_ops_writes_nothing_and_does_not_wedge_the_writer() { + let tmp = tempfile::tempdir().unwrap(); + let db_path = tmp.path().join("session.db"); + let writer = DbWriter::open(&db_path, 64).unwrap(); + let rt = tokio::runtime::Builder::new_current_thread().build().unwrap(); + + rt.block_on(async { + for _ in 0..3 { + writer + .write(file_event_with_credential( + "/bad/malformed", + Some("credential:blake3:not-a-real-digest"), + )) + .await; + } + writer.flush().await; + // The writer must still accept work after salvaging an all-bad batch. + writer.write(file_event_with_credential("/good/after", None)).await; + writer.flush().await; + }); + writer.shutdown_blocking(); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + let paths = conn + .prepare("SELECT path FROM fs_events ORDER BY id") + .unwrap() + .query_map([], |row| row.get::<_, String>(0)) + .unwrap() + .collect::, _>>() + .unwrap(); + + assert_eq!(paths, vec!["/good/after".to_string()]); +} diff --git a/crates/capsem-logger/src/writer/worker.rs b/crates/capsem-logger/src/writer/worker.rs new file mode 100644 index 000000000..c9d2bd365 --- /dev/null +++ b/crates/capsem-logger/src/writer/worker.rs @@ -0,0 +1,199 @@ +//! Dedicated SQLite writer-thread event loop. + +use super::*; + +pub(super) fn writer_loop( + conn: Connection, + rx: WriterReceiver, + db_path: Option, + batch_capacity: usize, + pending_body_bytes: &AtomicU64, + mut bodies: BodyArchive, + mut tally: LedgerTally, +) { + let mut flush_watermarks = + schema::with_memory_schema_lock(|| schema::initial_memory_flush_watermarks(&conn, schema::hot_ledger_tables())) + .unwrap_or_else(|error| { + warn!(error = %error, "db initial memory flush watermark load failed"); + schema::MemoryFlushWatermarks::new() + }); + // Rows at or below the open-time watermark were written by an earlier + // writer; exec ids restart with the process, so a completion must not + // reach one of those. + let exec_floor = flush_watermarks.get("exec_events").copied().unwrap_or(0); + let mut dirty_tables = BTreeSet::new(); + let mut dirty_ops = 0_usize; + let mut last_disk_flush = Instant::now(); + // 1. Block until at least one op arrives. Returns None when all + // Senders are dropped (clean shutdown) and ends the loop. + loop { + let first_message = if dirty_ops == 0 && !bodies.has_work() { + rx.recv().ok() + } else { + match rx.recv_timeout(DISK_FLUSH_INTERVAL) { + Ok(message) => Some(message), + Err(mpsc::RecvTimeoutError::Timeout) => { + if let Err(error) = flush_dirty_tables_to_disk_recorded( + &conn, + &mut dirty_tables, + &mut flush_watermarks, + db_path.as_deref(), + &mut bodies, + tally.counters(), + FlushRequest { + pending_ops: dirty_ops, + wait_for_body_sync: false, + }, + ) { + warn!(error = %error, "db interval flush failed"); + } else { + dirty_ops = 0; + last_disk_flush = Instant::now(); + } + ::metrics::gauge!(DB_MEMORY_UNFLUSHED_OPS).set(dirty_ops as f64); + continue; + } + Err(mpsc::RecvTimeoutError::Disconnected) => None, + } + }; + let Some(first_message) = first_message else { + break; + }; + + let mut batch = Vec::with_capacity(batch_capacity); + let mut barriers = Barriers::default(); + let mut at_barrier = barriers.accept(first_message.into_message(), &mut batch); + + // 2. Drain any ops already queued (non-blocking). + while !at_barrier && batch.len() < batch_capacity { + match rx.try_recv() { + Ok(message) => at_barrier = barriers.accept(message.into_message(), &mut batch), + Err(_) => break, + } + } + + // 3. Execute entire batch in a single transaction. + let batch_size = batch.len(); + let batch_bucket = batch_size_bucket(batch_size); + let span = tracing::debug_span!( + target: "capsem.db", + DB_WRITE_BATCH_SPAN, + batch_size_bucket = batch_bucket, + status = tracing::field::Empty, + ); + let started = Instant::now(); + let batch_capacity = batch.capacity(); + if batch.is_empty() { + record_batch(started, batch_size, batch_capacity, batch_bucket, "ok", &span); + } else { + match span.in_scope(|| execute_memory_batch(&conn, &batch, &mut bodies, exec_floor, &mut tally)) { + Ok(outcome) => { + dirty_tables.extend(outcome.tables); + dirty_ops += outcome.written; + record_batch(started, batch_size, batch_capacity, batch_bucket, "ok", &span); + } + Err(e) => { + record_batch(started, batch_size, batch_capacity, batch_bucket, "error", &span); + warn!( + error = %e, + count = batch.len(), + "db memory write batch failed; retrying its ops individually" + ); + let salvaged = span + .in_scope(|| retry_batch_ops_individually(&conn, &batch, &mut bodies, exec_floor, &mut tally)); + dirty_tables.extend(salvaged.tables); + dirty_ops += salvaged.written; + } + } + } + // A burst of large bodies closes its block as soon as the batch ends + // rather than growing it until the interval expires. Its index rows + // wait for the next transaction; its bytes are already on disk. + bodies.close_if_due(); + pending_body_bytes.store(bodies.pending_bytes() as u64, Ordering::Release); + let disk_flush_due = dirty_ops >= DISK_FLUSH_THRESHOLD_OPS + || last_disk_flush.elapsed() >= DISK_FLUSH_INTERVAL + || barriers.waiting(); + let mut barrier_outcome: FlushOutcome = Ok(()); + if disk_flush_due { + match flush_dirty_tables_to_disk_recorded( + &conn, + &mut dirty_tables, + &mut flush_watermarks, + db_path.as_deref(), + &mut bodies, + tally.counters(), + FlushRequest { + pending_ops: dirty_ops, + wait_for_body_sync: barriers.waiting(), + }, + ) { + Ok(()) => { + dirty_ops = 0; + last_disk_flush = Instant::now(); + pending_body_bytes.store(bodies.pending_bytes() as u64, Ordering::Release); + } + Err(error) => { + warn!(error = %error, "db dirty table flush failed"); + barrier_outcome = Err(format!("db dirty table flush failed: {error}")); + } + } + } + ::metrics::gauge!(DB_MEMORY_UNFLUSHED_OPS).set(dirty_ops as f64); + barriers.answer(&conn, &mut bodies, &barrier_outcome); + } + + // Test hook: lets `test_wal_absent_after_clean_shutdown`-style tests + // simulate a slow checkpoint so the explicit-cleanup path can be + // distinguished from implicit tokio-runtime-drop ordering. Gated on + // an env var so it's a no-op in production. + if let Ok(ms) = std::env::var("CAPSEM_TEST_SLOW_CHECKPOINT_MS") { + if let Ok(ms) = ms.parse::() { + std::thread::sleep(std::time::Duration::from_millis(ms)); + } + } + + // The open block ends with its FINAL segment, committed by this flush. + bodies.close_block(); + if let Err(error) = flush_dirty_tables_to_disk_recorded( + &conn, + &mut dirty_tables, + &mut flush_watermarks, + db_path.as_deref(), + &mut bodies, + tally.counters(), + FlushRequest { + pending_ops: dirty_ops, + wait_for_body_sync: true, + }, + ) { + warn!(error = %error, "db shutdown dirty table flush failed"); + // There is no next flush to retry into, so the bodies still waiting + // for an index row are lost here. Counted, so the drop counter closes + // over the session rather than ending on a number that is short by + // however much the last flush was carrying. + bodies.abandon_uncommitted("shutdown"); + } + bodies.sync(); + pending_body_bytes.store(bodies.pending_bytes() as u64, Ordering::Release); + + // All senders dropped -- checkpoint WAL before closing connection. + let span = tracing::debug_span!( + target: "capsem.db", + DB_SHUTDOWN_FLUSH_SPAN, + status = tracing::field::Empty, + ); + let started = Instant::now(); + let result = span.in_scope(|| conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")); + let elapsed_ms = started.elapsed().as_secs_f64() * 1000.0; + let status = if result.is_ok() { "ok" } else { "error" }; + ::metrics::histogram!(DB_SHUTDOWN_FLUSH_MS, "status" => status).record(elapsed_ms); + span.record("status", status); + // mmap coverage and file-size diagnostics query SQLite and the filesystem. + // Sampling them on every periodic flush once held this admission thread + // for more than a second under DNS load. Shutdown is outside the producer + // path and still records the final ledger size and mmap budget. + if let Some(path) = db_path.as_deref() { + schema::record_sqlite_mmap_telemetry(&conn, path, "writer", "shutdown"); + } +} From 595718653d048a93b059bcc08700744ef112e195 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 04:31:03 +0000 Subject: [PATCH 405/464] feat(bench): collect raw guest protocol samples --- crates/capsem-bench/src/cli.rs | 3 + crates/capsem-bench/src/main.rs | 6 ++ crates/capsem-bench/src/protocol.rs | 2 +- crates/capsem-bench/src/protocol_collector.rs | 67 +++++++++++++++++++ .../src/protocol_collector/tests.rs | 56 ++++++++++++++++ crates/capsem-bench/src/runtime_limits.rs | 8 +++ crates/capsem-bench/src/scenarios.rs | 1 - crates/capsem-bench/src/tests.rs | 5 -- 8 files changed, 141 insertions(+), 7 deletions(-) create mode 100644 crates/capsem-bench/src/protocol_collector.rs create mode 100644 crates/capsem-bench/src/protocol_collector/tests.rs diff --git a/crates/capsem-bench/src/cli.rs b/crates/capsem-bench/src/cli.rs index f19aed772..4c5c199dc 100644 --- a/crates/capsem-bench/src/cli.rs +++ b/crates/capsem-bench/src/cli.rs @@ -151,6 +151,9 @@ pub(crate) struct ProtocolArgs { pub(crate) lane: String, #[arg(long, default_value = "/tmp/capsem-benchmark.json")] pub(crate) json_out: PathBuf, + /// Write raw samples in the shared collector protocol. + #[arg(long)] + pub(crate) collector_out: Option, /// Also record into the host benchmark store. #[cfg(feature = "host")] #[arg(long)] diff --git a/crates/capsem-bench/src/main.rs b/crates/capsem-bench/src/main.rs index 50394ada9..d0387575e 100644 --- a/crates/capsem-bench/src/main.rs +++ b/crates/capsem-bench/src/main.rs @@ -11,6 +11,7 @@ mod http_load; mod machine; mod ping; mod protocol; +mod protocol_collector; #[cfg(feature = "host")] mod protocol_record; #[path = "stats/rates.rs"] @@ -54,6 +55,7 @@ async fn main() -> Result<()> { scenarios: None, lane: "host_direct".to_string(), json_out: PathBuf::from("/tmp/capsem-benchmark.json"), + collector_out: None, #[cfg(feature = "host")] record: None, #[cfg(feature = "host")] @@ -69,11 +71,15 @@ async fn main() -> Result<()> { Command::DnsLoad(args) => println!("{}", serde_json::to_string(&dns_load::run(args).await?)?), Command::RuntimeLimits => println!("{}", serde_json::to_string(&runtime_limits::report())?), Command::Protocol(args) => { + let collector_out = args.collector_out.clone(); #[cfg(feature = "host")] let destination = args.record.clone(); #[cfg(feature = "host")] let (channel, commit) = (args.channel.clone(), args.commit.clone()); let artifact = run_protocol(args).await?; + if let Some(path) = collector_out { + protocol_collector::write(&path, &artifact)?; + } #[cfg(feature = "host")] if let Some(root) = destination { let record = protocol_record::build(&artifact, &channel, &commit, machine::running_capsem_processes()?); diff --git a/crates/capsem-bench/src/protocol.rs b/crates/capsem-bench/src/protocol.rs index 2526e4315..6ed0c12c2 100644 --- a/crates/capsem-bench/src/protocol.rs +++ b/crates/capsem-bench/src/protocol.rs @@ -151,6 +151,7 @@ pub(crate) async fn run_protocol_delta(args: ProtocolDeltaArgs) -> Result, + sidecar: String, +} + +#[derive(Debug, Serialize)] +struct Metric { + unit: &'static str, + samples: Vec, +} + +pub(crate) fn build(artifact: &Artifact) -> Result { + let report = &artifact.mock_server_protocol; + let mut metrics = BTreeMap::new(); + for scenario in &report.scenarios { + let prefix = format!("{}.{}", report.lane, scenario.name); + metrics.insert( + format!("{prefix}.latency_ms"), + Metric { + unit: "milliseconds", + samples: scenario.latency_samples.clone(), + }, + ); + metrics.insert( + format!("{prefix}.requests_per_sec"), + Metric { + unit: "requests_per_second", + samples: vec![scenario.requests_per_sec], + }, + ); + metrics.insert( + format!("{prefix}.bytes_per_sec"), + Metric { + unit: "bytes", + samples: vec![scenario.bytes_per_sec], + }, + ); + metrics.insert( + format!("{prefix}.failed"), + Metric { + unit: "count", + samples: vec![scenario.failed as f64], + }, + ); + } + Ok(Document { + metrics, + sidecar: serde_json::to_string(artifact)?, + }) +} + +pub(crate) fn write(path: &Path, artifact: &Artifact) -> Result<()> { + std::fs::write(path, serde_json::to_vec(&build(artifact)?)?)?; + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-bench/src/protocol_collector/tests.rs b/crates/capsem-bench/src/protocol_collector/tests.rs new file mode 100644 index 000000000..62e6c70de --- /dev/null +++ b/crates/capsem-bench/src/protocol_collector/tests.rs @@ -0,0 +1,56 @@ +use super::*; +use crate::scenarios::{Artifact, LatencySummary, ProtocolReport, ScenarioResult}; +use std::collections::BTreeMap; + +#[test] +fn collector_keeps_raw_protocol_latencies() { + let artifact = Artifact { + version: "test".into(), + timestamp: 1.0, + hostname: "guest".into(), + benchmark: "capsem-bench-rs".into(), + mock_server_protocol: ProtocolReport { + version: "test".into(), + lane: "guest_capsem".into(), + base_url: "http://127.0.0.1:3713".into(), + dns_udp_addr: None, + total_requests: 3, + concurrency: 2, + timeout_ms: 1_000, + selected_scenarios: vec!["model_json_response".into()], + scenarios: vec![ScenarioResult { + name: "model_json_response".into(), + path: "/model/response".into(), + body_kind: "model_json".into(), + total_requests: 3, + concurrency: 2, + successful: 3, + failed: 0, + transport_retries: 0, + total_duration_ms: 4.0, + requests_per_sec: 750.0, + transfer_bytes: 120, + bytes_per_sec: 30_000.0, + latency_ms: LatencySummary { + min: 0.5, + max: 2.5, + mean: 1.5, + p50: 1.5, + p95: 2.4, + p99: 2.5, + }, + latency_samples: vec![0.5, 1.5, 2.5], + errors: BTreeMap::new(), + secret_shaped_fixture_seen: None, + raw_secret_stored_in_result: None, + }], + }, + }; + + let document = build(&artifact).unwrap(); + let latency = &document.metrics["guest_capsem.model_json_response.latency_ms"]; + assert_eq!(latency.unit, "milliseconds"); + assert_eq!(latency.samples, [0.5, 1.5, 2.5]); + assert!(document.sidecar.contains("model_json_response")); + assert!(!document.sidecar.contains("latency_samples")); +} diff --git a/crates/capsem-bench/src/runtime_limits.rs b/crates/capsem-bench/src/runtime_limits.rs index 3199dd3f5..ef7c610f9 100644 --- a/crates/capsem-bench/src/runtime_limits.rs +++ b/crates/capsem-bench/src/runtime_limits.rs @@ -5,9 +5,13 @@ use serde::Serialize; #[derive(Debug, Serialize)] pub(crate) struct Report { network_client_capacity: usize, + network_dns_sessions_per_vm: usize, + network_dns_max_udp_datagram_bytes: usize, + network_dns_udp_socket_buffer_bytes: usize, ledger_producer_queue_capacity: usize, ledger_admission_batch_capacity: usize, ledger_admission_coalesce_wait_microseconds: u64, + ledger_sqlite_mmap_bytes: i64, ledger_disk_flush_interval_milliseconds: u64, ledger_disk_flush_max_pending_events: usize, } @@ -15,9 +19,13 @@ pub(crate) struct Report { pub(crate) const fn report() -> Report { Report { network_client_capacity: capsem_proto::NETWORK_CLIENT_CAPACITY, + network_dns_sessions_per_vm: capsem_proto::NETWORK_DNS_SESSIONS_PER_VM, + network_dns_max_udp_datagram_bytes: capsem_proto::NETWORK_DNS_MAX_UDP_DATAGRAM_BYTES, + network_dns_udp_socket_buffer_bytes: capsem_proto::NETWORK_DNS_UDP_SOCKET_BUFFER_BYTES, ledger_producer_queue_capacity: capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, ledger_admission_batch_capacity: capsem_proto::LEDGER_ADMISSION_BATCH_CAPACITY, ledger_admission_coalesce_wait_microseconds: capsem_proto::LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS, + ledger_sqlite_mmap_bytes: capsem_proto::LEDGER_SQLITE_MMAP_BYTES, ledger_disk_flush_interval_milliseconds: capsem_proto::LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS, ledger_disk_flush_max_pending_events: capsem_proto::LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS, } diff --git a/crates/capsem-bench/src/scenarios.rs b/crates/capsem-bench/src/scenarios.rs index 7f6b3a20a..18475cb1e 100644 --- a/crates/capsem-bench/src/scenarios.rs +++ b/crates/capsem-bench/src/scenarios.rs @@ -251,7 +251,6 @@ pub(crate) struct ScenarioResult { /// Raw per-request latencies, kept for `capsem.bench.v2` and never /// serialized into this artifact: `capsem-bench` computes every statistic /// itself, so a collector that pre-summarizes hides the distribution. - #[cfg(feature = "host")] #[serde(skip)] pub(crate) latency_samples: Vec, pub(crate) errors: BTreeMap, diff --git a/crates/capsem-bench/src/tests.rs b/crates/capsem-bench/src/tests.rs index 5d1c3b93c..32783f8e0 100644 --- a/crates/capsem-bench/src/tests.rs +++ b/crates/capsem-bench/src/tests.rs @@ -107,7 +107,6 @@ fn delta_computes_abstraction_cost() { p95: 1.0, p99: 1.5, }, - #[cfg(feature = "host")] latency_samples: Vec::new(), errors: BTreeMap::new(), secret_shaped_fixture_seen: None, @@ -122,7 +121,6 @@ fn delta_computes_abstraction_cost() { p99: 9.5, ..host.latency_ms.clone() }, - #[cfg(feature = "host")] latency_samples: Vec::new(), ..host.clone() }; @@ -160,7 +158,6 @@ fn failed_scenarios_are_poisoned_benchmark_numbers() { p95: 1.0, p99: 1.5, }, - #[cfg(feature = "host")] latency_samples: Vec::new(), errors: BTreeMap::from([("request:connection refused".to_string(), 100)]), secret_shaped_fixture_seen: None, @@ -329,7 +326,6 @@ fn build_delta_report_keeps_inline_artifact_identity() { p95: 1.0, p99: 1.5, }, - #[cfg(feature = "host")] latency_samples: Vec::new(), errors: BTreeMap::new(), secret_shaped_fixture_seen: None, @@ -363,7 +359,6 @@ fn build_delta_report_keeps_inline_artifact_identity() { p99: 4.5, ..row.latency_ms.clone() }, - #[cfg(feature = "host")] latency_samples: Vec::new(), ..row }], From ef54dce7d71ab0497784452a93db36c648d88a2b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 04:31:15 +0000 Subject: [PATCH 406/464] test(bench): measure model and MCP VM pressure --- benchmarks/collectors/_guest | 154 ++++++------- benchmarks/collectors/protocol | 1 + benchmarks/helpers/guest_runtime.py | 103 +++++++++ build_system/builder/gate/benchmarkschema.py | 10 + config/gate.toml | 20 +- guest/artifacts/capsem_bench/__main__.py | 27 ++- tests/helpers/process_resources.py | 71 ++++++ tests/test_benchmark_guest_collector.py | 204 +++++++++++++++++- .../test_capsem_bench_mock_server_protocol.py | 40 ++++ tests/test_process_resource_sampler.py | 75 ++++++- 10 files changed, 605 insertions(+), 100 deletions(-) create mode 120000 benchmarks/collectors/protocol create mode 100644 benchmarks/helpers/guest_runtime.py diff --git a/benchmarks/collectors/_guest b/benchmarks/collectors/_guest index cd56ef194..3939c00c3 100755 --- a/benchmarks/collectors/_guest +++ b/benchmarks/collectors/_guest @@ -1,9 +1,9 @@ #!/usr/bin/env python3 """Measure one dimension inside a VM, via the guest `capsem-bench` package. -One program for eight dimensions: it reads the mode from the name it was +One program for nine dimensions: it reads the mode from the name it was invoked as, so `disk`, `rootfs`, `storage`, `startup`, `snapshot`, `mcp-load`, -`mitm-load` and `dns-load` are symlinks to this file. Eight copies of "boot a +`mitm-load`, `dns-load` and `protocol` are symlinks to this file. Nine copies of "boot a VM, run a mode, pull the JSON out" would drift apart, which is how the three load modes ended up with committed baselines and no caller. @@ -22,18 +22,22 @@ import argparse import contextlib import json import os -import shlex import sys import uuid from pathlib import Path +PROJECT_ROOT = Path(__file__).resolve().parents[2] + # Both, because the test helpers import each other as `helpers.*` and the # repository scripts as `scripts.*`; pytest puts the rootdir on the path and a # collector run outside pytest has to do the same. -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "tests")) -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +sys.path.insert(0, str(PROJECT_ROOT / "tests")) +sys.path.insert(0, str(PROJECT_ROOT)) + +from capsem_builder.gate import config as gate_config +from helpers.constants import DEFAULT_RAM_MB, EXEC_READY_TIMEOUT +from helpers.service import ServiceInstance, exec_output_text, wait_exec_ready -import psutil from benchmarks.helpers.guest_measurements import ( failed_load_levels, flatten, @@ -42,23 +46,34 @@ from benchmarks.helpers.guest_measurements import ( resource_samples, unit_for, ) -from helpers.constants import ( - DEFAULT_CPUS, - DEFAULT_RAM_MB, - EXEC_READY_TIMEOUT, +from benchmarks.helpers.guest_runtime import ( + LOAD_MODES, + MITM_FIXTURE_TARGET, + PROTOCOL_COLLECTOR_PATH, + PROXY_ERROR_MODES, + failed_load_diagnostics, + guest_command, + guest_cpus, + mitm_fixture_corp_toml, + process_tree_sampler, ) -from helpers.process_resources import ProcessTreeSampler -from helpers.service import ServiceInstance, exec_output_text, wait_exec_ready -FORWARDED_ENV = ( - "CAPSEM_BENCH_CONCURRENCY", - "CAPSEM_BENCH_DURATION_S", - "CAPSEM_BENCH_MITM_TARGET", - "CAPSEM_STORAGE_BENCH_PATHS", - "CAPSEM_STORAGE_BENCH_SIZE_MB", - "CAPSEM_STORAGE_IO_PROFILE_SIZE_MB", - "CAPSEM_STORAGE_IO_PROFILE_RANDOM_OPS", -) +PROTOCOL_SETTINGS = gate_config.load(PROJECT_ROOT).benchmark.protocol + + +def apply_protocol_defaults() -> None: + """Apply the config-owned routine model and MCP sample cohort.""" + os.environ.setdefault( + "CAPSEM_BENCH_TOTAL_REQUESTS", str(PROTOCOL_SETTINGS.requests) + ) + os.environ.setdefault( + "CAPSEM_BENCH_CONCURRENCY", str(PROTOCOL_SETTINGS.concurrency) + ) + os.environ.setdefault("CAPSEM_BENCH_TIMEOUT_MS", str(PROTOCOL_SETTINGS.timeout_ms)) + os.environ.setdefault( + "CAPSEM_BENCH_SCENARIOS", ",".join(PROTOCOL_SETTINGS.scenarios) + ) + # The mitm-load target: a name the mock server's DNS answers (with 127.0.0.1, # which the guest's NAT sends through capsem-net-proxy like any other address) @@ -72,47 +87,6 @@ FORWARDED_ENV = ( # Python threads doing TLS, plus the guest proxies, on the default two vCPUs # measured the generator's starvation, not the host rails. Give those guests # half the host's cores, at least the default and at most eight. -LOAD_MODES = ("dns-load", "mitm-load", "mcp-load") -LOAD_GUEST_CPUS_MAX = 8 - - -def guest_cpus(mode: str, host_cpus: int | None = None) -> int: - """vCPUs for the guest that runs `mode`.""" - if mode not in LOAD_MODES: - return DEFAULT_CPUS - host = host_cpus if host_cpus is not None else (os.cpu_count() or DEFAULT_CPUS) - return max(DEFAULT_CPUS, min(LOAD_GUEST_CPUS_MAX, host // 2)) - - -MITM_FIXTURE_HOST = "fixture.capsem.test" -MITM_FIXTURE_TARGET = f"https://{MITM_FIXTURE_HOST}/tiny" - - -def process_tree_sampler(service: ServiceInstance) -> ProcessTreeSampler: - """Measure the service and every confined per-VM descendant.""" - return ProcessTreeSampler(psutil.Process(service.proc.pid)) - - -def mitm_fixture_corp_toml(ready: dict) -> str: - """Corp config routing guest DNS and the fixture host to the mock server.""" - return ( - "[network.dns]\n" - f"upstreams = [{json.dumps(ready['dns_udp_addr'])}]\n" - "\n" - f'[network.upstream_overrides."{MITM_FIXTURE_HOST}:443"]\n' - f"dial = {json.dumps(ready['http_addr'])}\n" - 'protocol = "http"\n' - ) - - -def guest_command(mode: str) -> str: - """One safely quoted command with only documented benchmark controls.""" - assignments = [ - f"{name}={value}" - for name in FORWARDED_ENV - if (value := os.environ.get(name)) is not None - ] - return shlex.join(["env", *assignments, "capsem-bench", mode]) def main() -> int: @@ -138,6 +112,15 @@ def main() -> int: corp_path.write_text(mitm_fixture_corp_toml(ready), encoding="utf-8") os.environ["CAPSEM_CORP_CONFIG"] = str(corp_path) os.environ.setdefault("CAPSEM_BENCH_MITM_TARGET", MITM_FIXTURE_TARGET) + elif mode == "protocol": + from helpers.mock_server import start_mock_server + + mock_proc, ready = start_mock_server(capture_requests=False) + os.environ.setdefault("CAPSEM_MOCK_SERVER_BASE_URL", ready["base_url"]) + os.environ.setdefault( + "CAPSEM_MOCK_SERVER_DNS_UDP_ADDR", ready["dns_udp_addr"] + ) + apply_protocol_defaults() service.start() print(f"provisioning {name}", file=sys.stderr) created = client.post( @@ -176,11 +159,16 @@ def main() -> int: f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}" ) + artifact_path = ( + PROTOCOL_COLLECTOR_PATH + if mode == "protocol" + else "/tmp/capsem-benchmark.json" + ) artifact_result = client.post( f"/vms/{name}/exec", { "target": "vm", - "command": "cat /tmp/capsem-benchmark.json", + "command": f"cat {artifact_path}", "timeout_secs": 15, }, timeout=25, @@ -191,22 +179,33 @@ def main() -> int: f"exit={None if not artifact_result else artifact_result.get('exit_code')}\n" f"response={json.dumps(artifact_result)[:1000]}" ) - document = json.loads(exec_output_text(artifact_result)) + raw_document = json.loads(exec_output_text(artifact_result)) + protocol_collection = raw_document if mode == "protocol" else None + document = ( + json.loads(protocol_collection["sidecar"]) + if protocol_collection is not None + else raw_document + ) # Reject while the fixture is still live. ServiceInstance sees the # active exception during teardown and preserves the ledger/proxy # evidence instead of treating this as a clean benchmark shutdown. - section = document.get(mode.replace("-", "_"), document) + section_name = ( + "mock_server_protocol" if mode == "protocol" else mode.replace("-", "_") + ) + section = document.get(section_name, document) if failed := failed_load_levels(section): + diagnostics = failed_load_diagnostics(service, client, name, mode) raise RuntimeError( f"capsem-bench {mode}: requests failed at concurrency {failed}; " "the rail did not sustain the requested load, so the run is rejected; " f"artifact={json.dumps(document, sort_keys=True)}; " - f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}" + f"resources={json.dumps(resource_samples(resource_metrics), sort_keys=True)}; " + f"guest_diagnostics={diagnostics[-20_000:]!r}" ) finally: with contextlib.suppress(Exception): client.delete(f"/vms/{name}/delete") - if mode == "mitm-load": + if mode in PROXY_ERROR_MODES: service_log = service.stop_and_read_log() service.stop() else: @@ -218,7 +217,9 @@ def main() -> int: # The guest writes one document per run keyed by mode; take that section # when present so keys are not prefixed with the mode twice. - runtime_failures = mitm_runtime_failures(service_log) if mode == "mitm-load" else [] + runtime_failures = ( + mitm_runtime_failures(service_log) if mode in PROXY_ERROR_MODES else [] + ) if runtime_failures: raise RuntimeError( f"capsem-bench {mode}: {len(runtime_failures)} internal traffic " @@ -234,12 +235,17 @@ def main() -> int: f"exec_response={json.dumps(benchmark_result)[:1000]!r}" ) - measured = { - key: {"unit": unit, "samples": [value]} - for key, value in sorted(numbers.items()) - if (unit := unit_for(key)) is not None - } - measured.update(load_quality_metrics(section)) + measured = ( + dict(protocol_collection["metrics"]) + if protocol_collection is not None + else { + key: {"unit": unit, "samples": [value]} + for key, value in sorted(numbers.items()) + if (unit := unit_for(key)) is not None + } + ) + if protocol_collection is None: + measured.update(load_quality_metrics(section)) if mode == "mitm-load": measured["internal.proxy_traffic_refusals"] = { "unit": "count", diff --git a/benchmarks/collectors/protocol b/benchmarks/collectors/protocol new file mode 120000 index 000000000..08826b9bc --- /dev/null +++ b/benchmarks/collectors/protocol @@ -0,0 +1 @@ +_guest \ No newline at end of file diff --git a/benchmarks/helpers/guest_runtime.py b/benchmarks/helpers/guest_runtime.py new file mode 100644 index 000000000..2d7a1783a --- /dev/null +++ b/benchmarks/helpers/guest_runtime.py @@ -0,0 +1,103 @@ +"""Runtime controls shared by Capsem's in-guest benchmark collectors.""" + +from __future__ import annotations + +import json +import os +import shlex + +import psutil +from helpers.constants import DEFAULT_CPUS +from helpers.process_resources import ProcessTreeSampler +from helpers.service import ServiceInstance, exec_output_text + +FORWARDED_ENV = ( + "CAPSEM_BENCH_CONCURRENCY", + "CAPSEM_BENCH_DURATION_S", + "CAPSEM_BENCH_MITM_TARGET", + "CAPSEM_BENCH_SCENARIOS", + "CAPSEM_BENCH_TIMEOUT_MS", + "CAPSEM_BENCH_TOTAL_REQUESTS", + "CAPSEM_MOCK_SERVER_BASE_URL", + "CAPSEM_MOCK_SERVER_DNS_UDP_ADDR", + "CAPSEM_STORAGE_BENCH_PATHS", + "CAPSEM_STORAGE_BENCH_SIZE_MB", + "CAPSEM_STORAGE_IO_PROFILE_SIZE_MB", + "CAPSEM_STORAGE_IO_PROFILE_RANDOM_OPS", +) +LOAD_MODES = ("dns-load", "mitm-load", "mcp-load", "protocol") +PROXY_ERROR_MODES = ("mitm-load", "protocol") +LOAD_GUEST_CPUS_MAX = 8 +PROTOCOL_COLLECTOR_PATH = "/tmp/capsem-benchmark-collector.json" + +MITM_FIXTURE_HOST = "fixture.capsem.test" +MITM_FIXTURE_TARGET = f"https://{MITM_FIXTURE_HOST}/tiny" +DNS_FAILURE_DIAGNOSTIC_COMMAND = ( + "cat /run/capsem-dns-proxy.log /proc/sys/net/core/rmem_max " + "/proc/net/snmp /proc/net/udp /proc/net/softnet_stat" +) + + +def guest_cpus(mode: str, host_cpus: int | None = None) -> int: + """Give load generators enough guest CPU without consuming the host.""" + if mode not in LOAD_MODES: + return DEFAULT_CPUS + host = host_cpus if host_cpus is not None else (os.cpu_count() or DEFAULT_CPUS) + return max(DEFAULT_CPUS, min(LOAD_GUEST_CPUS_MAX, host // 2)) + + +def process_tree_sampler(service: ServiceInstance) -> ProcessTreeSampler: + """Measure the service and every confined per-VM descendant.""" + return ProcessTreeSampler(psutil.Process(service.proc.pid)) + + +def mitm_fixture_corp_toml(ready: dict) -> str: + """Route guest DNS and the fixture host to the mock server.""" + return ( + "[network.dns]\n" + f"upstreams = [{json.dumps(ready['dns_udp_addr'])}]\n" + "\n" + f'[network.upstream_overrides."{MITM_FIXTURE_HOST}:443"]\n' + f"dial = {json.dumps(ready['http_addr'])}\n" + 'protocol = "http"\n' + ) + + +def guest_command(mode: str) -> str: + """Build one safely quoted command with owned benchmark controls.""" + assignments = [ + f"{name}={value}" + for name in FORWARDED_ENV + if (value := os.environ.get(name)) is not None + ] + command = ["env", *assignments, "capsem-bench", mode] + if mode == "protocol": + command.extend( + ["--lane", "guest_capsem", "--collector-out", PROTOCOL_COLLECTOR_PATH] + ) + return shlex.join(command) + + +def failed_load_diagnostics(service, client, vm_id: str, mode: str) -> str: + """Capture guest state that teardown would otherwise discard.""" + if mode != "dns-load": + return "" + result = client.post( + f"/vms/{vm_id}/exec", + { + "target": "vm", + "command": DNS_FAILURE_DIAGNOSTIC_COMMAND, + "timeout_secs": 15, + }, + timeout=25, + ) + if not result or result.get("exit_code") != 0: + diagnostics = f"diagnostic command failed: {json.dumps(result)[:2000]}" + else: + diagnostics = exec_output_text(result) + diagnostic_path = getattr(service, "tmp_dir", None) + if diagnostic_path is not None: + (diagnostic_path / "dns-load-failure.txt").write_text( + diagnostics, encoding="utf-8" + ) + return diagnostics diff --git a/build_system/builder/gate/benchmarkschema.py b/build_system/builder/gate/benchmarkschema.py index dfe8e3331..e3f0dd023 100644 --- a/build_system/builder/gate/benchmarkschema.py +++ b/build_system/builder/gate/benchmarkschema.py @@ -122,6 +122,15 @@ class BenchmarkRunConfig(Strict): quick_timeout_secs: int +class BenchmarkProtocolConfig(Strict): + """Stable workload for the in-guest model and MCP protocol collector.""" + + requests: PositiveInt + concurrency: PositiveInt + timeout_ms: PositiveInt + scenarios: list[str] + + class DiskIopsConfig(Strict): """A floor that is not the same number on every platform. @@ -172,4 +181,5 @@ class BenchmarkConfig(Strict): route_load: RouteLoadConfig route_health: RouteHealthConfig run: BenchmarkRunConfig + protocol: BenchmarkProtocolConfig gates: BenchmarkGatesConfig diff --git a/config/gate.toml b/config/gate.toml index 275a64b3e..1768d575d 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3323,13 +3323,29 @@ store = "cache/target/tests/benchmarks/benchmarks.db" # under the project's interpreter rather than whatever the shebang resolves to. interpreter = "uv run --project build_system --frozen --active python" # A collector that never exits would hold the machine lock the whole gate runs -# under, so one is always bounded. -timeout_secs = 900 +# under, so one is always bounded. Criterion discovers every workspace target; +# a cold build plus the statistical runs exceeded the old 15-minute ceiling +# while still making progress, so the full lane gets a 30-minute bound. +timeout_secs = 1800 # The dev loop is only useful while it stays a dev loop. `quick` takes the two # dimensions that need no guest and finish in seconds; the ceiling is the # promise, not an estimate. quick_timeout_secs = 60 +[benchmark.protocol] +# Raw samples need enough p99 tail observations to expose drift while keeping +# this four-scenario VM lane routine. The 50,000-request release stress cohort +# remains a separate qualification fixture. +requests = 5000 +concurrency = 64 +timeout_ms = 30000 +scenarios = [ + "model_json_response", + "credential_response", + "mcp_tools_list", + "mcp_tool_call", +] + [benchmark_regression] # Product benchmarks use the latest checked-in result as evidence. A release # fails when a measured duration or allocated-byte metric grows by more than diff --git a/guest/artifacts/capsem_bench/__main__.py b/guest/artifacts/capsem_bench/__main__.py index c95fc2692..e803dc918 100644 --- a/guest/artifacts/capsem_bench/__main__.py +++ b/guest/artifacts/capsem_bench/__main__.py @@ -23,6 +23,12 @@ MOCK_SERVER_PROTOCOL_BASE_URL_ENV = "CAPSEM_MOCK_SERVER_BASE_URL" MOCK_SERVER_DNS_UDP_ADDR_ENV = "CAPSEM_MOCK_SERVER_DNS_UDP_ADDR" RUST_BENCH = "/usr/local/bin/capsem-bench-rs" +PROTOCOL_ENV_FLAGS = ( + ("CAPSEM_BENCH_TOTAL_REQUESTS", "--requests"), + ("CAPSEM_BENCH_CONCURRENCY", "--concurrency"), + ("CAPSEM_BENCH_TIMEOUT_MS", "--timeout-ms"), + ("CAPSEM_BENCH_SCENARIOS", "--scenarios"), +) def main(): @@ -157,10 +163,21 @@ def _run_rust_protocol(args): if not os.path.exists(RUST_BENCH): console.print(f"ERROR: {RUST_BENCH} is required for capsem-bench protocol") return 127 - completed = subprocess.run([RUST_BENCH, *args], check=False) + completed = subprocess.run([RUST_BENCH, *_protocol_args(args)], check=False) return completed.returncode +def _protocol_args(args): + """Translate the documented environment controls unless CLI wins.""" + args = list(args) + for variable, flag in PROTOCOL_ENV_FLAGS: + value = os.environ.get(variable) + supplied = any(arg == flag or arg.startswith(f"{flag}=") for arg in args) + if value and not supplied: + args.extend([flag, value]) + return args + + def _run_rust_protocol_artifact(scenarios=None, requests=None, concurrency=None): if not os.path.exists(RUST_BENCH): raise RuntimeError(f"{RUST_BENCH} is required for capsem-bench all protocol section") @@ -181,15 +198,9 @@ def _run_rust_protocol_artifact(scenarios=None, requests=None, concurrency=None) command.extend(["--dns-udp-addr", os.environ[MOCK_SERVER_DNS_UDP_ADDR_ENV]]) if requests is not None: command.extend(["--requests", str(requests)]) - elif os.environ.get("CAPSEM_BENCH_TOTAL_REQUESTS"): - command.extend(["--requests", os.environ["CAPSEM_BENCH_TOTAL_REQUESTS"]]) if concurrency is not None: command.extend(["--concurrency", str(concurrency)]) - elif os.environ.get("CAPSEM_BENCH_CONCURRENCY"): - command.extend(["--concurrency", os.environ["CAPSEM_BENCH_CONCURRENCY"]]) - if os.environ.get("CAPSEM_BENCH_TIMEOUT_MS"): - command.extend(["--timeout-ms", os.environ["CAPSEM_BENCH_TIMEOUT_MS"]]) - completed = subprocess.run(command, check=False, capture_output=True, text=True) + completed = subprocess.run(_protocol_args(command), check=False, capture_output=True, text=True) if completed.returncode != 0: raise RuntimeError( f"capsem-bench-rs protocol failed with {completed.returncode}: " diff --git a/tests/helpers/process_resources.py b/tests/helpers/process_resources.py index 7ad342f90..997eee6f6 100644 --- a/tests/helpers/process_resources.py +++ b/tests/helpers/process_resources.py @@ -7,6 +7,7 @@ import threading from dataclasses import dataclass from decimal import Decimal +from pathlib import Path import psutil @@ -35,6 +36,9 @@ class ProcessUsage: pss_bytes: int | None uss_bytes: int | None fds: int + rss_anon_bytes: int | None = None + rss_file_bytes: int | None = None + rss_shmem_bytes: int | None = None @dataclass(frozen=True) @@ -42,6 +46,27 @@ class TreeSnapshot: processes: dict[tuple[int, float], ProcessUsage] +def _linux_rss_breakdown(pid: int) -> tuple[int, int, int] | None: + """Return Linux RSS categories without walking every memory mapping.""" + try: + status = (Path(psutil.PROCFS_PATH) / str(pid) / "status").read_text( + encoding="utf-8" + ) + except (AttributeError, OSError): + return None + values = {} + for line in status.splitlines(): + name, separator, raw = line.partition(":") + if separator and name in {"RssAnon", "RssFile", "RssShmem"}: + fields = raw.split() + if not fields: + return None + values[name] = int(fields[0]) * 1024 + if values.keys() != {"RssAnon", "RssFile", "RssShmem"}: + return None + return values["RssAnon"], values["RssFile"], values["RssShmem"] + + def process_tree_snapshot(root: psutil.Process) -> TreeSnapshot: """Read one race-tolerant snapshot of a process and its descendants.""" with contextlib.suppress(psutil.Error): @@ -51,6 +76,7 @@ def process_tree_snapshot(root: psutil.Process) -> TreeSnapshot: with contextlib.suppress(psutil.Error, OSError): with process.oneshot(): memory = process.memory_full_info() + rss_breakdown = _linux_rss_breakdown(process.pid) usage = ProcessUsage( pid=process.pid, started_at=process.create_time(), @@ -60,6 +86,9 @@ def process_tree_snapshot(root: psutil.Process) -> TreeSnapshot: pss_bytes=getattr(memory, "pss", None), uss_bytes=getattr(memory, "uss", None), fds=process.num_fds(), + rss_anon_bytes=None if rss_breakdown is None else rss_breakdown[0], + rss_file_bytes=None if rss_breakdown is None else rss_breakdown[1], + rss_shmem_bytes=None if rss_breakdown is None else rss_breakdown[2], ) found[(usage.pid, usage.started_at)] = usage return TreeSnapshot(found) @@ -86,8 +115,10 @@ def __init__( self._observations = 0 self._baseline: dict[str, tuple[int, int, int, int]] = {} self._peak: dict[str, tuple[int, int, int, int]] = {} + self._peak_rss_breakdown: dict[str, tuple[int, int, int]] = {} self._pss_available = True self._uss_available = True + self._rss_breakdown_available = True self._peak_processes = 0 self._first_cpu: dict[tuple[int, float], Decimal] = {} self._last_cpu: dict[tuple[int, float], Decimal] = {} @@ -98,10 +129,19 @@ def observe(self, snapshot: TreeSnapshot) -> None: initial = self._observations == 0 self._observations += 1 totals: dict[str, tuple[int, int, int, int]] = {"total": (0, 0, 0, 0)} + rss_breakdowns: dict[str, tuple[int, int, int]] = {"total": (0, 0, 0)} for identity, usage in snapshot.processes.items(): name = _metric_name(usage.name) self._pss_available &= usage.pss_bytes is not None self._uss_available &= usage.uss_bytes is not None + self._rss_breakdown_available &= all( + value is not None + for value in ( + usage.rss_anon_bytes, + usage.rss_file_bytes, + usage.rss_shmem_bytes, + ) + ) pss = usage.pss_bytes or 0 uss = usage.uss_bytes or 0 rss_sum, pss_sum, uss_sum, fds_sum = totals.get(name, (0, 0, 0, 0)) @@ -118,6 +158,21 @@ def observe(self, snapshot: TreeSnapshot) -> None: total_uss + uss, total_fds + usage.fds, ) + rss_anon = usage.rss_anon_bytes or 0 + rss_file = usage.rss_file_bytes or 0 + rss_shmem = usage.rss_shmem_bytes or 0 + name_anon, name_file, name_shmem = rss_breakdowns.get(name, (0, 0, 0)) + rss_breakdowns[name] = ( + name_anon + rss_anon, + name_file + rss_file, + name_shmem + rss_shmem, + ) + total_anon, total_file, total_shmem = rss_breakdowns["total"] + rss_breakdowns["total"] = ( + total_anon + rss_anon, + total_file + rss_file, + total_shmem + rss_shmem, + ) if identity not in self._first_cpu: # A process born during the interval has spent all of its CPU # lifetime on this benchmark. Processes present at baseline @@ -131,6 +186,11 @@ def observe(self, snapshot: TreeSnapshot) -> None: for name, values in totals.items(): previous = self._peak.get(name, (0, 0, 0, 0)) self._peak[name] = tuple(max(old, new) for old, new in zip(previous, values, strict=True)) + for name, values in rss_breakdowns.items(): + previous = self._peak_rss_breakdown.get(name, (0, 0, 0)) + self._peak_rss_breakdown[name] = tuple( + max(old, new) for old, new in zip(previous, values, strict=True) + ) self._peak_processes = max(self._peak_processes, len(snapshot.processes)) def start(self) -> None: @@ -183,6 +243,17 @@ def metrics(self) -> dict[str, dict]: "unit": "bytes", "samples": [max(0, peak_rss - baseline_rss)], } + if self._rss_breakdown_available: + peak_anon, peak_file, peak_shmem = self._peak_rss_breakdown[name] + for category, value in ( + ("anon", peak_anon), + ("file", peak_file), + ("shmem", peak_shmem), + ): + metrics[f"{prefix}.peak_rss_{category}_bytes"] = { + "unit": "bytes", + "samples": [value], + } if self._pss_available: metrics[f"{prefix}.peak_pss_bytes"] = { "unit": "bytes", diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index addbfe834..d3916be3b 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -62,6 +62,49 @@ def test_storage_controls_allow_a_focused_workspace_measurement(monkeypatch) -> assert f"{name}={value}" in args +def test_protocol_command_forwards_fixture_controls_and_requests_raw_samples( + monkeypatch, +) -> None: + monkeypatch.setenv("CAPSEM_MOCK_SERVER_BASE_URL", "http://127.0.0.1:3713") + monkeypatch.setenv("CAPSEM_MOCK_SERVER_DNS_UDP_ADDR", "127.0.0.1:41000") + monkeypatch.setenv("CAPSEM_BENCH_TOTAL_REQUESTS", "100") + monkeypatch.setenv("CAPSEM_BENCH_SCENARIOS", "model_json_response,mcp_tool_call") + + args = shlex.split(collector()["guest_command"]("protocol")) + assert "CAPSEM_MOCK_SERVER_BASE_URL=http://127.0.0.1:3713" in args + assert "CAPSEM_MOCK_SERVER_DNS_UDP_ADDR=127.0.0.1:41000" in args + assert "CAPSEM_BENCH_TOTAL_REQUESTS=100" in args + assert "CAPSEM_BENCH_SCENARIOS=model_json_response,mcp_tool_call" in args + assert args[-5:] == [ + "protocol", + "--lane", + "guest_capsem", + "--collector-out", + "/tmp/capsem-benchmark-collector.json", + ] + + +def test_protocol_defaults_come_from_the_typed_gate_config(monkeypatch) -> None: + controls = ( + "CAPSEM_BENCH_TOTAL_REQUESTS", + "CAPSEM_BENCH_CONCURRENCY", + "CAPSEM_BENCH_TIMEOUT_MS", + "CAPSEM_BENCH_SCENARIOS", + ) + for name in controls: + monkeypatch.delenv(name, raising=False) + + module = collector() + module["apply_protocol_defaults"]() + settings = CONFIG.benchmark.protocol + + assert os.environ["CAPSEM_BENCH_TOTAL_REQUESTS"] == str(settings.requests) + assert os.environ["CAPSEM_BENCH_CONCURRENCY"] == str(settings.concurrency) + assert os.environ["CAPSEM_BENCH_TIMEOUT_MS"] == str(settings.timeout_ms) + assert os.environ["CAPSEM_BENCH_SCENARIOS"] == ",".join(settings.scenarios) + assert "dns_local_nxdomain" not in settings.scenarios + + def test_flatten_preserves_indexed_load_rows() -> None: flatten = collector()["flatten"] @@ -113,14 +156,12 @@ def test_a_level_where_any_request_failed_is_not_a_measurement() -> None: def test_mitm_runtime_refusals_are_fatal_even_when_the_client_retries() -> None: failures = collector()["mitm_runtime_failures"] - log = "\n".join( - [ - '{"fields":{"message":"service.start"}}', - '{"fields":{"message":"proxy worker refused traffic descriptor",' - '"error":"proxy worker rejected HttpTraffic grant 522: Capacity"}}', - '{"fields":{"message":"proxy worker refused traffic descriptor",' - '"error":"proxy worker rejected HttpTraffic grant 523: Capacity"}}', - ] + log = ( + '{"fields":{"message":"service.start"}}\n' + '{"fields":{"message":"proxy worker refused traffic descriptor",' + '"error":"proxy worker rejected HttpTraffic grant 522: Capacity"}}\n' + '{"fields":{"message":"proxy worker refused traffic descriptor",' + '"error":"proxy worker rejected HttpTraffic grant 523: Capacity"}}' ) assert failures(log) == [ @@ -176,7 +217,9 @@ def test_guest_result_rejects_a_hidden_proxy_refusal(monkeypatch, tmp_path) -> N ), ) monkeypatch.setattr(helpers.mock_server, "stop_process", lambda proc: None) - monkeypatch.setitem(main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True) + monkeypatch.setitem( + main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True + ) monkeypatch.setitem( main.__globals__, "process_tree_sampler", lambda service: ResourceSamplerStub() ) @@ -192,6 +235,7 @@ def test_load_dimensions_get_more_guest_cpus_bounded_by_the_host() -> None: assert guest_cpus("mitm-load", host_cpus=16) == 8 assert guest_cpus("dns-load", host_cpus=64) == 8, "capped" assert guest_cpus("mcp-load", host_cpus=6) == 3 + assert guest_cpus("protocol", host_cpus=16) == 8 assert guest_cpus("mitm-load", host_cpus=2) == 2, "never below the default" @@ -245,6 +289,82 @@ def test_guest_result_preserves_zero_error_proof(monkeypatch, capsys) -> None: assert stopped == [True] +def test_protocol_result_preserves_raw_model_samples(monkeypatch, capsys) -> None: + artifact = { + "mock_server_protocol": { + "total_requests": 3, + "scenarios": [ + { + "name": "model_json_response", + "successful": 3, + "failed": 0, + "requests_per_sec": 750.0, + } + ], + } + } + collection = { + "metrics": { + "guest_capsem.model_json_response.latency_ms": { + "unit": "milliseconds", + "samples": [0.5, 1.5, 2.5], + } + }, + "sidecar": json.dumps(artifact), + } + + def post(route, body=None, **kwargs): + if route == "/vms/create": + return {"id": "bench-test-vm"} + output = ( + json.dumps(collection) + if body["command"] == "cat /tmp/capsem-benchmark-collector.json" + else "benchmark output" + ) + return {"exit_code": 0, "stdout": {"encoding": "utf8", "data": output}} + + client = SimpleNamespace(post=post, delete=lambda *args: None) + main = collector()["main"] + monkeypatch.setitem( + main.__globals__, + "ServiceInstance", + lambda: SimpleNamespace( + client=lambda: client, + start=lambda: None, + stop=lambda: None, + stop_and_read_log=lambda: "", + ), + ) + import helpers.mock_server + + monkeypatch.setattr( + helpers.mock_server, + "start_mock_server", + lambda **kwargs: ( + object(), + { + "base_url": "http://127.0.0.1:3713", + "dns_udp_addr": "127.0.0.1:41000", + }, + ), + ) + monkeypatch.setattr(helpers.mock_server, "stop_process", lambda proc: None) + monkeypatch.setitem( + main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True + ) + monkeypatch.setitem( + main.__globals__, "process_tree_sampler", lambda service: ResourceSamplerStub() + ) + monkeypatch.setattr(sys, "argv", ["protocol"]) + + assert main() == 0 + result = json.loads(capsys.readouterr().out) + assert result["metrics"]["guest_capsem.model_json_response.latency_ms"][ + "samples" + ] == [0.5, 1.5, 2.5] + assert json.loads(result["sidecar"]) == artifact + + def test_guest_result_rejects_one_failed_request(monkeypatch, tmp_path) -> None: document = { "mitm_load": { @@ -290,6 +410,7 @@ def test_guest_result_rejects_one_failed_request(monkeypatch, tmp_path) -> None: monkeypatch.setitem( main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True ) + class FailedRunSampler(ResourceSamplerStub): def stop(self) -> dict: return { @@ -312,12 +433,73 @@ def stop(self) -> dict: main() message = str(failure.value) assert "requests failed at concurrency [200]" in message - assert f'artifact={json.dumps(document, sort_keys=True)}' in message + assert f"artifact={json.dumps(document, sort_keys=True)}" in message assert '"resources.total.cpu_seconds": 12.5' in message assert '"resources.total.peak_rss_bytes": 1048576' in message -def test_guest_command_failure_preserves_resource_pressure(monkeypatch, tmp_path) -> None: +def test_failed_dns_load_preserves_the_guest_proxy_log(monkeypatch) -> None: + document = { + "dns_load": { + "concurrency_levels": [ + {"concurrency": 512, "total_requests": 600, "errors": 1, "rps": 50.0}, + ] + } + } + exec_commands = [] + + def post(route, body=None, **kwargs): + if route == "/vms/create": + return {"id": "bench-test-vm"} + command = body["command"] + exec_commands.append(command) + if command == "cat /tmp/capsem-benchmark.json": + output = json.dumps(document) + elif command == ( + "cat /run/capsem-dns-proxy.log /proc/sys/net/core/rmem_max " + "/proc/net/snmp /proc/net/udp /proc/net/softnet_stat" + ): + output = "dns query unanswered: queue saturated" + else: + output = "benchmark output" + return { + "exit_code": 0, + "stdout": {"encoding": "utf8", "data": output}, + } + + client = SimpleNamespace(post=post, delete=lambda *args: None) + main = collector()["main"] + monkeypatch.setitem( + main.__globals__, + "ServiceInstance", + lambda: SimpleNamespace( + client=lambda: client, + start=lambda: None, + stop=lambda: None, + ), + ) + monkeypatch.setitem( + main.__globals__, "wait_exec_ready", lambda *args, **kwargs: True + ) + monkeypatch.setitem( + main.__globals__, "process_tree_sampler", lambda service: ResourceSamplerStub() + ) + monkeypatch.setattr(sys, "argv", ["dns-load"]) + + with pytest.raises(RuntimeError) as failure: + main() + assert "guest_diagnostics='dns query unanswered: queue saturated'" in str( + failure.value + ) + assert exec_commands[-1] == ( + "cat /run/capsem-dns-proxy.log /proc/sys/net/core/rmem_max " + "/proc/net/snmp /proc/net/udp /proc/net/softnet_stat" + ) + + +def test_guest_command_failure_preserves_resource_pressure( + monkeypatch, tmp_path +) -> None: client = SimpleNamespace( post=lambda route, *args, **kwargs: ( {"id": "bench-test-vm"} diff --git a/tests/test_capsem_bench_mock_server_protocol.py b/tests/test_capsem_bench_mock_server_protocol.py index ffe5ba3aa..8ebfb5262 100644 --- a/tests/test_capsem_bench_mock_server_protocol.py +++ b/tests/test_capsem_bench_mock_server_protocol.py @@ -263,6 +263,8 @@ def test_protocol_mode_delegates_to_rust_bench(monkeypatch, tmp_path): calls = [] rust_bench = tmp_path / "capsem-bench-rs" rust_bench.touch() + for variable, _ in bench_main.PROTOCOL_ENV_FLAGS: + monkeypatch.delenv(variable, raising=False) class Completed: returncode = 7 @@ -296,6 +298,44 @@ class Completed: ] +def test_protocol_mode_translates_documented_environment_controls(monkeypatch, tmp_path): + calls = [] + rust_bench = tmp_path / "capsem-bench-rs" + rust_bench.touch() + + class Completed: + returncode = 0 + + monkeypatch.setenv("CAPSEM_BENCH_TOTAL_REQUESTS", "1000") + monkeypatch.setenv("CAPSEM_BENCH_CONCURRENCY", "64") + monkeypatch.setenv("CAPSEM_BENCH_TIMEOUT_MS", "2500") + monkeypatch.setenv("CAPSEM_BENCH_SCENARIOS", "model_json_response,mcp_tool_call") + monkeypatch.setattr(bench_main, "RUST_BENCH", str(rust_bench)) + monkeypatch.setattr( + bench_main.subprocess, + "run", + lambda argv, check=False: calls.append((argv, check)) or Completed(), + ) + + assert bench_main._run_rust_protocol(["protocol", "--requests=7"]) == 0 + assert calls == [ + ( + [ + bench_main.RUST_BENCH, + "protocol", + "--requests=7", + "--concurrency", + "64", + "--timeout-ms", + "2500", + "--scenarios", + "model_json_response,mcp_tool_call", + ], + False, + ) + ] + + def test_all_mode_keeps_network_sections_by_merging_rust_protocol(monkeypatch): def fake_module(module_name, function_name): module = types.ModuleType(module_name) diff --git a/tests/test_process_resource_sampler.py b/tests/test_process_resource_sampler.py index 32db817de..c736ef4ae 100644 --- a/tests/test_process_resource_sampler.py +++ b/tests/test_process_resource_sampler.py @@ -18,9 +18,24 @@ def usage( pss: int, uss: int, fds: int, + rss_anon: int | None = None, + rss_file: int | None = None, + rss_shmem: int | None = None, started_at: float = 1.0, ) -> ProcessUsage: - return ProcessUsage(pid, started_at, name, Decimal(cpu), rss, pss, uss, fds) + return ProcessUsage( + pid, + started_at, + name, + Decimal(cpu), + rss, + pss, + uss, + fds, + rss_anon, + rss_file, + rss_shmem, + ) def test_sampler_reports_tree_and_named_process_pressure() -> None: @@ -124,8 +139,12 @@ def test_sampler_reports_tree_and_named_process_pressure() -> None: def test_process_that_exits_keeps_its_last_cpu_sample() -> None: sampler = ProcessTreeSampler(None) - sampler.observe(snapshot(usage(7, "capsem-proxy", cpu="3", rss=20, pss=15, uss=10, fds=2))) - sampler.observe(snapshot(usage(7, "capsem-proxy", cpu="4", rss=30, pss=25, uss=20, fds=3))) + sampler.observe( + snapshot(usage(7, "capsem-proxy", cpu="3", rss=20, pss=15, uss=10, fds=2)) + ) + sampler.observe( + snapshot(usage(7, "capsem-proxy", cpu="4", rss=30, pss=25, uss=20, fds=3)) + ) sampler.observe(snapshot()) assert sampler.metrics()["resources.capsem_proxy.cpu_s"]["samples"] == [1.0] @@ -133,8 +152,12 @@ def test_process_that_exits_keeps_its_last_cpu_sample() -> None: def test_sampler_attributes_dedicated_ledger_pressure() -> None: sampler = ProcessTreeSampler(None) - sampler.observe(snapshot(usage(8, "capsem-ledger", cpu="2", rss=100, pss=80, uss=70, fds=5))) - sampler.observe(snapshot(usage(8, "capsem-ledger", cpu="3", rss=250, pss=220, uss=200, fds=7))) + sampler.observe( + snapshot(usage(8, "capsem-ledger", cpu="2", rss=100, pss=80, uss=70, fds=5)) + ) + sampler.observe( + snapshot(usage(8, "capsem-ledger", cpu="3", rss=250, pss=220, uss=200, fds=7)) + ) metrics = sampler.metrics() assert metrics["resources.capsem_ledger.cpu_s"]["samples"] == [1.0] @@ -142,6 +165,48 @@ def test_sampler_attributes_dedicated_ledger_pressure() -> None: assert metrics["resources.capsem_ledger.peak_pss_delta_bytes"]["samples"] == [140] +def test_sampler_splits_anonymous_file_and_shared_resident_memory() -> None: + sampler = ProcessTreeSampler(None) + sampler.observe( + snapshot( + usage( + 8, + "capsem-ledger", + cpu="2", + rss=100, + pss=80, + uss=70, + fds=5, + rss_anon=30, + rss_file=60, + rss_shmem=10, + ) + ) + ) + sampler.observe( + snapshot( + usage( + 8, + "capsem-ledger", + cpu="3", + rss=250, + pss=220, + uss=200, + fds=7, + rss_anon=40, + rss_file=195, + rss_shmem=15, + ) + ) + ) + + metrics = sampler.metrics() + assert metrics["resources.capsem_ledger.peak_rss_anon_bytes"]["samples"] == [40] + assert metrics["resources.capsem_ledger.peak_rss_file_bytes"]["samples"] == [195] + assert metrics["resources.capsem_ledger.peak_rss_shmem_bytes"]["samples"] == [15] + assert metrics["resources.total.peak_rss_file_bytes"]["samples"] == [195] + + def test_platform_without_proportional_memory_omits_those_metrics() -> None: sampler = ProcessTreeSampler(None) sampler.observe( From a2498ee80ae5c871d5a7048d10b08e01c8a83877 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 04:42:40 +0000 Subject: [PATCH 407/464] test(bench): drive load rails to configured capacity --- guest/artifacts/capsem_bench/dns_load.py | 36 ++++--------------- guest/artifacts/capsem_bench/load_harness.py | 33 +++++++++++++++++ guest/artifacts/capsem_bench/mcp_load.py | 7 ++-- guest/artifacts/capsem_bench/mitm_load.py | 5 +-- tests/citadel/test_runtime_limit_authority.py | 18 +++++++++- .../test_capsem_bench_mock_server_protocol.py | 22 +++++++----- 6 files changed, 76 insertions(+), 45 deletions(-) diff --git a/guest/artifacts/capsem_bench/dns_load.py b/guest/artifacts/capsem_bench/dns_load.py index e2b90b7dc..15ca59ea7 100644 --- a/guest/artifacts/capsem_bench/dns_load.py +++ b/guest/artifacts/capsem_bench/dns_load.py @@ -5,7 +5,11 @@ import subprocess from .helpers import console -from .load_harness import DurationLoadConfig, render_load_table +from .load_harness import ( + DurationLoadConfig, + capacity_concurrency_levels, + render_load_table, +) DEFAULT_QNAME = "load-test.capsem-bogus" DEFAULT_QTYPE = 1 @@ -14,34 +18,6 @@ RUST_BENCH = "/usr/local/bin/capsem-bench-rs" -def _runtime_limits(): - completed = subprocess.run( - [RUST_BENCH, "runtime-limits"], - check=False, - capture_output=True, - text=True, - ) - if completed.returncode: - raise RuntimeError( - f"dns-load: read runtime limits failed with {completed.returncode}: " - f"{completed.stderr[-2000:].strip()}" - ) - try: - limits = json.loads(completed.stdout) - except json.JSONDecodeError as exc: - raise RuntimeError("dns-load: runtime limits returned invalid JSON") from exc - capacity = limits.get("network_client_capacity") - if type(capacity) is not int or capacity < 1: - raise RuntimeError("dns-load: runtime limits omitted network client capacity") - return limits - - -def _default_concurrency(): - capacity = _runtime_limits()["network_client_capacity"] - levels = (1, capacity // 32, capacity // 8, capacity // 4, capacity // 2, capacity) - return tuple(dict.fromkeys(max(1, level) for level in levels)) - - def _drive_at_concurrency(qname, qtype, concurrency, duration_s, timeout_s): """Run fixed async UDP clients and validate that every client ran.""" completed = subprocess.run( @@ -89,7 +65,7 @@ def dns_load_bench(qname=None, qtype=None, concurrency_levels=None, duration_s=N qname = qname or os.environ.get("CAPSEM_BENCH_DNS_QNAME", DEFAULT_QNAME) qtype = qtype or int(os.environ.get("CAPSEM_BENCH_DNS_QTYPE", DEFAULT_QTYPE)) timeout_s = float(os.environ.get("CAPSEM_BENCH_DNS_TIMEOUT", DEFAULT_TIMEOUT_S)) - default_concurrency = _default_concurrency() if concurrency_levels is None else () + default_concurrency = capacity_concurrency_levels() if concurrency_levels is None else () config = DurationLoadConfig.from_inputs( "dns-load", default_concurrency=default_concurrency, diff --git a/guest/artifacts/capsem_bench/load_harness.py b/guest/artifacts/capsem_bench/load_harness.py index f9337a437..a7fcc3a00 100644 --- a/guest/artifacts/capsem_bench/load_harness.py +++ b/guest/artifacts/capsem_bench/load_harness.py @@ -6,8 +6,10 @@ benchmarks cannot drift into incompatible result shapes. """ +import json import os import resource +import subprocess from dataclasses import dataclass GLOBAL_CONCURRENCY_ENV = "CAPSEM_BENCH_CONCURRENCY" @@ -15,6 +17,37 @@ GLOBAL_TOTAL_REQUESTS_ENV = "CAPSEM_BENCH_TOTAL_REQUESTS" GLOBAL_TIMEOUT_ENV = "CAPSEM_BENCH_TIMEOUT_S" GLOBAL_SCENARIOS_ENV = "CAPSEM_BENCH_SCENARIOS" +RUST_BENCH = "/usr/local/bin/capsem-bench-rs" + + +def runtime_limits(): + """Read the generated product capacity authority from the Rust harness.""" + completed = subprocess.run( + [RUST_BENCH, "runtime-limits"], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode: + raise RuntimeError( + "read runtime limits failed with " + f"{completed.returncode}: {completed.stderr[-2000:].strip()}" + ) + try: + limits = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise RuntimeError("runtime limits returned invalid JSON") from exc + capacity = limits.get("network_client_capacity") + if type(capacity) is not int or capacity < 1: + raise RuntimeError("runtime limits omitted network client capacity") + return limits + + +def capacity_concurrency_levels(): + """Exercise one client through the complete supported network capacity.""" + capacity = runtime_limits()["network_client_capacity"] + levels = (1, capacity // 32, capacity // 8, capacity // 4, capacity // 2, capacity) + return tuple(dict.fromkeys(max(1, level) for level in levels)) def _env_prefix(mode): diff --git a/guest/artifacts/capsem_bench/mcp_load.py b/guest/artifacts/capsem_bench/mcp_load.py index 2f235cfbc..d4b8d6704 100644 --- a/guest/artifacts/capsem_bench/mcp_load.py +++ b/guest/artifacts/capsem_bench/mcp_load.py @@ -26,13 +26,13 @@ from .helpers import console from .load_harness import ( DurationLoadConfig, + capacity_concurrency_levels, render_load_table, summarize_load_level, ) from .mcp_transport import StdioClient MCP_SERVER = "/run/capsem-mcp-server" -DEFAULT_CONCURRENCY = (1, 10, 50, 200) DEFAULT_DURATION_S = 10.0 DEFAULT_PAYLOAD = "ping" @@ -58,7 +58,7 @@ async def worker(): ms = (time.monotonic() - t0) * 1000 async with lat_lock: latencies.append(ms) - except Exception: + except Exception: # noqa: BLE001 -- every request failure is benchmark data. errors += 1 await asyncio.gather(*(worker() for _ in range(concurrency))) @@ -90,9 +90,10 @@ async def _run_async(concurrency_levels, duration_s, payload): def mcp_load_bench(concurrency_levels=None, duration_s=None, payload=None): """Drive local__echo at each concurrency level; return the result dict.""" + default_concurrency = capacity_concurrency_levels() if concurrency_levels is None else () config = DurationLoadConfig.from_inputs( "mcp-load", - default_concurrency=DEFAULT_CONCURRENCY, + default_concurrency=default_concurrency, default_duration_s=DEFAULT_DURATION_S, concurrency_levels=concurrency_levels, duration_s=duration_s, diff --git a/guest/artifacts/capsem_bench/mitm_load.py b/guest/artifacts/capsem_bench/mitm_load.py index 5d5a75d96..75b42b748 100644 --- a/guest/artifacts/capsem_bench/mitm_load.py +++ b/guest/artifacts/capsem_bench/mitm_load.py @@ -47,13 +47,13 @@ from .helpers import console from .load_harness import ( DurationLoadConfig, + capacity_concurrency_levels, render_load_table, ) # Served by capsem-mock-server through the benchmark collector's corp config # (DNS fixture + upstream override), configured by the shared guest collector. DEFAULT_TARGET = "https://fixture.capsem.test/tiny" -DEFAULT_CONCURRENCY = (1, 10, 50, 200) DEFAULT_DURATION_S = 10.0 RUST_BENCH = "/usr/local/bin/capsem-bench-rs" @@ -97,9 +97,10 @@ def _drive_at_concurrency(url, concurrency, duration_s): def mitm_load_bench(target=None, concurrency_levels=None, duration_s=None): """Drive the MITM proxy at each concurrency level; return the result dict.""" target = target or os.environ.get("CAPSEM_BENCH_MITM_TARGET", DEFAULT_TARGET) + default_concurrency = capacity_concurrency_levels() if concurrency_levels is None else () config = DurationLoadConfig.from_inputs( "mitm-load", - default_concurrency=DEFAULT_CONCURRENCY, + default_concurrency=default_concurrency, default_duration_s=DEFAULT_DURATION_S, concurrency_levels=concurrency_levels, duration_s=duration_s, diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 08b486dce..b5e5116a5 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -141,11 +141,14 @@ "LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS", "LEDGER_SQLITE_MMAP_BYTES", ), - "guest/artifacts/capsem_bench/dns_load.py": ( + "guest/artifacts/capsem_bench/load_harness.py": ( '"runtime-limits"', '"network_client_capacity"', "capacity // 2", ), + "guest/artifacts/capsem_bench/dns_load.py": ("capacity_concurrency_levels",), + "guest/artifacts/capsem_bench/mitm_load.py": ("capacity_concurrency_levels",), + "guest/artifacts/capsem_bench/mcp_load.py": ("capacity_concurrency_levels",), } @@ -158,6 +161,19 @@ def _numeric_limit_declarations(sources: dict[str, str]) -> list[str]: return violations +def test_guest_load_benchmarks_do_not_duplicate_a_concurrency_ceiling() -> None: + for relative in ( + "guest/artifacts/capsem_bench/dns_load.py", + "guest/artifacts/capsem_bench/mitm_load.py", + "guest/artifacts/capsem_bench/mcp_load.py", + ): + source = (ROOT / relative).read_text() + assert "DEFAULT_CONCURRENCY" not in source, ( + RUNTIME_LIMIT_RATIONALE + + f"\n{relative}: derive the load curve from NETWORK_CLIENT_CAPACITY" + ) + + def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: config = tomllib.loads(CONFIG.read_text()) capacity = config.get("network", {}).get("max_concurrent_clients") diff --git a/tests/test_capsem_bench_mock_server_protocol.py b/tests/test_capsem_bench_mock_server_protocol.py index 8ebfb5262..2b477bbab 100644 --- a/tests/test_capsem_bench_mock_server_protocol.py +++ b/tests/test_capsem_bench_mock_server_protocol.py @@ -8,7 +8,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent / "guest" / "artifacts")) from capsem_bench import __main__ as bench_main -from capsem_bench import dns_load, load_harness, mitm_load +from capsem_bench import dns_load, load_harness, mcp_load, mitm_load def test_mitm_load_delegates_to_async_rust_driver(monkeypatch): @@ -139,17 +139,21 @@ def run(argv, **kwargs): ] -def test_dns_load_curve_reaches_the_config_owned_client_ceiling(monkeypatch): +def test_load_curve_reaches_the_config_owned_client_ceiling(monkeypatch): monkeypatch.setattr( - dns_load, - "_runtime_limits", + load_harness, + "runtime_limits", lambda: {"network_client_capacity": 512}, ) - assert dns_load._default_concurrency() == (1, 16, 64, 128, 256, 512) + expected = (1, 16, 64, 128, 256, 512) + assert load_harness.capacity_concurrency_levels() == expected + assert dns_load.capacity_concurrency_levels() == expected + assert mitm_load.capacity_concurrency_levels() == expected + assert mcp_load.capacity_concurrency_levels() == expected -def test_dns_load_reads_runtime_limits_from_the_rust_harness(monkeypatch): +def test_load_harness_reads_runtime_limits_from_the_rust_harness(monkeypatch): calls = [] def run(argv, **kwargs): @@ -160,12 +164,12 @@ def run(argv, **kwargs): stderr="", ) - monkeypatch.setattr(dns_load.subprocess, "run", run) + monkeypatch.setattr(load_harness.subprocess, "run", run) - assert dns_load._runtime_limits()["network_client_capacity"] == 512 + assert load_harness.runtime_limits()["network_client_capacity"] == 512 assert calls == [ ( - [dns_load.RUST_BENCH, "runtime-limits"], + [load_harness.RUST_BENCH, "runtime-limits"], {"check": False, "capture_output": True, "text": True}, ) ] From 2934848dbaecdf312ff603df285e7783423a0ee5 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:13:07 +0000 Subject: [PATCH 408/464] feat(bench): register multi-VM MITM scaling --- crates/capsem-bench/src/schema.rs | 4 ++++ crates/capsem-bench/src/schema/tests.rs | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/capsem-bench/src/schema.rs b/crates/capsem-bench/src/schema.rs index d97998fa7..f50bd5bc2 100644 --- a/crates/capsem-bench/src/schema.rs +++ b/crates/capsem-bench/src/schema.rs @@ -50,6 +50,8 @@ pub enum Dimension { Startup, McpLoad, MitmLoad, + /// Aggregate HTTPS interception throughput as independent VMs fan out. + MitmScale, DnsLoad, } @@ -75,6 +77,7 @@ impl Dimension { Dimension::Startup, Dimension::McpLoad, Dimension::MitmLoad, + Dimension::MitmScale, Dimension::DnsLoad, ]; @@ -99,6 +102,7 @@ impl Dimension { Dimension::Startup => "startup", Dimension::McpLoad => "mcp-load", Dimension::MitmLoad => "mitm-load", + Dimension::MitmScale => "mitm-scale", Dimension::DnsLoad => "dns-load", } } diff --git a/crates/capsem-bench/src/schema/tests.rs b/crates/capsem-bench/src/schema/tests.rs index e0e896b66..371f7e31f 100644 --- a/crates/capsem-bench/src/schema/tests.rs +++ b/crates/capsem-bench/src/schema/tests.rs @@ -94,7 +94,15 @@ fn every_dimension_is_listed_once() { names.sort_unstable(); names.dedup(); assert_eq!(names.len(), listed, "a dimension is listed twice"); - assert_eq!(listed, 20, "ALL must carry every dimension"); + assert_eq!(listed, 21, "ALL must carry every dimension"); +} + +#[test] +fn mitm_scale_is_a_multi_vm_measurement_outside_the_quick_lane() { + let scale: Dimension = serde_json::from_str("\"mitm-scale\"").expect("parses"); + assert_eq!(scale, Dimension::MitmScale); + assert!(scale.needs_vm(), "it drives several live guests"); + assert!(!scale.in_quick_lane(), "it measures sustained VM fanout"); } #[test] From 97d1456622f87b1a488c5a427db1eadbd5a4df55 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:16:17 +0000 Subject: [PATCH 409/464] feat(bench): measure multi-VM MITM scaling --- benchmarks/collectors/mitm-scale | 248 +++++++++++++++++++ build_system/builder/gate/benchmarkschema.py | 18 ++ config/gate.toml | 8 + tests/test_benchmark_guest_collector.py | 7 + tests/test_benchmark_mitm_scale.py | 64 +++++ 5 files changed, 345 insertions(+) create mode 100755 benchmarks/collectors/mitm-scale create mode 100644 tests/test_benchmark_mitm_scale.py diff --git a/benchmarks/collectors/mitm-scale b/benchmarks/collectors/mitm-scale new file mode 100755 index 000000000..2925ac126 --- /dev/null +++ b/benchmarks/collectors/mitm-scale @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Measure aggregate HTTPS interception while independent VMs fan out.""" + +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import shlex +import sys +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(PROJECT_ROOT / "tests")) +sys.path.insert(0, str(PROJECT_ROOT)) + +from capsem_builder.gate import config as gate_config +from helpers.constants import DEFAULT_CPUS, DEFAULT_RAM_MB, EXEC_READY_TIMEOUT +from helpers.mock_server import start_mock_server, stop_process +from helpers.service import ServiceInstance, exec_output_text, wait_exec_ready + +from benchmarks.helpers.guest_measurements import ( + mitm_runtime_failures, + resource_samples, +) +from benchmarks.helpers.guest_runtime import ( + MITM_FIXTURE_TARGET, + mitm_fixture_corp_toml, + process_tree_sampler, +) + +ARTIFACT_PATH = "/tmp/capsem-benchmark.json" +SETTINGS = gate_config.load(PROJECT_ROOT).benchmark.mitm_scale + + +def guest_load_command( + concurrency: int, + duration_seconds: int, + target: str = MITM_FIXTURE_TARGET, +) -> str: + """Pin one identical load cohort inside each VM.""" + return shlex.join( + [ + "env", + f"CAPSEM_BENCH_CONCURRENCY={concurrency}", + f"CAPSEM_BENCH_DURATION_S={duration_seconds}", + f"CAPSEM_BENCH_MITM_TARGET={target}", + "capsem-bench", + "mitm-load", + ] + ) + + +def _single_row(document: dict, concurrency: int) -> dict: + section = document.get("mitm_load", document) + rows = section.get("concurrency_levels", []) + if len(rows) != 1 or rows[0].get("concurrency") != concurrency: + raise RuntimeError( + "MITM scale VM returned the wrong concurrency cohort: " + f"expected={concurrency} rows={rows!r}" + ) + row = rows[0] + if int(row.get("errors", 0)): + raise RuntimeError(f"MITM scale VM reported failed requests: {row!r}") + if int(row.get("total_requests", 0)) < concurrency: + raise RuntimeError( + "MITM scale VM starved client loops: " + f"concurrency={concurrency} row={row!r}" + ) + return row + + +def level_metrics( + vm_count: int, + concurrency: int, + documents: list[dict], +) -> dict[str, dict]: + """Aggregate throughput while retaining each VM's latency observation.""" + if len(documents) != vm_count: + raise RuntimeError( + f"MITM scale expected {vm_count} VM results, received {len(documents)}" + ) + rows = [_single_row(document, concurrency) for document in documents] + prefix = f"levels.{vm_count}" + metrics = { + f"{prefix}.vms": {"unit": "count", "samples": [vm_count]}, + f"{prefix}.aggregate_rps": { + "unit": "requests_per_second", + "samples": [sum(float(row["rps"]) for row in rows)], + }, + f"{prefix}.per_vm_rps": { + "unit": "requests_per_second", + "samples": [float(row["rps"]) for row in rows], + }, + f"{prefix}.requests": { + "unit": "count", + "samples": [sum(int(row["total_requests"]) for row in rows)], + }, + f"{prefix}.errors": { + "unit": "count", + "samples": [sum(int(row["errors"]) for row in rows)], + }, + } + for percentile in ("p50_ms", "p95_ms", "p99_ms", "p999_ms"): + metrics[f"{prefix}.per_vm_{percentile}"] = { + "unit": "milliseconds", + "samples": [float(row[percentile]) for row in rows], + } + return metrics + + +def _run_vm_load( + service: ServiceInstance, + vm_id: str, + command: str, + timeout: int, +) -> dict: + client = service.client() + result = client.post( + f"/vms/{vm_id}/exec", + {"target": "vm", "command": command, "timeout_secs": timeout}, + timeout=timeout + 10, + ) + if not result or result.get("exit_code") != 0: + raise RuntimeError( + f"MITM scale load failed in {vm_id}: " + f"exit={None if not result else result.get('exit_code')} " + f"response={json.dumps(result)[:2000]}" + ) + artifact = client.post( + f"/vms/{vm_id}/exec", + {"target": "vm", "command": f"cat {ARTIFACT_PATH}", "timeout_secs": 15}, + timeout=25, + ) + if not artifact or artifact.get("exit_code") != 0: + raise RuntimeError(f"read MITM scale artifact from {vm_id} failed: {artifact!r}") + return json.loads(exec_output_text(artifact)) + + +def _run_level( + service: ServiceInstance, + vm_ids: list[str], + concurrency: int, + duration_seconds: int, +) -> list[dict]: + command = guest_load_command(concurrency, duration_seconds) + timeout = max(60, duration_seconds + 30) + with ThreadPoolExecutor(max_workers=len(vm_ids)) as executor: + futures = [ + executor.submit(_run_vm_load, service, vm_id, command, timeout) + for vm_id in vm_ids + ] + return [future.result() for future in futures] + + +def _add_resource_metrics( + measured: dict[str, dict], + vm_count: int, + resources: dict[str, dict], +) -> None: + prefix = f"levels.{vm_count}" + measured.update( + {f"{prefix}.{name}": measurement for name, measurement in resources.items()} + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--quick", action="store_true", help="unused; this lane boots VMs") + parser.parse_args() + + service = ServiceInstance() + client = service.client() + mock_proc = None + vm_ids: list[str] = [] + measured: dict[str, dict] = {} + sidecar = {"settings": SETTINGS.model_dump(), "levels": {}} + service_log = "" + try: + mock_proc, ready = start_mock_server() + corp_path = service.tmp_dir / "bench-corp.toml" + corp_path.write_text(mitm_fixture_corp_toml(ready), encoding="utf-8") + os.environ["CAPSEM_CORP_CONFIG"] = str(corp_path) + service.start() + + for vm_count in SETTINGS.vm_counts: + while len(vm_ids) < vm_count: + name = f"bench-mitm-scale-{uuid.uuid4().hex[:8]}" + created = client.post( + "/vms/create", + {"name": name, "ram_mb": DEFAULT_RAM_MB, "cpus": DEFAULT_CPUS}, + ) + if not created or not created.get("id"): + raise RuntimeError(f"provision {name} failed: {created!r}") + vm_id = created["id"] + vm_ids.append(vm_id) + if not wait_exec_ready(client, vm_id, timeout=EXEC_READY_TIMEOUT): + raise RuntimeError(f"{vm_id} never became exec-ready") + + sampler = process_tree_sampler(service) + sampler.start() + resources = {} + try: + documents = _run_level( + service, + vm_ids, + SETTINGS.per_vm_concurrency, + SETTINGS.duration_seconds, + ) + finally: + resources = sampler.stop() + measured.update( + level_metrics(vm_count, SETTINGS.per_vm_concurrency, documents) + ) + _add_resource_metrics(measured, vm_count, resources) + sidecar["levels"][str(vm_count)] = { + "documents": documents, + "resources": resource_samples(resources), + } + finally: + for vm_id in reversed(vm_ids): + with contextlib.suppress(Exception): + client.delete(f"/vms/{vm_id}/delete") + with contextlib.suppress(Exception): + service_log = service.stop_and_read_log() + service.stop() + if mock_proc is not None: + stop_process(mock_proc) + + refusals = mitm_runtime_failures(service_log) + if refusals: + raise RuntimeError( + f"MITM scale saw {len(refusals)} hidden proxy traffic refusals: " + f"{refusals[:5]!r}" + ) + measured["internal.proxy_traffic_refusals"] = { + "unit": "count", + "samples": [0], + } + print(json.dumps({"metrics": measured, "sidecar": json.dumps(sidecar)})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/build_system/builder/gate/benchmarkschema.py b/build_system/builder/gate/benchmarkschema.py index e3f0dd023..f90131f34 100644 --- a/build_system/builder/gate/benchmarkschema.py +++ b/build_system/builder/gate/benchmarkschema.py @@ -131,6 +131,23 @@ class BenchmarkProtocolConfig(Strict): scenarios: list[str] +class BenchmarkMitmScaleConfig(Strict): + """Independent-VM fanout used to measure aggregate HTTPS interception.""" + + vm_counts: tuple[PositiveInt, ...] + per_vm_concurrency: PositiveInt + duration_seconds: PositiveInt + + @field_validator("vm_counts") + @classmethod + def fanout_starts_at_one_and_increases(cls, values: tuple[int, ...]) -> tuple[int, ...]: + if not values or values[0] != 1: + raise ValueError("MITM scale fanout must start at one VM") + if any(current <= previous for previous, current in zip(values, values[1:])): + raise ValueError("MITM scale VM counts must be strictly increasing") + return values + + class DiskIopsConfig(Strict): """A floor that is not the same number on every platform. @@ -182,4 +199,5 @@ class BenchmarkConfig(Strict): route_health: RouteHealthConfig run: BenchmarkRunConfig protocol: BenchmarkProtocolConfig + mitm_scale: BenchmarkMitmScaleConfig gates: BenchmarkGatesConfig diff --git a/config/gate.toml b/config/gate.toml index 1768d575d..69bfb7415 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3346,6 +3346,14 @@ scenarios = [ "mcp_tool_call", ] +[benchmark.mitm_scale] +# One low-contention HTTPS cohort per independent VM. Each level keeps the +# per-VM work fixed, so aggregate RPS and resident pressure reveal whether +# session isolation scales across host cores and memory. +vm_counts = [1, 2, 4] +per_vm_concurrency = 16 +duration_seconds = 10 + [benchmark_regression] # Product benchmarks use the latest checked-in result as evidence. A release # fails when a measured duration or allocated-byte metric grows by more than diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index d3916be3b..878afe7b9 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -105,6 +105,13 @@ def test_protocol_defaults_come_from_the_typed_gate_config(monkeypatch) -> None: assert "dns_local_nxdomain" not in settings.scenarios +def test_mitm_scale_settings_cover_independent_vm_fanout() -> None: + settings = CONFIG.benchmark.mitm_scale + assert tuple(settings.vm_counts) == (1, 2, 4) + assert settings.per_vm_concurrency == 16 + assert settings.duration_seconds == 10 + + def test_flatten_preserves_indexed_load_rows() -> None: flatten = collector()["flatten"] diff --git a/tests/test_benchmark_mitm_scale.py b/tests/test_benchmark_mitm_scale.py new file mode 100644 index 000000000..53e4335fb --- /dev/null +++ b/tests/test_benchmark_mitm_scale.py @@ -0,0 +1,64 @@ +"""The official MITM scaling collector measures independent VM fanout.""" + +from __future__ import annotations + +import runpy +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +COLLECTOR = PROJECT_ROOT / "benchmarks" / "collectors" / "mitm-scale" + + +def collector() -> dict: + return runpy.run_path(str(COLLECTOR)) + + +def load_row(*, concurrency: int = 16, rps: float = 700.0, errors: int = 0) -> dict: + return { + "mitm_load": { + "concurrency_levels": [ + { + "concurrency": concurrency, + "total_requests": 7000 + errors, + "errors": errors, + "rps": rps, + "p50_ms": 20.0, + "p95_ms": 30.0, + "p99_ms": 40.0, + "p999_ms": 50.0, + } + ] + } + } + + +def test_level_aggregates_rps_and_keeps_per_vm_latency_samples() -> None: + metrics = collector()["level_metrics"]( + 2, + 16, + [load_row(rps=680.0), load_row(rps=720.0)], + ) + + assert metrics["levels.2.aggregate_rps"]["samples"] == [1400.0] + assert metrics["levels.2.per_vm_rps"]["samples"] == [680.0, 720.0] + assert metrics["levels.2.per_vm_p95_ms"]["samples"] == [30.0, 30.0] + assert metrics["levels.2.requests"]["samples"] == [14000] + assert metrics["levels.2.errors"]["samples"] == [0] + + +def test_level_refuses_failed_or_wrong_concurrency_results() -> None: + level_metrics = collector()["level_metrics"] + + with pytest.raises(RuntimeError, match="failed requests"): + level_metrics(1, 16, [load_row(errors=1)]) + with pytest.raises(RuntimeError, match="concurrency"): + level_metrics(1, 16, [load_row(concurrency=8)]) + + +def test_guest_command_fixes_one_cohort_per_vm() -> None: + command = collector()["guest_load_command"](16, 10) + assert "CAPSEM_BENCH_CONCURRENCY=16" in command + assert "CAPSEM_BENCH_DURATION_S=10" in command + assert command.endswith("capsem-bench mitm-load") From a87034a8c0530db09347d5d4188a67486b497bf8 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:21:16 +0000 Subject: [PATCH 410/464] feat(bench): expose MITM fixture pressure --- benchmarks/collectors/mitm-scale | 22 ++++++++++++++++++++-- tests/test_benchmark_mitm_scale.py | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/benchmarks/collectors/mitm-scale b/benchmarks/collectors/mitm-scale index 2925ac126..aa2d9c8b1 100755 --- a/benchmarks/collectors/mitm-scale +++ b/benchmarks/collectors/mitm-scale @@ -13,6 +13,8 @@ import uuid from concurrent.futures import ThreadPoolExecutor from pathlib import Path +import psutil + PROJECT_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(PROJECT_ROOT / "tests")) sys.path.insert(0, str(PROJECT_ROOT)) @@ -20,6 +22,7 @@ sys.path.insert(0, str(PROJECT_ROOT)) from capsem_builder.gate import config as gate_config from helpers.constants import DEFAULT_CPUS, DEFAULT_RAM_MB, EXEC_READY_TIMEOUT from helpers.mock_server import start_mock_server, stop_process +from helpers.process_resources import ProcessTreeSampler from helpers.service import ServiceInstance, exec_output_text, wait_exec_ready from benchmarks.helpers.guest_measurements import ( @@ -156,12 +159,16 @@ def _run_level( return [future.result() for future in futures] -def _add_resource_metrics( +def add_resource_metrics( measured: dict[str, dict], vm_count: int, resources: dict[str, dict], + *, + namespace: str | None = None, ) -> None: prefix = f"levels.{vm_count}" + if namespace: + prefix = f"{prefix}.{namespace}" measured.update( {f"{prefix}.{name}": measurement for name, measurement in resources.items()} ) @@ -201,8 +208,11 @@ def main() -> int: raise RuntimeError(f"{vm_id} never became exec-ready") sampler = process_tree_sampler(service) + fixture_sampler = ProcessTreeSampler(psutil.Process(mock_proc.pid)) sampler.start() + fixture_sampler.start() resources = {} + fixture_resources = {} try: documents = _run_level( service, @@ -212,13 +222,21 @@ def main() -> int: ) finally: resources = sampler.stop() + fixture_resources = fixture_sampler.stop() measured.update( level_metrics(vm_count, SETTINGS.per_vm_concurrency, documents) ) - _add_resource_metrics(measured, vm_count, resources) + add_resource_metrics(measured, vm_count, resources) + add_resource_metrics( + measured, + vm_count, + fixture_resources, + namespace="fixture", + ) sidecar["levels"][str(vm_count)] = { "documents": documents, "resources": resource_samples(resources), + "fixture_resources": resource_samples(fixture_resources), } finally: for vm_id in reversed(vm_ids): diff --git a/tests/test_benchmark_mitm_scale.py b/tests/test_benchmark_mitm_scale.py index 53e4335fb..b8c69dfe3 100644 --- a/tests/test_benchmark_mitm_scale.py +++ b/tests/test_benchmark_mitm_scale.py @@ -62,3 +62,19 @@ def test_guest_command_fixes_one_cohort_per_vm() -> None: assert "CAPSEM_BENCH_CONCURRENCY=16" in command assert "CAPSEM_BENCH_DURATION_S=10" in command assert command.endswith("capsem-bench mitm-load") + + +def test_fixture_pressure_is_namespaced_away_from_capsem_pressure() -> None: + measured = {} + collector()["add_resource_metrics"]( + measured, + 4, + {"resources.total.cpu_s": {"unit": "seconds", "samples": [9.5]}}, + namespace="fixture", + ) + assert measured == { + "levels.4.fixture.resources.total.cpu_s": { + "unit": "seconds", + "samples": [9.5], + } + } From b40ac4c1064f5d85143be12b985a017b166c442c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:23:50 +0000 Subject: [PATCH 411/464] fix(bench): isolate MITM fanout cohorts --- benchmarks/collectors/mitm-scale | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/collectors/mitm-scale b/benchmarks/collectors/mitm-scale index aa2d9c8b1..fc94796ff 100755 --- a/benchmarks/collectors/mitm-scale +++ b/benchmarks/collectors/mitm-scale @@ -194,7 +194,10 @@ def main() -> int: service.start() for vm_count in SETTINGS.vm_counts: - while len(vm_ids) < vm_count: + for vm_id in reversed(vm_ids): + client.delete(f"/vms/{vm_id}/delete") + vm_ids.clear() + for _ in range(vm_count): name = f"bench-mitm-scale-{uuid.uuid4().hex[:8]}" created = client.post( "/vms/create", @@ -204,6 +207,7 @@ def main() -> int: raise RuntimeError(f"provision {name} failed: {created!r}") vm_id = created["id"] vm_ids.append(vm_id) + for vm_id in vm_ids: if not wait_exec_ready(client, vm_id, timeout=EXEC_READY_TIMEOUT): raise RuntimeError(f"{vm_id} never became exec-ready") From 97cad891fdf9abc44f359d29c7b3fb256457eb39 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:27:53 +0000 Subject: [PATCH 412/464] feat(bench): extend MITM scale to host saturation --- benchmarks/collectors/mitm-scale | 7 ++++++- config/gate.toml | 2 +- tests/test_benchmark_guest_collector.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/benchmarks/collectors/mitm-scale b/benchmarks/collectors/mitm-scale index fc94796ff..43d3d727b 100755 --- a/benchmarks/collectors/mitm-scale +++ b/benchmarks/collectors/mitm-scale @@ -183,7 +183,12 @@ def main() -> int: client = service.client() mock_proc = None vm_ids: list[str] = [] - measured: dict[str, dict] = {} + logical_cpus = os.cpu_count() or 1 + physical_cores = psutil.cpu_count(logical=False) or logical_cpus + measured: dict[str, dict] = { + "host.logical_cpus": {"unit": "count", "samples": [logical_cpus]}, + "host.physical_cores": {"unit": "count", "samples": [physical_cores]}, + } sidecar = {"settings": SETTINGS.model_dump(), "levels": {}} service_log = "" try: diff --git a/config/gate.toml b/config/gate.toml index 69bfb7415..38a8b70bc 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -3350,7 +3350,7 @@ scenarios = [ # One low-contention HTTPS cohort per independent VM. Each level keeps the # per-VM work fixed, so aggregate RPS and resident pressure reveal whether # session isolation scales across host cores and memory. -vm_counts = [1, 2, 4] +vm_counts = [1, 2, 4, 8] per_vm_concurrency = 16 duration_seconds = 10 diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index 878afe7b9..a0360502b 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -107,7 +107,7 @@ def test_protocol_defaults_come_from_the_typed_gate_config(monkeypatch) -> None: def test_mitm_scale_settings_cover_independent_vm_fanout() -> None: settings = CONFIG.benchmark.mitm_scale - assert tuple(settings.vm_counts) == (1, 2, 4) + assert tuple(settings.vm_counts) == (1, 2, 4, 8) assert settings.per_vm_concurrency == 16 assert settings.duration_seconds == 10 From f165d5128097d4e6c411289a9b475b48bc8f0b7a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:32:43 +0000 Subject: [PATCH 413/464] docs(skills): codify multi-VM capacity proof --- skills/dev-benchmark/SKILL.md | 11 +++++++++++ skills/dev-cache/SKILL.md | 12 ++++++++++++ skills/dev-mitm-proxy/SKILL.md | 8 ++++++++ 3 files changed, 31 insertions(+) diff --git a/skills/dev-benchmark/SKILL.md b/skills/dev-benchmark/SKILL.md index eaa224df5..2f2640065 100644 --- a/skills/dev-benchmark/SKILL.md +++ b/skills/dev-benchmark/SKILL.md @@ -119,6 +119,17 @@ capacity because summing RSS counts shared pages once per process; use USS to show the memory that disappears with that process. Compare two unchanged runs before and two changed runs after any performance work. +`just bench mitm-scale` is the aggregate HTTPS capacity proof. Its +`[benchmark.mitm_scale]` config creates a fresh identical cohort at each VM +count, fixes one keep-alive concurrency level per VM, samples the Capsem tree +and mock upstream separately, and rejects request errors or hidden proxy +refusals. It also records physical cores and logical CPUs. Judge scaling below +physical-core saturation first; once total CPU reaches the host budget, a flat +or falling aggregate RPS is host pressure rather than evidence of a shared +Capsem lock. Report aggregate RPS, RPS per consumed CPU, PSS per loaded VM and +per-VM latency together. Run the lane twice after changing VM ownership, +network workers, proxy concurrency, ledger admission or their limits. + ## Is the move real? `compare` and `verify` report `delta_abs`, `delta_pct`, `ratio`, and diff --git a/skills/dev-cache/SKILL.md b/skills/dev-cache/SKILL.md index 387c9f988..604f3db3b 100644 --- a/skills/dev-cache/SKILL.md +++ b/skills/dev-cache/SKILL.md @@ -21,6 +21,18 @@ Age and count fields are optional refinements. They never replace the common contract. Do not introduce storage-availability rails, warning thresholds, or a second config table. Cache decisions are based exclusively on owned usage. +This authority covers retained build and developer artifacts. Bounded runtime +maps, connection pools, socket buffers and admission queues have a different +lifetime: their shared capacity relationships live in +`config/data/runtime-limits.toml`, are validated and generated by +`capsem-proto/build.rs`, and are guarded by +`tests/citadel/test_runtime_limit_authority.py`. Do not copy a runtime ceiling +into cache code or put a runtime object cache in `config/cache.toml`. A runtime +cache must remain bounded by the generated capacity, expose hit, miss, +eviction and occupancy telemetry where those states exist, and preserve the +logger DB boundary instead of caching ledger projections in a route or UI +process. + ## Typed boundary Programs use `CacheRegistry`, `CacheRequest`, `CacheOperation`, and diff --git a/skills/dev-mitm-proxy/SKILL.md b/skills/dev-mitm-proxy/SKILL.md index 1c39e00b1..47e56caa0 100644 --- a/skills/dev-mitm-proxy/SKILL.md +++ b/skills/dev-mitm-proxy/SKILL.md @@ -89,6 +89,14 @@ contract tracked by issue #349. Extend and run `tests/citadel/test_runtime_limit_authority.py` whenever a new consumer or derived limit is added. +Use `just bench mitm-scale` when a one-VM curve plateaus while the host still +looks idle. The lane uses fresh 1/2/4/8-VM cohorts, keeps per-VM work fixed and +records the mock upstream outside the Capsem process tree. Compare the curve to +physical cores, not only logical CPUs: after the VM owners, confined proxies +and ledgers consume the physical-core budget, SMT contention is expected to +raise per-VM latency. A busy fixture, request error or hidden descriptor +refusal invalidates the run; do not explain it as product saturation. + ## Content-Encoding: the systemic rule The proxy MUST handle response decompression as a general capability. This is not optional, not per-feature. From 4bac11d8a7726af163d0cc44624691e09111f35c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:36:30 +0000 Subject: [PATCH 414/464] test(citadel): require one runtime cache authority --- tests/citadel/test_runtime_limit_authority.py | 103 ++++++++++++++++-- 1 file changed, 95 insertions(+), 8 deletions(-) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index b5e5116a5..0685f0c6d 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -12,19 +12,20 @@ RUNTIME_LIMIT_RATIONALE = """\ The proxy, trusted upstream broker, host DNS session, guest DNS forwarder and -security ledger must admit the same supported client burst. A pooled traffic -client can have an active, idle and retiring connection at once, and each -client can produce several audit rows, so downstream capacities must preserve -their config-owned multiples. Local numeric ceilings previously let one stage -accept work that the next stage shed, producing intermittent HTTP 502 and DNS -SERVFAIL responses under load. +security ledger must admit the same supported client burst. Runtime caches +must also keep their capacities and freshness windows under the same product +authority. A pooled traffic client can have an active, idle and retiring +connection at once, and each client can produce several audit rows, so +downstream capacities must preserve their config-owned multiples. Local +numeric ceilings previously let one stage accept work that the next stage +shed, producing intermittent HTTP 502 and DNS SERVFAIL responses under load. config/data/runtime-limits.toml is the authority; capsem-proto generates the shared constants and every consumer derives from them. See skills/dev-mitm-proxy/SKILL.md under Cross-process network limits. """ NUMERIC_CONSTANT = re.compile( - r"\b(?:pub(?:\([^)]*\))?\s+)?const\s+([A-Z][A-Z0-9_]*)\s*:\s*(?:usize|u64)\s*=\s*(\d+)\s*;" + r"\b(?:pub(?:\([^)]*\))?\s+)?const\s+([A-Z][A-Z0-9_]*)\s*:\s*(?:usize|u64|u32|i64)\s*=\s*(\d+)\s*;" ) CONFIG_OWNED_NAMES = { "NETWORK_CLIENT_CAPACITY", @@ -42,6 +43,18 @@ "LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS", "LEDGER_ARCHIVE_INDEX_ROWS_PER_FLUSH", "LEDGER_SQLITE_MMAP_BYTES", + "CACHE_DNS_ANSWER_ENTRIES", + "CACHE_DNS_MAX_TTL_SECONDS", + "CACHE_DNS_NEGATIVE_TTL_SECONDS", + "CACHE_TLS_LEAF_ENTRIES", + "CACHE_LEDGER_QUERY_BATCH_ENTRIES", + "CACHE_SQLITE_PREPARED_STATEMENTS", + "CACHE_SERVICE_STORAGE_DIAGNOSTICS_ENTRIES", + "CACHE_SERVICE_PERSISTENT_RESUME_ENTRIES", + "CACHE_UPDATE_CHECK_TTL_SECONDS", + "CACHE_IMAGE_CATALOG_REFRESH_SECONDS", + "CACHE_IMAGE_CATALOG_RETRY_SECONDS", + "CACHE_IMAGE_VERIFICATION_BATCH_KEYS", } REQUIRED_WIRING = { @@ -76,6 +89,20 @@ "pub const LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS: u64 = {ledger_disk_flush_interval_milliseconds}", 'get("disk_flush_max_pending_events")', "pub const LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS: usize = {ledger_disk_flush_max_pending_events}", + 'get("dns_answer_entries")', + 'get("dns_max_ttl_seconds")', + 'get("dns_negative_ttl_seconds")', + 'get("tls_leaf_entries")', + 'get("ledger_query_batch_entries")', + 'get("sqlite_prepared_statements")', + 'get("service_storage_diagnostics_entries")', + 'get("service_persistent_resume_entries")', + 'get("update_check_ttl_seconds")', + 'get("image_catalog_refresh_seconds")', + 'get("image_catalog_retry_seconds")', + 'get("image_verification_batch_keys")', + "pub const CACHE_DNS_ANSWER_ENTRIES: usize = {cache_dns_answer_entries}", + "pub const CACHE_UPDATE_CHECK_TTL_SECONDS: u64 = {cache_update_check_ttl_seconds}", ), "crates/capsem-proto/src/lib.rs": ( 'include!(concat!(env!("OUT_DIR"), "/runtime_limits.rs"))', @@ -117,6 +144,37 @@ "crates/capsem-logger/src/schema/pragmas.rs": ( "capsem_proto::LEDGER_SQLITE_MMAP_BYTES", ), + "crates/capsem-logger/src/db.rs": ( + "capsem_proto::CACHE_LEDGER_QUERY_BATCH_ENTRIES", + ), + "crates/capsem-logger/src/reader/open.rs": ( + "capsem_proto::CACHE_SQLITE_PREPARED_STATEMENTS", + ), + "crates/capsem-core/src/net/cert_authority.rs": ( + "capsem_proto::CACHE_TLS_LEAF_ENTRIES", + ), + "crates/capsem-core/src/net/dns/cache.rs": ( + "capsem_proto::CACHE_DNS_ANSWER_ENTRIES", + "capsem_proto::CACHE_DNS_MAX_TTL_SECONDS", + "capsem_proto::CACHE_DNS_NEGATIVE_TTL_SECONDS", + ), + "crates/capsem-service/src/service_runtime.rs": ( + "capsem_proto::CACHE_SERVICE_STORAGE_DIAGNOSTICS_ENTRIES", + "capsem_proto::CACHE_SERVICE_PERSISTENT_RESUME_ENTRIES", + ), + "crates/capsem-service/src/container_setup/images.rs": ( + "capsem_proto::CACHE_IMAGE_CATALOG_REFRESH_SECONDS", + "capsem_proto::CACHE_IMAGE_CATALOG_RETRY_SECONDS", + ), + "crates/capsem-service/src/container_setup/registry.rs": ( + "capsem_proto::CACHE_IMAGE_VERIFICATION_BATCH_KEYS", + ), + "crates/capsem-service/src/update_status.rs": ( + "capsem_proto::CACHE_UPDATE_CHECK_TTL_SECONDS", + ), + "crates/capsem/src/update.rs": ( + "capsem_proto::CACHE_UPDATE_CHECK_TTL_SECONDS", + ), "crates/capsem-core/src/net/dns/session/mod.rs": ("DNS_SESSION_MAX_IN_FLIGHT",), "crates/capsem-agent/src/dns_proxy/session.rs": ( "DNS_SESSION_MAX_IN_FLIGHT", @@ -271,6 +329,32 @@ def test_runtime_limit_config_is_valid_and_wired_to_every_consumer() -> None: and disk_flush_max_pending_events % admission_batch_capacity == 0 ), RUNTIME_LIMIT_RATIONALE + cache = config.get("cache", {}) + bounded_entries = ( + "dns_answer_entries", + "tls_leaf_entries", + "ledger_query_batch_entries", + "sqlite_prepared_statements", + "service_storage_diagnostics_entries", + "service_persistent_resume_entries", + "image_verification_batch_keys", + ) + for name in bounded_entries: + value = cache.get(name) + assert type(value) is int and 1 <= value <= 65_536, ( + RUNTIME_LIMIT_RATIONALE + f"\ncache.{name} must be an integer from 1 through 65536" + ) + dns_max_ttl = cache.get("dns_max_ttl_seconds") + dns_negative_ttl = cache.get("dns_negative_ttl_seconds") + assert type(dns_max_ttl) is int and 1 <= dns_max_ttl <= 86_400, RUNTIME_LIMIT_RATIONALE + assert type(dns_negative_ttl) is int and 1 <= dns_negative_ttl <= dns_max_ttl, RUNTIME_LIMIT_RATIONALE + update_ttl = cache.get("update_check_ttl_seconds") + assert type(update_ttl) is int and 60 <= update_ttl <= 604_800, RUNTIME_LIMIT_RATIONALE + catalog_refresh = cache.get("image_catalog_refresh_seconds") + catalog_retry = cache.get("image_catalog_retry_seconds") + assert type(catalog_refresh) is int and 60 <= catalog_refresh <= 86_400, RUNTIME_LIMIT_RATIONALE + assert type(catalog_retry) is int and 1 <= catalog_retry <= catalog_refresh, RUNTIME_LIMIT_RATIONALE + missing = [] for relative_path, required_fragments in REQUIRED_WIRING.items(): source = (ROOT / relative_path).read_text() @@ -300,6 +384,9 @@ def test_numeric_local_ceiling_is_rejected_adversarially() -> None: "process.rs": "const LEDGER_PRODUCER_QUEUE_CAPACITY: usize = 1024;", "logger.rs": "const LEDGER_ADMISSION_BATCH_CAPACITY: usize = 64;", "remote.rs": "const LEDGER_ADMISSION_COALESCE_WAIT_MICROSECONDS: u64 = 100;", + "dns.rs": "const CACHE_DNS_ANSWER_ENTRIES: usize = 1024;", + "cert.rs": "const CACHE_TLS_LEAF_ENTRIES: usize = 1024;", + "service.rs": "const CACHE_SERVICE_PERSISTENT_RESUME_ENTRIES: usize = 256;", } ) - assert len(violations) == 6, RUNTIME_LIMIT_RATIONALE + assert len(violations) == 9, RUNTIME_LIMIT_RATIONALE From d398591c72140f753c260c126bf5713d29370c2c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:37:25 +0000 Subject: [PATCH 415/464] feat(proto): centralize runtime cache limits --- config/README.md | 5 ++-- config/data/runtime-limits.toml | 17 +++++++++++ crates/capsem-proto/build.rs | 52 ++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/config/README.md b/config/README.md index efd9b4d8d..524c8feec 100644 --- a/config/README.md +++ b/config/README.md @@ -33,8 +33,9 @@ and the existing admin validation rail must learn it. are release policy, not test data. - `data/` contains project data embedded or loaded by code. Model pricing tables live here, and `runtime-limits.toml` is the one source for - cross-process runtime ceilings embedded by `capsem-proto`. Runtime consumers - use the generated protocol constants rather than copying numeric limits. + cross-process runtime ceilings, bounded in-memory cache capacities, and + cache freshness windows embedded by `capsem-proto`. Runtime consumers use + the generated protocol constants rather than copying numeric limits. ## Policy at Runtime diff --git a/config/data/runtime-limits.toml b/config/data/runtime-limits.toml index 4edba4daa..07892a975 100644 --- a/config/data/runtime-limits.toml +++ b/config/data/runtime-limits.toml @@ -42,3 +42,20 @@ sqlite_mmap_bytes = 33554432 disk_flush_interval_milliseconds = 1000 # A burst also flushes before the timer at this exact batch multiple. disk_flush_max_pending_events = 1048576 + +[cache] +# Bounded runtime maps and freshness windows are product capacities. Keep +# them here so every process sees the same policy and local literals cannot +# silently drift apart. +dns_answer_entries = 1024 +dns_max_ttl_seconds = 300 +dns_negative_ttl_seconds = 60 +tls_leaf_entries = 1024 +ledger_query_batch_entries = 8 +sqlite_prepared_statements = 64 +service_storage_diagnostics_entries = 256 +service_persistent_resume_entries = 256 +update_check_ttl_seconds = 86400 +image_catalog_refresh_seconds = 1800 +image_catalog_retry_seconds = 60 +image_verification_batch_keys = 1024 diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index 0bb6b339c..6745bb349 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -159,6 +159,32 @@ fn write_runtime_limits() { .unwrap_or_else(|| { panic!("runtime limits: ledger.disk_flush_max_pending_events must be a batch multiple between the batch capacity and 10000000") }); + let cache = config + .get("cache") + .and_then(toml::Value::as_table) + .unwrap_or_else(|| panic!("runtime limits: cache must be a table")); + let cache_integer = |name: &str, minimum: i64, maximum: i64| { + cache + .get(name) + .and_then(toml::Value::as_integer) + .filter(|value| (minimum..=maximum).contains(value)) + .unwrap_or_else(|| { + panic!("runtime limits: cache.{name} must be an integer between {minimum} and {maximum}") + }) + }; + let cache_dns_answer_entries = cache_integer("dns_answer_entries", 1, 65_536); + let cache_dns_max_ttl_seconds = cache_integer("dns_max_ttl_seconds", 1, 86_400); + let cache_dns_negative_ttl_seconds = cache_integer("dns_negative_ttl_seconds", 1, cache_dns_max_ttl_seconds); + let cache_tls_leaf_entries = cache_integer("tls_leaf_entries", 1, 65_536); + let cache_ledger_query_batch_entries = cache_integer("ledger_query_batch_entries", 1, 65_536); + let cache_sqlite_prepared_statements = cache_integer("sqlite_prepared_statements", 1, 65_536); + let cache_service_storage_diagnostics_entries = cache_integer("service_storage_diagnostics_entries", 1, 65_536); + let cache_service_persistent_resume_entries = cache_integer("service_persistent_resume_entries", 1, 65_536); + let cache_update_check_ttl_seconds = cache_integer("update_check_ttl_seconds", 60, 604_800); + let cache_image_catalog_refresh_seconds = cache_integer("image_catalog_refresh_seconds", 60, 86_400); + let cache_image_catalog_retry_seconds = + cache_integer("image_catalog_retry_seconds", 1, cache_image_catalog_refresh_seconds); + let cache_image_verification_batch_keys = cache_integer("image_verification_batch_keys", 1, 65_536); let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR not set"); std::fs::write( format!("{out_dir}/runtime_limits.rs"), @@ -191,7 +217,31 @@ fn write_runtime_limits() { /// Maximum time accepted rows stay only in the memory schema.\n\ pub const LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS: u64 = {ledger_disk_flush_interval_milliseconds};\n\ /// Pending events that force a disk flush before the timer.\n\ - pub const LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS: usize = {ledger_disk_flush_max_pending_events};\n" + pub const LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS: usize = {ledger_disk_flush_max_pending_events};\n\ + /// DNS answers retained per session.\n\ + pub const CACHE_DNS_ANSWER_ENTRIES: usize = {cache_dns_answer_entries};\n\ + /// Longest positive DNS answer reuse.\n\ + pub const CACHE_DNS_MAX_TTL_SECONDS: u32 = {cache_dns_max_ttl_seconds};\n\ + /// Longest negative DNS answer reuse.\n\ + pub const CACHE_DNS_NEGATIVE_TTL_SECONDS: u32 = {cache_dns_negative_ttl_seconds};\n\ + /// Minted TLS leaves retained per proxy.\n\ + pub const CACHE_TLS_LEAF_ENTRIES: usize = {cache_tls_leaf_entries};\n\ + /// Distinct ledger query batches retained per reader.\n\ + pub const CACHE_LEDGER_QUERY_BATCH_ENTRIES: usize = {cache_ledger_query_batch_entries};\n\ + /// Compiled SQLite statements retained per connection.\n\ + pub const CACHE_SQLITE_PREPARED_STATEMENTS: usize = {cache_sqlite_prepared_statements};\n\ + /// Service storage diagnostic results retained.\n\ + pub const CACHE_SERVICE_STORAGE_DIAGNOSTICS_ENTRIES: usize = {cache_service_storage_diagnostics_entries};\n\ + /// Persistent-resume states retained by the service.\n\ + pub const CACHE_SERVICE_PERSISTENT_RESUME_ENTRIES: usize = {cache_service_persistent_resume_entries};\n\ + /// Reuse period for release update checks.\n\ + pub const CACHE_UPDATE_CHECK_TTL_SECONDS: u64 = {cache_update_check_ttl_seconds};\n\ + /// Reuse period for the last good image catalog.\n\ + pub const CACHE_IMAGE_CATALOG_REFRESH_SECONDS: u64 = {cache_image_catalog_refresh_seconds};\n\ + /// Retry delay after an image catalog failure.\n\ + pub const CACHE_IMAGE_CATALOG_RETRY_SECONDS: u64 = {cache_image_catalog_retry_seconds};\n\ + /// Catalog keys admitted to one cache-verification request.\n\ + pub const CACHE_IMAGE_VERIFICATION_BATCH_KEYS: usize = {cache_image_verification_batch_keys};\n" ), ) .expect("runtime limits: write generated constants"); From b05701782397938a31be49c780a22046a69087ef Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:39:19 +0000 Subject: [PATCH 416/464] refactor(core): use shared runtime cache limits --- crates/capsem-core/src/net/cert_authority.rs | 7 +------ crates/capsem-core/src/net/dns/cache.rs | 6 +++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/crates/capsem-core/src/net/cert_authority.rs b/crates/capsem-core/src/net/cert_authority.rs index 66792c877..5a4b8c77c 100644 --- a/crates/capsem-core/src/net/cert_authority.rs +++ b/crates/capsem-core/src/net/cert_authority.rs @@ -9,11 +9,6 @@ use rustls::pki_types::CertificateDer; use rustls::server::ClientHello; use rustls::sign::CertifiedKey; -/// Most leaf certificates kept alive at once. The guest chooses the SNI, so -/// an unbounded map let a loop over random names grow host memory for the -/// life of the VM; a real workload talks to far fewer hosts than this. -const LEAF_CACHE_CAPACITY: usize = 1024; - /// Holds the static CA keypair and caches minted leaf certificates. pub struct CertAuthority { issuer: CertificateIssuer, @@ -28,7 +23,7 @@ impl CertAuthority { let issuer = CertificateIssuer::load(key_pem, cert_pem)?; Ok(Self { issuer, - cache: BoundedCache::new("core.tls_leaf", LEAF_CACHE_CAPACITY), + cache: BoundedCache::new("core.tls_leaf", capsem_proto::CACHE_TLS_LEAF_ENTRIES), }) } diff --git a/crates/capsem-core/src/net/dns/cache.rs b/crates/capsem-core/src/net/dns/cache.rs index 27d24f45f..da7bd1d33 100644 --- a/crates/capsem-core/src/net/dns/cache.rs +++ b/crates/capsem-core/src/net/dns/cache.rs @@ -41,19 +41,19 @@ use capsem_telemetry::dns as m; /// constraining real workloads (a single curl invocation typically /// resolves <= 5 names, so 1024 covers many minutes of agent /// activity). -pub const DEFAULT_CAPACITY: usize = 1024; +pub const DEFAULT_CAPACITY: usize = capsem_proto::CACHE_DNS_ANSWER_ENTRIES; /// Default upper bound on cache TTL, in seconds. DNS records can /// advertise TTLs up to 7 days; a long-lived cache entry would /// outlive most agent sessions and risk staleness on infrastructure /// change. Cap at 5 minutes -- aligns with the typical interactive /// agent loop and the existing CDN TTLs (Cloudflare default 5 min). -pub const DEFAULT_MAX_TTL_SECS: u32 = 300; +pub const DEFAULT_MAX_TTL_SECS: u32 = capsem_proto::CACHE_DNS_MAX_TTL_SECONDS; /// Ceiling on how long an upstream NXDOMAIN is remembered. RFC 2308 lets a /// negative answer live for the SOA minimum; a minute is enough to absorb a /// client retrying a dead name without amplifying a name that reappears. -pub const NEGATIVE_MAX_TTL_SECS: u32 = 60; +pub const NEGATIVE_MAX_TTL_SECS: u32 = capsem_proto::CACHE_DNS_NEGATIVE_TTL_SECONDS; #[derive(Clone)] struct Entry { From 371ce504ea96844837f26e503d31bb4a39449578 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:41:05 +0000 Subject: [PATCH 417/464] refactor(logger): use shared runtime cache limits --- crates/capsem-logger/src/db.rs | 19 ++++++------------- crates/capsem-logger/src/reader/open.rs | 2 +- crates/capsem-logger/src/writer.rs | 4 ++-- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/crates/capsem-logger/src/db.rs b/crates/capsem-logger/src/db.rs index 852d0c503..ecbc82e5b 100644 --- a/crates/capsem-logger/src/db.rs +++ b/crates/capsem-logger/src/db.rs @@ -73,20 +73,10 @@ fn query_cache_key(queries: &[DbQueryOwned]) -> DbQueryCacheKey { /// `serde_json::Value` is neither `Hash` nor `Ord`, so the shared bounded cache /// uses the complete canonical serialization as its collision-free key. The /// read epoch remains the freshness identity owned by this DB handle. +/// The generated product capacity leaves room for the route batches fetched +/// side by side while keeping every row-bounded result under one authority. type DbQueryManyCache = KeyedIdentityCache>; -/// How many distinct batches one handle keeps answers for. -/// -/// The route lists that read a session handle through this rail -- the -/// `stats/detail` batch, the timeline, history pages and the security window -/// -- are fetched side by side when a user opens a session, so the bound -/// leaves room for them without one evicting another. The bound is on memory -/// as much as on lookups: an entry holds a whole JSON result set, and every -/// batch that reaches here bounds its rows. Nothing in here outlives the -/// ledger state it was read from; the whole cache is dropped when the read -/// epoch moves. -const QUERY_MANY_CACHE_ENTRIES: usize = 8; - /// Typed invalidation domains for DB-owned data consumed by cached readers. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ReadCacheDomain { @@ -340,7 +330,10 @@ impl DbHandle { reader_join: Mutex::new(Some(reader_join)), writer: None, archive_blocks_inflated: AtomicU64::new(0), - query_many_cache: DbQueryManyCache::new("logger.query_many", QUERY_MANY_CACHE_ENTRIES), + query_many_cache: DbQueryManyCache::new( + "logger.query_many", + capsem_proto::CACHE_LEDGER_QUERY_BATCH_ENTRIES, + ), hot, reader_requests: AtomicU64::new(0), read_cache_epoch: AtomicU64::new(0), diff --git a/crates/capsem-logger/src/reader/open.rs b/crates/capsem-logger/src/reader/open.rs index 7fd58300b..e5f7c9e2c 100644 --- a/crates/capsem-logger/src/reader/open.rs +++ b/crates/capsem-logger/src/reader/open.rs @@ -41,7 +41,7 @@ impl DbReader { // Route SQL is a fixed set of statements polled on a timer; compiling // each one again on every poll was a measurable share of a route's // CPU. Room for every polled route's statements, as the writer has. - conn.set_prepared_statement_cache_capacity(64); + conn.set_prepared_statement_cache_capacity(capsem_proto::CACHE_SQLITE_PREPARED_STATEMENTS); Self { conn, sync: Default::default(), diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index 473598c07..b4b695372 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -591,7 +591,7 @@ impl DbWriter { conn.busy_timeout(Duration::from_secs(5))?; schema::apply_pragmas(&conn)?; // One statement per table per target; the default 16 would evict. - conn.set_prepared_statement_cache_capacity(64); + conn.set_prepared_statement_cache_capacity(capsem_proto::CACHE_SQLITE_PREPARED_STATEMENTS); schema::record_sqlite_mmap_telemetry(&conn, path, "writer", "open"); let bodies = match schema::archive_schema_status(&conn)? { schema::ArchiveSchemaStatus::Fresh => { @@ -646,7 +646,7 @@ impl DbWriter { let conn = Connection::open_in_memory()?; schema::apply_pragmas(&conn)?; // One statement per table per target; the default 16 would evict. - conn.set_prepared_statement_cache_capacity(64); + conn.set_prepared_statement_cache_capacity(capsem_proto::CACHE_SQLITE_PREPARED_STATEMENTS); schema::create_tables(&conn)?; let bodies = BodyArchive::disabled(SystemTime::now); let memory_uri = schema::memory_uri_for_name(&format!( From a5f8c2c540c8a1ce783af228f3b19fd6128b0fdb Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:43:45 +0000 Subject: [PATCH 418/464] refactor(service): use shared runtime cache limits --- crates/capsem-service/src/container_setup/images.rs | 4 ++-- crates/capsem-service/src/container_setup/registry.rs | 7 ++----- crates/capsem-service/src/service_runtime.rs | 10 ++++++++-- crates/capsem-service/src/tests.rs | 10 ++++++++-- crates/capsem-service/src/update_status.rs | 4 +--- 5 files changed, 21 insertions(+), 14 deletions(-) diff --git a/crates/capsem-service/src/container_setup/images.rs b/crates/capsem-service/src/container_setup/images.rs index dd95035f8..2876ddaeb 100644 --- a/crates/capsem-service/src/container_setup/images.rs +++ b/crates/capsem-service/src/container_setup/images.rs @@ -29,10 +29,10 @@ use std::collections::BTreeSet; use tokio::time::{Duration, Instant}; /// How long a catalog read is reused before the next request reads it again. -pub(crate) const CATALOG_REFRESH: Duration = Duration::from_secs(30 * 60); +pub(crate) const CATALOG_REFRESH: Duration = Duration::from_secs(capsem_proto::CACHE_IMAGE_CATALOG_REFRESH_SECONDS); /// How long a failed read waits before the next request tries again, so an /// offline host does not pay a registry timeout on every request. -pub(crate) const CATALOG_RETRY: Duration = Duration::from_secs(60); +pub(crate) const CATALOG_RETRY: Duration = Duration::from_secs(capsem_proto::CACHE_IMAGE_CATALOG_RETRY_SECONDS); pub(crate) type CatalogFuture = Pin> + Send>>; diff --git a/crates/capsem-service/src/container_setup/registry.rs b/crates/capsem-service/src/container_setup/registry.rs index d17b7db7e..ed96d9765 100644 --- a/crates/capsem-service/src/container_setup/registry.rs +++ b/crates/capsem-service/src/container_setup/registry.rs @@ -1,9 +1,6 @@ //! Request registry transports borrow persistent credential-free cache owners. use super::*; -// Bound queue metadata independently of the cache's content capacity. -const CACHE_VERIFICATION_BATCH_KEYS: usize = 1024; - #[derive(Clone, Default)] pub(crate) struct RegistryImages { cache: Arc>, @@ -51,7 +48,7 @@ impl ImageSource for RegistryImages { fn observe_cache(&self, keys: &[capsem_assets::oci::CacheKey], parent: &StdPath) -> anyhow::Result<()> { anyhow::ensure!( - keys.len() <= CACHE_VERIFICATION_BATCH_KEYS, + keys.len() <= capsem_proto::CACHE_IMAGE_VERIFICATION_BATCH_KEYS, "catalog cache observation exceeds capacity" ); if self.worker.get().is_none() { @@ -61,7 +58,7 @@ impl ImageSource for RegistryImages { let worker = self.cache()?.reconciler( stage::oci_architecture()?, parent.to_owned(), - CACHE_VERIFICATION_BATCH_KEYS, + capsem_proto::CACHE_IMAGE_VERIFICATION_BATCH_KEYS, )?; let _ = self.worker.set((parent.to_owned(), worker)); } diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index 7903dd99f..7c347a5a7 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -290,8 +290,14 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { host_stats: Mutex::new(Default::default()), last_defunct_reconcile_ms: AtomicU64::new(0), containers: Default::default(), - storage_diagnostics_cache: BoundedCache::new("service.storage_diagnostics", 256), - persistent_resume_state_cache: KeyedIdentityCache::new("service.persistent_resume", 256), + storage_diagnostics_cache: BoundedCache::new( + "service.storage_diagnostics", + capsem_proto::CACHE_SERVICE_STORAGE_DIAGNOSTICS_ENTRIES, + ), + persistent_resume_state_cache: KeyedIdentityCache::new( + "service.persistent_resume", + capsem_proto::CACHE_SERVICE_PERSISTENT_RESUME_ENTRIES, + ), asset_manifest_cache: IdentityCache::new("service.asset_manifest"), list_response_cache: IdentityCache::new("service.vm_list"), lifecycle: capsem_service::lifecycle::VmLifecycle::default(), diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 02591aaea..081e347bd 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -127,8 +127,14 @@ pub(crate) fn test_state( host_stats: Mutex::new(Default::default()), last_defunct_reconcile_ms: AtomicU64::new(0), containers: Default::default(), - storage_diagnostics_cache: BoundedCache::new("service.storage_diagnostics", 256), - persistent_resume_state_cache: KeyedIdentityCache::new("service.persistent_resume", 256), + storage_diagnostics_cache: BoundedCache::new( + "service.storage_diagnostics", + capsem_proto::CACHE_SERVICE_STORAGE_DIAGNOSTICS_ENTRIES, + ), + persistent_resume_state_cache: KeyedIdentityCache::new( + "service.persistent_resume", + capsem_proto::CACHE_SERVICE_PERSISTENT_RESUME_ENTRIES, + ), asset_manifest_cache: IdentityCache::new("service.asset_manifest"), list_response_cache: IdentityCache::new("service.vm_list"), lifecycle: capsem_service::lifecycle::VmLifecycle::default(), diff --git a/crates/capsem-service/src/update_status.rs b/crates/capsem-service/src/update_status.rs index cdffa7889..d95b9ad42 100644 --- a/crates/capsem-service/src/update_status.rs +++ b/crates/capsem-service/src/update_status.rs @@ -7,8 +7,6 @@ use crate::api; #[cfg(test)] mod tests; -const UPDATE_CACHE_TTL_SECS: u64 = 24 * 3600; - #[derive(Debug, Clone, Deserialize)] pub(super) struct UpdateCheckCache { #[serde(default)] @@ -57,7 +55,7 @@ pub(super) fn update_status_response_from_paths( }; let checked_at = cache.as_ref().map(|cache| cache.checked_at); let stale = checked_at - .map(|checked_at| now.saturating_sub(checked_at) > UPDATE_CACHE_TTL_SECS) + .map(|checked_at| now.saturating_sub(checked_at) > capsem_proto::CACHE_UPDATE_CHECK_TTL_SECONDS) .unwrap_or(true); let channel_url = cache .as_ref() From ab1a52b231eb45c6c64b0ba3bc705c6941e25b2a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:45:23 +0000 Subject: [PATCH 419/464] refactor(cli): use shared update cache lifetime --- crates/capsem/src/update.rs | 5 ++--- crates/capsem/src/update/tests.rs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/capsem/src/update.rs b/crates/capsem/src/update.rs index ce0f1baf5..63544f6a4 100644 --- a/crates/capsem/src/update.rs +++ b/crates/capsem/src/update.rs @@ -180,7 +180,6 @@ impl BinaryInstallerApplyCommand { } } -const CACHE_TTL_SECS: u64 = 24 * 3600; // 24 hours const DEFAULT_RELEASE_MANIFEST_URL: &str = "https://release.capsem.org/assets/stable/manifest.json"; const DEFAULT_RELEASE_CHANNELS_URL: &str = "https://release.capsem.org/channels.json"; const RELEASE_MANIFEST_URL_ENV: &str = "CAPSEM_RELEASE_MANIFEST_URL"; @@ -503,7 +502,7 @@ pub fn read_cached_update_notice() -> Option { // Only show if cache is still fresh let age = now_secs().saturating_sub(check.checked_at); - if age > CACHE_TTL_SECS { + if age > capsem_proto::CACHE_UPDATE_CHECK_TTL_SECONDS { return None; } @@ -657,7 +656,7 @@ pub async fn refresh_update_cache_if_stale() { .filter(|check| check.source.as_deref() == Some(manifest_url.as_str())); if let Some(check) = previous_check.as_ref() { let age = now_secs().saturating_sub(check.checked_at); - if age < CACHE_TTL_SECS { + if age < capsem_proto::CACHE_UPDATE_CHECK_TTL_SECONDS { return; // Still fresh } } diff --git a/crates/capsem/src/update/tests.rs b/crates/capsem/src/update/tests.rs index ccbb26885..3b86af3b5 100644 --- a/crates/capsem/src/update/tests.rs +++ b/crates/capsem/src/update/tests.rs @@ -371,7 +371,7 @@ fn update_channel_provenance_preserves_previous_cache_on_failure() { #[test] fn cache_ttl_constant() { - assert_eq!(CACHE_TTL_SECS, 86400); + assert_eq!(capsem_proto::CACHE_UPDATE_CHECK_TTL_SECONDS, 86400); } #[test] From 83cefbbcca3499a4f53054e6ef269eb77623df6c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:46:08 +0000 Subject: [PATCH 420/464] feat(bench): report runtime cache limits --- crates/capsem-bench/src/runtime_limits.rs | 24 +++++++++++ .../capsem-bench/src/runtime_limits/tests.rs | 42 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/crates/capsem-bench/src/runtime_limits.rs b/crates/capsem-bench/src/runtime_limits.rs index ef7c610f9..81e998372 100644 --- a/crates/capsem-bench/src/runtime_limits.rs +++ b/crates/capsem-bench/src/runtime_limits.rs @@ -14,6 +14,18 @@ pub(crate) struct Report { ledger_sqlite_mmap_bytes: i64, ledger_disk_flush_interval_milliseconds: u64, ledger_disk_flush_max_pending_events: usize, + cache_dns_answer_entries: usize, + cache_dns_max_ttl_seconds: u32, + cache_dns_negative_ttl_seconds: u32, + cache_tls_leaf_entries: usize, + cache_ledger_query_batch_entries: usize, + cache_sqlite_prepared_statements: usize, + cache_service_storage_diagnostics_entries: usize, + cache_service_persistent_resume_entries: usize, + cache_update_check_ttl_seconds: u64, + cache_image_catalog_refresh_seconds: u64, + cache_image_catalog_retry_seconds: u64, + cache_image_verification_batch_keys: usize, } pub(crate) const fn report() -> Report { @@ -28,6 +40,18 @@ pub(crate) const fn report() -> Report { ledger_sqlite_mmap_bytes: capsem_proto::LEDGER_SQLITE_MMAP_BYTES, ledger_disk_flush_interval_milliseconds: capsem_proto::LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS, ledger_disk_flush_max_pending_events: capsem_proto::LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS, + cache_dns_answer_entries: capsem_proto::CACHE_DNS_ANSWER_ENTRIES, + cache_dns_max_ttl_seconds: capsem_proto::CACHE_DNS_MAX_TTL_SECONDS, + cache_dns_negative_ttl_seconds: capsem_proto::CACHE_DNS_NEGATIVE_TTL_SECONDS, + cache_tls_leaf_entries: capsem_proto::CACHE_TLS_LEAF_ENTRIES, + cache_ledger_query_batch_entries: capsem_proto::CACHE_LEDGER_QUERY_BATCH_ENTRIES, + cache_sqlite_prepared_statements: capsem_proto::CACHE_SQLITE_PREPARED_STATEMENTS, + cache_service_storage_diagnostics_entries: capsem_proto::CACHE_SERVICE_STORAGE_DIAGNOSTICS_ENTRIES, + cache_service_persistent_resume_entries: capsem_proto::CACHE_SERVICE_PERSISTENT_RESUME_ENTRIES, + cache_update_check_ttl_seconds: capsem_proto::CACHE_UPDATE_CHECK_TTL_SECONDS, + cache_image_catalog_refresh_seconds: capsem_proto::CACHE_IMAGE_CATALOG_REFRESH_SECONDS, + cache_image_catalog_retry_seconds: capsem_proto::CACHE_IMAGE_CATALOG_RETRY_SECONDS, + cache_image_verification_batch_keys: capsem_proto::CACHE_IMAGE_VERIFICATION_BATCH_KEYS, } } diff --git a/crates/capsem-bench/src/runtime_limits/tests.rs b/crates/capsem-bench/src/runtime_limits/tests.rs index 335064ed6..b953b1c8a 100644 --- a/crates/capsem-bench/src/runtime_limits/tests.rs +++ b/crates/capsem-bench/src/runtime_limits/tests.rs @@ -24,4 +24,46 @@ fn benchmark_limits_are_the_protocol_authority() { limits.ledger_disk_flush_max_pending_events, capsem_proto::LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS ); + assert_eq!(limits.cache_dns_answer_entries, capsem_proto::CACHE_DNS_ANSWER_ENTRIES); + assert_eq!( + limits.cache_dns_max_ttl_seconds, + capsem_proto::CACHE_DNS_MAX_TTL_SECONDS + ); + assert_eq!( + limits.cache_dns_negative_ttl_seconds, + capsem_proto::CACHE_DNS_NEGATIVE_TTL_SECONDS + ); + assert_eq!(limits.cache_tls_leaf_entries, capsem_proto::CACHE_TLS_LEAF_ENTRIES); + assert_eq!( + limits.cache_ledger_query_batch_entries, + capsem_proto::CACHE_LEDGER_QUERY_BATCH_ENTRIES + ); + assert_eq!( + limits.cache_sqlite_prepared_statements, + capsem_proto::CACHE_SQLITE_PREPARED_STATEMENTS + ); + assert_eq!( + limits.cache_service_storage_diagnostics_entries, + capsem_proto::CACHE_SERVICE_STORAGE_DIAGNOSTICS_ENTRIES + ); + assert_eq!( + limits.cache_service_persistent_resume_entries, + capsem_proto::CACHE_SERVICE_PERSISTENT_RESUME_ENTRIES + ); + assert_eq!( + limits.cache_update_check_ttl_seconds, + capsem_proto::CACHE_UPDATE_CHECK_TTL_SECONDS + ); + assert_eq!( + limits.cache_image_catalog_refresh_seconds, + capsem_proto::CACHE_IMAGE_CATALOG_REFRESH_SECONDS + ); + assert_eq!( + limits.cache_image_catalog_retry_seconds, + capsem_proto::CACHE_IMAGE_CATALOG_RETRY_SECONDS + ); + assert_eq!( + limits.cache_image_verification_batch_keys, + capsem_proto::CACHE_IMAGE_VERIFICATION_BATCH_KEYS + ); } From 3de43e11158c7bab2c27963afa3af6466b18b5bc Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:46:29 +0000 Subject: [PATCH 421/464] test(citadel): guard runtime cache consumers --- tests/citadel/test_runtime_limit_authority.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 0685f0c6d..d1a8f10c6 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -89,18 +89,18 @@ "pub const LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS: u64 = {ledger_disk_flush_interval_milliseconds}", 'get("disk_flush_max_pending_events")', "pub const LEDGER_DISK_FLUSH_MAX_PENDING_EVENTS: usize = {ledger_disk_flush_max_pending_events}", - 'get("dns_answer_entries")', - 'get("dns_max_ttl_seconds")', - 'get("dns_negative_ttl_seconds")', - 'get("tls_leaf_entries")', - 'get("ledger_query_batch_entries")', - 'get("sqlite_prepared_statements")', - 'get("service_storage_diagnostics_entries")', - 'get("service_persistent_resume_entries")', - 'get("update_check_ttl_seconds")', - 'get("image_catalog_refresh_seconds")', - 'get("image_catalog_retry_seconds")', - 'get("image_verification_batch_keys")', + 'cache_integer("dns_answer_entries"', + 'cache_integer("dns_max_ttl_seconds"', + 'cache_integer("dns_negative_ttl_seconds"', + 'cache_integer("tls_leaf_entries"', + 'cache_integer("ledger_query_batch_entries"', + 'cache_integer("sqlite_prepared_statements"', + 'cache_integer("service_storage_diagnostics_entries"', + 'cache_integer("service_persistent_resume_entries"', + 'cache_integer("update_check_ttl_seconds"', + 'cache_integer("image_catalog_refresh_seconds"', + 'cache_integer("image_catalog_retry_seconds"', + 'cache_integer("image_verification_batch_keys"', "pub const CACHE_DNS_ANSWER_ENTRIES: usize = {cache_dns_answer_entries}", "pub const CACHE_UPDATE_CHECK_TTL_SECONDS: u64 = {cache_update_check_ttl_seconds}", ), From 71d94fa24f3a23e2dcba07b296f60375d1f3d514 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:47:13 +0000 Subject: [PATCH 422/464] docs(skills): enforce runtime cache authority --- skills/dev-cache/SKILL.md | 10 ++++++++++ skills/dev-mitm-proxy/SKILL.md | 6 +++++- skills/dev-rust-patterns/SKILL.md | 8 ++++++-- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/skills/dev-cache/SKILL.md b/skills/dev-cache/SKILL.md index 604f3db3b..237e2138c 100644 --- a/skills/dev-cache/SKILL.md +++ b/skills/dev-cache/SKILL.md @@ -33,6 +33,16 @@ eviction and occupancy telemetry where those states exist, and preserve the logger DB boundary instead of caching ledger projections in a route or UI process. +The `[cache]` table owns every in-memory cache count and freshness window: +DNS answers and positive/negative TTLs, TLS leaves, ledger query batches, +SQLite prepared statements, service diagnostics and resume state, update +checks, image-catalog refresh/retry and image-verification batching. Preserve +their current behavior by importing `capsem_proto::CACHE_*`; a local literal, +even when two crates currently agree, is a second authority. Add a new field, +build-time validation, generated constant, benchmark `runtime-limits` field +and Citadel consumer assertion together when a new bounded runtime cache is +introduced. User overrides remain the typed-settings follow-up in issue #349. + ## Typed boundary Programs use `CacheRegistry`, `CacheRequest`, `CacheOperation`, and diff --git a/skills/dev-mitm-proxy/SKILL.md b/skills/dev-mitm-proxy/SKILL.md index 47e56caa0..7bb2ccaed 100644 --- a/skills/dev-mitm-proxy/SKILL.md +++ b/skills/dev-mitm-proxy/SKILL.md @@ -52,7 +52,7 @@ Guest curl -> iptables REDIRECT -> capsem-net-proxy (guest, port 10443) | File | What | |------|------| | `crates/capsem-core/src/net/mitm_proxy.rs` | Async MITM proxy (rustls + hyper): TLS termination, HTTP inspection, upstream bridging | -| `crates/capsem-core/src/net/cert_authority.rs` | CA loader + on-demand domain cert minting with RwLock cache | +| `crates/capsem-core/src/net/cert_authority.rs` | CA loader + bounded single-flight domain certificate cache | | `crates/capsem-core/src/security_engine/` | Rule/plugin/decision rail over `SecurityEvent` | | `crates/capsem-core/src/net/sni.rs` | SNI parser for TLS ClientHello | | `crates/capsem-agent/src/net_proxy.rs` | Guest-side TCP-to-vsock relay | @@ -89,6 +89,10 @@ contract tracked by issue #349. Extend and run `tests/citadel/test_runtime_limit_authority.py` whenever a new consumer or derived limit is added. +The same file's `[cache]` table owns DNS answer count and TTLs plus the TLS +leaf count. Use its generated `capsem_proto::CACHE_*` constants; do not place a +second cache bound in the proxy or DNS modules. + Use `just bench mitm-scale` when a one-VM curve plateaus while the host still looks idle. The lane uses fresh 1/2/4/8-VM cohorts, keeps per-VM work fixed and records the mock upstream outside the Capsem process tree. Compare the curve to diff --git a/skills/dev-rust-patterns/SKILL.md b/skills/dev-rust-patterns/SKILL.md index a798be04e..65efb0580 100644 --- a/skills/dev-rust-patterns/SKILL.md +++ b/skills/dev-rust-patterns/SKILL.md @@ -195,10 +195,14 @@ struct FunctionCall { ## Concurrency patterns -- **RwLock for caches**: Cert authority uses `RwLock` -- many readers, rare writers. Use `read()` first, upgrade to `write()` only on cache miss. +- **Shared runtime caches**: Use `capsem-cache` primitives instead of a local + map and eviction policy. Import capacities and freshness windows from the + generated `capsem_proto::CACHE_*` constants; their only numeric source is + `[cache]` in `config/data/runtime-limits.toml`. - **Arc for shared state**: VM state, proxy config, and telemetry handles are `Arc`-wrapped for sharing across tasks. - **Per-connection tasks**: The MITM proxy spawns a new tokio task per connection. Each task owns its TLS state and upstream connection. No shared mutable state between connections. -- **Config-owned cross-process ceilings**: Network concurrency is owned by +- **Config-owned runtime ceilings**: Network concurrency and in-memory cache + policy are owned by `config/data/runtime-limits.toml` and generated into `capsem-proto`. Import the shared protocol constant in host and guest consumers; never duplicate a numeric capacity in individual crates. Ledger queue and batch capacities From 675aca10410ebc32922cd9e7da34d8099f4a64dd Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:47:58 +0000 Subject: [PATCH 423/464] docs(architecture): explain runtime capacity and scaling --- .../content/docs/architecture/mitm-proxy.md | 28 +++++++++++++++++ .../docs/architecture/runtime-caching.md | 25 ++++++++++++++++ .../docs/architecture/session-telemetry.md | 30 ++++++++++++++----- .../content/docs/development/benchmarking.md | 27 +++++++++++++++++ 4 files changed, 103 insertions(+), 7 deletions(-) diff --git a/web/docs/src/content/docs/architecture/mitm-proxy.md b/web/docs/src/content/docs/architecture/mitm-proxy.md index 23cb8b359..666a08ad5 100644 --- a/web/docs/src/content/docs/architecture/mitm-proxy.md +++ b/web/docs/src/content/docs/architecture/mitm-proxy.md @@ -41,6 +41,34 @@ graph TD The proxy uses hyper for HTTP parsing and tokio-rustls for TLS. Each vsock connection can carry multiple HTTP requests via keep-alive -- upstream connections are cached per-connection to avoid re-establishing TLS for each request. +## Capacity, reuse, and overload + +One session accepts up to 512 concurrent network clients in this release. The +guest relay, VM owner, upstream grant broker, proxy, DNS sessions, descriptor +floors, and ledger queues all derive their capacities from one embedded runtime +limits contract. This prevents one stage from accepting a burst that the next +stage cannot represent. + +An intercepted client connection reuses its upstream HTTP/TLS connection for +keep-alive requests. That sender stays inside the client connection because it +is pinned to the judged target and shares that connection's cancellation and +audit scope; unrelated clients do not borrow it. Independent connections and +VM sessions run concurrently across host cores. + +Queues and descriptor grants are bounded. If capacity or required audit +admission is unavailable, the affected operation waits within its deadline or +fails explicitly; it is never reported as a successful benchmark request. +The current limits are product defaults embedded in the binaries and are not +yet user-configurable. + +Use the official `mitm-load` curve to test one VM through the full supported +client range. Use `mitm-scale` to keep the per-VM work fixed while increasing +the number of independent VMs. The scale collector reports aggregate RPS, +per-VM latency, proportional memory, Capsem CPU, fixture CPU, physical cores, +logical CPUs, request errors, and hidden proxy refusals. A plateau after the +process tree consumes the host CPU budget is host saturation; a plateau before +that point requires investigation for shared serialization. + ## Configuration ```mermaid diff --git a/web/docs/src/content/docs/architecture/runtime-caching.md b/web/docs/src/content/docs/architecture/runtime-caching.md index 057618a5c..ad848220f 100644 --- a/web/docs/src/content/docs/architecture/runtime-caching.md +++ b/web/docs/src/content/docs/architecture/runtime-caching.md @@ -32,6 +32,31 @@ miss for the same bounded-cache key performs one load. Refreshing an existing key is distinct from a capacity eviction, so operational counters explain why an entry changed. +## Capacity authority + +`config/data/runtime-limits.toml` is the single source for bounded runtime +cache counts and freshness windows. `capsem-proto` validates it at build time +and generates constants consumed by every process. The benchmark +`runtime-limits` mode records the generated values, and a Citadel guard rejects +local numeric copies. + +| Policy | Shipped value | +|---|---:| +| DNS answer entries | 1,024 | +| Positive / negative DNS TTL ceiling | 300 s / 60 s | +| TLS leaf certificates | 1,024 | +| Ledger query batches per reader | 8 | +| SQLite prepared statements per connection | 64 | +| Storage diagnostics / persistent resume entries | 256 / 256 | +| Update-check reuse | 24 hours | +| Image catalog refresh / failed-read retry | 30 min / 60 s | +| Image verification keys per batch | 1,024 | + +These are product defaults embedded in this release. User settings do not yet +override them. Durable developer and build caches use the separate +`config/cache.toml` retention contract because their lifetime and enforcement +model are different. + ## Freshness owners | Cache | Freshness rule | Shared mechanism | diff --git a/web/docs/src/content/docs/architecture/session-telemetry.md b/web/docs/src/content/docs/architecture/session-telemetry.md index 9771533d6..e25b20c1f 100644 --- a/web/docs/src/content/docs/architecture/session-telemetry.md +++ b/web/docs/src/content/docs/architecture/session-telemetry.md @@ -929,13 +929,29 @@ Inside `capsem-ledger`, the ledger server gives `DbWriter` a dedicated thread that owns the SQLite connection. Callers see a typed remote handle and never a database file descriptor: -1. Async callers send `WriteOp` via `tx.send()` (non-blocking) -2. Writer thread blocks on `rx.blocking_recv()` for the first op -3. After receiving one op, drains the rest of the queue -4. Executes all drained ops in a single SQLite transaction -5. Repeats - -This **block-then-drain** pattern batches writes for efficiency while keeping the async callers non-blocking. The channel has configurable backpressure capacity. +1. A producer submits an event to a bounded, DB-owned admission queue. +2. Consecutive remote admissions coalesce for at most 100 microseconds and + cross IPC in batches of up to 256 events. +3. The writer thread blocks for the first message, drains up to the same + 256-event batch bound, and applies the batch to its in-memory SQLite tables + in one transaction. +4. Dirty tables publish to the disk-backed WAL on the one-second durability + interval or the configured pending-event threshold. Body archive segments + reach durable storage before their SQLite index rows become visible. +5. Readers cache complete typed query batches only until the logger-owned read + epoch or SQLite `data_version` changes. + +The producer queue holds 4,096 events: eight event slots for each of the 512 +supported network clients. `write(event).await` means the event was accepted +into this bounded queue. It may yield under backpressure, and it does not mean +the event is durable yet. Tests and lifecycle code that require durability use +the explicit flush barrier. The SQLite mapped window is 32 MiB per ledger +process, and archive index publication is capped at 8,192 rows per transaction +so a slow storage flush cannot turn into unbounded admission memory. + +All of these relationships come from `config/data/runtime-limits.toml` and are +generated through `capsem-proto`; no producer or route owns a second queue, +batch, cache, or flush limit. SQLite writer pragmas: WAL journal mode, FULL synchronous. Text fields are defensively capped at 256 KB, headers at 16 KB and display previews at 2 KB; full bodies go to the archive, capped at 10 MiB per direction. diff --git a/web/docs/src/content/docs/development/benchmarking.md b/web/docs/src/content/docs/development/benchmarking.md index 1d84e4b2a..e34253e9c 100644 --- a/web/docs/src/content/docs/development/benchmarking.md +++ b/web/docs/src/content/docs/development/benchmarking.md @@ -189,6 +189,33 @@ before its latency or throughput can become comparison data. Successful runs record `errors = 0` and `error_ratio = 0` for each level as the proof that the reported samples cover the complete load rather than only the successful tail. +### Multi-VM MITM scaling (`mitm-scale`) + +```bash +just bench mitm-scale +``` + +This host-side dimension creates a fresh identical cohort at each configured +VM count, starts one fixed-concurrency keep-alive HTTPS load in every VM, and +measures all VMs at the same time. Fresh cohorts keep memory and lifecycle +history from one level out of the next. The collector records: + +- aggregate and per-VM RPS; +- per-VM p50, p95, p99, and p99.9 latency; +- PSS, USS, RSS, CPU time, threads, descriptors, and I/O for the Capsem process + tree; +- the same resource pressure for the mock upstream in a separate namespace; +- host physical-core and logical-CPU counts; and +- request errors plus proxy traffic-grant refusals. + +Any request error, starved client loop, or hidden proxy refusal fails the +dimension. Compare two unchanged runs. Interpret scaling against physical +cores first and logical CPUs second: when total Capsem CPU reaches the host +budget, rising latency or falling aggregate RPS is a host-capacity result. +Before that point it can indicate a shared lock, serialized queue, or fixture +bottleneck. Fixture CPU is reported separately so upstream saturation cannot +be mistaken for a Capsem limit. + `capsem-bench protocol` runs deterministic local mock-server scenarios: tiny HTTP, 1 MiB body, gzip, SSE model stream, JSON model response, denied-target, credential-shaped response, and WebSocket control frames. When From f82051e4a6fa1f49db7ccea07918957057555d6e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:51:02 +0000 Subject: [PATCH 424/464] style(gate): iterate MITM fanout pairs --- build_system/builder/gate/benchmarkschema.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_system/builder/gate/benchmarkschema.py b/build_system/builder/gate/benchmarkschema.py index f90131f34..f8b44c180 100644 --- a/build_system/builder/gate/benchmarkschema.py +++ b/build_system/builder/gate/benchmarkschema.py @@ -8,6 +8,7 @@ from __future__ import annotations import math +from itertools import pairwise from pydantic import Field, PositiveFloat, PositiveInt, field_validator @@ -143,7 +144,7 @@ class BenchmarkMitmScaleConfig(Strict): def fanout_starts_at_one_and_increases(cls, values: tuple[int, ...]) -> tuple[int, ...]: if not values or values[0] != 1: raise ValueError("MITM scale fanout must start at one VM") - if any(current <= previous for previous, current in zip(values, values[1:])): + if any(current <= previous for previous, current in pairwise(values)): raise ValueError("MITM scale VM counts must be strictly increasing") return values From 7146f23bd5912dc5b2998716daac392b8cce51c4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:51:02 +0000 Subject: [PATCH 425/464] fix(bench): classify MCP load failures --- guest/artifacts/capsem_bench/mcp_load.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/guest/artifacts/capsem_bench/mcp_load.py b/guest/artifacts/capsem_bench/mcp_load.py index d4b8d6704..3323290cb 100644 --- a/guest/artifacts/capsem_bench/mcp_load.py +++ b/guest/artifacts/capsem_bench/mcp_load.py @@ -30,7 +30,7 @@ render_load_table, summarize_load_level, ) -from .mcp_transport import StdioClient +from .mcp_transport import McpError, StdioClient MCP_SERVER = "/run/capsem-mcp-server" DEFAULT_DURATION_S = 10.0 @@ -58,7 +58,7 @@ async def worker(): ms = (time.monotonic() - t0) * 1000 async with lat_lock: latencies.append(ms) - except Exception: # noqa: BLE001 -- every request failure is benchmark data. + except (McpError, OSError): errors += 1 await asyncio.gather(*(worker() for _ in range(concurrency))) From ea8b0c915d8a60eea1f3ab484de796110c20b407 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:51:02 +0000 Subject: [PATCH 426/464] test(citadel): merge runtime cache wiring guard --- tests/citadel/test_runtime_limit_authority.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index d1a8f10c6..1ab617e45 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -136,6 +136,7 @@ "crates/capsem-logger/src/db.rs": ( "LEDGER_PRODUCER_QUEUE_CAPACITY", "LEDGER_ADMISSION_BATCH_CAPACITY", + "capsem_proto::CACHE_LEDGER_QUERY_BATCH_ENTRIES", ), "crates/capsem-logger/src/writer.rs": ( "LEDGER_DISK_FLUSH_INTERVAL_MILLISECONDS", @@ -144,9 +145,6 @@ "crates/capsem-logger/src/schema/pragmas.rs": ( "capsem_proto::LEDGER_SQLITE_MMAP_BYTES", ), - "crates/capsem-logger/src/db.rs": ( - "capsem_proto::CACHE_LEDGER_QUERY_BATCH_ENTRIES", - ), "crates/capsem-logger/src/reader/open.rs": ( "capsem_proto::CACHE_SQLITE_PREPARED_STATEMENTS", ), From 32692ddfecd0af401838970a42c9f0569eaf13c9 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 05:53:24 +0000 Subject: [PATCH 427/464] style(citadel): match gate import order --- tests/citadel/test_runtime_limit_authority.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/citadel/test_runtime_limit_authority.py b/tests/citadel/test_runtime_limit_authority.py index 1ab617e45..b2e5f24aa 100644 --- a/tests/citadel/test_runtime_limit_authority.py +++ b/tests/citadel/test_runtime_limit_authority.py @@ -3,9 +3,8 @@ from __future__ import annotations import re -from pathlib import Path - import tomllib +from pathlib import Path ROOT = Path(__file__).resolve().parents[2] CONFIG = ROOT / "config/data/runtime-limits.toml" From e12dc65461dbb0e19c35a088767827d32080831c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 06:01:18 +0000 Subject: [PATCH 428/464] fix(proto): preserve shared build cache paths --- crates/capsem-proto/build.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index 6745bb349..a51ef1868 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -33,8 +33,7 @@ fn main() { } fn write_runtime_limits() { - let config_path = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR not set")) - .join("../../config/data/runtime-limits.toml"); + let config_path = PathBuf::from("../../config/data/runtime-limits.toml"); let source = std::fs::read_to_string(&config_path) .unwrap_or_else(|error| panic!("runtime limits: read {}: {error}", config_path.display())); let config: toml::Value = toml::from_str(&source) From 8aac3847e8b17b763ac56337fd58c3ca1b4f2147 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 06:01:19 +0000 Subject: [PATCH 429/464] test(bench): keep new load helpers type safe --- tests/helpers/process_resources.py | 11 ++++++++--- tests/test_benchmark_guest_collector.py | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/helpers/process_resources.py b/tests/helpers/process_resources.py index 997eee6f6..df4366003 100644 --- a/tests/helpers/process_resources.py +++ b/tests/helpers/process_resources.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from decimal import Decimal from pathlib import Path +from typing import cast import psutil @@ -185,11 +186,15 @@ def observe(self, snapshot: TreeSnapshot) -> None: self._baseline = dict(totals) for name, values in totals.items(): previous = self._peak.get(name, (0, 0, 0, 0)) - self._peak[name] = tuple(max(old, new) for old, new in zip(previous, values, strict=True)) + self._peak[name] = cast( + tuple[int, int, int, int], + tuple(max(old, new) for old, new in zip(previous, values, strict=True)), + ) for name, values in rss_breakdowns.items(): previous = self._peak_rss_breakdown.get(name, (0, 0, 0)) - self._peak_rss_breakdown[name] = tuple( - max(old, new) for old, new in zip(previous, values, strict=True) + self._peak_rss_breakdown[name] = cast( + tuple[int, int, int], + tuple(max(old, new) for old, new in zip(previous, values, strict=True)), ) self._peak_processes = max(self._peak_processes, len(snapshot.processes)) diff --git a/tests/test_benchmark_guest_collector.py b/tests/test_benchmark_guest_collector.py index a0360502b..4dde5d42b 100644 --- a/tests/test_benchmark_guest_collector.py +++ b/tests/test_benchmark_guest_collector.py @@ -323,6 +323,7 @@ def test_protocol_result_preserves_raw_model_samples(monkeypatch, capsys) -> Non def post(route, body=None, **kwargs): if route == "/vms/create": return {"id": "bench-test-vm"} + assert body is not None output = ( json.dumps(collection) if body["command"] == "cat /tmp/capsem-benchmark-collector.json" @@ -458,6 +459,7 @@ def test_failed_dns_load_preserves_the_guest_proxy_log(monkeypatch) -> None: def post(route, body=None, **kwargs): if route == "/vms/create": return {"id": "bench-test-vm"} + assert body is not None command = body["command"] exec_commands.append(command) if command == "cat /tmp/capsem-benchmark.json": From 24bd567f958abb2fbc62d7e610bc068630018c26 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 06:01:20 +0000 Subject: [PATCH 430/464] test(citadel): own benchmark helper sources --- tests/citadel/repository_surface_ownership.toml | 8 ++++---- tests/citadel/test_repository_surface_ownership.py | 1 + tests/citadel/test_web_benchmark_boundary.py | 5 +++-- tests/citadel/web_benchmark_boundary_debt.toml | 6 ++++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/citadel/repository_surface_ownership.toml b/tests/citadel/repository_surface_ownership.toml index 91e63b504..31115870c 100644 --- a/tests/citadel/repository_surface_ownership.toml +++ b/tests/citadel/repository_surface_ownership.toml @@ -139,14 +139,14 @@ reason = "Builder, gate, release, release-site, and script unit tests move with [[surface]] id = "benchmarks" state = "retained" -current = ["benchmarks/collectors/", "benchmarks/baselines/"] -targets = ["benchmarks/collectors/", "benchmarks/baselines/"] +current = ["benchmarks/collectors/", "benchmarks/baselines/", "benchmarks/helpers/"] +targets = ["benchmarks/collectors/", "benchmarks/baselines/", "benchmarks/helpers/"] lint = ["python.ruff", "python.ty.relaxed", "fast.audit.source-syntax"] tests = ["tests/test_benchmark_retention_contract.py", "tests/test_capsem_bench_gates.py"] ci = ["fast-gate", "test-linux", "test", "test-install", "pr-gate"] -coverage = "not-applicable: collectors and reviewed evidence are contract-tested; Rust harness coverage stays crate-owned" +coverage = "not-applicable: collectors, reusable helpers, and reviewed evidence are contract-tested; Rust harness coverage stays crate-owned" debt = "tests/citadel/repository_path_debt.toml" -reason = "Collectors and reviewed baselines converge under one benchmark owner." +reason = "Collectors, reusable helpers, and reviewed baselines converge under one benchmark owner." [[surface]] id = "product-config" diff --git a/tests/citadel/test_repository_surface_ownership.py b/tests/citadel/test_repository_surface_ownership.py index 1381aaf65..e75d95899 100644 --- a/tests/citadel/test_repository_surface_ownership.py +++ b/tests/citadel/test_repository_surface_ownership.py @@ -40,6 +40,7 @@ "build_system/tests/", "benchmarks/collectors/", "benchmarks/baselines/", + "benchmarks/helpers/", "config/", "crates/", "guest/", diff --git a/tests/citadel/test_web_benchmark_boundary.py b/tests/citadel/test_web_benchmark_boundary.py index fa8397c6d..d8bf31935 100644 --- a/tests/citadel/test_web_benchmark_boundary.py +++ b/tests/citadel/test_web_benchmark_boundary.py @@ -88,6 +88,7 @@ def _digest(records: tuple[str, ...]) -> str: #: matching, so a file is skipped only while its bytes match its pin; an #: edited one is scanned like any other source. VENDORED_TREES = ("crates/capsem-gateway/vendor/xpra-html5",) +BENCHMARK_SOURCE_ROOTS = ("benchmarks/collectors/", "benchmarks/helpers/") def _pinned_vendor_files(root: Path = ROOT) -> dict[str, str]: @@ -230,7 +231,7 @@ def _observe() -> Observed: path for path in tracked if path.startswith("benchmarks/") - and not path.startswith(("benchmarks/baselines/", "benchmarks/collectors/")) + and not path.startswith(("benchmarks/baselines/", *BENCHMARK_SOURCE_ROOTS)) ) graphics = tuple( f"{path}->web/graphics/{path.removeprefix('graphics/')}" @@ -241,7 +242,7 @@ def _observe() -> Observed: path for path in tracked if path.startswith("benchmarks/") - and not path.startswith("benchmarks/collectors/") + and not path.startswith(BENCHMARK_SOURCE_ROOTS) ) recordings = tuple( f"tracked generated recording: {path}" diff --git a/tests/citadel/web_benchmark_boundary_debt.toml b/tests/citadel/web_benchmark_boundary_debt.toml index 683434d19..4de22c385 100644 --- a/tests/citadel/web_benchmark_boundary_debt.toml +++ b/tests/citadel/web_benchmark_boundary_debt.toml @@ -15,5 +15,7 @@ lint_mapping_count = 4 lint_mapping_sha256 = "ac42e4aac5f97bab38e19bf3691343d0f79934dabef4137ac501baa6f164d538" coverage_mapping_count = 4 coverage_mapping_sha256 = "6e6d023b00ea5f0dcdb1f2373c3217907ef1b14075697ad50c4a3ebf5884aee0" -legacy_reference_count = 322 -legacy_reference_sha256 = "fe582d8926ad23876af1d5205e378e5cc907244591f21004a0408103cb820b27" +legacy_reference_count = 324 +# Includes the runtime benchmark store documented for users and the benchmark +# helper root recorded by repository surface ownership. +legacy_reference_sha256 = "1baa90f95ee3d7a7f122840ebb8308cf1f55ed1c0549aa79cf6e880793156120" From 2a3ec4128b1095b9eab0b869dc81f57d3f98f580 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 06:01:21 +0000 Subject: [PATCH 431/464] chore(gate): reconcile exact source identities --- config/gate.toml | 2 +- tests/citadel/cargo_lock_identity.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index 38a8b70bc..759a83e72 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -1116,7 +1116,7 @@ must_stay_below_lines = 3000 "crates/capsem-service/src/vm_files.rs" = 1031 "crates/capsem/src/main.rs" = 2025 "crates/capsem/src/tests.rs" = 1259 -"crates/capsem/src/update.rs" = 2143 +"crates/capsem/src/update.rs" = 2142 "crates/capsem/src/update/asset_install.rs" = 1125 "crates/capsem/src/update/tests.rs" = 2306 diff --git a/tests/citadel/cargo_lock_identity.toml b/tests/citadel/cargo_lock_identity.toml index 765465f5d..caa0e9884 100644 --- a/tests/citadel/cargo_lock_identity.toml +++ b/tests/citadel/cargo_lock_identity.toml @@ -7,5 +7,5 @@ # each moved the lock. `test_cargo_lock_changes_are_deliberate` fails until # this record matches the lock again, so the cost is chosen, reviewed and # batched rather than discovered twenty minutes into a gate. -sha256 = "9c77a9aa03c2ef6a6537b0903ea60c0bc7b3ac62c4ee0ca9b2511237d9cea58e" -reason = "The merged 0.7 line centralizes cryptographic dependencies in capsem-crypto while #206 adds isolated ledger and archive codecs; the host and sealed guest cohort will be rebuilt and qualified together." +sha256 = "31b8050cdf30b4c50c0365fa235a0bb6840e18b6cf40e48edefd1b7e9a9c2bca" +reason = "The unified observable cache adds quick_cache 0.7.0. The 0.7 crypto ownership retrofit centralizes first-party hashes, entropy, TLS keys, JWT verification and secret primitives; BLAKE2b/BLAKE2s SIMD 1.0.5 support normal-harness comparisons. Managed-session APIs add zeroizing request capabilities, and Rust managed scopes add SDK edges to existing capsem-crypto random/secrets, futures, humantime and tokio-util rt. Central runtime-limit generation adds capsem-proto's direct toml edge, and exact benchmark reporting adds capsem-bench's direct capsem-proto edge. The host and sealed guest cohort will be rebuilt and qualified together." From 748b90a0f316e106625a480119ca0a37920ad5af Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 06:12:00 +0000 Subject: [PATCH 432/464] test(proxy): enforce capacity invariant at compile time --- crates/capsem-proxy/src/tests.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index b2d334f91..3d03b2735 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -69,10 +69,13 @@ fn each_traffic_capability_keeps_its_full_bounded_capacity() { #[test] fn http_capacity_covers_active_idle_and_retiring_connections() { - assert!( - UPSTREAM_GRANT_CAPACITY - capsem_proto::NETWORK_CLIENT_CAPACITY >= capsem_proto::NETWORK_CLIENT_CAPACITY * 2, - "traffic capacity must reserve idle and retiring grants for every active client" - ); + const { + assert!( + UPSTREAM_GRANT_CAPACITY - capsem_proto::NETWORK_CLIENT_CAPACITY + >= capsem_proto::NETWORK_CLIENT_CAPACITY * 2, + "traffic capacity must reserve idle and retiring grants for every active client" + ); + } let active = (1..UPSTREAM_GRANT_CAPACITY as u64).collect::>(); let grants = BTreeMap::from([(ProxyCapability::HttpTraffic, active.clone())]); let grant_ids = active.into_iter().collect::>(); From e292cbc318f047b8008ee6d9b32387b2abaacc2c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 06:12:01 +0000 Subject: [PATCH 433/464] perf(service): release commitment locks promptly --- crates/capsem-service/src/ledger_commitment.rs | 1 + crates/capsem-service/src/ledger_commitment/tests.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/capsem-service/src/ledger_commitment.rs b/crates/capsem-service/src/ledger_commitment.rs index 5fa4ce461..498c598f8 100644 --- a/crates/capsem-service/src/ledger_commitment.rs +++ b/crates/capsem-service/src/ledger_commitment.rs @@ -157,6 +157,7 @@ impl CommitmentAuthority { state.next_global_sequence = global_sequence; state.global_sequences.extend(global_sequences.iter().copied()); state.pending.entry(key).or_default().extend(commitments); + drop(state); Ok(global_sequences) } diff --git a/crates/capsem-service/src/ledger_commitment/tests.rs b/crates/capsem-service/src/ledger_commitment/tests.rs index 7b853e475..1a15f7687 100644 --- a/crates/capsem-service/src/ledger_commitment/tests.rs +++ b/crates/capsem-service/src/ledger_commitment/tests.rs @@ -734,4 +734,5 @@ async fn reservation_batch_extends_one_trusted_tail_atomically() { assert_eq!(pending[0].producer_sequence(), 1); assert_eq!(pending[1].producer_sequence(), 2); assert_eq!(pending[1].previous_hash(), pending[0].commitment_hash()); + drop(state); } From ba38f9ff5522875879181f599cc09c38db950e9a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 06:51:14 +0000 Subject: [PATCH 434/464] fix(foundation): drain unrelated watch events --- .../capsem-foundation/src/unix/change_watch.rs | 8 +++++++- .../src/unix/change_watch/tests.rs | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/capsem-foundation/src/unix/change_watch.rs b/crates/capsem-foundation/src/unix/change_watch.rs index da49ebafd..e88ea47d2 100644 --- a/crates/capsem-foundation/src/unix/change_watch.rs +++ b/crates/capsem-foundation/src/unix/change_watch.rs @@ -17,6 +17,9 @@ use std::{ #[cfg(target_os = "linux")] use nix::sys::inotify::{AddWatchFlags, InitFlags, Inotify, WatchDescriptor}; + +#[cfg(target_os = "linux")] +const MAX_INOTIFY_DRAIN_BATCHES: usize = 64; #[cfg(target_os = "macos")] use nix::sys::{ event::{EventFilter, EventFlag, FilterFlag, KEvent, Kqueue}, @@ -211,7 +214,10 @@ impl ChangeWatch { #[cfg(target_os = "linux")] fn read_change(&self) -> io::Result { - for _ in 0..2 { + // Drain ordinary bursts completely so unrelated sibling churn cannot + // create a false invalidation. Retain a hard bound for a producer that + // continuously outruns polling; exhaustion remains conservative. + for _ in 0..MAX_INOTIFY_DRAIN_BATCHES { match self.watcher.read_events() { Ok(events) => { if events.is_empty() { diff --git a/crates/capsem-foundation/src/unix/change_watch/tests.rs b/crates/capsem-foundation/src/unix/change_watch/tests.rs index 75977a797..643e653e3 100644 --- a/crates/capsem-foundation/src/unix/change_watch/tests.rs +++ b/crates/capsem-foundation/src/unix/change_watch/tests.rs @@ -107,6 +107,23 @@ fn ancestor_notification_backlog_cannot_hide_rebinding() { assert!(watch.changed().unwrap()); } +#[cfg(target_os = "linux")] +#[test] +fn unrelated_ancestor_backlog_does_not_invalidate_the_binding() { + let parent = namespace_fixture(); + let path = parent.path().join("ancestor/cache"); + std::fs::create_dir_all(&path).unwrap(); + let mut watch = ChangeWatch::new().unwrap(); + let _held = watch.open_directory(&path).unwrap(); + for index in 0..400 { + std::fs::write(parent.path().join(format!("unrelated-{index}")), b"sibling").unwrap(); + } + assert!( + !watch.changed().unwrap(), + "a bounded burst of unrelated siblings must be fully drained" + ); +} + #[test] fn read_only_access_is_quiet_but_hardlink_writes_are_sticky() { let root = tempfile::tempdir().unwrap(); From 6f3a924bfaceaa320c6a3fbaa82d07fa4583a42f Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 06:54:07 +0000 Subject: [PATCH 435/464] test(core): retain coverage output for owner child --- .../src/session/spawn_owner/tests.rs | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/capsem-core/src/session/spawn_owner/tests.rs b/crates/capsem-core/src/session/spawn_owner/tests.rs index 7670eac17..43fb83dcc 100644 --- a/crates/capsem-core/src/session/spawn_owner/tests.rs +++ b/crates/capsem-core/src/session/spawn_owner/tests.rs @@ -5,6 +5,28 @@ fn new_binding() -> VmBinding { VmBinding::new("owner-vm".into(), capsem_crypto::uuid_v4()).unwrap() } +fn isolated_owner_command( + program: impl AsRef, + coverage_profile: Option<&std::ffi::OsStr>, +) -> std::process::Command { + let mut command = std::process::Command::new(program); + command.env_clear(); + if let Some(profile) = coverage_profile { + command.env("LLVM_PROFILE_FILE", profile); + } + command +} + +#[test] +fn isolated_owner_child_preserves_only_the_coverage_output_path() { + let profile = std::ffi::OsStr::new("/tmp/capsem-profile-%p-%m.profraw"); + let command = isolated_owner_command("true", Some(profile)); + assert_eq!( + command.get_envs().collect::>(), + [(std::ffi::OsStr::new("LLVM_PROFILE_FILE"), Some(profile))] + ); +} + #[test] fn spawn_owner_records_original_pid_and_excludes_duplicate_boot_or_recovery() { let root = tempfile::tempdir().unwrap(); @@ -135,14 +157,14 @@ fn spawn_owner_subprocess_holds_native_lifetime_until_actual_exit() { let binding = new_binding(); prepare(directory.path(), &binding, std::process::id()).unwrap(); let root = ContainedDir::open_root(directory.path()).unwrap(); + let coverage_profile = std::env::var_os("LLVM_PROFILE_FILE"); let mut child = Child( - std::process::Command::new(std::env::current_exe().unwrap()) + isolated_owner_command(std::env::current_exe().unwrap(), coverage_profile.as_deref()) .args([ "--exact", "session::spawn_owner::tests::spawn_owner_subprocess_fixture", "--nocapture", ]) - .env_clear() .env("CAPSEM_TEST_OWNER_ROOT", directory.path()) .env("CAPSEM_TEST_OWNER_BINDING", serde_json::to_string(&binding).unwrap()) .stdin(std::process::Stdio::piped()) From 46a45cfeb8b2dbab5621566918428aae47961674 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 07:22:24 +0000 Subject: [PATCH 436/464] test(service): preserve failures through owned child exit --- .../src/session_cleanup/tests.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/capsem-service/src/session_cleanup/tests.rs b/crates/capsem-service/src/session_cleanup/tests.rs index 08c23f259..62b50fdef 100644 --- a/crates/capsem-service/src/session_cleanup/tests.rs +++ b/crates/capsem-service/src/session_cleanup/tests.rs @@ -94,7 +94,23 @@ async fn failed_session_route_preserves_runtime_ipc_evidence() { let session_dir = state.run_dir.join("sessions").join(id); std::fs::create_dir_all(&session_dir).unwrap(); std::fs::write(session_dir.join("process.log"), b"process channel closed during doctor").unwrap(); - insert_fake_instance_with_session_dir(&state, id, 0, session_dir.clone()); + let child = crate::tests::isolated_test_command("sleep") + .arg("30") + .kill_on_drop(true) + .spawn() + .unwrap(); + insert_fake_instance_with_session_dir(&state, id, child.id().unwrap(), session_dir.clone()); + let generation = state.instances.lock().unwrap().get(id).unwrap().generation; + let retirement = state.retirements.register(id, generation).unwrap(); + let reaper = crate::instance_reaper::spawn_exit_reaper( + child, + id.into(), + id.into(), + Arc::clone(&state), + state.instance_socket_path(id).unwrap(), + session_dir.clone(), + retirement, + ); let (status, body) = route_request( build_service_router(Arc::clone(&state)), @@ -103,6 +119,7 @@ async fn failed_session_route_preserves_runtime_ipc_evidence() { None, ) .await; + reaper.await.unwrap(); assert_eq!(status, StatusCode::OK, "{body}"); assert_eq!(body["success"], true); From b932a57bc26bdb757179e31d85cdb65452386122 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 07:23:56 +0000 Subject: [PATCH 437/464] test(service): retain recovery child coverage output --- crates/capsem-service/src/managed_sessions/tests.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/capsem-service/src/managed_sessions/tests.rs b/crates/capsem-service/src/managed_sessions/tests.rs index 63ae028c9..37bf5c9a5 100644 --- a/crates/capsem-service/src/managed_sessions/tests.rs +++ b/crates/capsem-service/src/managed_sessions/tests.rs @@ -226,13 +226,12 @@ async fn recovery_fences_preboot_and_awaits_original_native_exit_before_closing( ); if admitted { let mut child = Child( - std::process::Command::new(std::env::current_exe().unwrap()) + crate::tests::isolated_std_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "managed_sessions::tests::recovery_native_child_fixture", "--nocapture", ]) - .env_clear() .env("CAPSEM_TEST_RECOVERY_ROOT", &session) .env("CAPSEM_TEST_RECOVERY_BINDING", serde_json::to_string(&binding).unwrap()) .stdin(std::process::Stdio::piped()) From 6442187ba3dc62ce564835471ab89a8df903e78c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 07:24:28 +0000 Subject: [PATCH 438/464] test(foundation): retain scope child coverage output --- crates/capsem-foundation/src/unix/process/scope/tests.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/capsem-foundation/src/unix/process/scope/tests.rs b/crates/capsem-foundation/src/unix/process/scope/tests.rs index 0adf3f20e..21f367955 100644 --- a/crates/capsem-foundation/src/unix/process/scope/tests.rs +++ b/crates/capsem-foundation/src/unix/process/scope/tests.rs @@ -37,12 +37,11 @@ fn native_process_scope_child_fixture() { #[test] fn native_process_scope_matches_a_real_child() { let scope = ProcessScope::current().unwrap(); - let status = std::process::Command::new(std::env::current_exe().unwrap()) + let status = crate::isolated_test_command(std::env::current_exe().unwrap()) .args([ "--exact", "unix::process::scope::tests::native_process_scope_child_fixture", ]) - .env_clear() .env("CAPSEM_TEST_PROCESS_SCOPE", serde_json::to_string(&scope).unwrap()) .stdin(std::process::Stdio::null()) .status() From 2df4c20f204d758f72319eae1ddf507391d3fde2 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 07:26:06 +0000 Subject: [PATCH 439/464] test(service): retain retirement child coverage output --- crates/capsem-service/src/instance/tests.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/capsem-service/src/instance/tests.rs b/crates/capsem-service/src/instance/tests.rs index b7245f6fe..7eb0c32af 100644 --- a/crates/capsem-service/src/instance/tests.rs +++ b/crates/capsem-service/src/instance/tests.rs @@ -89,14 +89,13 @@ async fn normal_native_exit_persists_retirement_before_next_host_boot_resume() { let generation = capsem_crypto::uuid_v4(); let binding = capsem_core::managed_sessions::VmBinding::new("receipt-vm".into(), generation).unwrap(); persist_spawn_identity(&session, binding.id(), generation).unwrap(); - let mut command = tokio::process::Command::new(std::env::current_exe().unwrap()); + let mut command = crate::tests::isolated_test_command(std::env::current_exe().unwrap()); command .args([ "--exact", "managed_sessions::tests::recovery_native_child_fixture", "--nocapture", ]) - .env_clear() .env("CAPSEM_TEST_RECOVERY_ROOT", &session) .env("CAPSEM_TEST_RECOVERY_BINDING", serde_json::to_string(&binding).unwrap()) .stdin(std::process::Stdio::piped()) From e0b41ac0f4c370bc9f114554e159378dbdf5a719 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 07:53:23 +0000 Subject: [PATCH 440/464] test(service): stop fixtures own their child exits --- .../capsem-service/src/tests/stop_cleanup.rs | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/capsem-service/src/tests/stop_cleanup.rs b/crates/capsem-service/src/tests/stop_cleanup.rs index 94f776b55..965775ba6 100644 --- a/crates/capsem-service/src/tests/stop_cleanup.rs +++ b/crates/capsem-service/src/tests/stop_cleanup.rs @@ -2,6 +2,26 @@ use super::*; +fn insert_owned_instance(state: &Arc, id: &str, session_dir: PathBuf) -> tokio::task::JoinHandle<()> { + let child = crate::tests::isolated_test_command("sleep") + .arg("30") + .kill_on_drop(true) + .spawn() + .unwrap(); + insert_fake_instance_with_session_dir(state, id, child.id().unwrap(), session_dir.clone()); + let generation = state.instances.lock().unwrap().get(id).unwrap().generation; + let retirement = state.retirements.register(id, generation).unwrap(); + crate::instance_reaper::spawn_exit_reaper( + child, + id.into(), + id.into(), + Arc::clone(state), + state.instance_socket_path(id).unwrap(), + session_dir, + retirement, + ) +} + #[tokio::test] async fn ephemeral_stop_propagates_session_directory_cleanup_failure() { let (state, _dir) = make_test_state_with_tempdir(); @@ -11,7 +31,7 @@ async fn ephemeral_stop_propagates_session_directory_cleanup_failure() { // A real filesystem error, independent of scheduling and effective uid: // this session's directory has been replaced by a regular file. std::fs::write(&session_dir, b"unremoved session state").unwrap(); - insert_fake_instance_with_session_dir(&state, id, 0, session_dir.clone()); + let reaper = insert_owned_instance(&state, id, session_dir.clone()); let (status, body) = route_request( build_service_router(Arc::clone(&state)), @@ -20,6 +40,7 @@ async fn ephemeral_stop_propagates_session_directory_cleanup_failure() { None, ) .await; + reaper.await.unwrap(); assert_eq!( status, @@ -44,7 +65,7 @@ async fn ephemeral_stop_uses_the_session_deletion_owner() { let session_dir = state.run_dir.join("sessions").join(id); std::fs::create_dir_all(session_dir.parent().unwrap()).unwrap(); std::os::unix::fs::symlink(&outside, &session_dir).unwrap(); - insert_fake_instance_with_session_dir(&state, id, 0, session_dir.clone()); + let reaper = insert_owned_instance(&state, id, session_dir.clone()); let (status, body) = route_request( build_service_router(Arc::clone(&state)), @@ -53,6 +74,7 @@ async fn ephemeral_stop_uses_the_session_deletion_owner() { None, ) .await; + reaper.await.unwrap(); assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}"); assert!(std::fs::symlink_metadata(&session_dir) @@ -74,7 +96,7 @@ async fn stop_success_removes_ephemeral_state_and_preserves_persistent_state() { let session_dir = state.run_dir.join("sessions").join(id); std::fs::create_dir_all(session_dir.join("guest")).unwrap(); std::fs::write(session_dir.join("guest/marker"), b"session state").unwrap(); - insert_fake_instance_with_session_dir(&state, id, 0, session_dir.clone()); + let reaper = insert_owned_instance(&state, id, session_dir.clone()); state.instances.lock().unwrap().get_mut(id).unwrap().persistent = persistent; let (status, body) = route_request( @@ -84,6 +106,7 @@ async fn stop_success_removes_ephemeral_state_and_preserves_persistent_state() { None, ) .await; + reaper.await.unwrap(); assert_eq!(status, StatusCode::OK, "{body}"); assert_eq!(body, json!({"success": true, "persistent": persistent})); From b7d291404a2e364cb10025a9b6cab20183aa43f4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 08:16:46 +0000 Subject: [PATCH 441/464] test(gate): ratchet improved Rust coverage --- config/gate.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/gate.toml b/config/gate.toml index 759a83e72..f3a83a690 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2425,7 +2425,7 @@ rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent # macOS compiles a different cfg-selected line inventory. Its stronger floors # must not raise Linux thresholds using code that Linux cannot compile or test. # Overrides may only increase the shared floors; no platform gets an exemption. -rust_coverage_platform_crate_floors = { Linux = { capsem = 61.69, capsem-agent = 53.91, capsem-api = 95.84, capsem-assets = 93.38, capsem-bench = 49.89, capsem-config = 90.19, capsem-core = 81.55, capsem-foundation = 81.75, capsem-gateway = 82.24, capsem-logger = 92.39, capsem-mcp-aggregator = 64.06, capsem-mcp-builtin = 57.06, capsem-mock-server = 71.1, capsem-network = 98.0, capsem-process = 49.39, capsem-proto = 90.98, capsem-router = 80.52, capsem-sdk = 97.39, capsem-service = 75.97, capsem-tray = 57.4, capsem-admin = 75.66 }, Darwin = { capsem = 51.27, capsem-agent = 53.13, capsem-api = 95.34, capsem-assets = 90.3, capsem-bench = 50.56, capsem-config = 89.69, capsem-core = 81.0, capsem-foundation = 83.8, capsem-logger = 91.88, capsem-mcp-builtin = 56.56, capsem-mock-server = 58.6, capsem-network = 97.5, capsem-process = 48.12, capsem-proto = 91.54, capsem-router = 79.8, capsem-service = 74.73 } } +rust_coverage_platform_crate_floors = { Linux = { capsem = 61.69, capsem-agent = 53.91, capsem-api = 95.84, capsem-assets = 93.38, capsem-bench = 55.85, capsem-config = 90.19, capsem-core = 81.60, capsem-foundation = 81.75, capsem-gateway = 82.24, capsem-logger = 92.39, capsem-mcp-aggregator = 64.06, capsem-mcp-builtin = 57.06, capsem-mock-server = 71.1, capsem-network = 98.0, capsem-process = 49.39, capsem-proto = 90.98, capsem-router = 80.52, capsem-sdk = 97.39, capsem-service = 75.97, capsem-tray = 57.4, capsem-admin = 75.66 }, Darwin = { capsem = 51.27, capsem-agent = 53.13, capsem-api = 95.34, capsem-assets = 90.3, capsem-bench = 50.56, capsem-config = 89.69, capsem-core = 81.0, capsem-foundation = 83.8, capsem-logger = 91.88, capsem-mcp-builtin = 56.56, capsem-mock-server = 58.6, capsem-network = 97.5, capsem-process = 48.12, capsem-proto = 91.54, capsem-router = 79.8, capsem-service = 74.73 } } # Nextest reads its profile from the environment; `--profile` on the command # line belongs to cargo-llvm-cov and would select a build profile instead. rust_test_profile_variable = "NEXTEST_PROFILE" From c773c996d64f4526cdbc55837b8f6c004ff49f84 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 09:19:13 +0000 Subject: [PATCH 442/464] fix(oci): bootstrap direct fixture launcher --- tests/fixtures/oci/pinned_image.py | 5 ++++- tests/fixtures/oci/test_pinned_image.py | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/oci/pinned_image.py b/tests/fixtures/oci/pinned_image.py index cfff159fd..8b2094d67 100644 --- a/tests/fixtures/oci/pinned_image.py +++ b/tests/fixtures/oci/pinned_image.py @@ -36,10 +36,13 @@ import urllib.request from pathlib import Path +ROOT = Path(__file__).resolve().parents[3] +if __package__ in (None, ""): + sys.path.insert(0, str(ROOT)) + from capsem_builder.cache.config import load_paths from capsem_builder.gate import config as gate_config -ROOT = Path(__file__).resolve().parents[3] MANIFEST_TYPES = ( "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json", diff --git a/tests/fixtures/oci/test_pinned_image.py b/tests/fixtures/oci/test_pinned_image.py index b7637843b..dfa8aa014 100644 --- a/tests/fixtures/oci/test_pinned_image.py +++ b/tests/fixtures/oci/test_pinned_image.py @@ -3,6 +3,9 @@ import hashlib import json import ssl +import subprocess +import sys +from pathlib import Path import pytest @@ -12,6 +15,25 @@ MANIFEST = "application/vnd.oci.image.manifest.v1+json" +def test_direct_launcher_can_import_repository_fixtures(tmp_path): + script = Path(__file__).with_name("pinned_image.py") + probe = ( + "import runpy; " + f"runpy.run_path({str(script)!r}); " + "import tests.fixtures.oci.published_root" + ) + + launched = subprocess.run( + [sys.executable, "-I", "-c", probe], + cwd=tmp_path, + capture_output=True, + text=True, + check=False, + ) + + assert launched.returncode == 0, launched.stderr + + def _layout(root, *, architecture="arm64", media=MANIFEST): """A one-layer OCI layout; returns (layout, manifest digest, blobs by digest).""" blobs = {} From bfdd068d3bc497a6a85121126ea5c6b3198498d4 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 09:19:13 +0000 Subject: [PATCH 443/464] fix(gate): forget retired artifact digests --- build_system/builder/gate/faultrules.py | 14 ++++- build_system/builder/gate/observation.py | 3 +- .../tests/gate/test_gate_observation.py | 56 +++++++++++++++++++ 3 files changed, 70 insertions(+), 3 deletions(-) diff --git a/build_system/builder/gate/faultrules.py b/build_system/builder/gate/faultrules.py index 37e433cbd..8509c7e7f 100644 --- a/build_system/builder/gate/faultrules.py +++ b/build_system/builder/gate/faultrules.py @@ -64,15 +64,25 @@ def judge(watch: Watch, event: Event) -> None: event, "over-permission", f"mode {event.mode:04o} is writable beyond its owner" ) + previous_digest = watch._path_digests.pop(event.path, None) + if previous_digest is not None: + previous_paths = watch._digests[previous_digest] + previous_paths.discard(event.path) + if not previous_paths: + del watch._digests[previous_digest] + exempt = duplication_expected( event.path, watch._source_root, (*watch._duplicate_exempt, *watch._source_replicas), ) if event.digest is not None and not exempt: - first = watch._digests.setdefault(event.digest, event.path) - if first != event.path and event.inode is not None: + matches = watch._digests.setdefault(event.digest, set()) + if matches and event.inode is not None: + first = min(matches, key=str) watch.fault(event, "duplicate-content", f"identical bytes already at {first}") + matches.add(event.path) + watch._path_digests[event.path] = event.digest if event.attribution == "exact" and len(event.steps) >= 2 and not source: shared = set.intersection( diff --git a/build_system/builder/gate/observation.py b/build_system/builder/gate/observation.py index 106bb4e6b..b98990918 100644 --- a/build_system/builder/gate/observation.py +++ b/build_system/builder/gate/observation.py @@ -95,7 +95,8 @@ def __init__( self._observer: BaseObserver | None = None self._metadata_survey: MetadataSurvey | None = None self._modes: dict[Path, list[int]] = {} - self._digests: dict[str, Path] = {} + self._digests: dict[str, set[Path]] = {} + self._path_digests: dict[Path, str] = {} self._reported: set[tuple[Path, str]] = set() self._refusal: str | None = None self._source_stamps: dict[Path, tuple[int, ...]] = {} diff --git a/build_system/tests/gate/test_gate_observation.py b/build_system/tests/gate/test_gate_observation.py index 5d07a9f90..5f66fe694 100644 --- a/build_system/tests/gate/test_gate_observation.py +++ b/build_system/tests/gate/test_gate_observation.py @@ -478,6 +478,62 @@ def test_identical_bytes_under_two_names_are_named(tmp_path: Path) -> None: assert "duplicate-content" in {fault.reason for fault in watch.faults} +def test_deleted_content_is_not_compared_with_a_later_artifact(tmp_path: Path) -> None: + watch = Watch([], source_root=tmp_path) + first = tmp_path / "cache" / "target" / "first" + second = tmp_path / "cache" / "target" / "second" + + watch._judge( + Event( + at=1.0, + kind="created", + path=first, + steps=(), + facts=Facts(inode=1, digest="deadbeef"), + ) + ) + watch._judge(Event(at=2.0, kind="deleted", path=first, steps=())) + watch._judge( + Event( + at=3.0, + kind="created", + path=second, + steps=(), + facts=Facts(inode=2, digest="deadbeef"), + ) + ) + + assert "duplicate-content" not in {fault.reason for fault in watch.faults} + + +def test_replaced_content_releases_its_old_digest(tmp_path: Path) -> None: + watch = Watch([], source_root=tmp_path) + replaced = tmp_path / "cache" / "target" / "replaced" + later = tmp_path / "cache" / "target" / "later" + + for at, digest in ((1.0, "old"), (2.0, "new")): + watch._judge( + Event( + at=at, + kind="modified", + path=replaced, + steps=(), + facts=Facts(inode=1, digest=digest), + ) + ) + watch._judge( + Event( + at=3.0, + kind="created", + path=later, + steps=(), + facts=Facts(inode=2, digest="old"), + ) + ) + + assert "duplicate-content" not in {fault.reason for fault in watch.faults} + + def test_copying_a_hardlinked_alias_tree_is_a_real_duplicate_fault(tmp_path: Path) -> None: """Canonical/hash aliases are zero-copy until a tree copy breaks them. From f33ae2763f0e95256f37d3e4f6f205680c589c38 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 09:27:34 +0000 Subject: [PATCH 444/464] fix(oci): authenticate private GHCR fixture pulls --- tests/fixtures/oci/pinned_image.py | 28 ++++++++++++++++++++-- tests/fixtures/oci/test_pinned_image.py | 32 +++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/fixtures/oci/pinned_image.py b/tests/fixtures/oci/pinned_image.py index 8b2094d67..76f62bc7f 100644 --- a/tests/fixtures/oci/pinned_image.py +++ b/tests/fixtures/oci/pinned_image.py @@ -22,6 +22,7 @@ from __future__ import annotations import argparse +import base64 import functools import hashlib import json @@ -42,6 +43,7 @@ from capsem_builder.cache.config import load_paths from capsem_builder.gate import config as gate_config +from capsem_builder.release.tools.fetch_channel_source_manifest import github_token MANIFEST_TYPES = ( "application/vnd.oci.image.manifest.v1+json", @@ -114,8 +116,17 @@ def __init__(self, context: ssl.SSLContext | None = None) -> None: _NoCrossHostAuth, urllib.request.HTTPSHandler(context=context) ) - def open(self, url: str, token: str | None = None, accept: str | None = None): + def open( + self, + url: str, + token: str | None = None, + accept: str | None = None, + basic: tuple[str, str] | None = None, + ): headers = {"Authorization": f"Bearer {token}"} if token else {} + if basic is not None: + encoded = base64.b64encode(f"{basic[0]}:{basic[1]}".encode()).decode() + headers["Authorization"] = f"Basic {encoded}" if accept: headers["Accept"] = accept return self._opener.open(urllib.request.Request(url, headers=headers), timeout=TIMEOUT) @@ -134,7 +145,20 @@ def _token(client: _Registry, registry: str, name: str) -> str | None: query = urllib.parse.urlencode( {"service": fields.get("service", registry), "scope": f"repository:{name}:pull"} ) - with client.open(f"{fields['realm']}?{query}") as response: + token_url = f"{fields['realm']}?{query}" + try: + response = client.open(token_url) + except urllib.error.HTTPError as error: + realm = urllib.parse.urlsplit(fields.get("realm", "")) + if error.code != 403 or registry != "ghcr.io" or realm != urllib.parse.urlsplit( + "https://ghcr.io/token" + ): + raise + credential = github_token() + if credential is None: + raise + response = client.open(token_url, basic=("x-access-token", credential)) + with response: body = json.load(response) return body.get("token") or body["access_token"] diff --git a/tests/fixtures/oci/test_pinned_image.py b/tests/fixtures/oci/test_pinned_image.py index dfa8aa014..62444a176 100644 --- a/tests/fixtures/oci/test_pinned_image.py +++ b/tests/fixtures/oci/test_pinned_image.py @@ -1,14 +1,18 @@ """capsem-debug's fixture: only the pinned bytes are ever served.""" import hashlib +import io import json import ssl import subprocess import sys +import urllib.error +from email.message import Message from pathlib import Path import pytest +from tests.fixtures.oci import pinned_image from tests.fixtures.oci.pinned_image import debug_image, reference_image from tests.fixtures.oci.registry import serve @@ -34,6 +38,34 @@ def test_direct_launcher_can_import_repository_fixtures(tmp_path): assert launched.returncode == 0, launched.stderr +def test_private_ghcr_pin_uses_the_existing_github_login(monkeypatch): + challenge = Message() + challenge["WWW-Authenticate"] = ( + 'Bearer realm="https://ghcr.io/token",service="ghcr.io"' + ) + + class Client: + def __init__(self): + self.calls = [] + + def open(self, url, token=None, accept=None, basic=None): + self.calls.append((url, basic)) + if url == "https://ghcr.io/v2/": + raise urllib.error.HTTPError(url, 401, "unauthorized", challenge, None) + if basic is None: + raise urllib.error.HTTPError(url, 403, "private", Message(), None) + return io.BytesIO(b'{"token":"registry-token"}') + + client = Client() + monkeypatch.setattr(pinned_image, "github_token", lambda: "github-token") + + assert pinned_image._token(client, "ghcr.io", "google/capsem/dev") == "registry-token" + assert client.calls[-1] == ( + "https://ghcr.io/token?service=ghcr.io&scope=repository%3Agoogle%2Fcapsem%2Fdev%3Apull", + ("x-access-token", "github-token"), + ) + + def _layout(root, *, architecture="arm64", media=MANIFEST): """A one-layer OCI layout; returns (layout, manifest digest, blobs by digest).""" blobs = {} From 79a9b5a2766b4dfb25489d64813a71500a3750fe Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 09:34:51 +0000 Subject: [PATCH 445/464] style(oci): preserve launcher import ordering --- tests/fixtures/oci/pinned_image.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fixtures/oci/pinned_image.py b/tests/fixtures/oci/pinned_image.py index 76f62bc7f..acb903081 100644 --- a/tests/fixtures/oci/pinned_image.py +++ b/tests/fixtures/oci/pinned_image.py @@ -37,14 +37,14 @@ import urllib.request from pathlib import Path -ROOT = Path(__file__).resolve().parents[3] -if __package__ in (None, ""): - sys.path.insert(0, str(ROOT)) - from capsem_builder.cache.config import load_paths from capsem_builder.gate import config as gate_config from capsem_builder.release.tools.fetch_channel_source_manifest import github_token +ROOT = Path(__file__).resolve().parents[3] +if __package__ in (None, ""): + sys.path.insert(0, str(ROOT)) + MANIFEST_TYPES = ( "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json", From 8a9a0e78d72fb8246cbcb4ff6209ade5c62ed51a Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 09:37:44 +0000 Subject: [PATCH 446/464] test(oci): type private registry fixture --- tests/fixtures/oci/test_pinned_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fixtures/oci/test_pinned_image.py b/tests/fixtures/oci/test_pinned_image.py index 62444a176..390b3f085 100644 --- a/tests/fixtures/oci/test_pinned_image.py +++ b/tests/fixtures/oci/test_pinned_image.py @@ -44,7 +44,7 @@ def test_private_ghcr_pin_uses_the_existing_github_login(monkeypatch): 'Bearer realm="https://ghcr.io/token",service="ghcr.io"' ) - class Client: + class Client(pinned_image._Registry): def __init__(self): self.calls = [] From 6412c2acfcd7b76a8eb3a106800cb7b0cb6a2ae1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 10:01:02 +0000 Subject: [PATCH 447/464] test(gate): clear core coverage headroom --- config/gate.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/gate.toml b/config/gate.toml index f3a83a690..1214b845f 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2425,7 +2425,7 @@ rust_coverage_crate_floors = { capsem = 50.0, capsem-admin = 71.0, capsem-agent # macOS compiles a different cfg-selected line inventory. Its stronger floors # must not raise Linux thresholds using code that Linux cannot compile or test. # Overrides may only increase the shared floors; no platform gets an exemption. -rust_coverage_platform_crate_floors = { Linux = { capsem = 61.69, capsem-agent = 53.91, capsem-api = 95.84, capsem-assets = 93.38, capsem-bench = 55.85, capsem-config = 90.19, capsem-core = 81.60, capsem-foundation = 81.75, capsem-gateway = 82.24, capsem-logger = 92.39, capsem-mcp-aggregator = 64.06, capsem-mcp-builtin = 57.06, capsem-mock-server = 71.1, capsem-network = 98.0, capsem-process = 49.39, capsem-proto = 90.98, capsem-router = 80.52, capsem-sdk = 97.39, capsem-service = 75.97, capsem-tray = 57.4, capsem-admin = 75.66 }, Darwin = { capsem = 51.27, capsem-agent = 53.13, capsem-api = 95.34, capsem-assets = 90.3, capsem-bench = 50.56, capsem-config = 89.69, capsem-core = 81.0, capsem-foundation = 83.8, capsem-logger = 91.88, capsem-mcp-builtin = 56.56, capsem-mock-server = 58.6, capsem-network = 97.5, capsem-process = 48.12, capsem-proto = 91.54, capsem-router = 79.8, capsem-service = 74.73 } } +rust_coverage_platform_crate_floors = { Linux = { capsem = 61.69, capsem-agent = 53.91, capsem-api = 95.84, capsem-assets = 93.38, capsem-bench = 55.85, capsem-config = 90.19, capsem-core = 81.61, capsem-foundation = 81.75, capsem-gateway = 82.24, capsem-logger = 92.39, capsem-mcp-aggregator = 64.06, capsem-mcp-builtin = 57.06, capsem-mock-server = 71.1, capsem-network = 98.0, capsem-process = 49.39, capsem-proto = 90.98, capsem-router = 80.52, capsem-sdk = 97.39, capsem-service = 75.97, capsem-tray = 57.4, capsem-admin = 75.66 }, Darwin = { capsem = 51.27, capsem-agent = 53.13, capsem-api = 95.34, capsem-assets = 90.3, capsem-bench = 50.56, capsem-config = 89.69, capsem-core = 81.0, capsem-foundation = 83.8, capsem-logger = 91.88, capsem-mcp-builtin = 56.56, capsem-mock-server = 58.6, capsem-network = 97.5, capsem-process = 48.12, capsem-proto = 91.54, capsem-router = 79.8, capsem-service = 74.73 } } # Nextest reads its profile from the environment; `--profile` on the command # line belongs to cargo-llvm-cov and would select a build profile instead. rust_test_profile_variable = "NEXTEST_PROFILE" From b0cf2838321f9ff3c4d595bbbadc59d8e2a68bbd Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 10:31:16 +0000 Subject: [PATCH 448/464] fix(gate): recognize mirrored architecture products --- build_system/builder/gate/faultrules.py | 9 +++-- build_system/builder/gate/faults.py | 18 ++++++++++ build_system/builder/gate/harnessschema.py | 2 ++ build_system/builder/gate/observation.py | 2 ++ build_system/builder/gate/observing.py | 1 + .../tests/gate/test_gate_observation.py | 36 +++++++++++++++++++ config/gate.toml | 10 ++++++ 7 files changed, 75 insertions(+), 3 deletions(-) diff --git a/build_system/builder/gate/faultrules.py b/build_system/builder/gate/faultrules.py index 8509c7e7f..3752474fa 100644 --- a/build_system/builder/gate/faultrules.py +++ b/build_system/builder/gate/faultrules.py @@ -16,7 +16,7 @@ from pathlib import Path from typing import TYPE_CHECKING -from .faults import duplication_expected +from .faults import duplication_expected, mirrored_paths #: The one fault reason a publishing run treats as fatal; `observing` reads it. SOURCE_TREE = "source-tree" @@ -78,8 +78,11 @@ def judge(watch: Watch, event: Event) -> None: ) if event.digest is not None and not exempt: matches = watch._digests.setdefault(event.digest, set()) - if matches and event.inode is not None: - first = min(matches, key=str) + unexpected = matches - mirrored_paths( + event.path, watch._source_root, watch._duplicate_mirrors + ) + if unexpected and event.inode is not None: + first = min(unexpected, key=str) watch.fault(event, "duplicate-content", f"identical bytes already at {first}") matches.add(event.path) watch._path_digests[event.path] = event.digest diff --git a/build_system/builder/gate/faults.py b/build_system/builder/gate/faults.py index de8516390..9e2921863 100644 --- a/build_system/builder/gate/faults.py +++ b/build_system/builder/gate/faults.py @@ -82,6 +82,24 @@ def duplication_expected(path: Path, root: Path, exempt: tuple[str, ...]) -> boo ) +def mirrored_paths( + path: Path, root: Path, mirrors: tuple[tuple[str, ...], ...] +) -> frozenset[Path]: + """Exact peer paths allowed to hold the same bytes in mirrored products.""" + try: + relative = path.resolve().relative_to(root) + except (ValueError, OSError): + return frozenset() + for group in mirrors: + for entry in group: + prefix = PurePosixPath(entry) + if relative.parts[: len(prefix.parts)] != prefix.parts: + continue + suffix = relative.parts[len(prefix.parts) :] + return frozenset((root / peer).joinpath(*suffix).resolve() for peer in group) + return frozenset() + + def is_source(path: Path, root: Path) -> bool: """Whether an absolute path is checked-in input rather than build output.""" # A relative path can come from an operation using dir_fd. Resolving it diff --git a/build_system/builder/gate/harnessschema.py b/build_system/builder/gate/harnessschema.py index 43411f8a9..1dd4ebc52 100644 --- a/build_system/builder/gate/harnessschema.py +++ b/build_system/builder/gate/harnessschema.py @@ -201,6 +201,8 @@ class RunLogConfig(Strict): observer_survey_seconds: PositiveFloat #: Trees where identical bytes are a third party's doing, not ours. duplicate_content_exempt: tuple[str, ...] + #: Product roots whose same relative files intentionally share source bytes. + duplicate_content_mirrors: tuple[tuple[str, ...], ...] #: Exact frozen source replicas: copied inputs, not authored artifacts. source_replica_roots: tuple[str, ...] #: How Linux names the path behind a file descriptor, so a `dir_fd`-relative diff --git a/build_system/builder/gate/observation.py b/build_system/builder/gate/observation.py index b98990918..cd70bf85c 100644 --- a/build_system/builder/gate/observation.py +++ b/build_system/builder/gate/observation.py @@ -71,6 +71,7 @@ def __init__( declared: Mapping[str, frozenset[str]] | None = None, on_fault: Callable[[Fault], None] | None = None, duplicate_content_exempt: Iterable[str] = (), + duplicate_content_mirrors: Iterable[tuple[str, ...]] = (), source_replica_roots: Iterable[str] = (), observer_timeout: float = 5.0, survey_interval: float = 1.0, @@ -83,6 +84,7 @@ def __init__( self._declared = dict(declared or {}) self._on_fault = on_fault self._duplicate_exempt = tuple(duplicate_content_exempt) + self._duplicate_mirrors = tuple(tuple(group) for group in duplicate_content_mirrors) self._source_replicas = tuple(source_replica_roots) self.faults: list[Fault] = [] self.events: list[Event] = [] diff --git a/build_system/builder/gate/observing.py b/build_system/builder/gate/observing.py index d7d4d6cf8..018776491 100644 --- a/build_system/builder/gate/observing.py +++ b/build_system/builder/gate/observing.py @@ -87,6 +87,7 @@ def record() -> None: declared=declared, on_fault=report, duplicate_content_exempt=config.runlog.duplicate_content_exempt, + duplicate_content_mirrors=config.runlog.duplicate_content_mirrors, source_replica_roots=config.runlog.source_replica_roots, observer_timeout=settings.observer_timeout_seconds, survey_interval=settings.observer_survey_seconds, diff --git a/build_system/tests/gate/test_gate_observation.py b/build_system/tests/gate/test_gate_observation.py index 5f66fe694..19c37c005 100644 --- a/build_system/tests/gate/test_gate_observation.py +++ b/build_system/tests/gate/test_gate_observation.py @@ -818,6 +818,42 @@ def test_the_exemption_does_not_blunt_the_rule_elsewhere(relative: str, tmp_path ) +def test_declared_mirror_roots_allow_only_matching_relative_files(tmp_path: Path) -> None: + """Architecture products may share inputs without hiding copies within one product.""" + arm = "cache/target/tests/ironbank/build-arm64/arm64" + x86 = "cache/target/tests/ironbank/build-x86_64/x86_64" + watch = Watch( + [], + source_root=tmp_path, + duplicate_content_mirrors=((arm, x86),), + ) + + for index, relative in enumerate((f"{arm}/staging/init", f"{x86}/staging/init")): + watch._judge( + Event( + at=1.0, + kind="created", + path=tmp_path / relative, + steps=("assets.pack-initrds",), + facts=Facts(inode=index + 1, digest="shared-input"), + ) + ) + + assert not [fault for fault in watch.faults if fault.reason == "duplicate-content"] + + watch._judge( + Event( + at=2.0, + kind="created", + path=tmp_path / arm / "staging" / "copied-init", + steps=("assets.pack-initrds",), + facts=Facts(inode=3, digest="shared-input"), + ) + ) + + assert [fault.reason for fault in watch.faults] == ["duplicate-content"] + + def test_a_declared_source_replica_is_not_judged_as_duplicate_or_empty_output( tmp_path: Path, ) -> None: diff --git a/config/gate.toml b/config/gate.toml index 1214b845f..405a8bcf7 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2923,6 +2923,16 @@ observer_survey_seconds = 1.0 # reported one filesystem fault, and a fault count that is never zero is a # fault count nobody reads. duplicate_content_exempt = ["crates/capsem-app/gen"] +# The dual-architecture asset proof builds one initrd product per architecture. +# Files at the same relative path may be architecture-independent inputs; a +# second copy anywhere else remains a fault. These exact product roots keep the +# observer useful without reporting every shared diagnostic and benchmark file. +duplicate_content_mirrors = [ + [ + "cache/target/tests/ironbank/build-arm64/arm64", + "cache/target/tests/ironbank/build-x86_64/x86_64", + ], +] # `source.record` intentionally copies every Git-visible byte into this exact # tree. Duplicate and empty bytes there are properties of checked-in input, # while hardlinks, mode churn, and concurrent writers remain faults. From f6d55e893c8e12680aab267f206c86995f8d57d6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 11:07:56 +0000 Subject: [PATCH 449/464] fix(cache): retain pinned fixture matrix --- build_system/tests/cache/test_budget.py | 13 +++++++++++++ config/cache.toml | 20 +++++++++----------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/build_system/tests/cache/test_budget.py b/build_system/tests/cache/test_budget.py index fe6b10070..aa814a82b 100644 --- a/build_system/tests/cache/test_budget.py +++ b/build_system/tests/cache/test_budget.py @@ -16,6 +16,7 @@ from capsem_builder.cache.budget import BudgetError, BudgetPolicy from capsem_builder.cache.config import load_policy from capsem_builder.cache.models import CachePolicy, CacheScope, PruneStrategy, StagePolicy +from capsem_builder.gate.config import load as load_gate_config ROOT = Path(__file__).resolve().parents[3] GIB = 1024**3 @@ -62,6 +63,18 @@ def test_the_checked_in_cargo_maximum_is_tens_of_gib() -> None: assert load_policy(ROOT).stages["cargo"].max_size_bytes <= 64 * GIB +def test_pinned_fixture_cache_retains_the_complete_platform_matrix() -> None: + """A gate needs both images locally before its network sandbox closes.""" + functional = load_gate_config(ROOT).functional + entries = sum( + len(image.digests) for image in (functional.debug_image, functional.reference_image) + ) + stage = load_policy(ROOT).stages["test-images"] + + assert stage.maximum_count is not None and stage.maximum_count >= entries + assert stage.warm_size_bytes >= entries * (3 * GIB // 2) + + def test_maxima_beyond_the_budget_are_refused_at_load() -> None: _policy(20 * GIB, 20 * GIB) # 40 + 10 <= 0.5 * 100 with pytest.raises(ValueError, match=r"cache maxima sum to 41\.0 GiB"): diff --git a/config/cache.toml b/config/cache.toml index 2e64be825..a4d9ad963 100644 --- a/config/cache.toml +++ b/config/cache.toml @@ -214,26 +214,24 @@ maximum_age_hours = 168 description = "Bounded diagnostics preserved from failed expensive tests." scope = "disk" path = "target/tests/evidence" -# Lowered from 4/8 GiB to fund `test-images` inside [budget]: the stage held -# 53 MB when measured on 2026-10-04, so 6 GiB still bounds a run of failures. -warm_size_bytes = 3221225472 # 3 GiB -max_size_bytes = 6442450944 # 6 GiB +# The stage held 53 MB when measured on 2026-10-04. Half a GiB retains several +# failed-run evidence sets while funding the complete pinned-image matrix. +warm_size_bytes = 268435456 # 256 MiB +max_size_bytes = 536870912 # 512 MiB prune_strategy = "lru" maximum_age_hours = 720 maximum_count = 20 [stages.test-images] -description = "Digest-pinned OCI layouts of test-fixture images (capsem-debug), shared by every checkout and gate prefix." +description = "Digest-pinned OCI layouts of the debug and reference images, shared by every checkout and gate prefix." scope = "disk" # One directory per image digest: a local build and a registry pull of the same # digest land in the same entry, so the harness pulls a pinned image once per -# machine rather than once per private checkout. An arm64 layout is ~430 MB. +# machine rather than once per private checkout. The gate needs both images for +# both supported architectures present before its network sandbox closes. path = "target/tests/images" -# A pinned 0.7 debug fixture is 1.15 GiB including its prepared EROFS. Retain -# the four-generation working set below; 1 GiB could not retain even one and -# an enforcement run discarded every fixture after crossing the old 2 GiB cap. -warm_size_bytes = 5368709120 # 5 GiB -max_size_bytes = 6442450944 # 6 GiB +warm_size_bytes = 6442450944 # 6 GiB +max_size_bytes = 8589934592 # 8 GiB prune_strategy = "lru" maximum_age_hours = 720 maximum_count = 4 From 3d10dde0cbbcbdd6ff1f7c07875eb767584b9d0c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 11:08:00 +0000 Subject: [PATCH 450/464] test(oci): refresh qualified amd64 fixtures --- config/gate.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index 405a8bcf7..c07d59811 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2345,7 +2345,7 @@ context = "images/capsem-debug" script = "tests/fixtures/oci/pinned_image.py" repository = "ghcr.io/google/capsem/capsem-debug" cache_stage = "test-images" -digests = { "linux/arm64" = "sha256:eb349ca15ee89715842252af6b253314d656ae41e9a685645393d8ec7a5666b6", "linux/amd64" = "sha256:3b0fd69477cfe0dccbad777608244664e72f828ba938db4a51ff7a2ac4635d95" } +digests = { "linux/arm64" = "sha256:15395ce8120d3b2a0962eaed621b071bb06cf51ad1d26375d027f266289333b2", "linux/amd64" = "sha256:0db9f0c9b973081a1c3342610b691538f4b6a6682671b39e6ccc0fda1da5554d" } # The reference image: a pinned digest of the official `dev` image. Every # runtime test that boots a workload boots this one, so main development @@ -2367,7 +2367,7 @@ cache_stage = "test-images" # replay) boot this image from the layout this variable names, mounted into # their container at its own path. layout_variable = "CAPSEM_REFERENCE_IMAGE_LAYOUT" -digests = { "linux/arm64" = "sha256:64729596d6cdf9e5005c6d0d4745c842b1d159eea95b7e4025350d1d830b80b1", "linux/amd64" = "sha256:aa823bfde4361684dc3f496bd591b2d10c64f8a2c2654204b98f57d479786b3d" } +digests = { "linux/arm64" = "sha256:206fb5037e1e3e25b798c00ad66867aaea67a667c169059f25b2cddcfd612719", "linux/amd64" = "sha256:d2ec6a385ec58e6fccbbf2860ed6944855b259c098a3ff04bd9a7d6b3bc26467" } [modules] # The build-chain suites that need built artifacts. The release-contracts From 79f647b897faaae3dea71cc408204d0f61fd630c Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 11:46:44 +0000 Subject: [PATCH 451/464] test(gateway): track consolidated status contract --- tests/capsem-gateway/conftest.py | 190 ++++++++++-------- tests/capsem-gateway/test_gw_status.py | 5 +- .../capsem-gateway/test_gw_status_advanced.py | 34 ++-- 3 files changed, 128 insertions(+), 101 deletions(-) diff --git a/tests/capsem-gateway/conftest.py b/tests/capsem-gateway/conftest.py index acdecf42e..0730bb47e 100644 --- a/tests/capsem-gateway/conftest.py +++ b/tests/capsem-gateway/conftest.py @@ -61,6 +61,96 @@ }, } +MOCK_UPDATE_STATUS = { + "checked_at": 1718444400, + "channel_url": "https://release.capsem.org/assets/stable/manifest.json", + "channel_hash": "0123456789abcdef" * 4, + "validation_status": "valid", + "stale": False, + "binary": { + "current": "1.3.1", + "latest": "1.3.2", + "update_available": True, + "state": "update_available", + "compatibility": "compatible", + }, + "assets": { + "current": "2026.0627.1", + "latest": "2026.0628.1", + "update_available": True, + "state": "update_available", + "compatibility": "compatible", + }, + "images": { + "update_available": False, + "state": "not_published", + "compatibility": "not_applicable", + }, + "supply_chain": { + "manifest": {"path": ""}, + "channel_index": {}, + "host_sbom": {"name": ""}, + "vm_obom": {"name": ""}, + "attestations": [], + }, +} + +MOCK_ASSET_STATUS = { + "ready": False, + "downloading": False, + "current_arch": "arm64", + "asset_version": "2026.0613.1", + "assets": [ + { + "kind": "kernel", + "name": "vmlinuz", + "path": "/Users/test/.capsem/assets/arm64/vmlinuz-0123456789abcdef", + "status": "present", + "expected_hash": "0123456789abcdef" * 4, + "expected_size": 3, + "actual_size": 3, + }, + { + "kind": "rootfs", + "name": "rootfs.erofs", + "path": "/Users/test/.capsem/assets/arm64/rootfs-fedcba9876543210.erofs", + "status": "missing", + "expected_hash": "fedcba9876543210" * 4, + "expected_size": 5, + }, + ], + "errors": ["rootfs.erofs is missing"], + "manifest": { + "origin": "package", + "path": "/Users/test/.capsem/assets/manifest.json", + "origin_path": "/Users/test/.capsem/assets/manifest-metadata.json", + "origin_source": "file:///tmp/corp/manifest.json", + "packaged_at": "2026-06-13T00:00:00Z", + "blake3": "0123456789abcdef", + "validation_status": "valid", + "refresh_policy": "24h", + "assets_current": "2026.0613.1", + "binaries_current": "1.3.0", + }, +} + + +def mock_list_response(): + return { + "sandboxes": [ + { + "id": vm["id"], + "pid": vm["pid"], + "status": vm["status"], + "persistent": vm["persistent"], + "ram_mb": vm["ram_mb"], + "cpus": vm["cpus"], + "available_actions": ["pause", "stop", "fork", "delete"], + } + for vm in MOCK_VMS.values() + ] + } + class MockServiceHandler(BaseHTTPRequestHandler): """HTTP handler mimicking capsem-service responses.""" @@ -72,8 +162,7 @@ def log_message(self, format, *args): def clean_path(self): """Strip http://localhost prefix that hyper sends over UDS.""" p = self.path - if p.startswith("http://localhost"): - p = p[len("http://localhost") :] + p = p.removeprefix("http://localhost") return p def _read_body(self): @@ -93,21 +182,12 @@ def _send_error(self, status, msg): def do_GET(self): path_only = self.clean_path.split("?", 1)[0] - if path_only == "/vms/list": - sandboxes = [] - for vm in MOCK_VMS.values(): - sandboxes.append( - { - "id": vm["id"], - "pid": vm["pid"], - "status": vm["status"], - "persistent": vm["persistent"], - "ram_mb": vm["ram_mb"], - "cpus": vm["cpus"], - "available_actions": ["pause", "stop", "fork", "delete"], - } - ) - self._send_json({"sandboxes": sandboxes}) + if path_only == "/internal/gateway/status": + response = mock_list_response() + response.update(assets=MOCK_ASSET_STATUS, updates=MOCK_UPDATE_STATUS) + self._send_json(response) + elif path_only == "/vms/list": + self._send_json(mock_list_response()) elif path_only == "/images": # Echoes the query, so a test can see the gateway kept it. query = urllib.parse.parse_qs(urllib.parse.urlsplit(self.clean_path).query) @@ -151,75 +231,9 @@ def do_GET(self): self.end_headers() self.wfile.write(body) elif path_only == "/update/status": - self._send_json( - { - "checked_at": 1718444400, - "channel_url": "https://release.capsem.org/assets/stable/manifest.json", - "channel_hash": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - "validation_status": "valid", - "stale": False, - "binary": { - "current": "1.3.1", - "latest": "1.3.2", - "update_available": True, - "state": "update_available", - "compatibility": "compatible", - }, - "assets": { - "current": "2026.0627.1", - "latest": "2026.0628.1", - "update_available": True, - "state": "update_available", - "compatibility": "compatible", - }, - "images": { - "update_available": False, - "state": "not_published", - "compatibility": "not_applicable", - }, - } - ) + self._send_json(MOCK_UPDATE_STATUS) elif path_only == "/assets/status": - self._send_json( - { - "ready": False, - "downloading": False, - "current_arch": "arm64", - "asset_version": "2026.0613.1", - "assets": [ - { - "kind": "kernel", - "name": "vmlinuz", - "path": "/Users/test/.capsem/assets/arm64/vmlinuz-0123456789abcdef", - "status": "present", - "expected_hash": "0123456789abcdef" * 4, - "expected_size": 3, - "actual_size": 3, - }, - { - "kind": "rootfs", - "name": "rootfs.erofs", - "path": "/Users/test/.capsem/assets/arm64/rootfs-fedcba9876543210.erofs", - "status": "missing", - "expected_hash": "fedcba9876543210" * 4, - "expected_size": 5, - }, - ], - "errors": ["rootfs.erofs is missing"], - "manifest": { - "origin": "package", - "path": "/Users/test/.capsem/assets/manifest.json", - "origin_path": "/Users/test/.capsem/assets/manifest-metadata.json", - "origin_source": "file:///tmp/corp/manifest.json", - "packaged_at": "2026-06-13T00:00:00Z", - "blake3": "0123456789abcdef", - "validation_status": "valid", - "refresh_policy": "24h", - "assets_current": "2026.0613.1", - "binaries_current": "1.3.0", - }, - } - ) + self._send_json(MOCK_ASSET_STATUS) else: self._send_error(404, f"unknown endpoint: {self.clean_path}") @@ -323,6 +337,12 @@ def mock_service(): svc.stop() +@pytest.fixture +def mock_vms(): + """Mutable service inventory for assertions about fresh gateway reads.""" + yield MOCK_VMS + + @pytest.fixture(scope="session") def gateway_env(mock_service): """Start capsem-gateway binary pointing at the mock UDS.""" diff --git a/tests/capsem-gateway/test_gw_status.py b/tests/capsem-gateway/test_gw_status.py index 5880c4f49..2b5158917 100644 --- a/tests/capsem-gateway/test_gw_status.py +++ b/tests/capsem-gateway/test_gw_status.py @@ -61,10 +61,9 @@ def test_status_includes_asset_readiness_and_manifest_provenance(self, gw_client assert manifest["assets_current"] == "2026.0613.1" assert manifest["binaries_current"] == "1.3.0" - def test_status_caches_within_ttl(self, gw_client): - """Two rapid calls return identical data (cache TTL is 2s).""" + def test_status_consecutive_reads_are_consistent(self, gw_client): + """Two rapid service snapshots return consistent inventory.""" resp1 = gw_client.get("/status") resp2 = gw_client.get("/status") - # Same data (cache hit) assert resp1["vm_count"] == resp2["vm_count"] assert resp1["service"] == resp2["service"] diff --git a/tests/capsem-gateway/test_gw_status_advanced.py b/tests/capsem-gateway/test_gw_status_advanced.py index 2ede6552e..b5e2d71f1 100644 --- a/tests/capsem-gateway/test_gw_status_advanced.py +++ b/tests/capsem-gateway/test_gw_status_advanced.py @@ -1,32 +1,40 @@ """Advanced gateway /status endpoint tests. -Tests cache TTL expiry, response schema, and edge cases through the -real gateway binary. +Tests refresh behavior, response schema, and edge cases through the real +gateway binary. """ -import time - import pytest pytestmark = pytest.mark.gateway -class TestStatusCacheBehavior: +class TestStatusRefreshBehavior: - def test_status_cache_expires_after_ttl(self, gw_client): - """After 2s TTL, /status fetches fresh data from service.""" + def test_status_reads_fresh_service_state(self, gw_client, mock_vms): + """Every /status request begins a fresh service read.""" resp1 = gw_client.get("/status") assert resp1 is not None - # Wait longer than the 2s cache TTL - time.sleep(2.5) + vm_id = "33333333-3333-4333-8333-333333333333" + mock_vms[vm_id] = { + "id": vm_id, + "pid": 300, + "name": "fresh", + "status": "Running", + "persistent": False, + "ram_mb": 1024, + "cpus": 1, + "version": "0.16.1", + } + try: + resp2 = gw_client.get("/status") + finally: + mock_vms.pop(vm_id) - resp2 = gw_client.get("/status") assert resp2 is not None - # Both should show the same data (mock doesn't change), - # but the second one was a fresh fetch, not a cache hit. assert resp2["service"] == "running" - assert resp2["vm_count"] == resp1["vm_count"] + assert resp2["vm_count"] == resp1["vm_count"] + 1 def test_status_version_is_semver(self, gw_client): """gateway_version field is a valid semver string.""" From 3db7266921976bb1bc0760cdeb5f95135d180957 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 11:50:22 +0000 Subject: [PATCH 452/464] style(gateway): satisfy strict fixture lint --- tests/capsem-gateway/conftest.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/capsem-gateway/conftest.py b/tests/capsem-gateway/conftest.py index 0730bb47e..476be39c7 100644 --- a/tests/capsem-gateway/conftest.py +++ b/tests/capsem-gateway/conftest.py @@ -161,9 +161,7 @@ def log_message(self, format, *args): @property def clean_path(self): """Strip http://localhost prefix that hyper sends over UDS.""" - p = self.path - p = p.removeprefix("http://localhost") - return p + return self.path.removeprefix("http://localhost") def _read_body(self): length = int(self.headers.get("Content-Length", 0)) From 2565e1b21f4816793e854c7a032a0cecf10871e6 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 12:52:14 +0000 Subject: [PATCH 453/464] test(ironbank): drain exact ledger fixtures --- .../capsem-service/test_dbwriter_contract.py | 7 +++++-- tests/helpers/image_session.py | 2 +- tests/ironbank/model_client_assertions.py | 5 ++++- .../test_model_client_ledger_contract.py | 21 ++++++++++++------- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/capsem-service/test_dbwriter_contract.py b/tests/capsem-service/test_dbwriter_contract.py index 8eab744a0..f50b24624 100644 --- a/tests/capsem-service/test_dbwriter_contract.py +++ b/tests/capsem-service/test_dbwriter_contract.py @@ -41,7 +41,10 @@ def test_dbwriter_source_boundaries_are_single_rail() -> None: assert "write_many" not in process_vsock assert "pub struct DbWriter" in logger_writer - assert "type WriterSender = mpsc::SyncSender;" in logger_writer - assert "mpsc::sync_channel(capacity.max(1))" in logger_writer + assert "struct WriterSender {" in logger_writer + assert "tx: mpsc::SyncSender," in logger_writer + assert "budget: Arc," in logger_writer + assert "let capacity = capacity.max(1);" in logger_writer + assert "let (tx, rx) = mpsc::sync_channel(capacity);" in logger_writer assert "mpsc::channel()" not in logger_writer assert '.name("capsem-db-writer".into())' in logger_writer diff --git a/tests/helpers/image_session.py b/tests/helpers/image_session.py index d677ff216..697bd534e 100644 --- a/tests/helpers/image_session.py +++ b/tests/helpers/image_session.py @@ -106,7 +106,7 @@ def session_of( with layout_registry(workdir, layout, image_name) as (reference, certificate, _): catalog = ( f"{reference.split('/', 1)[0]}/library/catalog:stable" - if (layout / "rootfs-v2").is_dir() or (layout / "rootfs").is_dir() + if (layout / "rootfs-v2").is_dir() else None ) grant_exact(service.home_dir, reference, catalog=catalog, certificate=certificate) diff --git a/tests/ironbank/model_client_assertions.py b/tests/ironbank/model_client_assertions.py index 8add64336..f90d12e5f 100644 --- a/tests/ironbank/model_client_assertions.py +++ b/tests/ironbank/model_client_assertions.py @@ -31,6 +31,8 @@ def log_paths(self) -> tuple[Path, ...]: ... def run_python(self, script: str, *, timeout_secs: int = 240) -> dict: ... + def settle_ledger(self) -> Path: ... + def assert_imported_script_contains( env: ModelClientEnvironment, @@ -166,8 +168,9 @@ def assert_two_turn_model_client(env: ModelClientEnvironment, script: str) -> di credential_provider=result.get("credential_provider") or result["provider"], ) raw_secret = "sk-" + result["credential_nonce"] + db_path = env.settle_ledger() run = ModelLedgerRun( - db_path=env.db_path, + db_path=db_path, upstream_transcript_path=env.upstream_transcript_path, log_paths=env.log_paths, raw_secrets=(raw_secret,), diff --git a/tests/ironbank/test_model_client_ledger_contract.py b/tests/ironbank/test_model_client_ledger_contract.py index ebaa9a927..ecbb94320 100644 --- a/tests/ironbank/test_model_client_ledger_contract.py +++ b/tests/ironbank/test_model_client_ledger_contract.py @@ -180,6 +180,18 @@ def run_python(self, script: str, *, timeout_secs: int = 240) -> dict: assert line is not None, stdout + stderr return json.loads(line.split("=", 1)[1]) + def settle_ledger(self) -> Path: + """Drain every producer before an exact direct-ledger assertion.""" + db_path = self.db_path + stopped = self.client.post( + f"/vms/{self.session_id}/stop", + {}, + timeout=60, + ) + assert stopped == {"success": True, "persistent": True}, stopped + assert db_path.exists(), db_path + return db_path + @pytest.fixture def model_client_env(): @@ -491,14 +503,7 @@ def test_openai_two_tool_calls_have_exact_item_cardinality( # a separate monitor. Stop the persistent VM first so the process drains # FsMonitor -> DbWriter and checkpoints the database before we reopen it. # A timing poll here would only hide a missing visibility barrier. - db_path = model_client_env.db_path - stopped = model_client_env.client.post( - f"/vms/{model_client_env.session_id}/stop", - {}, - timeout=60, - ) - assert stopped == {"success": True, "persistent": True}, stopped - assert db_path.exists(), db_path + db_path = model_client_env.settle_ledger() import sqlite3 From 65e9a333a7319c0afb73ee3b7218377cd126d741 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 13:40:58 +0000 Subject: [PATCH 454/464] test(ironbank): centralize ledger drain barrier --- tests/helpers/session_ledger.py | 28 +++++++++++++++++++ tests/ironbank/test_files_process_ledger.py | 14 ++++------ .../test_model_client_ledger_contract.py | 14 ++++------ 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/tests/helpers/session_ledger.py b/tests/helpers/session_ledger.py index c9070e376..f638e30c9 100644 --- a/tests/helpers/session_ledger.py +++ b/tests/helpers/session_ledger.py @@ -13,6 +13,17 @@ import sqlite3 from pathlib import Path +from typing import Protocol + + +class SessionLifecycleClient(Protocol): + def post( + self, + path: str, + body: dict, + *, + timeout: float, + ) -> dict | None: ... #: The occurrence columns a test reads; `run_id` is resolved, not shown. _EVENT_COLUMNS = ( @@ -66,6 +77,23 @@ def open_session_ledger(db_path: Path | str) -> sqlite3.Connection: raise +def stop_session_for_ledger_read( + client: SessionLifecycleClient, + session_id: str, + db_path: Path, +) -> Path: + """Stop a named VM and return its quiesced ledger path. + + A completed write accepts an event into the DB-owned producer buffer. The + stop barrier drains every producer and finalizes body archive indexes before + a black-box test opens the SQLite file directly. + """ + stopped = client.post(f"/vms/{session_id}/stop", {}, timeout=60) + assert stopped == {"success": True, "persistent": True}, stopped + assert db_path.exists(), db_path + return db_path + + #: `capsem_logger::counters::COUNTED_TOOL_ORIGINS`: the origins every surface #: counts as a tool call. `mcp_proxy` is a model naming an MCP tool that is #: recorded again as `mcp`, so it is not one. Held equal to the Rust constant diff --git a/tests/ironbank/test_files_process_ledger.py b/tests/ironbank/test_files_process_ledger.py index 4772b9aef..e9d9bc9b6 100644 --- a/tests/ironbank/test_files_process_ledger.py +++ b/tests/ironbank/test_files_process_ledger.py @@ -26,7 +26,7 @@ vm_session_dir, wait_exec_ready, ) -from helpers.session_ledger import open_session_ledger +from helpers.session_ledger import open_session_ledger, stop_session_for_ledger_read PROJECT_ROOT = Path(__file__).resolve().parents[2] @@ -104,13 +104,6 @@ } -def _connect_session_db(service: ServiceInstance, session_id: str) -> sqlite3.Connection: - db_path = vm_session_db_path(service.tmp_dir, service.client(), session_id) - conn = open_session_ledger(db_path) - conn.row_factory = sqlite3.Row - return conn - - def _table_columns(conn: sqlite3.Connection, table: str) -> set[str]: return {row[1] for row in conn.execute(f"PRAGMA table_info({table})").fetchall()} @@ -286,7 +279,10 @@ def test_file_process_routes_pay_full_ledger_debt_blackbox(): assert retired_status == 404, retired assert not (vm_session_dir(service.tmp_dir, client, session_id) / "auto_snapshots").exists() - conn = _connect_session_db(service, session_id) + db_path = vm_session_db_path(service.tmp_dir, client, session_id) + stop_session_for_ledger_read(client, session_id, db_path) + conn = open_session_ledger(db_path) + conn.row_factory = sqlite3.Row try: assert _table_columns(conn, "fs_events") == EXPECTED_FS_COLUMNS assert _table_columns(conn, "exec_events") == EXPECTED_EXEC_COLUMNS diff --git a/tests/ironbank/test_model_client_ledger_contract.py b/tests/ironbank/test_model_client_ledger_contract.py index ecbb94320..008178921 100644 --- a/tests/ironbank/test_model_client_ledger_contract.py +++ b/tests/ironbank/test_model_client_ledger_contract.py @@ -33,7 +33,7 @@ vm_session_db_path, vm_session_dir, ) -from helpers.session_ledger import open_session_ledger +from helpers.session_ledger import open_session_ledger, stop_session_for_ledger_read from ironbank.model_client_assertions import assert_one_model_client from ironbank.model_client_config import ( HERMETIC_OPENAI_PRICED_MODEL, @@ -182,15 +182,11 @@ def run_python(self, script: str, *, timeout_secs: int = 240) -> dict: def settle_ledger(self) -> Path: """Drain every producer before an exact direct-ledger assertion.""" - db_path = self.db_path - stopped = self.client.post( - f"/vms/{self.session_id}/stop", - {}, - timeout=60, + return stop_session_for_ledger_read( + self.client, + self.session_id, + self.db_path, ) - assert stopped == {"success": True, "persistent": True}, stopped - assert db_path.exists(), db_path - return db_path @pytest.fixture From e85ffcc5c51b0449f1e802f81577442c81ef24f1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 13:46:39 +0000 Subject: [PATCH 455/464] security(deps): update rustls for October advisories --- CHANGELOG.md | 3 +++ Cargo.lock | 4 ++-- tests/citadel/cargo_lock_identity.toml | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 655b3531f..a104c903b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Update rustls to 0.23.46, fixing unauthenticated certificate-decompression, + ECH-rejection, key-exchange downgrade, and session-secret exposure flaws. + - Fail DNS closed when any decision, rule or ask row is refused, and coalesce related supplemental admissions so concurrent queries do not serialize one ledger round trip per row. diff --git a/Cargo.lock b/Cargo.lock index 9f3d24ec6..9ea6c8046 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6029,9 +6029,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", diff --git a/tests/citadel/cargo_lock_identity.toml b/tests/citadel/cargo_lock_identity.toml index caa0e9884..b1b9d44f4 100644 --- a/tests/citadel/cargo_lock_identity.toml +++ b/tests/citadel/cargo_lock_identity.toml @@ -7,5 +7,5 @@ # each moved the lock. `test_cargo_lock_changes_are_deliberate` fails until # this record matches the lock again, so the cost is chosen, reviewed and # batched rather than discovered twenty minutes into a gate. -sha256 = "31b8050cdf30b4c50c0365fa235a0bb6840e18b6cf40e48edefd1b7e9a9c2bca" -reason = "The unified observable cache adds quick_cache 0.7.0. The 0.7 crypto ownership retrofit centralizes first-party hashes, entropy, TLS keys, JWT verification and secret primitives; BLAKE2b/BLAKE2s SIMD 1.0.5 support normal-harness comparisons. Managed-session APIs add zeroizing request capabilities, and Rust managed scopes add SDK edges to existing capsem-crypto random/secrets, futures, humantime and tokio-util rt. Central runtime-limit generation adds capsem-proto's direct toml edge, and exact benchmark reporting adds capsem-bench's direct capsem-proto edge. The host and sealed guest cohort will be rebuilt and qualified together." +sha256 = "bb76b3833746aa2a26714484474ceb59e004bd467097d6b9645c8415403ae7d4" +reason = "The unified observable cache adds quick_cache 0.7.0. The 0.7 crypto ownership retrofit centralizes first-party hashes, entropy, TLS keys, JWT verification and secret primitives; BLAKE2b/BLAKE2s SIMD 1.0.5 support normal-harness comparisons. Managed-session APIs add zeroizing request capabilities, and Rust managed scopes add SDK edges to existing capsem-crypto random/secrets, futures, humantime and tokio-util rt. Central runtime-limit generation adds capsem-proto's direct toml edge, and exact benchmark reporting adds capsem-bench's direct capsem-proto edge. rustls 0.23.46 fixes RUSTSEC-2026-0346, RUSTSEC-2026-0347, RUSTSEC-2026-0348 and RUSTSEC-2026-0349. The host and sealed guest cohort will be rebuilt and qualified together." From d1f988c5f1850702d8543fdd4333ac72758538af Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 13:48:21 +0000 Subject: [PATCH 456/464] test(ironbank): type model ledger client --- config/gate.toml | 2 +- tests/ironbank/test_model_client_ledger_contract.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/config/gate.toml b/config/gate.toml index c07d59811..e4d3ee364 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -888,7 +888,7 @@ invalid-assignment = 5 invalid-return-type = 2 no-matching-overload = 5 not-subscriptable = 11 -unresolved-attribute = 67 +unresolved-attribute = 62 unresolved-import = 1 unsupported-operator = 2 unused-ignore-comment = 3 diff --git a/tests/ironbank/test_model_client_ledger_contract.py b/tests/ironbank/test_model_client_ledger_contract.py index 008178921..1d452ada5 100644 --- a/tests/ironbank/test_model_client_ledger_contract.py +++ b/tests/ironbank/test_model_client_ledger_contract.py @@ -34,6 +34,7 @@ vm_session_dir, ) from helpers.session_ledger import open_session_ledger, stop_session_for_ledger_read +from helpers.uds_client import UdsHttpClient from ironbank.model_client_assertions import assert_one_model_client from ironbank.model_client_config import ( HERMETIC_OPENAI_PRICED_MODEL, @@ -133,7 +134,7 @@ def _credential_ref_for_secret(secret: str, *, provider: str = "openai") -> str: @dataclass class ModelClientEnv: service: ServiceInstance - client: object + client: UdsHttpClient session_id: str mock_base_url: str upstream_transcript_path: Path From ec000fdb7401d97c79d3b11bdcb1868b8f41c687 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 15:08:12 +0000 Subject: [PATCH 457/464] test(ironbank): retry live archive payload reads --- tests/helpers/body_archive.py | 40 +++++-- tests/ironbank/kingslanding/test_policy.py | 8 +- tests/ironbank/kingslanding/test_preview.py | 12 ++- tests/test_body_archive_reader.py | 110 +++++++++++++++++++- 4 files changed, 155 insertions(+), 15 deletions(-) diff --git a/tests/helpers/body_archive.py b/tests/helpers/body_archive.py index d55b32c12..65a86567c 100644 --- a/tests/helpers/body_archive.py +++ b/tests/helpers/body_archive.py @@ -508,6 +508,21 @@ def forensic_payload(body: bytes) -> dict[str, Any]: return payload +def _security_payload_from_response( + served: dict[str, Any], event_id: str, source_table: str +) -> dict[str, Any]: + for body in served["bodies"]: + if (body["source_table"], body["direction"]) != (source_table, "payload"): + continue + assert not body["truncated"] and not body["truncated_for_transport"], body + content = str(body["content"]) + raw = base64.b64decode(content) if body["encoding"] == "base64" else content.encode() + return forensic_payload(raw) + raise AssertionError( + f"{source_table} has no served payload for {event_id}: {served}" + ) + + def served_security_payload( client: Any, vm_id: str, event_id: str, source_table: str = "security_rule_events" ) -> dict[str, Any]: @@ -517,11 +532,20 @@ def served_security_payload( event, and its payload is an archived body like any other. """ served = client.get(f"/vms/{vm_id}/bodies/{event_id}") - for body in served["bodies"]: - if (body["source_table"], body["direction"]) != (source_table, "payload"): - continue - assert not body["truncated"] and not body["truncated_for_transport"], body - content = str(body["content"]) - raw = base64.b64decode(content) if body["encoding"] == "base64" else content.encode() - return forensic_payload(raw) - raise AssertionError(f"{source_table} has no served payload for {event_id}: {served}") + return _security_payload_from_response(served, event_id, source_table) + + +def served_security_payload_if_ready( + client: Any, vm_id: str, event_id: str, source_table: str = "security_rule_events" +) -> dict[str, Any] | None: + """Return a live event's payload once its archive index is visible. + + Live row queries include the writer's memory tables. Body reads use the + durable archive index, which lands with the next periodic flush, so an + event row may briefly precede its body. An empty response is retryable; + every other response retains the strict payload validation above. + """ + served = client.get(f"/vms/{vm_id}/bodies/{event_id}") + if not served["bodies"]: + return None + return _security_payload_from_response(served, event_id, source_table) diff --git a/tests/ironbank/kingslanding/test_policy.py b/tests/ironbank/kingslanding/test_policy.py index b0d8b5c91..b085623f4 100644 --- a/tests/ironbank/kingslanding/test_policy.py +++ b/tests/ironbank/kingslanding/test_policy.py @@ -6,7 +6,7 @@ import socket import pytest -from helpers.body_archive import served_security_payload, session_archive +from helpers.body_archive import served_security_payload_if_ready, session_archive from helpers.constants import DEFAULT_CPUS, DEFAULT_RAM_MB from helpers.session_ledger import open_session_ledger from helpers.settings_policy import apply_settings_rule, write_settings_rule @@ -195,11 +195,13 @@ def audited(): latest = client.get(f"/vms/{vm_id}/security/latest?limit=2000") for row in latest: if row["event_type"] == "network.connect" and row["event_id"] not in payloads: - payloads[row["event_id"]] = served_security_payload(client, vm_id, row["event_id"]) + payload = served_security_payload_if_ready(client, vm_id, row["event_id"]) + if payload is not None: + payloads[row["event_id"]] = payload rows[:] = [ {**row, "event": payloads[row["event_id"]]} for row in latest - if row["event_type"] == "network.connect" + if row["event_type"] == "network.connect" and row["event_id"] in payloads ] seen = {row["event"]["network"]["source"]["address"] for row in rows} return denied_peers <= seen diff --git a/tests/ironbank/kingslanding/test_preview.py b/tests/ironbank/kingslanding/test_preview.py index 2ada42722..0b0ce3371 100644 --- a/tests/ironbank/kingslanding/test_preview.py +++ b/tests/ironbank/kingslanding/test_preview.py @@ -8,7 +8,7 @@ from urllib.parse import urlencode, urlsplit import pytest -from helpers.body_archive import served_security_payload +from helpers.body_archive import served_security_payload_if_ready from helpers.settings_policy import apply_settings_rule from tests.ironbank.kingslanding.test_publish import redis @@ -278,8 +278,14 @@ def denied_audit(): rows[:] = client.get(f"/vms/{vm_id}/security/latest?limit=2000") for row in rows: if row["event_type"] == "network.connect" and row["event_id"] not in payloads: - payloads[row["event_id"]] = served_security_payload(client, vm_id, row["event_id"]) - events = (payloads[row["event_id"]] for row in rows if row["event_type"] == "network.connect") + payload = served_security_payload_if_ready(client, vm_id, row["event_id"]) + if payload is not None: + payloads[row["event_id"]] = payload + events = ( + payloads[row["event_id"]] + for row in rows + if row["event_type"] == "network.connect" and row["event_id"] in payloads + ) return any( ((event["network"] or {}).get("route") or {}).get("mode") == "preview" and (event["decision"] or {}).get("effective") == "block" diff --git a/tests/test_body_archive_reader.py b/tests/test_body_archive_reader.py index e60aa0108..9b05a3918 100644 --- a/tests/test_body_archive_reader.py +++ b/tests/test_body_archive_reader.py @@ -2,17 +2,40 @@ from __future__ import annotations +import base64 import os import shutil import sqlite3 from contextlib import closing from pathlib import Path -from helpers.body_archive import FILE_HEADER_BYTES, SessionArchive, archived_bodies +import pytest +import pyzstd +from blake3 import blake3 +from helpers.body_archive import ( + BLOCK_MAGIC, + CODEC_ZSTD, + FILE_HEADER_BYTES, + SEGMENT_FINAL, + SEGMENT_MAGIC, + SessionArchive, + archived_bodies, + served_security_payload, + served_security_payload_if_ready, +) FIXTURE_DB = Path(__file__).resolve().parent / "fixtures/session/test.db" +class _BodyClient: + def __init__(self, *responses: dict) -> None: + self.responses = iter(responses) + + def get(self, path: str) -> dict: + assert path == "/vms/vm-id/bodies/abcdef123456" + return next(self.responses) + + def _ledger(tmp_path: Path) -> Path: db = tmp_path / "session.db" shutil.copyfile(FIXTURE_DB, db) @@ -44,3 +67,88 @@ def test_a_body_committed_after_the_reader_opened_is_read(tmp_path: Path) -> Non _committed_end(db, committed) # the writer commits the rest read = {key: archive.read(key[1], key[0], key[2]) for key in expected} assert read == expected + + +def test_zstd_segments_share_one_bounded_decoder_stream(tmp_path: Path) -> None: + db = _ledger(tmp_path) + expected = archived_bodies(db) + with closing(sqlite3.connect(db)) as conn: + row = conn.execute( + "SELECT block_offset, disk_len, raw_len FROM body_blocks" + ).fetchone() + with SessionArchive(db) as archive: + block_offset, disk_len, raw_len = map(int, row) + raw = archive._span(block_offset, 0, raw_len, disk_len, raw_len) + generation = archive.generation_path + + split = len(raw) // 2 + compressor = pyzstd.ZstdCompressor(3) + compressed = ( + compressor.compress(raw[:split]) + + compressor.flush(pyzstd.ZstdCompressor.FLUSH_BLOCK), + compressor.compress(raw[split:]) + + compressor.flush(pyzstd.ZstdCompressor.FLUSH_FRAME), + ) + + def segment(start: int, body: bytes, encoded: bytes, flags: int) -> bytes: + return b"".join( + ( + SEGMENT_MAGIC, + bytes((flags, 0, 0, 0)), + start.to_bytes(4, "little"), + len(body).to_bytes(4, "little"), + len(encoded).to_bytes(4, "little"), + blake3(body).digest(), + encoded, + ) + ) + + block = b"".join( + ( + BLOCK_MAGIC, + bytes((CODEC_ZSTD, 0, 0, 0)), + segment(0, raw[:split], compressed[0], 0), + segment(split, raw[split:], compressed[1], SEGMENT_FINAL), + ) + ) + header = generation.read_bytes()[:FILE_HEADER_BYTES] + generation.write_bytes(header + block) + with closing(sqlite3.connect(db)) as conn: + conn.execute("UPDATE body_blocks SET disk_len = ?", (len(block),)) + conn.execute( + "UPDATE archive_state SET committed_end = ?", + (FILE_HEADER_BYTES + len(block),), + ) + conn.commit() + + assert archived_bodies(db) == expected + + +def test_a_live_security_payload_retries_until_the_archive_index_is_visible() -> None: + body = base64.b64encode(b"\x81\xaaevent_type\xa3dns").decode() + client = _BodyClient( + {"event_id": "abcdef123456", "bodies": []}, + { + "event_id": "abcdef123456", + "bodies": [ + { + "source_table": "security_rule_events", + "direction": "payload", + "truncated": False, + "truncated_for_transport": False, + "encoding": "base64", + "content": body, + } + ], + }, + ) + + assert served_security_payload_if_ready(client, "vm-id", "abcdef123456") is None + payload = served_security_payload_if_ready(client, "vm-id", "abcdef123456") + assert payload is not None and payload["event_type"] == "dns" + + +def test_the_strict_security_payload_reader_rejects_an_unindexed_body() -> None: + client = _BodyClient({"event_id": "abcdef123456", "bodies": []}) + with pytest.raises(AssertionError, match="has no served payload"): + served_security_payload(client, "vm-id", "abcdef123456") From 8a090110a71e142afe097ea2a2bdf34e51417037 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 15:45:32 +0000 Subject: [PATCH 458/464] merge(0.7): preserve reconciled worker, credential and benchmark contracts --- .github/workflows/ci.yaml | 7 + CHANGELOG.md | 279 +++++++++++++++++- Cargo.lock | 1 - Cargo.toml | 12 - benchmarks/collectors/routes | 86 ++---- benchmarks/helpers/route_measurements.py | 24 ++ build_system/builder/crypto.py | 34 +-- build_system/builder/gate/assetidentity.py | 2 +- build_system/builder/gate/buildschema.py | 5 + build_system/builder/gate/cachecontrol.py | 33 +++ build_system/builder/gate/config.py | 3 +- build_system/builder/gate/imagequalify.py | 17 ++ .../builder/gate/module_functional.py | 7 +- build_system/builder/gate/sdkchecks.py | 44 +++ .../tools/check_remote_release_readiness.py | 7 +- build_system/pyproject.toml | 1 + build_system/tests/gate/test_build_assets.py | 11 +- .../tests/gate/test_gate_cache_control.py | 41 ++- build_system/tests/gate/test_gate_config.py | 21 +- .../tests/gate/test_gate_functional_sdks.py | 14 + .../release/test_release_doctor_contract.py | 2 +- .../tests/sdkgen/test_python_operations.py | 29 +- build_system/uv.lock | 85 ++++++ config/cache.toml | 1 + config/gate.toml | 33 ++- config/public-surface.toml | 2 +- crates/capsem-api/src/document.rs | 2 +- crates/capsem-archive/src/reader.rs | 2 +- crates/capsem-archive/src/writer.rs | 2 +- .../capsem-assets/src/oci/cache/inventory.rs | 22 +- crates/capsem-assets/src/oci/worker/tests.rs | 18 +- .../src/security_rule_profile.rs | 2 - .../src/security_rule_profile/sigma.rs | 10 + .../src/security_rule_profile/tests.rs | 55 +++- crates/capsem-core/src/container/publish.rs | 99 ++++++- .../src/container/publish/saved.rs | 23 +- .../src/container/publish/tests.rs | 16 + .../src/container/publish/tests/saved.rs | 8 + .../src/container/publish/tests/security.rs | 25 +- crates/capsem-core/src/hypervisor/kvm/mod.rs | 2 +- .../src/hypervisor/kvm/virtio_fs/mod.rs | 12 + .../src/hypervisor/kvm/virtio_fs/ops_file.rs | 3 +- crates/capsem-core/src/net/ai_traffic/mod.rs | 38 ++- .../capsem-core/src/net/ai_traffic/tests.rs | 9 + crates/capsem-core/src/net/mitm_proxy/mod.rs | 39 +-- .../src/net/mitm_proxy/pipeline_factory.rs | 35 +++ .../src/net/mitm_proxy/telemetry_hook.rs | 64 +++- .../net/mitm_proxy/telemetry_hook/tests.rs | 5 +- .../capsem-core/src/net/mitm_proxy/upgrade.rs | 2 +- .../src/net/mitm_proxy/websocket.rs | 2 +- .../src/net/mitm_proxy/websocket/tests.rs | 31 +- .../src/net/mitm_proxy/websocket/turns.rs | 10 +- crates/capsem-core/src/net/upstream_grant.rs | 147 +++++++++ .../src/net/upstream_grant/tests.rs | 115 ++++++++ crates/capsem-core/src/security_engine/mod.rs | 1 + .../capsem-core/tests/mitm_integration/tls.rs | 2 +- crates/capsem-foundation/src/telemetry.rs | 44 ++- .../capsem-foundation/src/telemetry/tests.rs | 23 +- .../capsem-foundation/src/unix/contained.rs | 16 +- crates/capsem-foundation/src/unix/fd.rs | 53 ++++ crates/capsem-foundation/src/unix/fd/tests.rs | 54 ++++ crates/capsem-foundation/src/unix/fs.rs | 16 +- crates/capsem-foundation/src/unix/lock.rs | 5 +- crates/capsem-foundation/src/unix/process.rs | 20 +- .../src/unix/worker_sandbox.rs | 4 + .../src/unix/worker_sandbox/linux.rs | 9 + .../src/unix/worker_sandbox/seatbelt.rs | 2 +- .../src/unix/worker_sandbox/tests.rs | 7 + crates/capsem-gateway/src/main.rs | 23 +- crates/capsem-ledger/src/main.rs | 19 +- crates/capsem-ledger/src/tests.rs | 18 ++ crates/capsem-ledger/tests/subprocess.rs | 5 +- crates/capsem-logger/Cargo.toml | 1 - crates/capsem-logger/src/events/network.rs | 2 +- crates/capsem-logger/src/ledger_client.rs | 28 +- .../capsem-logger/src/ledger_client/tests.rs | 20 +- crates/capsem-logger/src/ledger_protocol.rs | 1 + crates/capsem-logger/src/ledger_server.rs | 15 +- crates/capsem-logger/src/writer.rs | 41 ++- .../src/writer/bodies/index_batch.rs | 62 ---- crates/capsem-logger/src/writer/operation.rs | 2 +- crates/capsem-logger/src/writer/remote.rs | 29 +- .../capsem-logger/src/writer/remote/tests.rs | 4 + crates/capsem-logger/src/writer/retention.rs | 2 +- crates/capsem-logger/src/writer/worker.rs | 27 +- crates/capsem-mock-server/src/main.rs | 99 +------ crates/capsem-process/src/ipc.rs | 6 +- crates/capsem-process/src/ipc/tests.rs | 22 +- .../src/ipc/tests/classification.rs | 2 +- crates/capsem-process/src/ipc/tests/reload.rs | 9 +- crates/capsem-process/src/main.rs | 106 +++++-- crates/capsem-process/src/owner.rs | 14 +- crates/capsem-process/src/owner_sandbox.rs | 26 +- .../capsem-process/src/owner_sandbox/tests.rs | 80 +++++ crates/capsem-process/src/runtime_config.rs | 33 ++- crates/capsem-process/src/tests.rs | 51 +++- crates/capsem-process/src/trace_hints.rs | 39 +++ .../capsem-process/src/trace_hints/tests.rs | 51 ++++ crates/capsem-proto/build.rs | 9 +- crates/capsem-proto/src/ipc.rs | 8 +- crates/capsem-proto/src/ipc/tests.rs | 10 +- .../capsem-proto/src/ipc/tests/correlation.rs | 8 +- crates/capsem-proto/src/ipc/tests/wire.rs | 5 +- crates/capsem-proto/src/ledger.rs | 4 +- crates/capsem-proto/src/ledger_commitment.rs | 2 +- crates/capsem-proto/src/lib.rs | 7 +- crates/capsem-proto/src/proxy_control.rs | 4 +- .../capsem-proto/src/proxy_control/tests.rs | 1 + crates/capsem-proto/src/proxy_trace_hints.rs | 122 ++++++++ .../src/proxy_trace_hints/tests.rs | 44 +++ crates/capsem-proto/src/upstream_grant.rs | 94 +++++- .../capsem-proto/src/upstream_grant/tests.rs | 32 ++ crates/capsem-proxy/src/credential_client.rs | 9 +- .../src/credential_client/tests.rs | 41 +++ crates/capsem-proxy/src/main.rs | 45 ++- crates/capsem-proxy/src/tests.rs | 13 + crates/capsem-proxy/src/trace_hint_client.rs | 58 ++++ .../src/trace_hint_client/tests.rs | 28 ++ crates/capsem-proxy/tests/subprocess.rs | 31 +- crates/capsem-router/README.md | 28 +- crates/capsem-service/src/active_policy.rs | 20 +- .../src/connection_routes/policy.rs | 34 ++- .../src/connection_routes/policy/tests.rs | 19 +- .../capsem-service/src/credential_routes.rs | 2 +- .../src/credential_routes/tests.rs | 19 +- crates/capsem-service/src/instance/tests.rs | 8 +- crates/capsem-service/src/instance_reaper.rs | 92 +++--- .../src/instance_reaper/retirement.rs | 29 +- .../src/instance_reaper/retirement/tests.rs | 28 ++ .../capsem-service/src/ledger_commitment.rs | 6 +- crates/capsem-service/src/ledger_worker.rs | 70 ++++- .../capsem-service/src/ledger_worker/tests.rs | 54 ++++ crates/capsem-service/src/main.rs | 1 + .../src/managed_routes/tests.rs | 2 +- crates/capsem-service/src/managed_sessions.rs | 4 +- .../src/owner_connection/tests.rs | 6 +- crates/capsem-service/src/proxy_worker.rs | 4 +- crates/capsem-service/src/router_runtime.rs | 2 + .../src/router_runtime/exposures/tests.rs | 2 +- .../src/router_runtime/streams/tests/flow.rs | 2 +- crates/capsem-service/src/service_runtime.rs | 22 +- crates/capsem-service/src/session_cleanup.rs | 40 +-- .../src/session_cleanup/tests.rs | 34 +++ .../capsem-service/src/session_db_handles.rs | 38 +-- .../src/session_housekeeping.rs | 35 ++- crates/capsem-service/src/standalone_proxy.rs | 4 +- .../src/standalone_proxy/tests.rs | 132 +++++++++ crates/capsem-service/src/switches.rs | 2 +- crates/capsem-service/src/tests.rs | 28 +- .../src/tests/assets_registry.rs | 16 +- crates/capsem-service/src/tests/files_api.rs | 2 +- .../src/tests/instance_reaper.rs | 147 ++++++--- .../capsem-service/src/tests/ipc_command.rs | 2 +- .../capsem-service/src/tests/policy_push.rs | 26 +- .../src/tests/session_housekeeping.rs | 61 ++-- .../src/tests/settings_files.rs | 4 +- crates/capsem-service/src/upstream_broker.rs | 64 +++- .../src/upstream_broker/tests.rs | 86 +++++- crates/capsem-service/src/vm_files.rs | 103 +------ crates/capsem-service/src/vm_files/fork.rs | 2 +- .../src/vm_files/provision/tests.rs | 15 + .../capsem-service/src/vm_files/readiness.rs | 2 +- crates/capsem-service/src/vm_lifecycle.rs | 158 ++-------- .../src/vm_lifecycle/provision.rs | 14 +- .../capsem-service/src/vm_lifecycle/resume.rs | 10 +- .../src/vm_lifecycle/resume_process.rs | 14 +- crates/capsem-service/src/vm_lifecycle/run.rs | 134 +++++++++ .../src/vm_lifecycle/session_dirs.rs | 12 +- mcp/typescript/tests/image-tools.test.ts | 2 +- mcp/typescript/tests/packed-package.test.ts | 15 +- mcp/typescript/tools/prewarm-package.mjs | 11 +- sdk/python/capsem/_operations/__init__.py | 5 + sdk/python/capsem/models/__init__.py | 4 + sdk/python/pyproject.toml | 3 +- sdk/python/tests/image_package_acceptance.py | 1 + sdk/python/uv.lock | 4 + sdk/rust/src/operations/generated/mod.rs | 6 + sdk/rust/src/operations/tests.rs | 3 + sdk/specification/openapi.json | 63 +++- sdk/typescript/src/models/index.ts | 2 + sdk/typescript/src/operations/index.ts | 3 + sdk/typescript/src/validation/index.ts | 2 + sdk/typescript/tests/package-install.test.ts | 2 +- sdk/typescript/tools/prewarm-package.mjs | 25 +- skills/dev-capsem/SKILL.md | 2 +- skills/dev-testing/references/test-matrix.md | 2 +- .../references/crate-and-privilege-model.md | 27 +- .../test_process_reads_only_active_policy.py | 11 +- tests/capsem-release/test_binary_lane_gate.py | 7 +- .../test_release_test_composition.py | 20 ++ tests/capsem-sdk/test_mcp_cli_parity.py | 1 + .../test_companion_lifecycle.py | 51 +++- .../capsem-service/test_dbwriter_contract.py | 5 +- .../test_settings_security_routes.py | 28 +- .../test_svc_startup_failure_logged.py | 18 +- tests/citadel/cargo_lock_identity.toml | 14 +- tests/citadel/fixture_ownership.toml | 2 +- tests/citadel/python_boundary_debt.toml | 4 +- .../citadel/repository_surface_ownership.toml | 12 - tests/citadel/test_bench_route_coverage.py | 7 +- .../test_exposure_forwarding_boundary.py | 16 +- .../citadel/test_generated_output_boundary.py | 9 +- tests/citadel/test_hot_build_contract.py | 2 + .../test_repository_surface_ownership.py | 2 +- tests/citadel/test_sdk_ci.py | 9 +- tests/citadel/test_sdk_rust_quality.py | 9 +- tests/citadel/test_sdk_typescript_ci.py | 12 +- tests/citadel/unix_boundary_debt.toml | 18 +- tests/fixtures/oci/README.md | 14 + tests/fixtures/session/test.db | Bin 585728 -> 589824 bytes tests/frontend/test_ui_contract.py | 1 + tests/helpers/body_archive.py | 108 +++++-- tests/helpers/image_session.py | 2 +- tests/ironbank/kingslanding/test_publish.py | 5 +- tests/ironbank/model_ledger.py | 22 +- tests/ironbank/test_doctor_ledger.py | 21 +- tests/ironbank/test_mcp_settings_ledger.py | 5 + .../test_model_client_ledger_contract.py | 42 ++- tests/ironbank/test_model_sdk_ledger.py | 7 +- tests/ironbank/test_standalone_proxy.py | 4 + tests/test_benchmark_retention_contract.py | 2 +- tests/test_benchmark_route_collector.py | 4 +- tests/test_benchmark_routes.py | 15 +- tests/test_body_archive_reader.py | 20 +- .../session-language-contract.test.ts | 8 +- .../docs/architecture/host-isolation.md | 58 +++- .../docs/architecture/session-telemetry.md | 25 +- 227 files changed, 4356 insertions(+), 1352 deletions(-) create mode 100644 benchmarks/helpers/route_measurements.py create mode 100644 crates/capsem-core/src/container/publish/tests/saved.rs create mode 100644 crates/capsem-core/src/net/mitm_proxy/pipeline_factory.rs delete mode 100644 crates/capsem-logger/src/writer/bodies/index_batch.rs create mode 100644 crates/capsem-process/src/owner_sandbox/tests.rs create mode 100644 crates/capsem-process/src/trace_hints.rs create mode 100644 crates/capsem-process/src/trace_hints/tests.rs create mode 100644 crates/capsem-proto/src/proxy_trace_hints.rs create mode 100644 crates/capsem-proto/src/proxy_trace_hints/tests.rs create mode 100644 crates/capsem-proxy/src/trace_hint_client.rs create mode 100644 crates/capsem-proxy/src/trace_hint_client/tests.rs create mode 100644 crates/capsem-service/src/vm_lifecycle/run.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9bdf6fcd1..4c193babd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -308,6 +308,11 @@ jobs: CAPSEM_FRONTEND_JUNIT: ${{ github.workspace }}/cache/target/coverage/junit/frontend.xml run: bash build_system/scripts/web/check-web-surface.sh frontend-verify + - name: Build Python SDK distributions + run: | + uv sync --project sdk/python --frozen --no-install-project + uv run --project sdk/python --frozen --no-sync python -m build --no-isolation --outdir cache/target/sdk/python sdk/python + # Python schema tests with coverage - name: Python SDK tests with coverage working-directory: sdk/python @@ -324,6 +329,7 @@ jobs: working-directory: sdk/typescript run: | pnpm install --frozen-lockfile + pnpm run prewarm:package pnpm test --reporter=default --reporter=junit --outputFile=../../cache/target/coverage/junit/typescript-sdk.xml # Builds against the SDK package the step above just built. @@ -331,6 +337,7 @@ jobs: working-directory: mcp/typescript run: | pnpm install --frozen-lockfile + pnpm run prewarm:packed pnpm test --reporter=default --reporter=junit --outputFile=../../cache/target/coverage/junit/mcp-typescript.xml - name: Cross-system Python schema tests with coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index a104c903b..8ca26f432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fail DNS closed when any decision, rule or ask row is refused, and coalesce related supplemental admissions so concurrent queries do not serialize one ledger round trip per row. +- Ledger event identifiers reject NUL suffixes, and host audit-chain hashes + require binary storage. Stale declarations fail integrity readiness. + +- Ledger digest and credential-reference checks reject nonhex characters and + NUL suffixes. Readers and writers refuse older checks without rewriting the ledger. + +- Session credential grants require the original runtime's acknowledged policy + and current policy; applied-policy receipts retire with stopped sessions. + +- Credential target preflight uses the existing security rules and plugins, + including corporate ask/block and method, query and port constraints. + +- Delayed process exit cleanup fences its exact old credential runtime even + after a resumed replacement has taken over the session registry. + +- Session stop, suspend, exit and terminal cleanup fence credential grants. + Ready resume rebinds valid approval, and deletion refuses a resumed replacement. + +- Credential runtime fences preserve finite same-session approval on retained + stops and retire terminal session approvals, including recovered metadata. + +- Invalid corporate policy now refuses security admission instead of being + treated as empty constraints; the settings UI remains available for repair. + +- Credential grant issuance checks current policy and runtime ownership inside + the same admission lock as terminal retirement, preventing stale permission + checks from inserting authority after its owner has retired. - VM boot admission retains an exclusive generation lifetime fence. Service restart cleanup confirms exit in the original kernel boot and PID namespace @@ -65,6 +92,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 parent watcher, ledger client, filesystem monitor, or metric workers; pre-bind only the exact IPC and VSOCK descriptors those workers inherit. +- Keep dynamic port publication inside VM-owner confinement by having the + generation-bound coordinator channel grant an exact loopback listener. The + owner validates and admits that descriptor while retaining no ambient bind + authority, grants only `/dev/null` for the confined router's closed output, + persists restored mappings without mode-change authority, and preserves TCP + resets through descriptor closure without restoring outbound connect. The + service never accepts or forwards workload bytes. + - Keep host-worker confinement fail-closed across supported Linux Landlock ABIs, including single-threaded startup on ABI 6 and 7, and canonicalize macOS Seatbelt grants so system path aliases cannot silently deny or widen @@ -109,6 +144,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the matching checkpoint outside the ledger directory. Startup verification now rejects altered, substituted, reordered, or omitted checkpointed rows. +- Split high-volume ledger flushes across bounded commitment checkpoints, so + filesystem and package-manager audit bursts cannot stop the fail-closed + security ledger and refuse later VM commands. + +- Retire trusted ledger checkpoints with every permanent session deletion and + move them with retained failure evidence, so a reused ephemeral session name + cannot inherit an earlier ledger's authority. + - Pass the packaged ledger and proxy worker paths explicitly to installed and direct services, refuse incomplete confined-worker cohorts, and retire both children during reinstall or uninstall. @@ -126,6 +169,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 HTTP routing, security plugins, provider identity, response telemetry and guest DNS, preventing a request from combining authority from two revisions. +- Deliver VM-owner policy reloads as exact authenticated IPC bytes and retain + the last valid snapshot for MCP refreshes, so atomic policy publication + cannot replace the file inode granted at confinement and leave a running VM + stuck on stale policy; drop that post-boot path grant. + +- Keep credential broker calls valid on the confined proxy's single-threaded + runtime, preventing credential capture during model requests from panicking + the proxy and tearing down its VM session. + - Route proxy policy evaluation, credential capture, upstream substitution and redaction through a transport-independent engine with typed ledger and credential capabilities, so a confined proxy needs no credential-store or @@ -208,6 +260,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 authenticate the preview owner's kernel identity before transferring a browser connection descriptor. +- Generated Python requests redact nested credential validation inputs, + including raw parent inputs retained when another required field is missing. + +- Vhost-vsock validates complete descriptor and ring ranges, queue sizes, and + index-cell arithmetic before exposing guest-controlled addresses to the + kernel backend. + +- Virtio descriptor and ring metadata validate their complete guest RAM span + and address arithmetic before host access, including PCI-hole crossings. + +- OAuth disconnect joins an in-flight refresh and retires the latest rotated + token under denial, including material whose durable publication failed. + +- Disconnect admission and reconnect publication share the credential owner's + state lock, so concurrent reauthorization cannot clear a pending denial. + +- Responses WebSocket messages cross the model security boundary before + forwarding requests or returning output, including changes to rules on an + open connection. Completed turns retain model, usage and caller telemetry. + ### Fixed - The guest DNS listener sizes its UDP receive queue from the supported client, @@ -251,6 +323,156 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - The authenticated gateway forwards standalone proxy create, heartbeat and stop operations instead of returning 404 for documented SDK endpoints. +- Staged release verification includes the shared crypto owner and can verify + SHA-256 without optional BLAKE3 packages. Missing BLAKE3 reports a setup hint. + +- Decision ledger flushes use the full snapshot index when recording + occurrences, avoiding repeated scans as distinct decisions accumulate. + +- Idle ledger writers honor the original flush deadline when a late event + arrives, and failed idle flushes retain their normal retry interval. + +- Connection inventory supports bounded `limit` and `cursor` pages through the + service and SDKs, with malformed queries refused. + +- Interrupted credential disconnects resume bounded provider revocation after + service restart; maintenance retries unresolved retirement while grants stay revoked. + +- Archive index flushes batch SQLite inserts within the existing atomic + archive/ledger transaction, reducing repeated statement execution. + +- Stopped named sessions offer a read-only Credentials view with explicit + revocation; inspecting approvals does not wait for the connection catalog. + +- Named stopped sessions expose retained approval inventory and revocation + through canonical session IDs; grant creation still requires a live runtime. + +- Default session grant expiry fits the trusted remaining owner lifetime, + including time spent checking policy; explicit expiry overruns remain refused. + +- Retained credential approvals can be inspected and durably revoked using + their original session owner without creating or rebinding runtime authority. + +- Credential settings show account sign-in, cancellation and provider retirement; + running sessions can explicitly grant and revoke selected account capabilities. + Unsupported sign-in profiles remain disabled with an explanation. + +- Frontend connection and session-grant requests use authenticated SDK calls, + validate returned ownership, and never replay consent or grant mutations. + +- Grant polling retires authority when its consent is unavailable and refuses + exhausted revisions instead of publishing numbers clients cannot represent. + +- Credential grant retries normalize capability sets and exact RFC3339 expiry + instants, including offsets, and share Google's documented scope aliases. + Broker input budgets and timeouts come from one checked-in policy; completed + retry records retire after its 24-hour window without evicting active work. + +- Sigma YAML imports use a maintained safe Rust parser, reject duplicate + selection keys, and bound document, nesting and alias expansion. + +- Sigma selection lists and negated selections retain their logical grouping, + so an alternative cannot bypass another required security rule field. + Compiling grouped alternatives rejects excessive expansion before allocation. + +- Test-image cache retention keeps the OCI and prepared EROFS working set for + concurrent checkouts instead of reclaiming every fixture during enforcement. + +- Development builds optimize SQLite, its adapter and the ledger writer, and + archive index flushes reuse their prepared statement to reduce HTTP completion + stalls under load. + +- Protocol throughput benchmarks explicitly identify guest execution and reject + host/guest comparisons with incorrect execution labels. + +- The shared gateway OpenAPI contract correctly declares network creation's + existing HTTP 201 response, with matching generated SDK contracts. + +- Managed close retains original exit and grant authority when network cleanup + fails. Already-retired switches are awaited even if another departure fails; + a retry completes networking before session deletion and ownership closure. + +- Last-member network departure and retirement commit atomically. Failed + retirement preserves the membership and address for retry instead of + leaving an empty active network that cleanup can no longer find. + +- Logger durability barriers report accepted operations rejected by SQLite + during batch salvage, preventing failed registry writes from returning a + successful flush while preserving valid rows in the same producer batch. + +- Managed policy reload narrows active leases from their original renewal time + and updates expiry workers without extending ownership. Stale admission or + claim completions cannot overwrite a tighter timer; responses report the + enforced lease and valid renewal timing. Invalid policy refuses admission + and renewal while original cleanup authority remains available. + +- Managed close retains its confirmed original native exit outside ephemeral + session files, so restart can finish cleanup after directory deletion and + before the ownership close commit. Receipts survive unresolved cleanup and + retire with ownership tombstones. + +- Normal VM exit persists its native retirement receipt before moving a named + session, preserving confirmed stop authority for resume after host reboot. + +- Registered VM launches release shutdown admission before ongoing exec or + image setup, so workload duration does not hold the launch drain. + +- Failed VM launches retain original child retirement authority. Observed + native spawn failures prove no child exists; failures after spawn kill and + reap that child before publishing retirement to managed cleanup. + +- Managed startup recovery closes durable reservations that never prepared a + VM target, preserving capability authentication without replaying creation. + +- Service shutdown seals launch admission and joins already admitted workers + before selecting children for teardown, including cancelled HTTP callers. + +- Managed ownership stops renewing after ordinary stop/delete or loss of its + original VM spawn. Authenticated claims join the existing cleanup barrier; + changed spawns and failed cleanup cannot regain active ownership. + +- Managed VM spawns cannot transfer to persistence through ordinary save. + Admission uses original spawn ownership and refuses uncertain storage before + any persistence side effect. + +- TypeScript managed scopes cancel the owned callback and join cleanup at + validated lease expiry, including while renewal is stalled. + +- Python managed scopes stop at the validated lease expiry, including while + a renewal request is stalled, and join original-generation cleanup. + +- The Rust SDK provides managed ephemeral scopes with finite renewal, + cancellation-surviving cleanup, original-generation validation and explicit + retry authority when the service cannot confirm closure. + +- Preview and app-surface routes validate VM identifiers using the same rule + as the service and streaming gateway, rejecting malformed leading characters. + +- Linux warm resume restores allocated checkpoint extents into lazy zeroed RAM + and immediately resets stale guest vsock connections. Checkpoints preserve + page-sized sparse holes and existing file and directory handles. + +- The pinned guest kernel honors negotiated FUSE I/O sizes for large explicit + reads while retaining the existing speculative read-ahead limits. + +- Linux VM RAM stays aligned for transparent huge pages, accelerating boot and + cold filesystem reads without eagerly allocating guest memory. VirtioFS + supports full one-MiB requests, event-index wakeups on fresh and resumed VMs, + and bounded reusable I/O buffers through the shared Unix owner. + +- Prepared EROFS images use a catalog-bound BLAKE3 integrity annotation for + accelerated host and guest verification. The version 2 artifact contract + retains OCI transport SHA-256 and exact size, subject and descriptor checks. + +- OCI cache and prepared filesystem verification use the existing accelerated + crypto backend on CPUs without dedicated SHA instructions, retaining SHA-256 + identities and full integrity checks. + +- Explicitly admitted image references select their catalog-bound EROFS + filesystem instead of unnecessarily extracting OCI layers at startup. + +- CLI history and MCP tool tables truncate Unicode text at character + boundaries instead of panicking on a split UTF-8 character. - The authenticated gateway forwards host credential injection, so desktop settings and TCP SDK clients can use file and memory storage. @@ -341,10 +563,60 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- TypeScript SDK connection and credential grant resources preserve explicit + consent, typed refusal codes, ownership checks, deadlines and cancellation. + +- Python SDK connection and credential grant resources support explicit consent, + typed refusal codes, ownership checks and deadlines during VM name resolution. + +- Service-owned connection metadata and explicit credential grant routes share + managed ownership fences and enforce the current broker policy. Native AGY + enrollment remains unavailable pending its supported handoff integration. + +- Connection authorization and generation-bound credential grant operations in + the authenticated API contract and generated Rust, Python and TypeScript SDKs. + +- Credential-owned coordination for finite enrollment, redacted connection + state and explicit session grants, including durable request records and + same-owner resume fencing. Service activation and the supported native AGY + adapter remain under integration. + +- Rust SDK connection and session credential-grant resources implement the + nine approved calls with explicit consent, ownership checks and no automatic + replay. Live service activation and native AGY handoff remain pending. + +- TypeScript SDK managed scopes renew finite ownership and await cleanup + through caller abort, lease failure and uncertain creation without replay. + +- Python SDK managed scopes renew finite ownership leases and await the + original session's cleanup on exit, failure and cancellation without replaying create. + - `capsem proxy` starts a leased, VM-free OpenAI-compatible endpoint for a configured provider, prints its base URL, keeps it alive while the command runs and tears down its confined worker and ledger on exit. +- Ephemeral managed-session creation, capability-authenticated claim/renewal, + and awaited close use a finite service-owned lease and durable replay protection. + +- Desktop credential settings accept explicit API keys or tokens with file or + memory storage, clear submitted input, and return a copyable reference. + +- Rust SDK exposes the matching credential injection resource with file or + memory storage and opaque reference responses. + +- TypeScript SDK exposes credential injection with explicit storage, deadlines + and cancellation through the authenticated gateway. + +- Python SDK exposes credential injection with file or memory storage and + redacted request diagnostics. + +- Credential injection accepts explicitly selected host startup variables, + private file inputs, or an authenticated API. File and memory storage return + opaque references; memory material reaches VM owners only through host IPC. + +- The shared gateway contract describes credential injection with file or + memory storage and responses containing only opaque references. + - Rust SDK exposes the registry image catalog and prefetch through `Hypervisor.images()`, with typed disk states and per-call registry access. @@ -2028,9 +2300,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 A half-close on a published connection is carried as a signal: a slow reply after the client stops sending, and an upload after the peer stops sending, both arrive in full. -- Published TCP listeners stay with the VM owner. The confined router receives - only connected descriptor pairs; bounded acknowledgements and control failure - close both endpoints even when the router retains duplicate descriptors. +- Published TCP listeners are granted to and held by the VM owner. The confined + router receives only connected descriptor pairs; bounded acknowledgements and + control failure close both endpoints even when the router retains duplicate + descriptors. - The confined network companion is now named `capsem-router`; package signing continues to exclude virtualization authority. - Shell runs flush captured output before exiting, preserving short output diff --git a/Cargo.lock b/Cargo.lock index 9ea6c8046..371d0dcba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1280,7 +1280,6 @@ dependencies = [ "async-trait", "axum", "base64 0.22.1", - "blake3", "bytes", "capsem-api", "capsem-assets", diff --git a/Cargo.toml b/Cargo.toml index b86fcbe2a..cbc83c9a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,18 +122,6 @@ opt-level = 3 [profile.dev.package.ryu] opt-level = 3 -# Periodic ledger flushes drain archive indexes and copy hot tables on the -# writer actor. Unoptimized SQLite/adapter work filled its bounded producer -# queue and held HTTP response completion for seconds (#350). -[profile.dev.package.rusqlite] -opt-level = 3 - -[profile.dev.package.libsqlite3-sys] -opt-level = 3 - -[profile.dev.package.capsem-logger] -opt-level = 3 - [profile.release] lto = "thin" strip = "symbols" diff --git a/benchmarks/collectors/routes b/benchmarks/collectors/routes index 8bb18ced6..9e00ad1e1 100755 --- a/benchmarks/collectors/routes +++ b/benchmarks/collectors/routes @@ -46,10 +46,13 @@ from helpers.stopped_workspace import ( seed_stopped_workspace, ) +from benchmarks.helpers.route_measurements import route_metrics + ROOT = Path(__file__).resolve().parents[2] WARMUP = 2 MANAGED_REQUEST_ID = "fc120782-2272-4b3d-a2e6-a26ee2f8ab61" MANAGED_TOKEN = "af" * 32 +LIVE_ROUTES = {"/vms/{id}/exposures", "/vms/{id}/security/status"} def measured_routes() -> list[str]: @@ -62,11 +65,17 @@ def route_load_config() -> dict[str, int]: return dict(config["benchmark"]["route_load"]) -def request_path(route: str) -> str: +def request_path(route: str, live_session_id: str | None = None) -> str: """Resolve a configured route template against the stopped VM fixture.""" if route == BODY_ROUTE: return BODY_URL - return route.replace("{id}", VM_ID).replace("{request_id}", MANAGED_REQUEST_ID) + if route in LIVE_ROUTES and live_session_id is None: + raise ValueError(f"measured route requires a live owner fixture: {route}") + subject = live_session_id if route in LIVE_ROUTES else VM_ID + path = route.replace("{id}", subject).replace("{request_id}", MANAGED_REQUEST_ID) + if "{" in path or "}" in path: + raise ValueError(f"measured route has no successful fixture: {route}") + return path def request_input(path: str) -> tuple[str, bytes | None]: @@ -219,8 +228,9 @@ def main() -> int: metrics: dict[str, dict] = {} try: seed_stopped_workspace(service.tmp_dir, service.assets_dir) - service.start() - gateway.start() + with redirect_stdout(sys.stderr): + service.start() + gateway.start() service_proc = psutil.Process(service.proc.pid) gateway_proc = psutil.Process(gateway.proc.pid) targets = ( @@ -242,61 +252,23 @@ def main() -> int: }, ), ) - for path in routes: - concrete_path = request_path(path) - for target in targets: - print(f"{target.label} {path}", file=sys.stderr) - latencies, cpu = sample_warm_route(target, concrete_path, samples) - cold_latencies = sample_cold_route(target, concrete_path, samples) - metrics[f"{target.label}.{path}.latency_ms"] = { - "unit": "milliseconds", - "samples": latencies, - } - metrics[f"{target.label}.{path}.cold_latency_ms"] = { - "unit": "milliseconds", - "samples": cold_latencies, - } - for process_name, cpu_seconds in cpu.items(): - suffix = ( - "cpu_s" - if process_name == target.label - else f"{process_name}_cpu_s" - ) - metrics[f"{target.label}.{path}.{suffix}"] = { - "unit": "seconds", - "samples": [cpu_seconds], - } - concurrent_latencies, concurrent_cpu, rps = sample_concurrent_route( - target, concrete_path, load_samples, concurrency + with ExitStack() as fixtures: + live_id = None + if LIVE_ROUTES.intersection(routes): + live_id = fixtures.enter_context( + debug_session(service, service.home_dir / "route-registry", "route-live-owner") ) - metrics[f"{target.label}.{path}.concurrent_latency_ms"] = { - "unit": "milliseconds", - "samples": concurrent_latencies, - } - metrics[f"{target.label}.{path}.concurrent_rps"] = { - "unit": "requests_per_second", - "samples": [rps], - } - for process_name, cpu_seconds in concurrent_cpu.items(): - suffix = ( - "concurrent_cpu_s" - if process_name == target.label - else f"concurrent_{process_name}_cpu_s" + for path in routes: + concrete_path = request_path(path, live_id) + for target in targets: + print(f"{target.label} {path}", file=sys.stderr) + latencies, cpu = sample_warm_route(target, concrete_path, samples) + cold_latencies = sample_cold_route(target, concrete_path, samples) + metrics.update(route_metrics(target.label, path, latencies, cold_latencies, cpu)) + concurrent_latencies, concurrent_cpu, rps = sample_concurrent_route( + target, concrete_path, load_samples, concurrency ) - metrics[f"{target.label}.{path}.{suffix}"] = { - "unit": "seconds", - "samples": [cpu_seconds], - } - metrics[f"{target.label}.{path}.cold_latency_ms"] = { - "unit": "milliseconds", - "samples": cold_latencies, - } - for process_name, cpu_seconds in cpu.items(): - suffix = "cpu_s" if process_name == target.label else f"{process_name}_cpu_s" - metrics[f"{target.label}.{path}.{suffix}"] = { - "unit": "seconds", - "samples": [cpu_seconds], - } + metrics.update(route_metrics(target.label, path, concurrent_latencies, None, concurrent_cpu, rps)) finally: gateway.stop() service.stop() diff --git a/benchmarks/helpers/route_measurements.py b/benchmarks/helpers/route_measurements.py new file mode 100644 index 000000000..a3776c6b4 --- /dev/null +++ b/benchmarks/helpers/route_measurements.py @@ -0,0 +1,24 @@ +"""Raw route timing, throughput and per-process CPU series.""" + + +def route_metrics( + label: str, + path: str, + latencies: list[float], + cold_latencies: list[float] | None, + cpu: dict[str, float], + rps: float | None = None, +) -> dict[str, dict]: + concurrent = "concurrent_" if rps is not None else "" + prefix = f"{label}.{path}." + metrics = { + f"{prefix}{concurrent}latency_ms": {"unit": "milliseconds", "samples": latencies}, + } + if cold_latencies is not None: + metrics[f"{prefix}cold_latency_ms"] = {"unit": "milliseconds", "samples": cold_latencies} + if rps is not None: + metrics[f"{prefix}concurrent_rps"] = {"unit": "requests_per_second", "samples": [rps]} + for name, seconds in cpu.items(): + suffix = "cpu_s" if name == label else f"{name}_cpu_s" + metrics[f"{prefix}{concurrent}{suffix}"] = {"unit": "seconds", "samples": [seconds]} + return metrics diff --git a/build_system/builder/crypto.py b/build_system/builder/crypto.py index 7927f6a45..ad8a05181 100644 --- a/build_system/builder/crypto.py +++ b/build_system/builder/crypto.py @@ -12,17 +12,11 @@ import re import secrets as _secrets import uuid as _uuid -from collections.abc import Callable from pathlib import Path -from typing import Any, BinaryIO, Protocol +from typing import TYPE_CHECKING, Any, BinaryIO, Protocol -try: - from blake3 import blake3 as _blake3 -except ModuleNotFoundError as error: - _blake3 = None - _BLAKE3_IMPORT_ERROR = error -else: - _BLAKE3_IMPORT_ERROR = None +if TYPE_CHECKING: + from blake3 import blake3 as Blake3 sha256 = _hashlib.sha256 sha1 = _hashlib.sha1 # Protocol-specific handshake checks only. @@ -35,25 +29,21 @@ uuid_v4 = _uuid.uuid4 -class Digest(Protocol): - """Hash state shared by native digest implementations.""" +class Hasher(Protocol): + """Incremental digest contract shared by the accelerated bindings.""" - def update(self, data: bytes | bytearray | memoryview) -> None: ... + def update(self, data: bytes | bytearray | memoryview, /) -> object: ... - def hexdigest(self) -> str: ... + def digest(self) -> bytes: ... + def hexdigest(self) -> str: ... -def require_blake3() -> Callable[..., Any]: - """Return the optional BLAKE3 binding or preserve its import failure.""" - if _blake3 is None: - assert _BLAKE3_IMPORT_ERROR is not None - raise _BLAKE3_IMPORT_ERROR - return _blake3 +def blake3(*args: Any, **kwargs: Any) -> Blake3: + """Load the SIMD binding only for BLAKE3; staged SHA verification is stdlib-only.""" + from blake3 import blake3 as constructor -def blake3(*args: Any, **kwargs: Any) -> Any: - """Construct a BLAKE3 hasher without burdening SHA-only consumers.""" - return require_blake3()(*args, **kwargs) + return constructor(*args, **kwargs) def digest_file(path: Path, *, algorithm: str = "blake3") -> str: diff --git a/build_system/builder/gate/assetidentity.py b/build_system/builder/gate/assetidentity.py index 1fbfb36aa..6148e51fa 100644 --- a/build_system/builder/gate/assetidentity.py +++ b/build_system/builder/gate/assetidentity.py @@ -80,7 +80,7 @@ def _inputs(root: Path, relatives: tuple[str, ...]) -> Iterator[tuple[str, Path] yield path.relative_to(root).as_posix(), path -def _feed(digest: blake3.Digest, path: Path) -> None: +def _feed(digest: blake3.Hasher, path: Path) -> None: mode = path.lstat().st_mode digest.update(f"{stat.S_IMODE(mode):04o}".encode("ascii")) digest.update(b"\0") diff --git a/build_system/builder/gate/buildschema.py b/build_system/builder/gate/buildschema.py index 5b9eb9afc..298c02d58 100644 --- a/build_system/builder/gate/buildschema.py +++ b/build_system/builder/gate/buildschema.py @@ -37,6 +37,11 @@ class SdkConfig(SourcePackageConfig): specification: str +class PythonSdkConfig(SdkConfig): + environment_python: str + package_build_requirements: str + + class NodePackageConfig(Strict): project: str diff --git a/build_system/builder/gate/cachecontrol.py b/build_system/builder/gate/cachecontrol.py index f8011de79..a7147ce4c 100644 --- a/build_system/builder/gate/cachecontrol.py +++ b/build_system/builder/gate/cachecontrol.py @@ -3,10 +3,13 @@ from __future__ import annotations import os +from pathlib import Path from ..cache import budget from ..cache.config import load_policy from ..cache.controlmodels import ImageCachePolicy +from ..cache.generations import managed +from ..cache.leases import release_path, retain_generation from . import cachelayout from .config import GateConfig, for_root from .errors import GateError @@ -105,6 +108,36 @@ def enforce(self, cache_id: str, label: str | None = None) -> None: self._run("enforce", cache_id, "--reason", reason) +class CacheGenerationLease(Resource, name="cache-generation"): + """Keep an explicit managed input alive for one gate command.""" + + def __init__(self, config: GateConfig, stage_id: str, generation: Path) -> None: + self._config = config + self._stage_id = stage_id + self._generation = generation + self._lease: Path | None = None + + def acquire(self) -> None: + paths = cachelayout.cache_paths(self._config) + root = paths.stage(self._stage_id).resolve() + try: + relative = self._generation.resolve().relative_to(root) + except ValueError: + return + policy = paths.policy.stages[self._stage_id] + if len(relative.parts) != 1 or not managed(policy, relative.name): + return + if policy.lease_template is None: + raise GateError(f"cache stage {self._stage_id!r} cannot lease explicit inputs") + retain_generation(paths, self._stage_id, relative.name) + self._lease = root / policy.lease_template.format(key=relative.name) + + def release(self) -> None: + if self._lease is not None: + release_path(self._lease) + self._lease = None + + class CargoCacheBound(Resource, name="cargo-cache"): """Hold the shared Cargo target to its contract around a compiling command. diff --git a/build_system/builder/gate/config.py b/build_system/builder/gate/config.py index 25eb6add2..ea1a2fe14 100644 --- a/build_system/builder/gate/config.py +++ b/build_system/builder/gate/config.py @@ -34,6 +34,7 @@ LogsConfig, ModulesConfig, NodePackageConfig, + PythonSdkConfig, SbomConfig, SdkConfig, SigningConfig, @@ -118,7 +119,7 @@ class GateConfig(Strict): toolchain: ToolchainConfig functional: FunctionalConfig modules: ModulesConfig - sdk_python: SdkConfig + sdk_python: PythonSdkConfig sdk_typescript: SdkConfig mcp_typescript: NodePackageConfig sdk_rust: SdkConfig diff --git a/build_system/builder/gate/imagequalify.py b/build_system/builder/gate/imagequalify.py index 5a49939f8..5da039774 100644 --- a/build_system/builder/gate/imagequalify.py +++ b/build_system/builder/gate/imagequalify.py @@ -15,9 +15,12 @@ from pathlib import Path from . import pytestsuite, referenceimage, runtimeprepare +from .cachecontrol import CacheGenerationLease from .command import GateCommand from .errors import GateError +from .lifecycle import Resource from .plan import Plan +from .proc import Runner from .testmodules import InWorkspace @@ -35,6 +38,20 @@ def add_arguments(cls, parser: argparse.ArgumentParser) -> None: parser.add_argument("--image", help="the candidate's catalog name (with --layout)") parser.add_argument("--layout", type=Path, help="the candidate's OCI layout (with --image)") + def resources(self, runner: Runner) -> tuple[Resource, ...]: + resources = super().resources(runner) + layout = getattr(self._args, "layout", None) + if layout is None: + return resources + return ( + *resources, + CacheGenerationLease( + self._config, + self._config.functional.debug_image.cache_stage, + Path(layout), + ), + ) + def plan(self) -> Plan: image, layout = getattr(self._args, "image", None), getattr(self._args, "layout", None) if (image is None) != (layout is None): diff --git a/build_system/builder/gate/module_functional.py b/build_system/builder/gate/module_functional.py index 96c54c01e..4b328a1f2 100644 --- a/build_system/builder/gate/module_functional.py +++ b/build_system/builder/gate/module_functional.py @@ -111,9 +111,12 @@ def functional( # SDK producers need their toolchain, not VM content. In a composed plan # they already belong to the fast phase; adding the later runtime-content # edge to those shared producers would close a dependency cycle. - prepared: tuple[Step, ...] = ( - sdk_node, *checked, *sdkchecks.braavos(plan, phase, config, after=(sdk_node,)), + sdk_ready = sdkchecks.braavos(plan, phase, config, after=(sdk_node,)) + python_consumer = phase.add( + sdkchecks.python_package_prewarm(config), + after=(sdk_ready[0], *checked), ) + prepared: tuple[Step, ...] = (sdk_node, *checked, *sdk_ready, python_consumer) # The generated mock is gitignored, so it is never part of the source a run # is given, and the broad suite checks it for staleness. Made here when # this module runs alone, and handed over when a composed run has already diff --git a/build_system/builder/gate/sdkchecks.py b/build_system/builder/gate/sdkchecks.py index d2e1fb6a2..5f6c2174e 100644 --- a/build_system/builder/gate/sdkchecks.py +++ b/build_system/builder/gate/sdkchecks.py @@ -55,6 +55,50 @@ def python_package(config: GateConfig) -> Step: ) +def python_package_prewarm(config: GateConfig) -> Step: + """Rehydrate build backends after the candidate cache-enforcement phase. + + Clean consumers install the sdist offline, so uv must still have its + PEP 517 backend after the gate has enforced the shared cache budget. + """ + settings = config.sdk_python + return step( + "sdk.python.package-prewarm", + Run( + [ + "uv", + "export", + "--project", + settings.project, + "--frozen", + "--only-group", + "package-build", + "--no-emit-project", + "--format", + "requirements.txt", + "--output-file", + settings.package_build_requirements, + ], + ), + Run( + [ + "uv", + "pip", + "install", + "--python", + settings.environment_python, + "--reinstall", + "--requirements", + settings.package_build_requirements, + ], + outside_sandbox=True, + ), + kind=Kind.COMPILE, + needs=frozenset({Needs.DISK, Needs.NETWORK}), + speed=Speed.FAST, + ) + + def typescript_package(config: GateConfig) -> Step: settings = config.sdk_typescript return step( diff --git a/build_system/builder/release/tools/check_remote_release_readiness.py b/build_system/builder/release/tools/check_remote_release_readiness.py index 0d8c74412..e1835b506 100644 --- a/build_system/builder/release/tools/check_remote_release_readiness.py +++ b/build_system/builder/release/tools/check_remote_release_readiness.py @@ -15,7 +15,6 @@ import urllib.request from dataclasses import dataclass from pathlib import Path -from types import ModuleType from typing import Any, cast from urllib.parse import quote, urlparse @@ -28,15 +27,11 @@ from ... import crypto as hashlib -blake3: ModuleType | None try: - from ... import crypto as blake3_binding - - blake3_binding.require_blake3() + hashlib.blake3() except ModuleNotFoundError as error: BLAKE3_IMPORT_ERROR = error else: - blake3 = blake3_binding BLAKE3_IMPORT_ERROR = None diff --git a/build_system/pyproject.toml b/build_system/pyproject.toml index 517a18a3a..c2ceaa421 100644 --- a/build_system/pyproject.toml +++ b/build_system/pyproject.toml @@ -109,6 +109,7 @@ dev = [ "pytest>=9.0.3", "pytest-cov>=6.0", "pytest-xdist>=3.8.0", + "pyzstd>=0.17.0", "requests>=2.34.2", "rich>=15.0.0", "shellcheck-py>=0.11.0.1", diff --git a/build_system/tests/gate/test_build_assets.py b/build_system/tests/gate/test_build_assets.py index 3b419823b..690713a1b 100644 --- a/build_system/tests/gate/test_build_assets.py +++ b/build_system/tests/gate/test_build_assets.py @@ -383,7 +383,16 @@ def test_in_container_commands_write_only_where_the_container_user_owns() -> Non # walking unrelated Cargo or release output. assert "settings.layout.owned_parent_paths(settings.mount)" in container assert "claim_owned_paths(self._docker, self.name, self._settings)" in container - assert '["chown", f"{guest}:{guest}", *parents]' in container + runner = RecordingRunner(PROJECT_ROOT) + claim_owned_paths(Docker(runner), config.install.container, config.install) + assert list(runner.commands[-1].argv) == [ + "docker", + "exec", + config.install.container, + "chown", + f"{guest.name}:{guest.name}", + *config.install.layout.owned_parent_paths(config.install.mount), + ] assert '["chown", "-R", f"{guest}:{guest}", *parents]' not in container # Every path this user writes has to live off the bind mount. diff --git a/build_system/tests/gate/test_gate_cache_control.py b/build_system/tests/gate/test_gate_cache_control.py index 9bc196e7f..c2285ce3a 100644 --- a/build_system/tests/gate/test_gate_cache_control.py +++ b/build_system/tests/gate/test_gate_cache_control.py @@ -1,10 +1,15 @@ """The gate is a thin caller of the typed cache control plane.""" +import argparse from pathlib import Path +from types import SimpleNamespace from capsem_builder.cache.config import load_policy +from capsem_builder.cache.leases import active_path +from capsem_builder.gate import cachelayout from capsem_builder.gate import config as gate_config -from capsem_builder.gate.cachecontrol import CacheControl +from capsem_builder.gate.cachecontrol import CacheControl, CacheGenerationLease +from capsem_builder.gate.imagequalify import ImageQualifyModule from capsem_builder.gate.sourcecommit import SourceCommit from helpers.gate import RecordingRunner @@ -58,6 +63,40 @@ def test_receipt_policy_comes_from_validated_cache_policy() -> None: assert policy.max_size_bytes == 12 * 1024**3 +def test_a_managed_candidate_layout_is_leased_for_the_command_lifetime( + monkeypatch, tmp_path: Path +) -> None: + stage = tmp_path / "images" + layout = stage / "capsem-debug-candidate" + layout.mkdir(parents=True) + policy = SimpleNamespace( + stages={ + "test-images": SimpleNamespace( + lease_template=".{key}.lock", managed_globs=("*",) + ) + } + ) + paths = SimpleNamespace(policy=policy, stage=lambda _stage_id: stage) + monkeypatch.setattr(cachelayout, "cache_paths", lambda _config: paths) + lease = CacheGenerationLease(CONFIG, "test-images", layout) + + lease.acquire() + lease_path = stage / ".capsem-debug-candidate.lock" + assert active_path(lease_path) + lease.release() + assert not active_path(lease_path) + + +def test_image_qualification_holds_its_explicit_candidate_layout(tmp_path: Path) -> None: + runner = RecordingRunner(PROJECT_ROOT) + command = ImageQualifyModule( + runner, + argparse.Namespace(image="capsem-debug", layout=tmp_path / "layout", sandbox=None), + ) + + assert any(isinstance(resource, CacheGenerationLease) for resource in command.resources(runner)) + + def test_private_gate_controls_outer_cache_with_private_policy(monkeypatch, tmp_path: Path) -> None: prefix = tmp_path / "prefix" prefix.mkdir() diff --git a/build_system/tests/gate/test_gate_config.py b/build_system/tests/gate/test_gate_config.py index a83e41e99..710921d78 100644 --- a/build_system/tests/gate/test_gate_config.py +++ b/build_system/tests/gate/test_gate_config.py @@ -502,7 +502,7 @@ def _source_package_type(): def test_source_package_and_sdk_keep_required_fields_and_reviewed_order() -> None: - from capsem_builder.gate.buildschema import SdkConfig + from capsem_builder.gate.buildschema import PythonSdkConfig, SdkConfig source = _source_package_type() fields = ("project", "manifest", "source", "tests", "build_output") @@ -513,23 +513,30 @@ def test_source_package_and_sdk_keep_required_fields_and_reviewed_order() -> Non # last, while remaining mandatory and preserving every SDK owner type. assert tuple(SdkConfig.model_fields) == (*fields, "specification") assert all(field.is_required() for field in SdkConfig.model_fields.values()) - for owner in ("sdk_python", "sdk_typescript", "sdk_rust"): + for owner in ("sdk_typescript", "sdk_rust"): assert gate_config.GateConfig.model_fields[owner].annotation is SdkConfig assert isinstance(getattr(CONFIG, owner), SdkConfig) + assert gate_config.GateConfig.model_fields["sdk_python"].annotation is PythonSdkConfig + assert tuple(PythonSdkConfig.model_fields) == ( + *SdkConfig.model_fields, + "environment_python", + "package_build_requirements", + ) + assert all(field.is_required() for field in PythonSdkConfig.model_fields.values()) def test_hand_written_source_package_needs_no_codegen_specification() -> None: source = _source_package_type() - values = CONFIG.sdk_python.model_dump(exclude={"specification"}) + values = CONFIG.sdk_rust.model_dump(exclude={"specification"}) assert source.model_validate(values).model_dump() == values with pytest.raises(ValidationError, match="extra_forbidden"): - source.model_validate(CONFIG.sdk_python.model_dump()) + source.model_validate(CONFIG.sdk_rust.model_dump()) @pytest.mark.parametrize("field", ("project", "manifest", "source", "tests", "build_output")) def test_source_package_cannot_omit_a_required_input(field: str) -> None: source = _source_package_type() - values = CONFIG.sdk_python.model_dump(exclude={"specification", field}) + values = CONFIG.sdk_rust.model_dump(exclude={"specification", field}) with pytest.raises(ValidationError, match="Field required"): source.model_validate(values) @@ -538,12 +545,12 @@ def test_sdk_cannot_lose_its_generated_source_specification() -> None: from capsem_builder.gate.buildschema import SdkConfig with pytest.raises(ValidationError, match="specification"): - SdkConfig.model_validate(CONFIG.sdk_python.model_dump(exclude={"specification"})) + SdkConfig.model_validate(CONFIG.sdk_rust.model_dump(exclude={"specification"})) def test_source_package_retains_strict_frozen_configuration() -> None: source = _source_package_type() - values = CONFIG.sdk_python.model_dump(exclude={"specification"}) + values = CONFIG.sdk_rust.model_dump(exclude={"specification"}) with pytest.raises(ValidationError, match="extra_forbidden"): source.model_validate({**values, "unknown": "input"}) package = source.model_validate(values) diff --git a/build_system/tests/gate/test_gate_functional_sdks.py b/build_system/tests/gate/test_gate_functional_sdks.py index f9b02f6f3..dd17a0c65 100644 --- a/build_system/tests/gate/test_gate_functional_sdks.py +++ b/build_system/tests/gate/test_gate_functional_sdks.py @@ -17,6 +17,7 @@ PREPARED = ( "sdk.python.sync", "functional.sdk.rust.example", "fast.sdk.python.build", "fast.sdk.typescript.build", "fast.sdk.typescript.package-prewarm", + "functional.sdk.python.package-prewarm", ) @@ -75,6 +76,19 @@ def test_functional_installed_sdks_use_current_archives_and_declared_network_pre assert "--no-isolation" in plan.step_named(python).actions[0].render() +def test_candidate_rehydrates_python_build_backends_after_cache_enforcement() -> None: + from capsem_builder.gate.execution import Needs + + plan = gate_plan("candidate") + label = "functional.sdk.python.package-prewarm" + warmed = plan.step_named(label) + assert {"fast.sdk.python.build", "prepare.cache-enforcement"} <= _ancestors(plan, label) + assert Needs.NETWORK in warmed.needs + exported, installed = (action.render() for action in warmed.actions) + assert "--only-group package-build" in exported + assert "--reinstall" in installed and "[outside kernel sandbox]" in installed + + def test_composed_sdk_packages_have_one_producer_and_no_dependency_cycle() -> None: plan = gate_plan("candidate") labels = plan.labels diff --git a/build_system/tests/release/test_release_doctor_contract.py b/build_system/tests/release/test_release_doctor_contract.py index b8e028237..6eb191a9e 100644 --- a/build_system/tests/release/test_release_doctor_contract.py +++ b/build_system/tests/release/test_release_doctor_contract.py @@ -1315,7 +1315,7 @@ def test_asset_channel_deploy_smoke_verifies_public_evidence_artifacts() -> None in workflow ) assert "from ... import crypto as hashlib" in script - assert "from ... import crypto as blake3" in script + assert "hashlib.blake3()" in script assert "def fetch_and_verify_evidence_artifact" in script assert 'site, sbom, "sha256", "host SBOM evidence", "spdx"' in script assert 'site, obom, "blake3", "VM OBOM evidence", "rootfs_cyclonedx"' in script diff --git a/build_system/tests/sdkgen/test_python_operations.py b/build_system/tests/sdkgen/test_python_operations.py index 86c3d13e6..df9d890ac 100644 --- a/build_system/tests/sdkgen/test_python_operations.py +++ b/build_system/tests/sdkgen/test_python_operations.py @@ -26,29 +26,12 @@ ROUTES = read_operations(SPEC) -def sample(schema: Schema) -> object: - if schema.ref: - return sample(SCHEMAS[schema.ref.rsplit("/", 1)[1]]) - if schema.type is None and not schema.one_of: - return {"nested": [True, None, {"number": 3}]} - if schema.one_of: - return sample(schema.one_of[0]) - if schema.enum: - return schema.enum[0] - if isinstance(schema.type, list) or schema.type == "null": - return None - if schema.type == "object": - return {key: sample(schema.properties[key]) for key in schema.required} - if schema.type == "array": - return [] - assert schema.type is not None - if schema.format == "ipv4": - return "10.128.0.2" - if schema.format == "uuid": - return "fc120782-2272-4b3d-a2e6-a26ee2f8ab61" - if schema.pattern == "^[0-9a-f]{64}$": - return "af" * 32 - return {"string": "value", "integer": 0, "number": 0.5, "boolean": True}[schema.type] +def test_uuid_query_uses_the_shared_alias_and_refuses_invalid_material(package: ModuleType) -> None: + async def run() -> None: + async with package._transport.Transport("http://127.0.0.1:1", "secret") as transport: + with pytest.raises(ValidationError): + await package.operations.list_connections(transport, limit=1, cursor="private-query") + asyncio.run(run()) @pytest.fixture diff --git a/build_system/uv.lock b/build_system/uv.lock index 81850e7d4..f002aef03 100644 --- a/build_system/uv.lock +++ b/build_system/uv.lock @@ -160,6 +160,76 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "backports-zstd" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/f0/9ba1b05811aa5f5434f69768253129460a5744e1814f359efba39a01ce20/backports_zstd-1.7.0.tar.gz", hash = "sha256:1a967189c1822b6e85a2e550fdfc88a3272c17633ea0a4732dac5911a8034f2b", size = 1003722, upload-time = "2026-08-15T17:26:43.96Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/14/416e2e75d434bf2b8433ba54f10e5ec01a63bb1dfc7c6a82151faa735b50/backports_zstd-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:165a8898c5514b69533edf4ab1f4f4b4bacc62a137a76f36889b473150ec28a5", size = 438790, upload-time = "2026-08-15T17:25:16.171Z" }, + { url = "https://files.pythonhosted.org/packages/08/0d/97e70a1d47d660c3854dfbbbd8a8ea9a98a0993976d9b0e0da07525dcff1/backports_zstd-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:700ebb797956767679dbca38e45eaa5c21630e460e31ef53bb4b849125bb5d87", size = 367595, upload-time = "2026-08-15T17:25:17.382Z" }, + { url = "https://files.pythonhosted.org/packages/5c/8f/9b09bc4d29c2b697e9557a54e1e52b264a1ca3babd36542e7be6a0609cf6/backports_zstd-1.7.0-cp311-cp311-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:47f14a24428a2bc070e26c402f8d6d25676345c32fa116b16b60167a2925df2a", size = 507936, upload-time = "2026-08-15T17:25:18.695Z" }, + { url = "https://files.pythonhosted.org/packages/89/e6/7eb513bb06fb2733e71cf358f227969996b74883de86458935c09f08d1c4/backports_zstd-1.7.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c358e72e5ff8f23e9f3ec778be4d67ddc5ced3e6d8f03521db29d7357a773fc3", size = 477864, upload-time = "2026-08-15T17:25:19.833Z" }, + { url = "https://files.pythonhosted.org/packages/3f/68/fe0e57f2c8e04560eecc106bba18ad62d0576001722e8c5b619bb4517991/backports_zstd-1.7.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6c8c183027eae38f5b0643d153f7f91e569d22ee8db25639aea0745677a38ed8", size = 583229, upload-time = "2026-08-15T17:25:21.144Z" }, + { url = "https://files.pythonhosted.org/packages/e3/d3/5944fbdfc8c03b8ef72c73c36652a32bb251b1b8ccefab07a8a8fbf202cb/backports_zstd-1.7.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5d8493f71d9c5c05d18554afc6bb9a319a6674478e8f3042c7e22900c3a06f4d", size = 642898, upload-time = "2026-08-15T17:25:22.379Z" }, + { url = "https://files.pythonhosted.org/packages/96/f2/661bd15e062ceb1a20a78e40598fee599a31472e60a7961dcd75b467c94f/backports_zstd-1.7.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b2e505d8923e1e9224cf249b99c92cf728e9eb91fbd1e07a9c2816013621fad3", size = 493145, upload-time = "2026-08-15T17:25:23.833Z" }, + { url = "https://files.pythonhosted.org/packages/00/6b/454369a552a3d114b293706441dc43412639a30665b9551959f0773e9b62/backports_zstd-1.7.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5d1bdc293267200e31baf35aa142c6d0ac3e8cce650f79c84e6a32980dfbfd5c", size = 567043, upload-time = "2026-08-15T17:25:25.104Z" }, + { url = "https://files.pythonhosted.org/packages/bc/f2/e7af20299deb43f52aaf24f74e60b994201aea6d22c8a2adaaa13dc4b109/backports_zstd-1.7.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d85c18170e8cdba339edc67a5021cf79ccc858f5fda6aeae71f9015c5e463f6", size = 483881, upload-time = "2026-08-15T17:25:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c4/0882cfac8714345cfcc5ba139e16c7b64aee9f2fec3ebff9de77131f1d14/backports_zstd-1.7.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:96a6f8d3f4cefb6b11ebfc30fc0d970430ecfb169a6555990734a2a46977ec4b", size = 511455, upload-time = "2026-08-15T17:25:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/be/85/01cfbb2f07475ac1091ea93fbd04762c95ee6d82c937e2508072e12a9eb6/backports_zstd-1.7.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c2c01cb823ed1b2422905a9791759bdc986e44e7a12b4661e9d712d5c8946016", size = 587585, upload-time = "2026-08-15T17:25:29.012Z" }, + { url = "https://files.pythonhosted.org/packages/61/2f/f378daf513ca0feb5740aa4b1291c5133e5095830a7052da6088974f477b/backports_zstd-1.7.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:86785aef2b4663a97c932d829ddc9565354cc628e2ae61764d9d93c8b186d65f", size = 564894, upload-time = "2026-08-15T17:25:30.28Z" }, + { url = "https://files.pythonhosted.org/packages/34/ef/cb031d27b06863aa666d49dada3c1010151306b98861b8b826ae722af1a6/backports_zstd-1.7.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:633ceee3ba86f696fc4e992f7bce558c132c26d04d64d0bb8c2f5d487d5b3aee", size = 633546, upload-time = "2026-08-15T17:25:31.652Z" }, + { url = "https://files.pythonhosted.org/packages/f0/44/b5f1480f6c250ed72f22e8682d6532f992aef0e2033b21b8d8bff96be034/backports_zstd-1.7.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a80bc6a8c9aeaad76cc3ecd58067ec038a764807186b0df970c760df39b89c7", size = 497152, upload-time = "2026-08-15T17:25:32.957Z" }, + { url = "https://files.pythonhosted.org/packages/db/30/13f0447faef940a763dfddf6ba2d4941bb45a350bba8c9ba56a22e933dfd/backports_zstd-1.7.0-cp311-cp311-win32.whl", hash = "sha256:1713271e2faea852a1682a6143c19c3506cd2e1b71f60a8924c59a9d2554d2b2", size = 292182, upload-time = "2026-08-15T17:25:34.262Z" }, + { url = "https://files.pythonhosted.org/packages/99/04/8f67d5436f7ef4b1d286b8b186fb4a3e371416921110f8dc0f6c4d9e497d/backports_zstd-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:ae840be71108f6020567dd389c973e70a4374a6c0b03c02d3242c8a98a9b3cdb", size = 329717, upload-time = "2026-08-15T17:25:35.517Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/e446bb3d4520e618571a929ceb7776124d6f8491ea17d3355a8867deb031/backports_zstd-1.7.0-cp311-cp311-win_arm64.whl", hash = "sha256:8827a5601c749a986faa163f3b59d59eedc5947812be114f7132c3d4ad153fee", size = 292294, upload-time = "2026-08-15T17:25:36.797Z" }, + { url = "https://files.pythonhosted.org/packages/df/23/240495dec973dcfb34816248956ca8d05b32fb75936c226c1cf497b83b83/backports_zstd-1.7.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b5548a857bb0fcc5449cc3687353547396c6b1ecd4bd882f1cd34fa8d29e70ca", size = 439047, upload-time = "2026-08-15T17:25:38.084Z" }, + { url = "https://files.pythonhosted.org/packages/6d/24/5556959c7d03bfee5ff14d7f07dd9bf8de737c69f81d823a32784ab39c34/backports_zstd-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bab192b934fdf5a03df4752556d9c8af2d058163fdfbafd4a253cdfe25449a6f", size = 367666, upload-time = "2026-08-15T17:25:39.233Z" }, + { url = "https://files.pythonhosted.org/packages/b2/cb/557db98001c4a7202beed19e8bd42603a2315b80fd5def7e21a0b048ec3b/backports_zstd-1.7.0-cp312-cp312-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:8344260bed9842c415a93d9bfe23ea834e5f27758827d56933d8c0d06db507a2", size = 508475, upload-time = "2026-08-15T17:25:40.367Z" }, + { url = "https://files.pythonhosted.org/packages/40/4b/820acbc2c1d1d945aedca0c0d22546a948630ffb186df523098fbd669a95/backports_zstd-1.7.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c55e55e1e9dee312bc5e186386e6aa5207482a6d2242bd7c14709ded254f87f", size = 478240, upload-time = "2026-08-15T17:25:41.806Z" }, + { url = "https://files.pythonhosted.org/packages/f9/76/77fa9b385e79d4c106ce15d66681978f39a844b0eb5db02682687246b716/backports_zstd-1.7.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cf609af3735c7e697ccd13f6b0c88da57c201b6ea63c6afbfe81d6f9b50e298c", size = 583611, upload-time = "2026-08-15T17:25:43.104Z" }, + { url = "https://files.pythonhosted.org/packages/95/a4/fbb7c73336f3279dad36da94382a59755100b656301ea836ebaa42736581/backports_zstd-1.7.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:676a37971f676830d4f90cee8fdf4e438781596fb2f2d1984ac76c9b3eb39a69", size = 642496, upload-time = "2026-08-15T17:25:44.322Z" }, + { url = "https://files.pythonhosted.org/packages/1e/40/121917bd2671bc3f1507c25503c0554f0b52483edcca4e6210e6d22228df/backports_zstd-1.7.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:470895d0bcddc850766e593d1b26764fb138c2feed149f515a2627ef9587d54c", size = 496195, upload-time = "2026-08-15T17:25:45.503Z" }, + { url = "https://files.pythonhosted.org/packages/de/c7/c6379a0d734bea1c7f14d07c23258108cc92b994654e25cfe3a3e88cd785/backports_zstd-1.7.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:02f2f6649a342d0901ddb35596ddadb7c3bb1cf6bb54d691e5e0285f1fa0674f", size = 570964, upload-time = "2026-08-15T17:25:46.648Z" }, + { url = "https://files.pythonhosted.org/packages/7a/a4/372c3dd3017c3f93cda0acbc282f8073b70efdc1b56d1fdeebe023660725/backports_zstd-1.7.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:132ba81fad59d44958b7d10da31545e7128c469cfbc2e268d0eaab96daa64175", size = 484276, upload-time = "2026-08-15T17:25:48.129Z" }, + { url = "https://files.pythonhosted.org/packages/f5/50/83fa7bdd5e1d808203b9143848fdf7e15de399b8119a0d4378b2aea9be78/backports_zstd-1.7.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a3e1c6ce0b232ee6703ed24ee126e8186107f5a4e56edbd21cd1aa5a8c6bfd12", size = 511963, upload-time = "2026-08-15T17:25:49.666Z" }, + { url = "https://files.pythonhosted.org/packages/56/d2/d4ed32c353148acc18f3b665ab24a677b9c49d3640244424c5d6046400c5/backports_zstd-1.7.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:d7a7cb964eb8d1bb5d039970b16fe54802ea47dc935ae96d9874844a126bf8ff", size = 588025, upload-time = "2026-08-15T17:25:51.288Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5a/df8b5b848e8dfdec6edca55f22067ffbafa081d81aec1313e28155c3fea3/backports_zstd-1.7.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:12a9842a2ec2854cbec7f252ab29d44c2b772788a9bbafded743ca4bf73b115f", size = 568223, upload-time = "2026-08-15T17:25:52.633Z" }, + { url = "https://files.pythonhosted.org/packages/43/8c/f970f15e7fdbf8a251f121c91364fa68bbc2dfab4d5eca058427dec63397/backports_zstd-1.7.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:138154eea8ced84394991bf0e819dba6b690306a178dd528c28eee724b7d4aec", size = 632937, upload-time = "2026-08-15T17:25:54.511Z" }, + { url = "https://files.pythonhosted.org/packages/6a/d0/e36c18c87a74421954502d123ff7027e0a63a7624dffa99ec0f7474deff9/backports_zstd-1.7.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:468b636ed365627b364c94be1c35a52858e13b5bc1fa3f068bbc71b1af65f3d7", size = 500735, upload-time = "2026-08-15T17:25:56.064Z" }, + { url = "https://files.pythonhosted.org/packages/1c/57/fc72280334d2aa94238c5882052263bd7796c1fa924044353c30d058e0c3/backports_zstd-1.7.0-cp312-cp312-win32.whl", hash = "sha256:f026fe2e89b7ff01ba6ebec6abaff34c6063919151a32afb68714cf139e17c50", size = 292394, upload-time = "2026-08-15T17:25:57.469Z" }, + { url = "https://files.pythonhosted.org/packages/71/89/6cea747bdeef34cd12482b17e604b832fdb0962987132b99496f1a6c3f82/backports_zstd-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:2ea62ba2f1a6e6c9e6dc108921f9ae881969ca72e073162fa488d0de3eb2713f", size = 329876, upload-time = "2026-08-15T17:25:58.798Z" }, + { url = "https://files.pythonhosted.org/packages/64/28/b4a17c07d5a50a45cb04592960d1593cdf3b3728968371f332aa3643b804/backports_zstd-1.7.0-cp312-cp312-win_arm64.whl", hash = "sha256:cefb983345c55ccaa20423a4eb96434730e6d640ffa2db9b60e5bedb0fbef94e", size = 292461, upload-time = "2026-08-15T17:25:59.928Z" }, + { url = "https://files.pythonhosted.org/packages/6f/24/32b3358ae3a4df0ebad85ebbce721818c6d76a836119bee76089d103e951/backports_zstd-1.7.0-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:a3fbcbf819bee2b06b8666b13742098d0f40663ee34e64a12bc360ec0f5e3d89", size = 400913, upload-time = "2026-08-15T17:26:01.089Z" }, + { url = "https://files.pythonhosted.org/packages/af/f3/39ef7dd75eb1e699e25a19212737a73d3c030a0c9fd1d0ed1572b5f8e493/backports_zstd-1.7.0-cp313-cp313-android_24_x86_64.whl", hash = "sha256:efee02f18e04c2e9e6d694c5cf9b7457c4bda3ea96f48b1ee69769e06bb9d89f", size = 454915, upload-time = "2026-08-15T17:26:02.294Z" }, + { url = "https://files.pythonhosted.org/packages/76/e8/8209081e094aa98b2f28bac388619c85b1a44aed813d6b3c54d1da79d19a/backports_zstd-1.7.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:ecc95fa0e91d92951d74468e7789afdf91d9e702f40af2d0fcbf0ded4d0f650a", size = 357992, upload-time = "2026-08-15T17:26:03.552Z" }, + { url = "https://files.pythonhosted.org/packages/b3/65/64025302bae4ba924d613e404c6120bf194b5636786960ece274622a4a3e/backports_zstd-1.7.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34154d82fc0246738159084d146401073f9ac9cfd755b66bb8853ca06037810c", size = 366686, upload-time = "2026-08-15T17:26:04.812Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b9/c4d24d113d28b774662152c462d38d28109741d6d45c1aea7834371741dc/backports_zstd-1.7.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:44b687b1c0be5cb279693d2682f91ff84c559d679b2ef2fbe501fe4b2db2c4bb", size = 447221, upload-time = "2026-08-15T17:26:05.979Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9f/8db55c7f77aec60879844a879ac026065d8f03aab74080701acc060c4168/backports_zstd-1.7.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:dcdbd368659f46b570114eeea36b75347716523870d71f6bc5d7801862aefd6e", size = 438571, upload-time = "2026-08-15T17:26:07.421Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f8/72930ae4bb7bf6b9d6c7c31bce7b3e5751c062269a4ee718066e25f1973b/backports_zstd-1.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eda97fa535d4651a4ccdeed4ee7dde3978369046abc8a7456a7117d4271f9333", size = 367041, upload-time = "2026-08-15T17:26:08.537Z" }, + { url = "https://files.pythonhosted.org/packages/17/9b/7289dc191b34279d8f176bf5b181c3b26f8e049d14a2c0a2637650f034e5/backports_zstd-1.7.0-cp313-cp313-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:7e3999b5141d7f85171822d06112f70f7f317d162f0120530dd2c7a28dbf8add", size = 507676, upload-time = "2026-08-15T17:26:09.909Z" }, + { url = "https://files.pythonhosted.org/packages/7c/4d/6dd730b79ab96532e23fe851003545b4cc79e50c5b4c79ffcbe1b724eec4/backports_zstd-1.7.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:69367726f4075c2574746f5883b0dc045805c5b02a81fdf8c829c26d33969de3", size = 477744, upload-time = "2026-08-15T17:26:11.038Z" }, + { url = "https://files.pythonhosted.org/packages/e1/53/11687e5019d56ea47893cf2ba59a6b4884a4e2d1496d0e653aed373b973f/backports_zstd-1.7.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:15e97edfd173ade365c01bac7d9d297fa906686015cdbcb5f32a0d410887826b", size = 583215, upload-time = "2026-08-15T17:26:12.379Z" }, + { url = "https://files.pythonhosted.org/packages/aa/c7/5a8c58542469ab31680c403b844770c119a976fd4cf1000fd7d53e7d0f77/backports_zstd-1.7.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:32a94cdcf16b44395cd55086ea38877395ca6bf3362cb507b0eb86db2a45a6a4", size = 644125, upload-time = "2026-08-15T17:26:13.651Z" }, + { url = "https://files.pythonhosted.org/packages/11/35/be5485e65df95b86c4981ad4a577b505cfeec6b700a46a86e2e3175ac718/backports_zstd-1.7.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3f4887a8a1fd1290017fe5a1d29a7d1dc5c57f9477fbd64f119316a7e3ae769", size = 492714, upload-time = "2026-08-15T17:26:14.838Z" }, + { url = "https://files.pythonhosted.org/packages/96/8b/a0603458ca08e4a56f09ae58588ce3c0453425e753df704d9aeaabb66ae5/backports_zstd-1.7.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e590313ce156f1d8986dff3107e8ed1651d6d106a56b3a95f965ff8d845ba979", size = 567953, upload-time = "2026-08-15T17:26:16.276Z" }, + { url = "https://files.pythonhosted.org/packages/02/87/2296db4c3c578947c35ccd8dcdf7992316d7e1f5f43cc829c062b3ed9319/backports_zstd-1.7.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:565270b0d6497970fa97a0df59593ae0d225e4176678bbce851d39e5f8aa422b", size = 483564, upload-time = "2026-08-15T17:26:17.493Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d8/f53a79e6bf3cdb7ae08f95220c80bd0d606f3d6c3482995deaf21d024fb9/backports_zstd-1.7.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:37ef23c6c522fe935726c8fba6344350973c4a23b06d10194d90d0868b09ff7a", size = 511193, upload-time = "2026-08-15T17:26:18.7Z" }, + { url = "https://files.pythonhosted.org/packages/31/ea/d4e2eb159cd5813debd5a34d0644caff5fe7cf2e569bf5b02a82934aeee7/backports_zstd-1.7.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b3975330159f1efdd1fba76afe1c7b84f66f26e2bf209b32630fb148d647e0d5", size = 587748, upload-time = "2026-08-15T17:26:20.148Z" }, + { url = "https://files.pythonhosted.org/packages/81/d2/b5ec9709660fb1c193508215d9c30e781fac406183faac7c3c36b1c583a9/backports_zstd-1.7.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b40bc8cd0a86cbbe8263a9c3a2bf2e34897483516c6d799725412a19524c32e3", size = 565808, upload-time = "2026-08-15T17:26:21.349Z" }, + { url = "https://files.pythonhosted.org/packages/bd/13/004735cc4536483cbd973a60346a9dbc7bb977b13c28b55a11da14bb0a1e/backports_zstd-1.7.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f37e12ef10747f76901b1f20ef70d33221e861de177dba5ba08552242c6fd4bd", size = 634520, upload-time = "2026-08-15T17:26:22.944Z" }, + { url = "https://files.pythonhosted.org/packages/a3/28/05b11f7084d1100491cf7c60962aafd900c3dd01b1fc1ce85914476cdae0/backports_zstd-1.7.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5992143b2a8b71b4d17afed20cce2df50f8718228e31d6e716493b1fe9201712", size = 497098, upload-time = "2026-08-15T17:26:24.181Z" }, + { url = "https://files.pythonhosted.org/packages/f5/a5/bdc98d039ddbd5815fc1dd71912bbfb9f820a46ec12004ead51c8d60ea50/backports_zstd-1.7.0-cp313-cp313-win32.whl", hash = "sha256:31ae30d216ffae9243dfa607bcb995f94a70de5765bb8fae1e35ea1ad6497959", size = 291974, upload-time = "2026-08-15T17:26:25.512Z" }, + { url = "https://files.pythonhosted.org/packages/13/7d/fb0da7351e8b152d5149127594972922829281c316618df37a7e724f2eb9/backports_zstd-1.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:8086b4a7443bb2863f7ef8edb317b715d5f3ccec6c5512619bd23d57661ba1b7", size = 329550, upload-time = "2026-08-15T17:26:26.683Z" }, + { url = "https://files.pythonhosted.org/packages/37/f9/109ac272d650483fbdfa611c0040253a405f640604fbc90acc8076c6d37f/backports_zstd-1.7.0-cp313-cp313-win_arm64.whl", hash = "sha256:7eaceeec75e1dbdce40b81fb0ed1ffdb7ce492d970db7f8aabd6a95ccd6c3dd3", size = 292174, upload-time = "2026-08-15T17:26:27.819Z" }, + { url = "https://files.pythonhosted.org/packages/7a/ea/42fe3258e02a65603d1eab26200712e37bef6ea408e7f9dbfd6858bc036a/backports_zstd-1.7.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:de58be0a3109cfb83b4e61e2b6eb770201cc132ee5a7c677cd8e0140ad2be80c", size = 413302, upload-time = "2026-08-15T17:26:36.286Z" }, + { url = "https://files.pythonhosted.org/packages/37/00/486044556d715a7b1a41e9cd69544bf8cb3988b383453657c021d24c5c27/backports_zstd-1.7.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:c13f73d0389cdc88b02c05e8175d8ad3030e9e70ee079748763166aa843b647d", size = 343983, upload-time = "2026-08-15T17:26:37.603Z" }, + { url = "https://files.pythonhosted.org/packages/02/f8/f078a32c80ef7546ec2d1206a38bedf4d150cbaac653f8f32d7329f987ff/backports_zstd-1.7.0-pp311-pypy311_pp73-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:a2e30ea49c673533d40eb73d0f7abc0ebe9d2e4fc6dbada5ad60b42ff98ffa86", size = 422218, upload-time = "2026-08-15T17:26:38.911Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e2/e5151b85ca9ddfce58388f0fb0316adaacda25d494d2a668842e09f02063/backports_zstd-1.7.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3e3f760ee9d16378e3cde9d862e1c9ced577a86736763fb486b9f731d5116807", size = 395760, upload-time = "2026-08-15T17:26:40.09Z" }, + { url = "https://files.pythonhosted.org/packages/78/bf/fd7d55452431d836b3ae81170689f19ddab210fa6c385a72e22006320afe/backports_zstd-1.7.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25caf23dc36de3b839d16c25893751323cf51a8c986f2d01478c16b25133e2e8", size = 415725, upload-time = "2026-08-15T17:26:41.322Z" }, + { url = "https://files.pythonhosted.org/packages/f9/fe/f30ad42bd082b9c6d419c23311a8904a55e248e07c61bf6b91e1691188aa/backports_zstd-1.7.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:a64e796c7eee69dfe45827b2e003b7731785ec890c73ea5f5fbc30a1c362fcad", size = 316244, upload-time = "2026-08-15T17:26:42.614Z" }, +] + [[package]] name = "blake3" version = "1.0.9" @@ -261,6 +331,7 @@ dev = [ { name = "pytest-cov" }, { name = "pytest-xdist" }, { name = "pyyaml" }, + { name = "pyzstd" }, { name = "requests" }, { name = "rich" }, { name = "ruff" }, @@ -294,6 +365,7 @@ dev = [ { name = "pytest-cov", specifier = ">=6.0" }, { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "pyyaml", specifier = ">=6.0.3" }, + { name = "pyzstd", specifier = ">=0.17.0" }, { name = "requests", specifier = ">=2.34.2" }, { name = "rich", specifier = ">=15.0.0" }, { name = "ruff", specifier = ">=0.15.16" }, @@ -1388,6 +1460,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "pyzstd" +version = "0.19.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backports-zstd", marker = "python_full_version < '3.14'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/66/59fed71d0d2065e02974b40296f836a237c364c8bbe07295f2d0dc33c278/pyzstd-0.19.1.tar.gz", hash = "sha256:36723d3c915b3981de9198d0a2c82b2f5fe3eaa36e4d8d586937830a8afc7d72", size = 69531, upload-time = "2025-12-13T08:15:33.455Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/62/f55d1363512d1b1cc122af6ca5951008e42605b63675c7d6781affc7c475/pyzstd-0.19.1-py3-none-any.whl", hash = "sha256:267e2eb0de0291dfcb7ccfebc4ffe75b2f9d84e7007ac38844f6ccafc6dce081", size = 23779, upload-time = "2025-12-13T08:15:31.979Z" }, +] + [[package]] name = "requests" version = "2.34.2" diff --git a/config/cache.toml b/config/cache.toml index a4d9ad963..7821b8b8e 100644 --- a/config/cache.toml +++ b/config/cache.toml @@ -235,6 +235,7 @@ max_size_bytes = 8589934592 # 8 GiB prune_strategy = "lru" maximum_age_hours = 720 maximum_count = 4 +lease_template = ".{key}.lock" [stages.test-benchmarks] description = "Local benchmark recordings used for performance regression checks." diff --git a/config/gate.toml b/config/gate.toml index e4d3ee364..971a55775 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -845,9 +845,12 @@ key = "version" # --------------------------------------------------------------------------- [lint] -python_roots = ["build_system/builder", "build_system/sdist_command.py", "build_system/packaging", "build_system/scripts", "build_system/tests", "tests", "guest", "images", "integrations"] +python_roots = ["build_system/builder", "build_system/sdist_command.py", "build_system/packaging", "build_system/scripts", "build_system/tests", "tests", "guest", "images", "integrations/inspect-ai"] # Already passes every rule, and must keep doing so. -strict_roots = ["build_system/builder", "build_system/sdist_command.py", "integrations"] +strict_roots = ["build_system/builder", "build_system/sdist_command.py", "integrations/inspect-ai"] +# Resolve the separate carried namespace without treating builder/ as a +# top-level package root and shadowing its installed capsem_builder identity. +strict_ty_search_paths = ["integrations/inspect-ai"] # Match the import roots exercised by the sole pytest project plus the guest # trees and script bootstrap paths that are materialized at runtime. Keeping # these explicit lets Ty inspect a conventional capsem-builder install without @@ -1104,7 +1107,7 @@ must_stay_below_lines = 3000 "crates/capsem-core/src/hypervisor/kvm/virtio_fs/tests.rs" = 2041 "crates/capsem-core/src/net/mitm_proxy/mod.rs" = 2528 "crates/capsem-core/src/net/policy_config/security_rule_profile/tests.rs" = 1549 -"crates/capsem-core/src/security_engine/mod.rs" = 2048 +"crates/capsem-core/src/security_engine/mod.rs" = 2046 "crates/capsem-core/tests/mitm_integration.rs" = 2087 "crates/capsem-logger/src/writer/tests.rs" = 1237 "crates/capsem-mock-server/src/main.rs" = 1167 @@ -1112,7 +1115,7 @@ must_stay_below_lines = 3000 "crates/capsem-proto/src/lib.rs" = 1008 "crates/capsem-proto/src/tests.rs" = 1446 "crates/capsem-service/src/tests/assets_registry.rs" = 1111 -"crates/capsem-service/src/tests/lifecycle.rs" = 1404 +"crates/capsem-service/src/tests/lifecycle.rs" = 1367 "crates/capsem-service/src/vm_files.rs" = 1031 "crates/capsem/src/main.rs" = 2025 "crates/capsem/src/tests.rs" = 1259 @@ -1127,6 +1130,7 @@ must_stay_below_lines = 3000 # # They carry no suffix (`benchmarks/collectors/routes` is run by name), so they are # found by shebang, the same way suffix-less packaging helpers are. + [boundary.bench] roots = ["benchmarks/collectors"] suffixes = [".py"] @@ -1138,7 +1142,7 @@ max_lines = 300 "build_system/packaging/macos/build-pkg.sh" = 317 "build_system/packaging/macos/macos_release_glowup.py" = 360 "build_system/packaging/macos/macos_tart_glowup.py" = 519 -"build_system/packaging/macos/macos_tart_guest.sh" = 310 +"build_system/packaging/macos/macos_tart_guest.sh" = 312 # --------------------------------------------------------------------------- # What may not run beside what @@ -2023,6 +2027,7 @@ building = ["frontend-build", "docs", "site"] # `tests/test_gate_*.py` to appear here, since by construction those need # neither. source_contract = [ + "tests/test_benchmark_routes.py", "integrations/inspect-ai/tests/containers/test_compose_fields.py", "integrations/inspect-ai/tests/containers/test_compose_inputs.py", "integrations/inspect-ai/tests/containers/test_compose_interpolation.py", @@ -2030,6 +2035,7 @@ source_contract = [ "integrations/inspect-ai/tests/containers/test_dockerfile_ca.py", "integrations/inspect-ai/tests/containers/test_dockerfile_heredoc.py", "integrations/inspect-ai/tests/containers/test_dockerfile_metadata.py", + "tests/qualification/test_candidate.py", "build_system/tests/gate/test_gate_observation.py", "build_system/tests/gate/test_gate_observation_startup.py", "build_system/tests/gate/test_gate_observation_propagation.py", @@ -3026,8 +3032,11 @@ internal = { "/vms/{id}/preserve-failure" = "CLI-only doctor failure preservatio measured = [ "/vms/managed/{request_id}/close", "/vms/managed/{request_id}/claim", + "/connections", + "/vms/{id}/credential-grants", "/vms/{id}/bodies/{event_id}", "/vms/{id}/status", + "/vms/{id}/exposures", "/vms/{id}/security/status", "/status", "/version", @@ -3043,15 +3052,19 @@ measured = [ "/update/status", ] unmeasured = [ - # Exposure inventory is owned by a live VM process. The quick routes - # collector uses a stopped workspace so it can stay host-only; Ironbank's - # workload-surface fixture measures the real owner round trip. - "/vms/{id}/exposures", # Standalone proxy lifecycle mutations require a supervised worker fixture; # their functional acceptance owns successful create, lease, and teardown. "/proxies", "/proxies/{id}/heartbeat", "/proxies/{id}/stop", + # Provider authorization/poll/disconnect and concrete connection records + # need genuine provider lifecycle fixtures, not timed error responses. + "/connections/google/authorize", + "/connections/authorizations/{transaction_id}", + "/connections/{connection_id}", + # Claim/close mutate owned session lifetimes; grant revocation mutates + # credential authority. Their native acceptance owns the effects. + "/vms/{id}/credential-grants/{grant_id}", # Restart needs a disposable supervisor; the read-only route collector # cannot time accepted restarts. Its functional proof owns that lifecycle. "/restart", @@ -3723,6 +3736,8 @@ specification = "sdk/specification/openapi.json" source = "sdk/python/capsem" tests = "sdk/python/tests" build_output = "cache/target/sdk/python" +environment_python = "sdk/python/.venv/bin/python" +package_build_requirements = "cache/target/sdk/python/package-build-requirements.txt" [sdk_rust] project = "sdk/rust" diff --git a/config/public-surface.toml b/config/public-surface.toml index fe8013dbf..136a0557f 100644 --- a/config/public-surface.toml +++ b/config/public-surface.toml @@ -86,7 +86,7 @@ approved = [ ] [http] -count = 100 +count = 111 approved = [ # All nine connection/grant routes approved by the user on 2026-10-14. "GET /connections", diff --git a/crates/capsem-api/src/document.rs b/crates/capsem-api/src/document.rs index 336b66cca..3e9650c33 100644 --- a/crates/capsem-api/src/document.rs +++ b/crates/capsem-api/src/document.rs @@ -99,7 +99,7 @@ struct Document { /// force every generated SDK to be regenerated, and an SDK built against the /// contract keeps working across binary releases. Raise it when the contract /// changes in a way clients must notice. -pub const CONTRACT_VERSION: &str = "3.1.0"; +pub const CONTRACT_VERSION: &str = "3.3.0"; impl Document { fn connections(&mut self) { diff --git a/crates/capsem-archive/src/reader.rs b/crates/capsem-archive/src/reader.rs index 890658b19..b50684d62 100644 --- a/crates/capsem-archive/src/reader.rs +++ b/crates/capsem-archive/src/reader.rs @@ -362,7 +362,7 @@ fn inflate_segment(cursor: &mut Cursor, header: &SegmentHeader, comp: &[u8], at: if raw.len() != want { return Err(ArchiveError::Integrity(at)); } - if blake3::hash(&raw).as_bytes() != &header.hash { + if capsem_crypto::blake3::hash(&raw).as_bytes() != &header.hash { return Err(ArchiveError::Integrity(at)); } cursor.raw.extend_from_slice(&raw); diff --git a/crates/capsem-archive/src/writer.rs b/crates/capsem-archive/src/writer.rs index 7a56ce19e..28d7cb0d5 100644 --- a/crates/capsem-archive/src/writer.rs +++ b/crates/capsem-archive/src/writer.rs @@ -325,7 +325,7 @@ impl BodyLogWriter { raw_len: 0, flushed_raw: 0, disk_len: 0, - hasher: blake3::Hasher::new(), + hasher: capsem_crypto::blake3::Hasher::new(), out: Vec::new(), }); } diff --git a/crates/capsem-assets/src/oci/cache/inventory.rs b/crates/capsem-assets/src/oci/cache/inventory.rs index 35281fcca..3b65bc833 100644 --- a/crates/capsem-assets/src/oci/cache/inventory.rs +++ b/crates/capsem-assets/src/oci/cache/inventory.rs @@ -104,8 +104,8 @@ fn watch_tree(root: &ContainedDir, watch: &mut ChangeWatch, maximum_entries: usi let mut remaining = maximum_entries .checked_sub(1) .context("inventory entry budget exhausted")?; - let mut directories = vec![root.try_clone()?]; - while let Some(directory) = directories.pop() { + let mut directories = vec![(root.try_clone()?, PathBuf::new())]; + while let Some((directory, relative)) = directories.pop() { directory.visit_entries(|entry| { remaining = remaining .checked_sub(1) @@ -113,10 +113,24 @@ fn watch_tree(root: &ContainedDir, watch: &mut ChangeWatch, maximum_entries: usi match entry.kind { EntryKind::Directory => { let child = directory.descend(&entry.name)?; - watch.add(child.as_fd())?; - directories.push(child); + let child_relative = relative.join(&entry.name); + // kqueue reports opens of flock control children as + // directory activity too. Inventory epochs already own + // those transitions, so neither the directory nor its + // files belong to the filesystem-change proof. + if !child_relative.starts_with("locks") { + watch.add(child.as_fd())?; + } + directories.push((child, child_relative)); } EntryKind::File => { + // Materialization flock transitions are represented by + // explicit inventory epochs. Watching their control files + // makes macOS kqueue report our own lock probes as cache + // mutations, so no quiet inventory can ever publish. + if relative.starts_with("locks") { + return Ok(true); + } let file = directory.open_file(&entry.name, ContainedOpenOptions::read_only())?; let current = file.metadata()?; if (current.dev(), current.ino()) != (entry.identity.dev, entry.identity.ino) { diff --git a/crates/capsem-assets/src/oci/worker/tests.rs b/crates/capsem-assets/src/oci/worker/tests.rs index 154bd71f8..e9a5c373c 100644 --- a/crates/capsem-assets/src/oci/worker/tests.rs +++ b/crates/capsem-assets/src/oci/worker/tests.rs @@ -5,6 +5,11 @@ use super::super::{ use capsem_foundation::poll::{poll_until, PollOpts}; use std::time::Duration; +// Receipt proof hashes cache content on a background worker. Full coverage +// runs can leave that worker CPU-starved well beyond the subsecond focused +// runtime; keep cancellation tests on their tighter deadlines below. +const CACHE_PROOF_TIMEOUT: Duration = Duration::from_secs(5); + #[tokio::test] async fn inventory_worker_publishes_only_quiet_bounded_observations_and_tracks_nested_changes() { let root = super::super::tests::private_dir(); @@ -210,13 +215,10 @@ async fn incompatible_receipts_are_observed_once_until_their_metadata_changes() std::fs::write(&path, &bytes).unwrap(); let worker = cache.reconciler("arm64", parent.path().to_owned(), 1).unwrap(); assert!(worker.request(std::slice::from_ref(&key)).unwrap()); - let observed = poll_until( - PollOpts::new("foreign-cache-observed", Duration::from_secs(2)), - || async { - let observed = cache.snapshot(&key).unwrap(); - (!observed.verification_pending).then_some(observed) - }, - ) + let observed = poll_until(PollOpts::new("foreign-cache-observed", CACHE_PROOF_TIMEOUT), || async { + let observed = cache.snapshot(&key).unwrap(); + (!observed.verification_pending).then_some(observed) + }) .await .unwrap(); assert_eq!(observed.state, CacheState::Unknown); @@ -239,7 +241,7 @@ async fn incompatible_receipts_are_observed_once_until_their_metadata_changes() .await .unwrap(); poll_until( - PollOpts::new("foreign-cache-reobserved", Duration::from_secs(2)), + PollOpts::new("foreign-cache-reobserved", CACHE_PROOF_TIMEOUT), || async { (!cache.snapshot(&key).unwrap().verification_pending).then_some(()) }, ) .await diff --git a/crates/capsem-config/src/security_rule_profile.rs b/crates/capsem-config/src/security_rule_profile.rs index 1580f48a6..6ae2377d2 100644 --- a/crates/capsem-config/src/security_rule_profile.rs +++ b/crates/capsem-config/src/security_rule_profile.rs @@ -7,8 +7,6 @@ use crate::types::{default_true, PolicySubject}; use crate::validation::{validate_identifier, validate_non_empty, validate_policy_target}; mod sigma; -#[cfg(test)] -mod tests; use sigma::SigmaRule; #[cfg(test)] diff --git a/crates/capsem-config/src/security_rule_profile/sigma.rs b/crates/capsem-config/src/security_rule_profile/sigma.rs index 951b071c0..ae5ce575a 100644 --- a/crates/capsem-config/src/security_rule_profile/sigma.rs +++ b/crates/capsem-config/src/security_rule_profile/sigma.rs @@ -97,6 +97,7 @@ impl SigmaRule { let mut positive = Vec::new(); let mut negative = Vec::new(); for (field, expected) in mapping { + let field = field.as_str(); validate_security_event_field(field)?; let clause = sigma_field_clause(field, expected)?; positive.push(clause.positive); @@ -224,6 +225,15 @@ fn sigma_scalar_compare(field: &str, operator: &str, expected: &serde_json::Valu Ok(format!("{field} {operator} {}", cel_string_literal(&expected))) } +fn join_grouped(clauses: &[String], operator: &str) -> String { + let expression = clauses.join(operator); + if clauses.len() > 1 { + format!("({expression})") + } else { + expression + } +} + fn sigma_scalar_to_string(value: &serde_json::Value) -> Option { match value { serde_json::Value::String(value) => Some(value.clone()), diff --git a/crates/capsem-config/src/security_rule_profile/tests.rs b/crates/capsem-config/src/security_rule_profile/tests.rs index d9802de69..aa724220b 100644 --- a/crates/capsem-config/src/security_rule_profile/tests.rs +++ b/crates/capsem-config/src/security_rule_profile/tests.rs @@ -1,4 +1,57 @@ -use super::SecurityRuleProfile; +use serde_json::json; + +use super::*; + +mod yaml_migration; + +#[test] +fn sigma_sequence_alternatives_still_require_every_selection_field() { + let profile = SecurityRuleProfile::parse_sigma_yaml( + "title: Grouped Rule\nlogsource:\n product: capsem\n service: security_event\ndetection:\n selection:\n http.host: approved.example\n http.method: [GET, POST]\n condition: selection\nlevel: high\n", + ).unwrap(); + let rule = &profile.profiles.rules["grouped_rule"]; + for method in ["GET", "POST"] { + assert!(evaluate_security_event_match( + &rule.condition, + &json!({"http": {"host": "approved.example", "method": method}}) + ) + .unwrap()); + assert!( + !evaluate_security_event_match( + &rule.condition, + &json!({"http": {"host": "unexpected.example", "method": method}}) + ) + .unwrap(), + "{method} cannot bypass the required host" + ); + } +} + +#[test] +fn sigma_negated_selection_alternatives_still_require_the_guard() { + let profile = SecurityRuleProfile::parse_sigma_yaml( + "title: Guarded Rule\nlogsource:\n product: capsem\n service: security_event\ndetection:\n guard:\n tcp.port: 443\n selection:\n http.host: approved.example\n http.method: [GET, POST]\n condition: guard and not selection\nlevel: high\n", + ).unwrap(); + let rule = &profile.profiles.rules["guarded_rule"]; + for (port, host, method, expected) in [ + ("443", "approved.example", "GET", false), + ("443", "approved.example", "POST", false), + ("443", "approved.example", "DELETE", true), + ("443", "unexpected.example", "GET", true), + ("80", "approved.example", "DELETE", false), + ("80", "unexpected.example", "GET", false), + ] { + assert_eq!( + evaluate_security_event_match( + &rule.condition, + &json!({"tcp": {"port": port}, "http": {"host": host, "method": method}}) + ) + .unwrap(), + expected, + "{port} {host} {method}" + ); + } +} const SIGMA_HEADER: &str = r#" logsource: diff --git a/crates/capsem-core/src/container/publish.rs b/crates/capsem-core/src/container/publish.rs index f0cf0d546..b037b2f25 100644 --- a/crates/capsem-core/src/container/publish.rs +++ b/crates/capsem-core/src/container/publish.rs @@ -1,4 +1,4 @@ -//! VM owner: bind declared listeners and broker data fds, never route TCP bytes. +//! VM owner: admit granted listeners and broker data fds, never route TCP bytes. use crate::hypervisor::VsockConnection; use anyhow::{ensure, Context, Result}; use capsem_proto::ipc::ServiceToProcess; @@ -29,6 +29,7 @@ pub use security::{AuditFlow, ContainerPullRefused, ExposureRefused}; pub struct Publisher { security: Option>, + listener_authority: Option>, control_lease: Mutex>, budgets: capsem_config::router::RouterConfig, /// Nonzero by type: audit identities and guest flow keys both require it, @@ -49,6 +50,46 @@ pub struct Publisher { declared: registry::Registry, } +pub type LoopbackListenerFuture<'a> = + std::pin::Pin> + Send + 'a>>; + +pub struct LoopbackListenerGrant { + listener: std::net::TcpListener, + lease: Option>, +} + +impl std::fmt::Debug for LoopbackListenerGrant { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("LoopbackListenerGrant") + .field("listener", &self.listener) + .finish_non_exhaustive() + } +} + +impl LoopbackListenerGrant { + pub fn untracked(listener: std::net::TcpListener) -> Self { + Self { listener, lease: None } + } + + pub(crate) fn tracked(listener: std::net::TcpListener, lease: impl Send + Sync + 'static) -> Self { + Self { + listener, + lease: Some(Box::new(lease)), + } + } + + pub(crate) fn into_parts(self) -> (std::net::TcpListener, Option>) { + (self.listener, self.lease) + } +} + +/// Opens an exact loopback listener without granting the confined VM owner an +/// ambient bind syscall. +pub trait LoopbackListenerAuthority: Send + Sync { + fn open(&self, port: u16) -> LoopbackListenerFuture<'_>; +} + type GuestClose = (capsem_proto::router::FlowKey, capsem_proto::router::CloseReport); /// Where a flow's bytes come from: a host client on a published port. @@ -72,9 +113,19 @@ impl Source { Ok(()) } - /// End the flow now, discarding what the peer has not read. + /// End the flow, discarding what the peer has not read. + /// + /// Linux's immediate reset uses `connect(AF_UNSPEC)`, which the confined + /// VM owner cannot call. Linger-zero is already armed, so keeping it armed + /// makes the last parent/router descriptor close deliver the same reset + /// without restoring ambient connect authority. fn reset(&self) -> std::io::Result<()> { - capsem_foundation::unix::fd::reset_tcp(self.as_fd()).map(|_| ()) + match capsem_foundation::unix::fd::reset_tcp(self.as_fd()) { + Ok(_) => Ok(()), + #[cfg(target_os = "linux")] + Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied => Ok(()), + Err(error) => Err(error), + } } /// End the flow after what was written has been delivered. @@ -118,6 +169,7 @@ impl Publisher { budgets.validate().map_err(anyhow::Error::msg)?; Ok(Self { security: None, + listener_authority: None, control_lease: Mutex::new(None), // The UUID variant bits make its low half nonzero; the fallback is // unreachable and exists so that nothing here can panic. @@ -141,6 +193,11 @@ impl Publisher { budgets, }) } + + pub fn with_listener_authority(mut self, authority: Arc) -> Self { + self.listener_authority = Some(authority); + self + } } pub struct Publication { @@ -152,6 +209,7 @@ pub struct Publication { task: tokio::task::AbortHandle, cancellation: CancellationToken, preview: Option>, + _listener_lease: Option>, } const PREVIEW_BOOTSTRAP_LIFETIME: Duration = Duration::from_secs(capsem_proto::PREVIEW_BOOTSTRAP_LIFETIME_SECS as u64); @@ -436,11 +494,18 @@ impl Publisher { .clone() .try_acquire_owned() .context("VM publication limit reached")?; - let listener = - std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, host_port)).context("bind publication listener")?; - let host_address = listener.local_addr()?; + let listener_authority = self + .listener_authority + .as_ref() + .context("publication listener authority missing")?; + let listener = listener_authority + .open(host_port) + .await + .context("request publication listener")?; + let (listener, listener_lease) = listener.into_parts(); + let host_address = listener.local_addr().context("inspect publication listener")?; let host_port = host_address.port(); - // Bound, so the audited listener is the real one, but not accepting: + // Granted, so the audited listener is the real one, but not accepting: // dropping it on refusal serves nothing. let publication_id = capsem_crypto::uuid_v4(); authority @@ -452,12 +517,12 @@ impl Publisher { capsem_proto::PublicationAccess::LoopbackTcp, action, ) - .await?; - listener.set_nonblocking(true)?; - let listener = tokio::net::TcpListener::from_std(listener)?; + .await + .context("record publication admission")?; + let listener = tokio::net::TcpListener::from_std(listener).context("register publication listener")?; let mut current = self.router.lock().await; if current.as_ref().is_none_or(|router| router.closed.is_cancelled()) { - *current = Some(companion::start(self).await?); + *current = Some(companion::start(self).await.context("start publication router")?); } let router = current.as_ref().unwrap().clone(); drop(current); @@ -465,7 +530,9 @@ impl Publisher { let owner = self.clone(); let cancellation = self.cancellation.child_token(); let stop = cancellation.clone(); - let incoming = self.accept_publication(listener, publication_id, guest_port, target, stop.clone())?; + let incoming = self + .accept_publication(listener, publication_id, guest_port, target, stop.clone()) + .context("start publication listener")?; let task = self.spawn(async move { let _permit = permit; if let Err(error) = broker::serve(owner, incoming, control, router, stop).await { @@ -481,6 +548,7 @@ impl Publisher { task, cancellation, preview: None, + _listener_lease: listener_lease, }) } @@ -557,9 +625,9 @@ impl Publisher { } } drop(pending); - // Apply the reset before the control actor ACKs. The guest keeps its - // VSOCK endpoint open until that ACK, preventing EOF from racing a FIN - // through the confined copier ahead of this TCP reset. + // Arm the reset before the control actor ACKs. The trusted owner can + // revoke it immediately; under Linux confinement the parent and router + // retain linger-zero until their last descriptor closes. if report.reason != capsem_proto::router::CloseReason::Complete { if let Some(source) = source { source.reset()?; @@ -701,6 +769,7 @@ impl Publisher { task, cancellation, preview: Some(preview), + _listener_lease: None, }; let info = capsem_proto::ipc::PublicationInfo { id: id.clone(), diff --git a/crates/capsem-core/src/container/publish/saved.rs b/crates/capsem-core/src/container/publish/saved.rs index 2c32bd878..34429ad7d 100644 --- a/crates/capsem-core/src/container/publish/saved.rs +++ b/crates/capsem-core/src/container/publish/saved.rs @@ -54,8 +54,11 @@ fn forget(path: &Path, host: u16) -> Result { } impl Publisher { - pub fn for_session(session_dir: &Path, budgets: capsem_config::router::RouterConfig) -> Result { - let state_dir = crate::session::prepare_owner_state_dir(session_dir)?; + /// Materialize and migrate publication state while the coordinator still + /// permits changes to the session root. The confined VM owner may mutate + /// only the resulting owner directory. + pub fn prepare_session(session_dir: &Path) -> Result<()> { + crate::session::prepare_owner_state_dir(session_dir)?; let root = capsem_foundation::unix::contained::ContainedDir::open_root(session_dir)?; let state = root.descend(OsStr::new(crate::session::OWNER_STATE_DIR))?; let legacy = OsStr::new(SAVED_PUBLICATIONS_FILE); @@ -66,14 +69,28 @@ impl Publisher { ); root.rename_to(legacy, &state, legacy)?; } + Ok(()) + } + + /// Attach saved-publication state after [`Self::prepare_session`] ran. + /// This performs no access to the session root, which is outside the VM + /// owner's post-confinement authority. + pub fn for_prepared_session(session_dir: &Path, budgets: capsem_config::router::RouterConfig) -> Result { let mut publisher = Self::configured(budgets)?; publisher.saved = Some(Mappings { - path: state_dir.join(SAVED_PUBLICATIONS_FILE), + path: session_dir + .join(crate::session::OWNER_STATE_DIR) + .join(SAVED_PUBLICATIONS_FILE), lock: tokio::sync::Mutex::new(()), }); Ok(publisher) } + pub fn for_session(session_dir: &Path, budgets: capsem_config::router::RouterConfig) -> Result { + Self::prepare_session(session_dir)?; + Self::for_prepared_session(session_dir, budgets) + } + /// Re-open every saved publication; returns how many were restored. pub async fn restore(self: &Arc, control: mpsc::Sender) -> Result { let Some(saved) = &self.saved else { diff --git a/crates/capsem-core/src/container/publish/tests.rs b/crates/capsem-core/src/container/publish/tests.rs index 6b0521023..99931d214 100644 --- a/crates/capsem-core/src/container/publish/tests.rs +++ b/crates/capsem-core/src/container/publish/tests.rs @@ -1,11 +1,27 @@ use super::*; +use std::net::Ipv4Addr; use std::os::fd::AsRawFd; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; +mod saved; mod security; mod sessions; +struct LocalListenerAuthority; + +impl LoopbackListenerAuthority for LocalListenerAuthority { + fn open(&self, port: u16) -> LoopbackListenerFuture<'_> { + Box::pin(async move { + std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, port)).map(LoopbackListenerGrant::untracked) + }) + } +} + +fn with_local_listener_authority(publisher: Publisher) -> Publisher { + publisher.with_listener_authority(Arc::new(LocalListenerAuthority)) +} + fn source_fixture() -> Arc { let listener = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); let _client = std::net::TcpStream::connect(listener.local_addr().unwrap()).unwrap(); diff --git a/crates/capsem-core/src/container/publish/tests/saved.rs b/crates/capsem-core/src/container/publish/tests/saved.rs new file mode 100644 index 000000000..a0187f6ee --- /dev/null +++ b/crates/capsem-core/src/container/publish/tests/saved.rs @@ -0,0 +1,8 @@ +use super::*; + +#[test] +fn prepared_publication_state_does_not_reopen_the_session_root() { + let directory = tempfile::tempdir().unwrap(); + let absent_session = directory.path().join("absent-session"); + Publisher::for_prepared_session(&absent_session, capsem_config::router::RouterConfig::default()).unwrap(); +} diff --git a/crates/capsem-core/src/container/publish/tests/security.rs b/crates/capsem-core/src/container/publish/tests/security.rs index 4a5a7dc08..8ee66792f 100644 --- a/crates/capsem-core/src/container/publish/tests/security.rs +++ b/crates/capsem-core/src/container/publish/tests/security.rs @@ -39,10 +39,11 @@ fn engine(action: &str, plugin_block: bool) -> Arc { } pub(super) fn authorized_publisher(budgets: capsem_config::router::RouterConfig) -> Publisher { - let owner = - Publisher::configured(budgets) - .unwrap() - .with_security("vm-id".into(), "redis".into(), engine("allow", false)); + let owner = with_local_listener_authority(Publisher::configured(budgets).unwrap()).with_security( + "vm-id".into(), + "redis".into(), + engine("allow", false), + ); owner.control_ready().unwrap(); owner } @@ -537,7 +538,11 @@ async fn a_refused_exposure_is_audited_and_leaves_its_port_unbound() { for (rules, refusal) in [(BLOCK_REDIS, "blocked by policy"), (ASK_REDIS, "needs approval")] { let dir = tempfile::tempdir().unwrap(); let (engine, path) = lifecycle_engine(&dir, rules); - let owner = Arc::new(Publisher::default().with_security("vm-id".into(), "redis".into(), engine.clone())); + let owner = Arc::new(with_local_listener_authority(Publisher::default()).with_security( + "vm-id".into(), + "redis".into(), + engine.clone(), + )); let (control, _requests) = mpsc::channel(8); let port = free_port(); let error = owner @@ -546,7 +551,7 @@ async fn a_refused_exposure_is_audited_and_leaves_its_port_unbound() { .err() .expect("the rules refuse this exposure"); assert!(error.is::(), "{error:#}"); - assert!(error.to_string().contains(refusal), "{error:#}"); + assert!(format!("{error:#}").contains(refusal), "{error:#}"); std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, port)).expect("a refused exposure keeps no listener"); assert!(owner.publications().is_empty()); let rows = lifecycle_rows(&engine, &path).await; @@ -560,7 +565,11 @@ async fn an_exposure_whose_audit_cannot_be_admitted_is_not_opened() { let dir = tempfile::tempdir().unwrap(); let (engine, _path) = lifecycle_engine(&dir, ""); engine.db.shutdown_blocking(); - let owner = Arc::new(Publisher::default().with_security("vm-id".into(), "redis".into(), engine)); + let owner = Arc::new(with_local_listener_authority(Publisher::default()).with_security( + "vm-id".into(), + "redis".into(), + engine, + )); let (control, _requests) = mpsc::channel(8); let port = free_port(); let error = owner @@ -598,6 +607,7 @@ async fn a_revoked_exposure_closes_then_is_audited_under_its_publication() { task: task.abort_handle(), cancellation: cancellation.clone(), preview: None, + _listener_lease: None, }, }); assert!( @@ -627,6 +637,7 @@ async fn a_saved_exposure_the_rules_now_refuse_is_forgotten_on_restore() { let owner = Arc::new( Publisher::for_session(dir.path(), capsem_config::router::RouterConfig::default()) .unwrap() + .with_listener_authority(Arc::new(LocalListenerAuthority)) .with_security("vm-id".into(), "redis".into(), engine.clone()), ); let (control, _requests) = mpsc::channel(8); diff --git a/crates/capsem-core/src/hypervisor/kvm/mod.rs b/crates/capsem-core/src/hypervisor/kvm/mod.rs index 89dd70ba0..9b47ed579 100644 --- a/crates/capsem-core/src/hypervisor/kvm/mod.rs +++ b/crates/capsem-core/src/hypervisor/kvm/mod.rs @@ -56,7 +56,7 @@ fn require_metadata_authority(config: &VmConfig) -> Result<()> { } pub(crate) fn kvm_vsock_seed(config: &VmConfig) -> u32 { - let mut hasher = blake3::Hasher::new(); + let mut hasher = capsem_crypto::blake3::Hasher::new(); hasher.update(config.kernel_path.to_string_lossy().as_bytes()); if let Some(path) = config.scratch_disk_path.as_ref().or(config.disk_path.as_ref()) { hasher.update(path.to_string_lossy().as_bytes()); diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs index 2aebb1374..5ce99a1dc 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs @@ -91,6 +91,18 @@ impl FuseProcessor { authority.set_times(relative.as_os_str().as_bytes(), atime, mtime) } + fn handle_request_into(&mut self, request_buf: &[u8], response: &mut Vec) { + if let Some(header) = fuse::read_struct::(request_buf) { + if header.opcode == FUSE_READ { + self.do_read_into(&header, &request_buf[std::mem::size_of::()..], response); + return; + } + } + response.clear(); + response.extend_from_slice(&self.handle_non_read_request(request_buf)); + } + + #[cfg(test)] fn handle_request(&mut self, request_buf: &[u8]) -> Vec { let mut response = Vec::new(); self.handle_request_into(request_buf, &mut response); diff --git a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs index f119b8b83..b46cfd0f0 100644 --- a/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs +++ b/crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs @@ -2,8 +2,9 @@ use capsem_foundation::unix::fd; use std::io::{Seek, SeekFrom, Write}; -use std::os::unix::fs::FileExt; +use std::os::fd::AsFd; use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::fs::PermissionsExt; use super::FuseProcessor; use crate::hypervisor::fuse::{self, *}; diff --git a/crates/capsem-core/src/net/ai_traffic/mod.rs b/crates/capsem-core/src/net/ai_traffic/mod.rs index 1cfca56ed..e4ae3fbf1 100644 --- a/crates/capsem-core/src/net/ai_traffic/mod.rs +++ b/crates/capsem-core/src/net/ai_traffic/mod.rs @@ -29,6 +29,8 @@ pub mod provider; pub mod request_parser; use std::collections::{HashMap, VecDeque}; +use std::future::Future; +use std::pin::Pin; pub use provider::{ModelProtocol, Provider, ProviderKind}; @@ -46,6 +48,14 @@ pub struct TraceState { file_hint_order: VecDeque<(String, String)>, } +/// Narrow one-way authority for delivering model/file correlation hints to +/// the process that owns the workspace monitor. +pub type TraceHintFuture<'a> = Pin> + Send + 'a>>; + +pub trait TraceHintSink: Send + Sync { + fn register<'a>(&'a self, trace_id: &'a str, relative_paths: &'a [String]) -> TraceHintFuture<'a>; +} + const MAX_FILE_HINTS: usize = 4096; impl Default for TraceState { @@ -85,14 +95,38 @@ impl TraceState { /// Register workspace file paths found in model-emitted tool-call /// arguments. The fs monitor later uses this to attribute ordinary /// workspace writes to the model/tool trace that caused them. - pub fn register_tool_file_hints<'a>(&mut self, trace_id: &str, arguments: impl IntoIterator) { + pub fn register_tool_file_hints<'a>( + &mut self, + trace_id: &str, + arguments: impl IntoIterator, + ) -> Vec { + let mut registered = Vec::new(); for arguments in arguments { for path in extract_workspace_file_hints(arguments) { self.file_hints.insert(path.clone(), trace_id.to_string()); - self.file_hint_order.push_back((path, trace_id.to_string())); + self.file_hint_order.push_back((path.clone(), trace_id.to_string())); self.trim_file_hints(); + registered.push(path); } } + registered.sort(); + registered.dedup(); + registered + } + + /// Register one already-normalized hint received over the proxy's narrow + /// trace-hint capability. Invalid or escaping paths are rejected. + pub fn register_file_hint(&mut self, trace_id: &str, relative_path: &str) -> bool { + if trace_id.is_empty() || trace_id.len() > 64 { + return false; + } + let Some(path) = normalize_workspace_path_hint(relative_path) else { + return false; + }; + self.file_hints.insert(path.clone(), trace_id.to_string()); + self.file_hint_order.push_back((path, trace_id.to_string())); + self.trim_file_hints(); + true } /// Look up a trace_id for a workspace-relative file path. diff --git a/crates/capsem-core/src/net/ai_traffic/tests.rs b/crates/capsem-core/src/net/ai_traffic/tests.rs index 5bce269b8..cd2d88a3a 100644 --- a/crates/capsem-core/src/net/ai_traffic/tests.rs +++ b/crates/capsem-core/src/net/ai_traffic/tests.rs @@ -76,6 +76,15 @@ fn trace_state_registers_workspace_file_hints_from_tool_arguments() { assert!(state.lookup_file_path("../escape.txt").is_none()); } +#[test] +fn trace_state_accepts_only_contained_hints_from_an_external_proxy() { + let mut state = TraceState::new(); + assert!(state.register_file_hint("trace_file", "/root/direct.txt")); + assert_eq!(state.lookup_file_path("direct.txt").as_deref(), Some("trace_file")); + assert!(!state.register_file_hint("trace_file", "../escape.txt")); + assert!(!state.register_file_hint("", "empty-trace.txt")); +} + #[test] fn trace_state_keeps_file_hints_after_tool_trace_completes() { let mut state = TraceState::new(); diff --git a/crates/capsem-core/src/net/mitm_proxy/mod.rs b/crates/capsem-core/src/net/mitm_proxy/mod.rs index bc370b858..a9f423449 100644 --- a/crates/capsem-core/src/net/mitm_proxy/mod.rs +++ b/crates/capsem-core/src/net/mitm_proxy/mod.rs @@ -23,6 +23,7 @@ mod mcp_frame; mod mcp_http; mod mcp_observe; pub mod pipeline; +mod pipeline_factory; pub mod protocol; pub mod spans; pub mod sse_parser_hook; @@ -72,6 +73,7 @@ use util::{ pub use config::MitmProxyConfig; pub use mcp_endpoint::{McpEndpointState, McpTimeouts, ScopedMcpTools}; pub use mcp_frame::{dispatch_logged_mcp_request, McpTransport}; +pub use pipeline_factory::{make_production_pipeline, make_production_pipeline_with_trace_hints}; pub use upstream::{ GrantedTcpStream, TcpConnectGrantFuture, TcpGrantSelection, TcpResolveGrantFuture, TcpUpstreamGrants, UpstreamTarget, @@ -89,10 +91,7 @@ const CREDENTIAL_BODY_CAPTURE_LIMIT: usize = HTTP_BODY_CAPTURE_LIMIT; static FIRST_NETWORK_READY_EMITTED: AtomicBool = AtomicBool::new(false); -/// Build the default (empty) hook pipeline. T1 slices 2 + 3 will -/// extend this to register the production hook set; until then the -/// pipeline is wired through `MitmProxyConfig` but no dispatch -/// happens from `handle_request`. +/// Build an empty pipeline for callers that do not install production hooks. pub fn make_default_pipeline() -> Arc { Arc::new(pipeline::Pipeline::builder().build()) } @@ -109,38 +108,6 @@ impl Drop for ConnectionGauge { } } -/// Build the production hook pipeline. Registers the full sync ChunkHook chain -/// (decompression → SSE parse → provider interpreters → telemetry). -/// -/// All four ChunkHook stages are pure-sync: per-chunk work runs -/// inline from `poll_frame` with no `.await`, no channel hop, no -/// async wrapper. Header mutations needed for decompression -/// (Content-Encoding / Content-Length strip) happen inline in -/// `handle_request` before chunk dispatch begins -- the chunk hooks -/// themselves never see the head. -pub fn make_production_pipeline(telemetry: Arc) -> Arc { - let p = pipeline::Pipeline::builder() - // Chunk-hook order is load-bearing: - // 1. DecompressionHook -- gzip detection on first chunk's - // magic; subsequent chunks fed through flate2::Decompress. - // 2. SseParserHook -- needs decompressed bytes for AI - // domains. - // 3. Interpreter hooks -- drain SseParserHook's queue and - // build LlmEvents. Three providers; only the matching - // one runs. - // 4. TelemetryHook -- counts response bytes, captures - // preview, fires NetEvent + optional ModelCall on - // on_response_end. - .register_chunk(Arc::new(decompression_hook::DecompressionHook::new())) - .register_chunk(Arc::new(sse_parser_hook::SseParserHook::new())) - .register_chunk(Arc::new(interpreter_hook::AnthropicInterpreterHook::new())) - .register_chunk(Arc::new(interpreter_hook::OpenAiInterpreterHook::new())) - .register_chunk(Arc::new(interpreter_hook::GoogleInterpreterHook::new())) - .register_chunk(Arc::new(telemetry_hook::TelemetryHook::new(telemetry))) - .build(); - Arc::new(p) -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct ModelTrafficIdentity { /// Endpoint owner used for policy/logging. Example: `ollama` for diff --git a/crates/capsem-core/src/net/mitm_proxy/pipeline_factory.rs b/crates/capsem-core/src/net/mitm_proxy/pipeline_factory.rs new file mode 100644 index 000000000..46851a959 --- /dev/null +++ b/crates/capsem-core/src/net/mitm_proxy/pipeline_factory.rs @@ -0,0 +1,35 @@ +//! Production hook pipeline assembly. + +use std::sync::Arc; + +use crate::net::ai_traffic::TraceHintSink; + +use super::{decompression_hook, interpreter_hook, pipeline, sse_parser_hook, telemetry_hook}; + +/// Build the production hook pipeline. All chunk stages are synchronous; +/// header mutations needed for decompression happen before chunk dispatch. +pub fn make_production_pipeline(telemetry: Arc) -> Arc { + make(telemetry_hook::TelemetryHook::new(telemetry)) +} + +pub fn make_production_pipeline_with_trace_hints( + telemetry: Arc, + trace_hints: Arc, +) -> Arc { + make(telemetry_hook::TelemetryHook::new(telemetry).with_trace_hint_sink(trace_hints)) +} + +fn make(telemetry: telemetry_hook::TelemetryHook) -> Arc { + // Order is load-bearing: decompress, parse SSE, interpret provider + // events, then record the completed request and optional model call. + Arc::new( + pipeline::Pipeline::builder() + .register_chunk(Arc::new(decompression_hook::DecompressionHook::new())) + .register_chunk(Arc::new(sse_parser_hook::SseParserHook::new())) + .register_chunk(Arc::new(interpreter_hook::AnthropicInterpreterHook::new())) + .register_chunk(Arc::new(interpreter_hook::OpenAiInterpreterHook::new())) + .register_chunk(Arc::new(interpreter_hook::GoogleInterpreterHook::new())) + .register_chunk(Arc::new(telemetry)) + .build(), + ) +} diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs index 7a8eea44b..463cdee86 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook.rs @@ -38,7 +38,7 @@ use crate::net::ai_traffic::events::{ }; use crate::net::ai_traffic::pricing::PricingTable; use crate::net::ai_traffic::provider::{extract_model_from_path, tool_origin, ModelProtocol, ProviderKind}; -use crate::net::ai_traffic::{request_parser, TraceState}; +use crate::net::ai_traffic::{request_parser, TraceHintSink, TraceState}; use crate::net::policy_config::{PluginPolicySnapshot, SecurityRuleSet}; use crate::net::proxy_engine::{ProxyCredentials, ProxyPolicySnapshot}; use crate::security_engine::{ @@ -119,6 +119,7 @@ pub struct TelemetryDeps { /// `ModelCall` for the request just completed. pub struct TelemetryHook { deps: Arc, + trace_hint_sink: Option>, } struct PendingTelemetryCompletion { @@ -131,6 +132,7 @@ struct PendingTelemetryCompletion { model_call: Option, credential_observations: Vec, credential_injections: Vec, + trace_hints: Option<(Arc, String, Vec)>, } impl PendingTelemetryCompletion { @@ -145,8 +147,15 @@ impl PendingTelemetryCompletion { model_call, credential_observations, credential_injections, + trace_hints, } = self; + if let Some((sink, trace_id, paths)) = trace_hints { + if let Err(error) = sink.register(&trace_id, &paths).await { + warn!(%error, "proxy trace-hint capability stopped"); + } + } + let model_was_denied = net_event.decision == Decision::Denied; if let Some(event_id) = emit_security_write(&db, WriteOp::NetEvent(net_event)).await { emit_evaluated_security_rules( @@ -190,7 +199,15 @@ impl PendingTelemetryCompletion { impl TelemetryHook { pub fn new(deps: Arc) -> Self { - Self { deps } + Self { + deps, + trace_hint_sink: None, + } + } + + pub fn with_trace_hint_sink(mut self, trace_hint_sink: Arc) -> Self { + self.trace_hint_sink = Some(trace_hint_sink); + self } } @@ -293,13 +310,25 @@ impl ChunkHook for TelemetryHook { record_telemetry_stage(stage_started, "credential_detect_and_redact"); let stage_started = Instant::now(); - let model_call = maybe_build_model_call( + let model_call = maybe_build_model_call_with_hints( &req_ctx, &resp_stats, &llm_events, &self.deps.pricing, &self.deps.trace_state, ); + let (model_call, trace_hints) = match model_call { + Some((model_call, paths)) => { + let trace_hints = self + .trace_hint_sink + .as_ref() + .zip(model_call.trace_id.clone()) + .filter(|_| !paths.is_empty()) + .map(|(sink, trace_id)| (Arc::clone(sink), trace_id, paths)); + (Some(model_call), trace_hints) + } + None => (None, None), + }; record_telemetry_stage(stage_started, "model_call_build"); let stage_started = Instant::now(); @@ -342,6 +371,7 @@ impl ChunkHook for TelemetryHook { model_call, credential_observations, credential_injections, + trace_hints, }); record_telemetry_stage(stage_started, "ledger_handoff"); record_telemetry_response_end(response_end_started, has_model_call); @@ -514,6 +544,17 @@ pub fn maybe_build_model_call( pricing: &PricingTable, trace_state: &Arc>, ) -> Option { + maybe_build_model_call_with_hints(req_ctx, resp_stats, llm_events, pricing, trace_state) + .map(|(model_call, _)| model_call) +} + +fn maybe_build_model_call_with_hints( + req_ctx: &TelemetryRequestContext, + resp_stats: &TelemetryResponseStats, + llm_events: &[crate::net::ai_traffic::events::LlmEvent], + pricing: &PricingTable, + trace_state: &Arc>, +) -> Option<(ModelCall, Vec)> { let provider = req_ctx.ai_provider?; let protocol = req_ctx.ai_protocol?; if req_ctx.method == "HEAD" @@ -647,7 +688,7 @@ pub fn maybe_build_model_call( // non-tool-use stop completes the trace. let tool_response_ids: Vec = req_meta.tool_results.iter().map(|tr| tr.call_id.clone()).collect(); let tool_call_ids: Vec = tool_calls.iter().map(|tc| tc.call_id.clone()).collect(); - let trace_id = { + let (trace_id, file_hints) = { let mut state = trace_state.lock().unwrap_or_else(|e| e.into_inner()); let tid = state.lookup(&tool_response_ids).unwrap_or_else(|| { if tool_call_ids.is_empty() { @@ -662,16 +703,21 @@ pub fn maybe_build_model_call( .as_deref() .map(|r| r.contains("tool") || r == "tool_use") .unwrap_or(false); - if is_tool_use && !tool_call_ids.is_empty() { + let correlation = if is_tool_use && !tool_call_ids.is_empty() { state.register_tool_calls(&tid, &tool_call_ids); - state.register_tool_file_hints( + let hints = state.register_tool_file_hints( &tid, tool_calls.iter().filter_map(|tool_call| tool_call.arguments.as_deref()), ); + (tid, hints) } else if !is_tool_use { state.complete_trace(&tid); - } - tid + (tid, Vec::new()) + } else { + (tid, Vec::new()) + }; + drop(state); + correlation }; for tool_call in &mut tool_calls { if tool_call.trace_id.is_none() { @@ -745,7 +791,7 @@ pub fn maybe_build_model_call( ); } - Some(model_call) + Some((model_call, file_hints)) } /// Per-request log line, mirrors what `TelemetryEmitter::emit` does. diff --git a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs index 4609740ba..fe4c209e6 100644 --- a/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/telemetry_hook/tests.rs @@ -3,7 +3,7 @@ use super::super::hooks::{ChunkCtx, ChunkHook, ConnMeta, HookState}; use super::*; use crate::credential_broker::{CredentialInjection, CredentialObservation, CredentialProvider}; use crate::net::policy_config::{SecurityRuleProfile, SecurityRuleSet, SecurityRuleSource}; -use capsem_logger::{credential_reference, Decision}; +use capsem_logger::{credential_reference, DbWriter, Decision}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::Instant; @@ -180,10 +180,9 @@ async fn websocket_upgrade_is_network_admission_and_not_model_inference() { let trace = Arc::new(Mutex::new(TraceState::new())); let hook = TelemetryHook::new(Arc::new(TelemetryDeps { db: Arc::clone(&db), + credentials: Arc::new(crate::net::proxy_engine::LocalProxyCredentials), pricing: Arc::clone(&pricing), trace_state: Arc::clone(&trace), - security_rules: empty_security_rules(), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), })); let mut request = anthropic_req_ctx(); request.domain = "api.openai.com".into(); diff --git a/crates/capsem-core/src/net/mitm_proxy/upgrade.rs b/crates/capsem-core/src/net/mitm_proxy/upgrade.rs index 7970703af..686acdbd1 100644 --- a/crates/capsem-core/src/net/mitm_proxy/upgrade.rs +++ b/crates/capsem-core/src/net/mitm_proxy/upgrade.rs @@ -312,7 +312,7 @@ pub(super) async fn handle_upgrade( let tunnel_span = ws_span.clone(); let model_context = responses_websocket.then(|| websocket::Context { pipeline: Arc::clone(&config.pipeline), - telemetry: Arc::clone(&config.telemetry), + engine: Arc::clone(&config.engine), conn: hooks::ConnMeta { domain: domain.to_string(), process_name: process_name.clone(), diff --git a/crates/capsem-core/src/net/mitm_proxy/websocket.rs b/crates/capsem-core/src/net/mitm_proxy/websocket.rs index c966e4fed..6ef5e6f76 100644 --- a/crates/capsem-core/src/net/mitm_proxy/websocket.rs +++ b/crates/capsem-core/src/net/mitm_proxy/websocket.rs @@ -14,7 +14,7 @@ mod turns; pub(super) struct Context { pub pipeline: Arc, - pub telemetry: Arc, + pub engine: Arc, pub conn: ConnMeta, pub ip: Option, pub method: String, diff --git a/crates/capsem-core/src/net/mitm_proxy/websocket/tests.rs b/crates/capsem-core/src/net/mitm_proxy/websocket/tests.rs index 6419d3eb0..207777f74 100644 --- a/crates/capsem-core/src/net/mitm_proxy/websocket/tests.rs +++ b/crates/capsem-core/src/net/mitm_proxy/websocket/tests.rs @@ -1,6 +1,10 @@ use super::*; use crate::net::ai_traffic::{pricing::PricingTable, TraceState}; use crate::net::policy_config::{SecurityRuleProfile, SecurityRuleSet, SecurityRuleSource}; +use crate::net::proxy_engine::{ + LocalProxyCredentials, ProxyCredentials, ProxyEngine, ProxyPolicyHandle, ProxyPolicySnapshot, +}; +use capsem_logger::DbWriter; use futures::{SinkExt, StreamExt}; use serde_json::{json, Value}; use std::collections::BTreeMap; @@ -15,17 +19,18 @@ type Socket = WebSocketStream; fn context(db: Arc, rules: &str) -> Context { let profile = SecurityRuleProfile::parse_toml(rules).unwrap(); let rules = SecurityRuleSet::compile_profile(&profile, SecurityRuleSource::Corp).unwrap(); + let policy = ProxyPolicyHandle::new(policy_snapshot(rules)); + let credentials: Arc = Arc::new(LocalProxyCredentials); + let engine = Arc::new(ProxyEngine::new(policy, db.clone(), Arc::clone(&credentials))); let telemetry = Arc::new(telemetry_hook::TelemetryDeps { db, + credentials, pricing: Arc::new(PricingTable::load()), trace_state: Arc::new(Mutex::new(TraceState::new())), - security_rules: Arc::new(std::sync::RwLock::new(Arc::new(rules))), - plugin_policy: Arc::new(std::sync::RwLock::new(BTreeMap::new().into())), }); - let policy = Arc::new(std::sync::RwLock::new(Arc::new(NetworkMechanics::new()))); Context { - pipeline: make_production_pipeline(policy, Arc::clone(&telemetry)), - telemetry, + pipeline: make_production_pipeline(telemetry), + engine, conn: ConnMeta { domain: "api.openai.com".into(), process_name: Some("codex".into()), @@ -45,6 +50,16 @@ fn context(db: Arc, rules: &str) -> Context { } } +fn policy_snapshot(rules: SecurityRuleSet) -> ProxyPolicySnapshot { + ProxyPolicySnapshot::new( + "blake3:websocket-test".into(), + crate::net::policy::NetworkMechanics::new(), + rules, + BTreeMap::new(), + crate::net::policy_config::ModelEndpointRegistry::default(), + ) +} + const ALLOW: &str = "[corp.rules.allow]\nname = 'allow'\naction = 'allow'\npriority = -10\nmatch = 'model.provider == \"openai\"'\n"; @@ -138,7 +153,7 @@ async fn websocket_models_enforce_live_request_rules_before_forwarding() { let dir = tempfile::tempdir().unwrap(); let db = Arc::new(DbWriter::open(&dir.path().join("session.db"), 16).unwrap()); let ctx = context(Arc::clone(&db), ALLOW); - let rules = Arc::clone(&ctx.telemetry.security_rules); + let policy = ctx.engine.policy().clone(); let (mut client, mut server, job) = sockets(ctx).await; exchange( &mut client, @@ -152,7 +167,9 @@ async fn websocket_models_enforce_live_request_rules_before_forwarding() { "[corp.rules.block]\nname='block'\naction='block'\npriority=-100\nmatch='model.provider == \"openai\"'\n", ) .unwrap(); - *rules.write().unwrap() = Arc::new(SecurityRuleSet::compile_profile(&profile, SecurityRuleSource::Corp).unwrap()); + policy.replace(policy_snapshot( + SecurityRuleSet::compile_profile(&profile, SecurityRuleSource::Corp).unwrap(), + )); client .send(Message::Text( json!({"type":"response.create","model":"gpt-fixture","input":[]}) diff --git a/crates/capsem-core/src/net/mitm_proxy/websocket/turns.rs b/crates/capsem-core/src/net/mitm_proxy/websocket/turns.rs index 0fa81abda..12e06e2b4 100644 --- a/crates/capsem-core/src/net/mitm_proxy/websocket/turns.rs +++ b/crates/capsem-core/src/net/mitm_proxy/websocket/turns.rs @@ -29,6 +29,7 @@ struct Turn { request: String, response: Vec, response_bytes: usize, + policy_snapshot: Arc, state: HookState, } @@ -206,8 +207,10 @@ impl Turns { impl Turn { fn new(text: &str, lane: Option, inference: bool, ctx: &Context) -> Self { + let policy_snapshot = ctx.engine.policy().snapshot(); let mut state = HookState::default(); state.set::>(Some(TelemetryRequestContext { + policy_snapshot: Arc::clone(&policy_snapshot), domain: ctx.conn.domain.clone(), process_name: ctx.conn.process_name.clone(), ai_provider: ctx.conn.ai_provider, @@ -249,6 +252,7 @@ impl Turn { request: text.to_owned(), response: vec![], response_bytes: 0, + policy_snapshot, state, } } @@ -284,11 +288,7 @@ impl Turn { ..Default::default() }); let event = security_event_with_transport(event, ctx.ip, ctx.conn.port); - let rules = ctx.telemetry.security_rules.read().unwrap().clone(); - let plugins = ctx.telemetry.plugin_policy.read().unwrap().clone(); - Ok(crate::security_engine::evaluate_security_boundary( - &rules, plugins, event, - )?) + Ok(ctx.engine.evaluate(&self.policy_snapshot, event)?) } fn decision(&mut self, decision: &crate::security_engine::SecurityEnforcementDecision) { diff --git a/crates/capsem-core/src/net/upstream_grant.rs b/crates/capsem-core/src/net/upstream_grant.rs index ad23dc82e..d422ef907 100644 --- a/crates/capsem-core/src/net/upstream_grant.rs +++ b/crates/capsem-core/src/net/upstream_grant.rs @@ -30,10 +30,18 @@ type WireSender = DescriptorSender; enum Command { + OpenLoopbackListener { + port: u16, + reply: oneshot::Sender>, + }, AttachProxyMcp { descriptor: OwnedFd, reply: oneshot::Sender>, }, + AttachProxyTraceHints { + descriptor: OwnedFd, + reply: oneshot::Sender>, + }, AttachProxyTraffic { service: ProxyTrafficService, descriptor: OwnedFd, @@ -72,6 +80,17 @@ enum Command { }, } +struct ListenerLease { + grant_id: u64, + releases: mpsc::UnboundedSender, +} + +impl Drop for ListenerLease { + fn drop(&mut self) { + let _ = self.releases.send(self.grant_id); + } +} + /// Serializes grant protocol traffic over the inherited generation channel. #[derive(Clone)] pub struct UpstreamGrantClient { @@ -107,6 +126,18 @@ impl UpstreamGrantClient { self.stopped.clone() } + pub async fn open_loopback_listener(&self, port: u16) -> Result { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::OpenLoopbackListener { port, reply }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + } + pub async fn open_ledger(&self) -> Result<(UnixStream, UnixStream, LedgerChannelGrant)> { let (reply, result) = oneshot::channel(); self.commands @@ -146,6 +177,24 @@ impl UpstreamGrantClient { .map_err(|_| anyhow!("upstream grant channel stopped"))? .map_err(anyhow::Error::msg) } + + pub async fn attach_proxy_trace_hints(&self, descriptor: OwnedFd) -> Result<()> { + let (reply, result) = oneshot::channel(); + self.commands + .send(Command::AttachProxyTraceHints { descriptor, reply }) + .await + .map_err(|_| anyhow!("upstream grant channel is closed"))?; + result + .await + .map_err(|_| anyhow!("upstream grant channel stopped"))? + .map_err(anyhow::Error::msg) + } +} + +impl crate::container::publish::LoopbackListenerAuthority for UpstreamGrantClient { + fn open(&self, port: u16) -> crate::container::publish::LoopbackListenerFuture<'_> { + Box::pin(async move { self.open_loopback_listener(port).await.map_err(io::Error::other) }) + } } impl crate::GuestMetadataAuthority for UpstreamGrantClient { @@ -285,9 +334,17 @@ async fn dispatch( command: Command, ) -> Result<(), String> { match command { + Command::OpenLoopbackListener { port, reply } => finish( + reply, + open_loopback_listener(sender, receiver, releases, request_id, port).await, + ), Command::AttachProxyMcp { descriptor, reply } => { finish(reply, attach_proxy_mcp(sender, receiver, request_id, descriptor).await) } + Command::AttachProxyTraceHints { descriptor, reply } => finish( + reply, + attach_proxy_trace_hints(sender, receiver, request_id, descriptor).await, + ), Command::AttachProxyTraffic { service, descriptor, @@ -353,6 +410,67 @@ async fn dispatch( } } +async fn open_loopback_listener( + sender: &WireSender, + receiver: &WireReceiver, + releases: &mpsc::UnboundedSender, + request_id: u64, + requested_port: u16, +) -> Result, String> { + send_request( + sender, + &UpstreamGrantRequest::OpenLoopbackListener { + request_id, + port: requested_port, + }, + ) + .await?; + let (response, mut fds) = receive_response(receiver).await?; + match response { + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("loopback listener grant denied: {reason:?}"))), + UpstreamGrantResponse::LoopbackListenerGranted { + request_id: response_id, + grant_id, + port: granted_port, + } if response_id == request_id => { + let descriptor = fds.pop().ok_or("loopback listener grant omitted its descriptor")?; + send_request(sender, &UpstreamGrantRequest::Adopted { grant_id }).await?; + let listener = std::net::TcpListener::from(descriptor); + let address = listener + .local_addr() + .map_err(|error| format!("inspect granted loopback listener: {error}"))?; + if address.ip() != std::net::Ipv4Addr::LOCALHOST || address.port() != granted_port { + send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; + return Err(format!( + "granted listener address {address} does not match 127.0.0.1:{granted_port}" + )); + } + if requested_port != 0 && requested_port != granted_port { + send_request(sender, &UpstreamGrantRequest::Release { resource_id: grant_id }).await?; + return Err(format!( + "granted listener port {granted_port} does not match requested port {requested_port}" + )); + } + listener + .set_nonblocking(true) + .map_err(|error| format!("prepare granted loopback listener: {error}"))?; + Ok(Ok(crate::container::publish::LoopbackListenerGrant::tracked( + listener, + ListenerLease { + grant_id, + releases: releases.clone(), + }, + ))) + } + response => Err(format!( + "unexpected loopback listener response for request {request_id}: {response:?}" + )), + } +} + async fn attach_proxy_mcp( sender: &WireSender, receiver: &WireReceiver, @@ -382,6 +500,35 @@ async fn attach_proxy_mcp( } } +async fn attach_proxy_trace_hints( + sender: &WireSender, + receiver: &WireReceiver, + request_id: u64, + descriptor: OwnedFd, +) -> Result, String> { + let bytes = encode_upstream_grant_request(&UpstreamGrantRequest::AttachProxyTraceHints { request_id }) + .map_err(|error| format!("encode proxy trace-hint handoff: {error:#}"))?; + tokio::time::timeout(WIRE_TIMEOUT, sender.send(&bytes, &[descriptor.as_raw_fd()])) + .await + .map_err(|_| "send proxy trace-hint handoff timed out".to_string())? + .map_err(|error| format!("send proxy trace-hint handoff: {error}"))?; + drop(descriptor); + let (response, fds) = receive_response(receiver).await?; + debug_assert!(fds.is_empty()); + match response { + UpstreamGrantResponse::ProxyTraceHintsAdopted { + request_id: response_id, + } if response_id == request_id => Ok(Ok(())), + UpstreamGrantResponse::Denied { + request_id: response_id, + reason, + } if response_id == request_id => Ok(Err(format!("proxy trace-hint handoff denied: {reason:?}"))), + response => Err(format!( + "unexpected proxy trace-hint response for request {request_id}: {response:?}" + )), + } +} + async fn attach_proxy_traffic( sender: &WireSender, receiver: &WireReceiver, diff --git a/crates/capsem-core/src/net/upstream_grant/tests.rs b/crates/capsem-core/src/net/upstream_grant/tests.rs index 3def410bc..a1ffacc3b 100644 --- a/crates/capsem-core/src/net/upstream_grant/tests.rs +++ b/crates/capsem-core/src/net/upstream_grant/tests.rs @@ -17,6 +17,91 @@ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; const POLICY_A: &str = "blake3:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; const POLICY_B: &str = "blake3:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +#[tokio::test] +async fn loopback_listener_is_adopted_only_when_its_descriptor_matches_the_grant() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenLoopbackListener { request_id: 1, port: 0 } + ); + let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + send_response( + &responses, + &UpstreamGrantResponse::LoopbackListenerGranted { + request_id: 1, + grant_id: 21, + port, + }, + Some(listener.as_raw_fd()), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { grant_id: 21 } + ); + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { resource_id: 21 } + ); + }); + + let grant = client.open_loopback_listener(0).await.unwrap(); + let (listener, lease) = grant.into_parts(); + let address = listener.local_addr().unwrap(); + assert_eq!(address.ip(), std::net::Ipv4Addr::LOCALHOST); + let listener = tokio::net::TcpListener::from_std(listener).unwrap(); + let connection = tokio::net::TcpStream::connect(address).await.unwrap(); + let (accepted, peer) = listener.accept().await.unwrap(); + assert_eq!(accepted.local_addr().unwrap(), address); + assert_eq!(peer, connection.local_addr().unwrap()); + drop(listener); + drop(lease); + broker_task.await.unwrap(); +} + +#[tokio::test] +async fn wildcard_listener_descriptor_revokes_the_grant_channel() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let broker_task = tokio::spawn(async move { + assert!(matches!( + receive_request(&requests).await, + UpstreamGrantRequest::OpenLoopbackListener { request_id: 1, .. } + )); + let listener = std::net::TcpListener::bind((std::net::Ipv4Addr::UNSPECIFIED, 0)).unwrap(); + let port = listener.local_addr().unwrap().port(); + send_response( + &responses, + &UpstreamGrantResponse::LoopbackListenerGranted { + request_id: 1, + grant_id: 22, + port, + }, + Some(listener.as_raw_fd()), + ) + .await; + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Adopted { grant_id: 22 } + ); + assert_eq!( + receive_request(&requests).await, + UpstreamGrantRequest::Release { resource_id: 22 } + ); + }); + + let error = client.open_loopback_listener(0).await.unwrap_err().to_string(); + assert!(error.contains("does not match 127.0.0.1"), "{error}"); + broker_task.await.unwrap(); + let error = client.open_loopback_listener(0).await.unwrap_err().to_string(); + assert!(error.contains("closed") || error.contains("stopped"), "{error}"); +} + #[tokio::test] async fn ledger_channel_preserves_grant_and_is_adopted_before_use() { let grant = LedgerChannelGrant::new(LedgerGeneration::new([0x6c; 16]), 44, LedgerClientRole::VmOwner).unwrap(); @@ -122,6 +207,36 @@ async fn proxy_mcp_descriptor_is_surrendered_before_adoption_returns() { broker_task.await.unwrap(); } +#[tokio::test] +async fn proxy_trace_hint_descriptor_is_surrendered_before_adoption_returns() { + let (broker, worker) = UnixStream::pair().unwrap(); + let client = UpstreamGrantClient::start(worker).unwrap(); + let (requests, responses) = channels(broker); + let (capability, mut proxy) = UnixStream::pair().unwrap(); + let broker_task = tokio::spawn(async move { + let frame = requests.recv().await.unwrap(); + assert_eq!(frame.fds.len(), 1); + assert_eq!( + decode_upstream_grant_request(&frame.bytes).unwrap(), + UpstreamGrantRequest::AttachProxyTraceHints { request_id: 1 } + ); + let mut adopted = UnixStream::from(frame.fds.into_iter().next().unwrap()); + adopted.write_all(b"hint").unwrap(); + send_response( + &responses, + &UpstreamGrantResponse::ProxyTraceHintsAdopted { request_id: 1 }, + None, + ) + .await; + }); + + client.attach_proxy_trace_hints(capability.into()).await.unwrap(); + let mut proof = [0; 4]; + proxy.read_exact(&mut proof).unwrap(); + assert_eq!(&proof, b"hint"); + broker_task.await.unwrap(); +} + #[tokio::test] async fn ledger_denial_and_missing_descriptor_fail_closed() { let (broker, worker) = UnixStream::pair().unwrap(); diff --git a/crates/capsem-core/src/security_engine/mod.rs b/crates/capsem-core/src/security_engine/mod.rs index c8e680314..0d7d969c6 100644 --- a/crates/capsem-core/src/security_engine/mod.rs +++ b/crates/capsem-core/src/security_engine/mod.rs @@ -23,6 +23,7 @@ mod container; mod credential_grants; pub use container::ContainerSecurityEvent; mod engine; +pub use credential_grants::CredentialGrantAdmission; pub use engine::*; mod event_id; pub use event_id::SecurityEventId; diff --git a/crates/capsem-core/tests/mitm_integration/tls.rs b/crates/capsem-core/tests/mitm_integration/tls.rs index 2bce74a43..1ab3e3d70 100644 --- a/crates/capsem-core/tests/mitm_integration/tls.rs +++ b/crates/capsem-core/tests/mitm_integration/tls.rs @@ -13,7 +13,7 @@ pub(super) fn make_tls_client_config() -> rustls::ClientConfig { for cert in certs { root_store.add(cert).unwrap(); } - let provider = Arc::new(rustls::crypto::aws_lc_rs::default_provider()); + let provider = Arc::new(capsem_crypto::tls::provider()); let mut config = rustls::ClientConfig::builder_with_provider(provider) .with_safe_default_protocol_versions() .unwrap() diff --git a/crates/capsem-foundation/src/telemetry.rs b/crates/capsem-foundation/src/telemetry.rs index 09e7587cd..a01e525b0 100644 --- a/crates/capsem-foundation/src/telemetry.rs +++ b/crates/capsem-foundation/src/telemetry.rs @@ -391,6 +391,12 @@ pub struct TelemetryGuard { /// var at startup. W4/W5 read this for in-band propagation. Empty when /// unset (CLI invocations and top-level binaries). static PARENT_TRACEPARENT: OnceLock = OnceLock::new(); +/// Process identity delivered over a trusted non-environment channel. +/// +/// Confined workers clear their entire inherited environment before parsing +/// coordinator-minted arguments. They install the resulting trace identity +/// here so telemetry keeps correlation without reopening ambient authority. +static IN_BAND_TRACE_ID: OnceLock = OnceLock::new(); /// Initialize tracing. Call exactly once per binary, in `main()`, before /// any `tracing::info!` macro fires. @@ -538,10 +544,44 @@ fn env_truthy(value: &str) -> bool { /// representation to remember when grepping. pub fn ambient_capsem_trace_id() -> Option { let env = std::env::var("CAPSEM_TRACE_ID").ok(); - resolve_ambient_capsem_trace_id(env.as_deref(), PARENT_TRACEPARENT.get().map(String::as_str)) + resolve_ambient_capsem_trace_id( + IN_BAND_TRACE_ID.get().map(String::as_str), + env.as_deref(), + PARENT_TRACEPARENT.get().map(String::as_str), + ) +} + +/// Install a trace identity received through a trusted in-band channel. +/// +/// The compact 16-lowercase-hex form is the same one used by +/// `CAPSEM_TRACE_ID`. A process may install it once, before telemetry starts. +pub fn install_in_band_trace_id(trace_id: String) -> Result<(), &'static str> { + if !valid_compact_trace_id(&trace_id) { + return Err("trace id must be 16 lowercase hexadecimal digits"); + } + IN_BAND_TRACE_ID + .set(trace_id) + .map_err(|_| "in-band trace id is already installed") +} + +fn valid_compact_trace_id(trace_id: &str) -> bool { + trace_id.len() == 16 + && trace_id + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + && trace_id.bytes().any(|byte| byte != b'0') } -fn resolve_ambient_capsem_trace_id(capsem_trace_id: Option<&str>, parent_traceparent: Option<&str>) -> Option { +fn resolve_ambient_capsem_trace_id( + in_band_trace_id: Option<&str>, + capsem_trace_id: Option<&str>, + parent_traceparent: Option<&str>, +) -> Option { + if let Some(in_band) = in_band_trace_id { + if valid_compact_trace_id(in_band) { + return Some(in_band.to_string()); + } + } if let Some(env) = capsem_trace_id { if !env.is_empty() { return Some(env.to_string()); diff --git a/crates/capsem-foundation/src/telemetry/tests.rs b/crates/capsem-foundation/src/telemetry/tests.rs index eebbd0c30..1ac998207 100644 --- a/crates/capsem-foundation/src/telemetry/tests.rs +++ b/crates/capsem-foundation/src/telemetry/tests.rs @@ -5,6 +5,7 @@ use super::*; #[test] fn ambient_trace_id_from_capsem_env_takes_precedence() { let id = resolve_ambient_capsem_trace_id( + None, Some("deadbeefcafef00d"), Some("00-11111111111111112222222222222222-3333333333333333-01"), ); @@ -13,16 +14,34 @@ fn ambient_trace_id_from_capsem_env_takes_precedence() { #[test] fn ambient_trace_id_returns_none_without_env() { - let id = resolve_ambient_capsem_trace_id(None, None); + let id = resolve_ambient_capsem_trace_id(None, None, None); assert_eq!(id, None); } #[test] fn ambient_trace_id_extracts_lower_half_from_traceparent() { - let id = resolve_ambient_capsem_trace_id(None, Some("00-11111111111111112222222222222222-3333333333333333-01")); + let id = resolve_ambient_capsem_trace_id( + None, + None, + Some("00-11111111111111112222222222222222-3333333333333333-01"), + ); assert_eq!(id.as_deref(), Some("2222222222222222")); } +#[test] +fn trusted_in_band_trace_id_survives_an_empty_environment() { + let id = resolve_ambient_capsem_trace_id(Some("0707070707070707"), None, None); + assert_eq!(id.as_deref(), Some("0707070707070707")); +} + +#[test] +fn trusted_in_band_trace_id_must_use_the_compact_canonical_form() { + for invalid in ["", "0707", "070707070707070G", "070707070707070A", "0000000000000000"] { + assert!(!valid_compact_trace_id(invalid), "accepted {invalid}"); + } + assert!(valid_compact_trace_id("070707070707070a")); +} + #[test] fn debug_telemetry_policy_is_local_only_by_default() { let policy = debug_telemetry_policy_from_pairs([ diff --git a/crates/capsem-foundation/src/unix/contained.rs b/crates/capsem-foundation/src/unix/contained.rs index f2ff187da..46dabf8ff 100644 --- a/crates/capsem-foundation/src/unix/contained.rs +++ b/crates/capsem-foundation/src/unix/contained.rs @@ -527,10 +527,20 @@ impl ContainedDir { | OFlag::O_NONBLOCK; let fd = openat(Some(self.fd.as_raw_fd()), name, flags, permission_mode(0o600))?; let file = File::from(owned(fd)); - file.set_permissions(std::fs::Permissions::from_mode(0o600))?; - let metadata = file.metadata()?; + let mut metadata = file.metadata()?; + // Normal umasks leave an explicitly requested 0600 mode unchanged. + // Avoid a redundant fchmod so sandboxed workers can create private + // files while permission changes remain outside their authority. + if metadata.permissions().mode() & 0o777 != 0o600 { + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + metadata = file.metadata()?; + } let uid = super::process::current_uid(); - if !metadata.is_file() || metadata.uid() != uid || metadata.nlink() != 1 { + if !metadata.is_file() + || metadata.uid() != uid + || metadata.nlink() != 1 + || metadata.permissions().mode() & 0o777 != 0o600 + { return Err(io::Error::new( io::ErrorKind::PermissionDenied, format!("refusing newly created private file {}", Path::new(name).display()), diff --git a/crates/capsem-foundation/src/unix/fd.rs b/crates/capsem-foundation/src/unix/fd.rs index e19f43a43..de238d70a 100644 --- a/crates/capsem-foundation/src/unix/fd.rs +++ b/crates/capsem-foundation/src/unix/fd.rs @@ -140,6 +140,59 @@ pub fn set_close_on_exec(fd: BorrowedFd<'_>) -> io::Result { Ok(was_enabled) } +/// Read at a checked file offset without changing the descriptor's cursor. +/// Interrupted operations are retried; EOF and short reads remain visible. +pub fn read_at(fd: BorrowedFd<'_>, bytes: &mut [u8], offset: u64) -> io::Result { + let offset = i64::try_from(offset).map_err(|_| io::Error::from_raw_os_error(Errno::EINVAL as i32))?; + retry_eintr(|| nix::sys::uio::pread(fd, bytes, offset)).map_err(errno::io) +} + +/// Write at a checked file offset without changing the descriptor's cursor. +/// Callers own any short-write loop so their protocol can account for progress. +pub fn write_at(fd: BorrowedFd<'_>, bytes: &[u8], offset: u64) -> io::Result { + let offset = i64::try_from(offset).map_err(|_| io::Error::from_raw_os_error(Errno::EINVAL as i32))?; + retry_eintr(|| nix::sys::uio::pwrite(fd, bytes, offset)).map_err(errno::io) +} + +/// Exact logical size of an already-open regular file, retaining descriptor identity. +pub fn regular_file_size(fd: BorrowedFd<'_>) -> io::Result { + regular_file_metadata(fd).map(|metadata| metadata.len) +} + +/// Descriptor-owned logical and allocated sizes for a regular file. +#[derive(Debug, Clone, Copy)] +pub struct RegularFileMetadata { + pub len: u64, + pub allocated_bytes: u64, +} + +pub fn regular_file_metadata(fd: BorrowedFd<'_>) -> io::Result { + use nix::sys::stat::{fstat, SFlag}; + let stat = retry_eintr(|| fstat(fd.as_raw_fd())).map_err(errno::io)?; + if SFlag::from_bits_truncate(stat.st_mode) & SFlag::S_IFMT != SFlag::S_IFREG { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "descriptor is not a regular file", + )); + } + let len = u64::try_from(stat.st_size) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "file length is negative"))?; + let allocated_bytes = u64::try_from(stat.st_blocks) + .ok() + .and_then(|blocks| blocks.checked_mul(512)) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "file allocation is invalid"))?; + Ok(RegularFileMetadata { len, allocated_bytes }) +} + +/// Signal one eventfd completion without borrowing raw Unix mechanics at callers. +pub fn signal_counter(fd: BorrowedFd<'_>) -> io::Result<()> { + let count = retry_eintr(|| nix::unistd::write(fd, &1u64.to_ne_bytes())).map_err(errno::io)?; + if count != std::mem::size_of::() { + return Err(io::Error::new(io::ErrorKind::WriteZero, "short eventfd counter write")); + } + Ok(()) +} + /// Set or clear `O_NONBLOCK`, returning its previous state. /// /// All unrelated descriptor status flags are preserved. An interrupted diff --git a/crates/capsem-foundation/src/unix/fd/tests.rs b/crates/capsem-foundation/src/unix/fd/tests.rs index 2538e19df..f97c7340a 100644 --- a/crates/capsem-foundation/src/unix/fd/tests.rs +++ b/crates/capsem-foundation/src/unix/fd/tests.rs @@ -60,6 +60,60 @@ fn isolated_exec_child_closes_ambient_descriptor_and_keeps_declared_grant() { assert!(child.wait().unwrap().success()); } +#[cfg(target_os = "linux")] +#[test] +fn counter_signals_are_owned_and_accumulate_completions() { + use nix::sys::eventfd::{EfdFlags, EventFd}; + let counter = EventFd::from_flags(EfdFlags::EFD_CLOEXEC | EfdFlags::EFD_NONBLOCK).unwrap(); + super::signal_counter(counter.as_fd()).unwrap(); + assert_eq!(counter.read().unwrap(), 1); + super::signal_counter(counter.as_fd()).unwrap(); + super::signal_counter(counter.as_fd()).unwrap(); + assert_eq!(counter.read().unwrap(), 2); +} + +#[test] +fn positioned_io_preserves_cursor_and_reports_short_reads_and_eof() { + use std::io::{Seek, SeekFrom}; + let mut file = tempfile::tempfile().unwrap(); + file.write_all(b"abcdef").unwrap(); + assert_eq!(super::regular_file_size(file.as_fd()).unwrap(), 6); + file.seek(SeekFrom::Start(2)).unwrap(); + assert_eq!(super::write_at(file.as_fd(), b"XY", 4).unwrap(), 2); + assert_eq!(file.stream_position().unwrap(), 2); + let mut bytes = [0; 8]; + assert_eq!(super::read_at(file.as_fd(), &mut bytes, 3).unwrap(), 3); + assert_eq!(&bytes[..3], b"dXY"); + assert_eq!(super::read_at(file.as_fd(), &mut bytes, 6).unwrap(), 0); + assert_eq!(file.stream_position().unwrap(), 2); +} + +#[test] +fn positioned_io_rejects_unrepresentable_offsets_and_preserves_os_errors() { + let file = tempfile::tempfile().unwrap(); + assert_eq!( + super::read_at(file.as_fd(), &mut [0], u64::MAX) + .unwrap_err() + .raw_os_error(), + Some(22) + ); + assert_eq!( + super::write_at(file.as_fd(), b"x", u64::MAX) + .unwrap_err() + .raw_os_error(), + Some(22) + ); + let (socket, _peer) = UnixStream::pair().unwrap(); + assert_eq!( + super::regular_file_size(socket.as_fd()).unwrap_err().kind(), + std::io::ErrorKind::InvalidInput + ); + assert_eq!( + super::read_at(socket.as_fd(), &mut [0], 0).unwrap_err().kind(), + std::io::ErrorKind::NotSeekable + ); +} + #[test] fn tcp_reset_waits_for_the_last_owned_descriptor_then_reaches_the_peer() { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/crates/capsem-foundation/src/unix/fs.rs b/crates/capsem-foundation/src/unix/fs.rs index 2d41b9476..34409f0a6 100644 --- a/crates/capsem-foundation/src/unix/fs.rs +++ b/crates/capsem-foundation/src/unix/fs.rs @@ -316,12 +316,16 @@ pub fn create_private_sibling(path: &Path) -> io::Result { path: temporary, kept: false, }; - // Through the handle, and before the guard could hand it out: - // a mode the filesystem refuses fails the create, and the - // guard removes the file on the way out. - sibling - .file - .set_permissions(std::fs::Permissions::from_mode(PRIVATE_FILE_MODE))?; + // The create mode is already private under every umask. Avoid + // an unnecessary fchmod when it is exactly right: confined + // workers deliberately cannot change file modes. A stricter + // process umask is repaired before the guard can hand the + // sibling out. + if sibling.file.metadata()?.permissions().mode() & 0o777 != PRIVATE_FILE_MODE { + sibling + .file + .set_permissions(std::fs::Permissions::from_mode(PRIVATE_FILE_MODE))?; + } return Ok(sibling); } Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} diff --git a/crates/capsem-foundation/src/unix/lock.rs b/crates/capsem-foundation/src/unix/lock.rs index bf3aaccdf..0bd1d8f6f 100644 --- a/crates/capsem-foundation/src/unix/lock.rs +++ b/crates/capsem-foundation/src/unix/lock.rs @@ -201,7 +201,10 @@ fn acquire_reserved(path: &Path, mode: LockMode, create: bool, after_open: impl )); } let permissions = metadata.permissions().mode() & 0o777; - if create { + // A freshly opened 0600 lock already has the required mode under normal + // umasks. Avoid a redundant fchmod so confined workers can create locks + // while permission changes remain denied by their sandbox. + if create && permissions != 0o600 { file.set_permissions(std::fs::Permissions::from_mode(0o600))?; } else if permissions != 0o600 { return Err(io::Error::new( diff --git a/crates/capsem-foundation/src/unix/process.rs b/crates/capsem-foundation/src/unix/process.rs index 6ae4ff99d..aaa329e68 100644 --- a/crates/capsem-foundation/src/unix/process.rs +++ b/crates/capsem-foundation/src/unix/process.rs @@ -12,16 +12,6 @@ use super::errno; mod scope; pub use scope::ProcessScope; -#[cfg(target_os = "linux")] -mod pinned; -#[cfg(target_os = "linux")] -pub use pinned::ProcessHandle; - -#[cfg(target_os = "macos")] -mod pinned_macos; -#[cfg(target_os = "macos")] -pub use pinned_macos::ProcessHandle; - /// Erase the inherited environment and detach it from the C process image. /// /// `remove_var` alone leaves the original stack bytes visible through Linux @@ -71,6 +61,16 @@ fn inherited_environment() -> *mut *mut libc::c_char { std::ptr::null_mut() } +#[cfg(target_os = "linux")] +mod pinned; +#[cfg(target_os = "linux")] +pub use pinned::ProcessHandle; + +#[cfg(target_os = "macos")] +mod pinned_macos; +#[cfg(target_os = "macos")] +pub use pinned_macos::ProcessHandle; + /// A positive process identifier representable by the host kernel ABI. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] pub struct ProcessId(NonZeroI32); diff --git a/crates/capsem-foundation/src/unix/worker_sandbox.rs b/crates/capsem-foundation/src/unix/worker_sandbox.rs index 18e17f1d1..b9ae122c9 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox.rs @@ -17,6 +17,10 @@ pub enum Role { pub enum Access { ReadOnly, ReadWrite, + /// Read, write, and issue device ioctls. Keep this narrower than a normal + /// writable path so a filesystem grant cannot accidentally gain device + /// control authority. + ReadWriteDevice, Executable, } diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs index 4c943644f..2b9272bb8 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/linux.rs @@ -145,6 +145,15 @@ fn add_path_rule(ruleset: &OwnedFd, path: &std::path::Path, access: Access) -> i (Access::ReadOnly, false) => FS_READ_FILE, (Access::ReadWrite, true) => FS_WRITE, (Access::ReadWrite, false) => FS_READ_FILE | FS_WRITE_FILE | FS_TRUNCATE, + (Access::ReadWriteDevice, false) if kind.contains(SFlag::S_IFCHR) || kind.contains(SFlag::S_IFBLK) => { + FS_READ_FILE | FS_WRITE_FILE | FS_IOCTL_DEV + } + (Access::ReadWriteDevice, _) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("sandbox device grant is not a device: {}", path.display()), + )); + } (Access::Executable, false) => FS_READ_FILE | FS_EXECUTE, (Access::Executable, true) => { return Err(io::Error::new( diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs b/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs index 830541e2f..c6fde401b 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/seatbelt.rs @@ -38,7 +38,7 @@ pub(super) fn compile(policy: &Policy) -> io::Result { )); match rule.access() { Access::ReadOnly => source.push_str(&format!("(allow file-read* {filter})\n")), - Access::ReadWrite => { + Access::ReadWrite | Access::ReadWriteDevice => { source.push_str(&format!("(allow file-read* file-write* {filter})\n")); } Access::Executable => { diff --git a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs index a832889f2..6187f131f 100644 --- a/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs +++ b/crates/capsem-foundation/src/unix/worker_sandbox/tests.rs @@ -246,6 +246,13 @@ fn linux_sandbox_child() { assert_eq!(std::fs::read(exact).unwrap(), b"one file"); assert!(std::fs::write(readonly.join("value"), b"changed").is_err()); std::fs::write(allowed.join("created"), b"ok").unwrap(); + let atomic = allowed.join("atomic"); + super::super::fs::atomic_write_private(&atomic, b"private").unwrap(); + assert_eq!(std::fs::read(&atomic).unwrap(), b"private"); + assert_eq!( + std::fs::symlink_metadata(&atomic).unwrap().permissions().mode() & 0o777, + 0o600 + ); assert!(std::fs::read(denied.join("secret")).is_err()); assert!(UnixStream::connect(control).is_err()); assert!(UnixStream::pair().is_err()); diff --git a/crates/capsem-gateway/src/main.rs b/crates/capsem-gateway/src/main.rs index 93a9054c1..b50bc07d2 100644 --- a/crates/capsem-gateway/src/main.rs +++ b/crates/capsem-gateway/src/main.rs @@ -86,10 +86,6 @@ fn main() -> Result<()> { } async fn run(args: Args) -> Result<()> { - anyhow::ensure!( - args.service_grant_stdin, - "capsem-gateway requires a coordinator descriptor grant channel" - ); let run_dir = gateway_run_dir(&args); let _ = std::fs::create_dir_all(&run_dir); let log_path = run_dir.join("gateway.log"); @@ -109,6 +105,10 @@ async fn run(args: Args) -> Result<()> { let Some(_singleton) = capsem_guard::Singleton::try_acquire(&lock_path)? else { return Ok(()); }; + anyhow::ensure!( + args.service_grant_stdin, + "capsem-gateway requires a coordinator descriptor grant channel" + ); let uds_path = args.uds_path.unwrap_or_else(|| run_dir.join("service.sock")); let service_socket_missing = !uds_path.exists(); @@ -303,6 +303,21 @@ fn service_proxy_routes() -> Router> { .route("/proxies", post(proxy::handle_proxy)) .route("/proxies/{id}/heartbeat", post(proxy::handle_proxy)) .route("/proxies/{id}/stop", post(proxy::handle_proxy)) + .route("/connections", get(proxy::handle_proxy)) + .route("/connections/google/authorize", post(proxy::handle_proxy)) + .route( + "/connections/authorizations/{transaction_id}", + get(proxy::handle_proxy).delete(proxy::handle_proxy), + ) + .route( + "/connections/{connection_id}", + get(proxy::handle_proxy).delete(proxy::handle_proxy), + ) + .route( + "/vms/{id}/credential-grants", + get(proxy::handle_proxy).post(proxy::handle_proxy), + ) + .route("/vms/{id}/credential-grants/{grant_id}", delete(proxy::handle_proxy)) .route("/vms/create", post(proxy::handle_proxy)) .route("/vms/managed/{request_id}/claim", post(proxy::handle_proxy)) .route("/vms/managed/{request_id}/close", post(proxy::handle_proxy)) diff --git a/crates/capsem-ledger/src/main.rs b/crates/capsem-ledger/src/main.rs index 1a608f694..6ad9700b6 100644 --- a/crates/capsem-ledger/src/main.rs +++ b/crates/capsem-ledger/src/main.rs @@ -55,11 +55,14 @@ fn main() -> Result<()> { let args = Args::parse(); let stdin = io::stdin(); let control = UnixStream::from(fd::duplicate(stdin.as_fd())?); - let session_dir = session_directory(&args.database)?; - validate_session_directory(&session_dir)?; + let session_dir = resolved_session_directory(&args.database)?; + // Seatbelt grants canonical path spellings. Keep every post-confinement + // open on that same spelling when an ancestor such as macOS `/var` + // resolves through a symlink. + let database = session_dir.join("session.db"); let denied_file = std::env::current_exe().context("resolve ledger executable before confinement")?; let codecs = codec::archive_codecs().context("initialize confined archive codecs")?; - let prepared = LedgerServer::prepare_with_codecs(&args.database, codecs).context("prepare session ledger")?; + let prepared = LedgerServer::prepare_with_codecs(&database, codecs).context("prepare session ledger")?; // The current-thread runtime prepares Tokio's internal descriptors without // creating sibling threads. This keeps Landlock ABI 6 and 7 safe while // allowing tasks and blocking workers created after confinement to inherit @@ -95,6 +98,12 @@ fn validate_session_directory(directory: &Path) -> Result<()> { Ok(()) } +fn resolved_session_directory(database: &Path) -> Result { + let directory = session_directory(database)?; + validate_session_directory(&directory)?; + std::fs::canonicalize(&directory).context("resolve ledger session directory") +} + fn attest_confinement( session_dir: &Path, denied_file: &Path, @@ -120,6 +129,10 @@ fn attest_confinement( std::fs::remove_file(&marker)?; require_denied(std::fs::read(denied_file), "read outside its session")?; + // Linux seccomp denies socket(2) itself. Seatbelt denies endpoint bind, + // listen and connect operations but permits authority-free private + // socketpairs, which Tokio and worker-internal channels may use. + #[cfg(target_os = "linux")] require_denied(UnixStream::pair(), "create a Unix socket")?; require_denied( std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)), diff --git a/crates/capsem-ledger/src/tests.rs b/crates/capsem-ledger/src/tests.rs index bc35ee55d..e06c03495 100644 --- a/crates/capsem-ledger/src/tests.rs +++ b/crates/capsem-ledger/src/tests.rs @@ -263,6 +263,24 @@ fn database_path_is_one_absolute_session_directory() { } } +#[test] +fn database_path_uses_the_resolved_session_directory() { + use std::os::unix::fs::symlink; + + let root = tempfile::tempdir().unwrap(); + let real_parent = root.path().join("real"); + let session = real_parent.join("session"); + std::fs::create_dir_all(&session).unwrap(); + let alias = root.path().join("alias"); + symlink(&real_parent, &alias).unwrap(); + + let database = alias.join("session/session.db"); + assert_eq!( + resolved_session_directory(&database).unwrap(), + session.canonicalize().unwrap() + ); +} + #[test] fn inherited_authority_is_removed_and_confinement_precedes_readiness() { let source = include_str!("main.rs"); diff --git a/crates/capsem-ledger/tests/subprocess.rs b/crates/capsem-ledger/tests/subprocess.rs index 64e9e88ba..7bfec7975 100644 --- a/crates/capsem-ledger/tests/subprocess.rs +++ b/crates/capsem-ledger/tests/subprocess.rs @@ -106,7 +106,10 @@ async fn executable_opens_one_ledger_and_reports_durable_stop() { .env("CAPSEM_LEDGER_SECRET", "must-not-survive") .stdin(Stdio::from(OwnedFd::from(child_control))) .stdout(Stdio::null()) - .stderr(Stdio::piped()) + // Startup failures happen before the worker can send Ready. Keep + // its exact confinement error in the test log instead of reducing + // every refusal to an opaque control-channel EOF. + .stderr(Stdio::inherit()) .spawn() .unwrap(), )); diff --git a/crates/capsem-logger/Cargo.toml b/crates/capsem-logger/Cargo.toml index 231da71b2..6d821d375 100644 --- a/crates/capsem-logger/Cargo.toml +++ b/crates/capsem-logger/Cargo.toml @@ -25,7 +25,6 @@ serde = { workspace = true } serde_bytes = { workspace = true } serde_json = { workspace = true } rmp-serde = { workspace = true } -blake3 = "1" uuid = { version = "1", features = ["v4"] } metrics = "0.24" tempfile = "3" diff --git a/crates/capsem-logger/src/events/network.rs b/crates/capsem-logger/src/events/network.rs index 2255559cf..e39160ebb 100644 --- a/crates/capsem-logger/src/events/network.rs +++ b/crates/capsem-logger/src/events/network.rs @@ -38,7 +38,7 @@ pub enum MembershipState { /// A detached member and its optional last-member network retirement must /// commit in one database transaction. Neither half is independently durable. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct NetworkDeparture { pub(crate) membership: NetworkMembership, pub(crate) retired: Option, diff --git a/crates/capsem-logger/src/ledger_client.rs b/crates/capsem-logger/src/ledger_client.rs index 12c4b7e80..250548775 100644 --- a/crates/capsem-logger/src/ledger_client.rs +++ b/crates/capsem-logger/src/ledger_client.rs @@ -213,17 +213,8 @@ async fn run_actor( read_cache_epoch: Arc, ) { let mut request_id = 1_u64; - let mut counters = None; while let Some(command) = commands.recv().await { - let result = dispatch( - &sender, - &receiver, - request_id, - command, - &mut counters, - &read_cache_epoch, - ) - .await; + let result = dispatch(&sender, &receiver, request_id, command, &read_cache_epoch).await; if let Err(error) = result { tracing::error!(%error, "ledger client operation failed"); fail_pending(&mut commands, &error); @@ -242,7 +233,6 @@ async fn dispatch( receiver: &ClientReceiver, request_id: u64, command: Command, - previous_counters: &mut Option>, read_cache_epoch: &AtomicU64, ) -> Result<(), String> { match command { @@ -259,15 +249,15 @@ async fn dispatch( let result = one_reply(sender, receiver, request_id, LedgerCommand::Counters) .await .and_then(|response| match response { - LedgerReply::Counters { counters } => Ok(Arc::new(*counters)), + LedgerReply::Counters { + counters, + read_cache_epoch: remote_epoch, + } => { + read_cache_epoch.store(remote_epoch, Ordering::Release); + Ok(Arc::new(*counters)) + } response => Err(unexpected("counters", response)), }); - if let Ok(current) = &result { - if previous_counters.as_deref() != Some(current.as_ref()) { - *previous_counters = Some(Arc::clone(current)); - read_cache_epoch.fetch_add(1, Ordering::AcqRel); - } - } finish(reply, result) } Command::ReadBodies { event_id, reply } => { @@ -439,7 +429,7 @@ fn finish_bodies(bodies: BTreeMap) -> Result, if body.bytes.len() as u64 != body.metadata.stored_bytes { return Err("ledger body ended before its declared length".to_string()); } - let hash = format!("blake3:{}", blake3::hash(&body.bytes).to_hex()); + let hash = format!("blake3:{}", capsem_crypto::blake3::hash(&body.bytes).to_hex()); if hash != body.metadata.body_hash { return Err("ledger body hash does not match its bytes".to_string()); } diff --git a/crates/capsem-logger/src/ledger_client/tests.rs b/crates/capsem-logger/src/ledger_client/tests.rs index 498fe2966..a390ccffe 100644 --- a/crates/capsem-logger/src/ledger_client/tests.rs +++ b/crates/capsem-logger/src/ledger_client/tests.rs @@ -13,6 +13,7 @@ const EVENT_ID: &str = "0123456789ab"; struct Fixture { _dir: tempfile::TempDir, + server: Arc, client: LedgerClient, writer: crate::DbWriter, producer_task: tokio::task::JoinHandle>, @@ -47,6 +48,7 @@ impl Fixture { .unwrap(); Self { _dir: dir, + server, client, writer, producer_task, @@ -144,9 +146,10 @@ async fn reader_queries_counters_bodies_and_warc_through_one_serial_channel() { let counters = fixture.client.counters().await.unwrap(); assert_eq!(counters.net.total, 1); - assert_eq!(fixture.client.read_cache_epoch(), 1); + let epoch = fixture.client.read_cache_epoch(); + assert!(epoch > 0); assert_eq!(fixture.client.counters().await.unwrap().net.total, 1); - assert_eq!(fixture.client.read_cache_epoch(), 1); + assert_eq!(fixture.client.read_cache_epoch(), epoch); let sets = fixture.client.query(LedgerQuery::StatsDetail).await.unwrap(); assert_eq!(sets.len(), 11); @@ -168,13 +171,18 @@ async fn reader_queries_counters_bodies_and_warc_through_one_serial_channel() { } #[tokio::test] -async fn counters_advance_the_cache_epoch_only_when_the_snapshot_changes() { +async fn counters_carry_the_db_owned_cache_epoch_even_when_the_snapshot_is_unchanged() { let fixture = Fixture::start(LedgerClientRole::Reader).await; - assert_eq!(fixture.client.counters().await.unwrap().net.total, 0); - assert_eq!(fixture.client.read_cache_epoch(), 1); + let counters = fixture.client.counters().await.unwrap(); + let initial_epoch = fixture.client.read_cache_epoch(); + fixture.server.invalidate_read_cache_for_tests(); + assert_eq!(fixture.client.counters().await.unwrap().as_ref(), counters.as_ref()); + assert!(fixture.client.read_cache_epoch() > initial_epoch); + + let invalidated_epoch = fixture.client.read_cache_epoch(); fixture.write_network_body().await; assert_eq!(fixture.client.counters().await.unwrap().net.total, 1); - assert_eq!(fixture.client.read_cache_epoch(), 2); + assert!(fixture.client.read_cache_epoch() > invalidated_epoch); fixture.stop().await; } diff --git a/crates/capsem-logger/src/ledger_protocol.rs b/crates/capsem-logger/src/ledger_protocol.rs index 58e07562a..eec71a444 100644 --- a/crates/capsem-logger/src/ledger_protocol.rs +++ b/crates/capsem-logger/src/ledger_protocol.rs @@ -351,6 +351,7 @@ pub enum LedgerReply { }, Counters { counters: Box, + read_cache_epoch: u64, }, BodyStart { body_id: u32, diff --git a/crates/capsem-logger/src/ledger_server.rs b/crates/capsem-logger/src/ledger_server.rs index e3f3540fc..87cd18a41 100644 --- a/crates/capsem-logger/src/ledger_server.rs +++ b/crates/capsem-logger/src/ledger_server.rs @@ -16,7 +16,7 @@ use crate::ledger_protocol::{ LedgerBodyMetadata, LedgerClientMessage, LedgerCommand, LedgerExportSummary, LedgerReply, LedgerServerMessage, MAX_LEDGER_STREAM_CHUNK_BYTES, }; -use crate::{DbHandle, StoredBody}; +use crate::{DbHandle, ReadCacheDomain, StoredBody}; mod queries; @@ -214,12 +214,13 @@ impl LedgerServer { Err(error) => Err(storage(error)), }, LedgerCommand::Counters => match self.current_counters().await { - Ok(counters) => { + Ok((counters, read_cache_epoch)) => { send_success( sender, request_id, LedgerReply::Counters { counters: Box::new((*counters).clone()), + read_cache_epoch, }, ) .await @@ -260,9 +261,15 @@ impl LedgerServer { Ok(false) } - async fn current_counters(&self) -> Result, String> { + async fn current_counters(&self) -> Result<(Arc, u64), String> { self.db.ready().await?; - self.db.ledger_counters().await + let counters = self.db.ledger_counters().await?; + Ok((counters, self.db.read_cache_epoch(ReadCacheDomain::All))) + } + + #[cfg(test)] + pub(crate) fn invalidate_read_cache_for_tests(&self) { + self.db.invalidate_read_cache(); } async fn send_bodies(&self, sender: &ServerSender, request_id: u64, bodies: Vec) -> Result<(), String> { diff --git a/crates/capsem-logger/src/writer.rs b/crates/capsem-logger/src/writer.rs index b4b695372..814333837 100644 --- a/crates/capsem-logger/src/writer.rs +++ b/crates/capsem-logger/src/writer.rs @@ -503,13 +503,26 @@ pub(crate) struct PreparedDbWriter { held: capsem_foundation::unix::lock::FileLock, conn: Connection, db_path: PathBuf, - batch_capacity: usize, + schedule: WriterSchedule, pending_body_bytes: Arc, bodies: BodyArchive, tally: LedgerTally, } impl DbWriter { + #[cfg(test)] + fn open_once(path: &Path, schedule: WriterSchedule, now: LedgerClock) -> rusqlite::Result { + let mut prepared = Self::prepare_once( + path, + schedule.capacity, + schedule.capacity, + now, + ArchiveCodecs::default(), + )?; + prepared.schedule = schedule; + Ok(prepared.start()) + } + /// Spawn a dedicated writer thread that owns the DB connection. /// `capacity` controls the mpsc channel size (backpressure). pub fn open(path: &Path, capacity: usize) -> rusqlite::Result { @@ -634,7 +647,7 @@ impl DbWriter { held, conn, db_path, - batch_capacity, + schedule: WriterSchedule::new(batch_capacity), pending_body_bytes, bodies, tally, @@ -660,13 +673,27 @@ impl DbWriter { })?; let tally = LedgerTally::restored(counters::load(&conn)?); - let schedule = WriterSchedule::new(capacity); - let (tx, rx) = writer_channel(schedule.capacity); + let batch_capacity = if capacity == 0 { + DEFAULT_BATCH_CAPACITY + } else { + capacity + }; + let (tx, rx) = writer_channel(batch_capacity); let pending_body_bytes = Arc::new(AtomicU64::new(0)); let loop_pending_body_bytes = Arc::clone(&pending_body_bytes); let join_handle = std::thread::Builder::new() .name("capsem-db-writer".into()) - .spawn(move || writer_loop(conn, rx, None, schedule, &loop_pending_body_bytes, bodies, tally)) + .spawn(move || { + writer_loop( + conn, + rx, + None, + WriterSchedule::new(batch_capacity), + &loop_pending_body_bytes, + bodies, + tally, + ) + }) .expect("failed to spawn db writer thread"); Ok(Self { @@ -821,7 +848,7 @@ impl PreparedDbWriter { held, conn, db_path, - batch_capacity, + schedule, pending_body_bytes, bodies, tally, @@ -836,7 +863,7 @@ impl PreparedDbWriter { conn, rx, writer_loop_db_path, - batch_capacity, + schedule, &loop_pending_body_bytes, bodies, tally, diff --git a/crates/capsem-logger/src/writer/bodies/index_batch.rs b/crates/capsem-logger/src/writer/bodies/index_batch.rs deleted file mode 100644 index d33f6b36a..000000000 --- a/crates/capsem-logger/src/writer/bodies/index_batch.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Finite parameter batches reduce SQLite executions without splitting a flush. -use super::*; -use rusqlite::{params_from_iter, ToSql}; - -// Fifteen fields × 64 rows = 960 parameters, below even SQLite's older 999 -// default. Only references are collected; body bytes and transaction ownership -// stay with the existing archive and writer. -const ROWS: usize = 64; -const INSERT: &str = "INSERT OR REPLACE INTO event_body_blobs ( - event_id, event_type, source_table, direction, content_type, - original_bytes, stored_bytes, truncated, body_hash, - block_offset, body_offset, body_len, trace_id, turn_id, created_at -) VALUES "; -const VALUES: &str = "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"; - -fn sql(rows: usize) -> String { - let mut statement = String::from(INSERT); - for index in 0..rows { - if index != 0 { - statement.push(','); - } - statement.push_str(VALUES); - } - statement -} -fn values(row: &BodyIndexRow) -> [&dyn ToSql; 15] { - [ - &row.event_id, - &row.event_type, - &row.source_table, - &row.direction, - &row.content_type, - &row.original_bytes, - &row.stored_bytes, - &row.truncated, - &row.body_hash, - &row.block_offset, - &row.body_offset, - &row.stored_bytes, - &row.trace_id, - &row.turn_id, - &row.created_at, - ] -} -pub(super) fn insert<'a>(conn: &Connection, mut rows: impl Iterator) -> rusqlite::Result<()> { - let mut batch = Vec::with_capacity(ROWS); - let mut full = conn.prepare_cached(&sql(ROWS))?; - loop { - batch.clear(); - batch.extend(rows.by_ref().take(ROWS)); - if batch.is_empty() { - return Ok(()); - } - let parameters = params_from_iter(batch.iter().flat_map(|row| values(row))); - if batch.len() == ROWS { - full.execute(parameters)?; - } else { - conn.prepare_cached(&sql(batch.len()))?.execute(parameters)?; - return Ok(()); - } - } -} diff --git a/crates/capsem-logger/src/writer/operation.rs b/crates/capsem-logger/src/writer/operation.rs index b0aed90c6..0ec9f0474 100644 --- a/crates/capsem-logger/src/writer/operation.rs +++ b/crates/capsem-logger/src/writer/operation.rs @@ -82,7 +82,7 @@ pub fn commitment_event_hash(op: &WriteOp) -> Result<[u8; 32], String> { pub(crate) fn commitment_event_fingerprint(op: &WriteOp) -> Result<([u8; 32], usize), String> { let encoded = serde_json::to_vec(op).map_err(|error| format!("encode ledger event commitment: {error}"))?; - let mut hash = blake3::Hasher::new(); + let mut hash = capsem_crypto::blake3::Hasher::new(); hash.update(b"capsem-ledger-event-v1\0"); hash.update(&encoded); Ok((*hash.finalize().as_bytes(), encoded.len())) diff --git a/crates/capsem-logger/src/writer/remote.rs b/crates/capsem-logger/src/writer/remote.rs index 15cb27f92..7ca36d059 100644 --- a/crates/capsem-logger/src/writer/remote.rs +++ b/crates/capsem-logger/src/writer/remote.rs @@ -658,25 +658,26 @@ impl CommitmentState { } async fn anchor_through(&mut self, through_admission_id: u64) -> Result<(), String> { - let count = self + let mut remaining = self .pending .iter() .take_while(|(admission_id, _)| *admission_id <= through_admission_id) .count(); - if count == 0 { - return Ok(()); - } - let commitments = self.pending[..count] - .iter() - .map(|(_, commitment)| commitment.clone()) - .collect(); - let reply = self.request(CommitmentCommand::Anchor { commitments }).await?; - if !matches!(reply, CommitmentReply::Anchored { .. }) { - return Err(format!( - "ledger commitment coordinator returned {reply:?} for checkpoint" - )); + while remaining > 0 { + let count = remaining.min(MAX_COMMITMENTS_PER_CHECKPOINT); + let commitments = self.pending[..count] + .iter() + .map(|(_, commitment)| commitment.clone()) + .collect(); + let reply = self.request(CommitmentCommand::Anchor { commitments }).await?; + if !matches!(reply, CommitmentReply::Anchored { .. }) { + return Err(format!( + "ledger commitment coordinator returned {reply:?} for checkpoint" + )); + } + self.pending.drain(..count); + remaining -= count; } - self.pending.drain(..count); Ok(()) } } diff --git a/crates/capsem-logger/src/writer/remote/tests.rs b/crates/capsem-logger/src/writer/remote/tests.rs index ce4f8db14..9b286ce94 100644 --- a/crates/capsem-logger/src/writer/remote/tests.rs +++ b/crates/capsem-logger/src/writer/remote/tests.rs @@ -3,6 +3,10 @@ use std::sync::Arc; use std::time::SystemTime; use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; +use capsem_proto::ledger_commitment::{ + CommitmentClientMessage, CommitmentReply, CommitmentServerMessage, LedgerCommitment, + MAX_COMMITMENTS_PER_CHECKPOINT, ZERO_COMMITMENT_HASH, +}; use crate::ledger_protocol::{LedgerClientMessage, LedgerResponse, LedgerServerMessage}; use crate::ledger_server::LedgerServer; diff --git a/crates/capsem-logger/src/writer/retention.rs b/crates/capsem-logger/src/writer/retention.rs index e891d6634..26c6365ce 100644 --- a/crates/capsem-logger/src/writer/retention.rs +++ b/crates/capsem-logger/src/writer/retention.rs @@ -118,7 +118,7 @@ pub(super) fn retain_bodies( return Ok(RetainOutcome::default()); } - let generation_id = GenerationId::new_v4(); + let generation_id = GenerationId::generate(); let mut candidate = BodyLogWriter::create_generation_with_codecs( &directory, state.header.archive_id, diff --git a/crates/capsem-logger/src/writer/worker.rs b/crates/capsem-logger/src/writer/worker.rs index c9d2bd365..0693866ce 100644 --- a/crates/capsem-logger/src/writer/worker.rs +++ b/crates/capsem-logger/src/writer/worker.rs @@ -6,11 +6,15 @@ pub(super) fn writer_loop( conn: Connection, rx: WriterReceiver, db_path: Option, - batch_capacity: usize, + schedule: WriterSchedule, pending_body_bytes: &AtomicU64, mut bodies: BodyArchive, mut tally: LedgerTally, ) { + let WriterSchedule { + capacity: batch_capacity, + interval, + } = schedule; let mut flush_watermarks = schema::with_memory_schema_lock(|| schema::initial_memory_flush_watermarks(&conn, schema::hot_ledger_tables())) .unwrap_or_else(|error| { @@ -23,6 +27,7 @@ pub(super) fn writer_loop( let exec_floor = flush_watermarks.get("exec_events").copied().unwrap_or(0); let mut dirty_tables = BTreeSet::new(); let mut dirty_ops = 0_usize; + let mut rejected_ops = 0_usize; let mut last_disk_flush = Instant::now(); // 1. Block until at least one op arrives. Returns None when all // Senders are dropped (clean shutdown) and ends the loop. @@ -30,9 +35,10 @@ pub(super) fn writer_loop( let first_message = if dirty_ops == 0 && !bodies.has_work() { rx.recv().ok() } else { - match rx.recv_timeout(DISK_FLUSH_INTERVAL) { + match rx.recv_timeout(interval.saturating_sub(last_disk_flush.elapsed())) { Ok(message) => Some(message), Err(mpsc::RecvTimeoutError::Timeout) => { + last_disk_flush = Instant::now(); if let Err(error) = flush_dirty_tables_to_disk_recorded( &conn, &mut dirty_tables, @@ -103,6 +109,7 @@ pub(super) fn writer_loop( .in_scope(|| retry_batch_ops_individually(&conn, &batch, &mut bodies, exec_floor, &mut tally)); dirty_tables.extend(salvaged.tables); dirty_ops += salvaged.written; + rejected_ops = rejected_ops.saturating_add(salvaged.rejected); } } } @@ -111,11 +118,15 @@ pub(super) fn writer_loop( // wait for the next transaction; its bytes are already on disk. bodies.close_if_due(); pending_body_bytes.store(bodies.pending_bytes() as u64, Ordering::Release); - let disk_flush_due = dirty_ops >= DISK_FLUSH_THRESHOLD_OPS - || last_disk_flush.elapsed() >= DISK_FLUSH_INTERVAL - || barriers.waiting(); - let mut barrier_outcome: FlushOutcome = Ok(()); + let disk_flush_due = + dirty_ops >= DISK_FLUSH_THRESHOLD_OPS || last_disk_flush.elapsed() >= interval || barriers.waiting(); + let mut barrier_outcome: FlushOutcome = if rejected_ops == 0 { + Ok(()) + } else { + Err(format!("db rejected {rejected_ops} accepted write operations")) + }; if disk_flush_due { + last_disk_flush = Instant::now(); match flush_dirty_tables_to_disk_recorded( &conn, &mut dirty_tables, @@ -140,7 +151,11 @@ pub(super) fn writer_loop( } } ::metrics::gauge!(DB_MEMORY_UNFLUSHED_OPS).set(dirty_ops as f64); + let explicit_boundary = barriers.waiting(); barriers.answer(&conn, &mut bodies, &barrier_outcome); + if explicit_boundary { + rejected_ops = 0; + } } // Test hook: lets `test_wal_absent_after_clean_shutdown`-style tests diff --git a/crates/capsem-mock-server/src/main.rs b/crates/capsem-mock-server/src/main.rs index 2d4ff99e0..24586cf03 100644 --- a/crates/capsem-mock-server/src/main.rs +++ b/crates/capsem-mock-server/src/main.rs @@ -454,15 +454,7 @@ event: model.done\ndata: {\"finish_reason\":\"stop\"}\n\n", if let Some(response) = openai_sdk_control(&payload).await { return response; } - if payload.get("stream").and_then(Value::as_bool) == Some(true) { - response( - StatusCode::OK, - responses_stream(&payload, payload_has_function_call_output(&payload)), - "text/event-stream", - ) - } else { - json_response(responses_response(payload)) - } + responses::reply(payload) } (&Method::POST, "/model/shape") => json_response(json!({ "id": "chatcmpl_shape_fixture", @@ -790,95 +782,6 @@ data: [DONE]\n\n", ) } -fn responses_response(payload: Value) -> Value { - let model = payload.get("model").and_then(Value::as_str).unwrap_or("mock-local"); - let has_tool_output = serde_json::to_string(&payload) - .map(|raw| raw.contains("function_call_output")) - .unwrap_or(false); - if !has_tool_output { - return json!({ - "id": "resp_ironbank_tool_01", - "object": "response", - "created_at": 1781205836_u64, - "status": "completed", - "model": model, - "output": [{ - "id": "fc_codex_write_poem", - "type": "function_call", - "status": "completed", - "call_id": "call_codex_write_poem", - "name": "exec_command", - "arguments": "{\"cmd\":\"printf '%s\\\\n' 'Capsem ironbank poem\\\\nledgers count the sparks\\\\nno secret crosses raw' > /root/codex-cli-output.txt\",\"yield_time_ms\":1000,\"max_output_tokens\":2000}" - }], - "usage": {"input_tokens": 31, "output_tokens": 17, "total_tokens": 48} - }); - } - json!({ - "id": "resp_ironbank_01", - "object": "response", - "created_at": 1781205836_u64, - "status": "completed", - "model": model, - "output": [{ - "id": "msg_ironbank_01", - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "text": EXPECTED_POEM, "annotations": []}] - }], - "output_text": EXPECTED_POEM, - "usage": { - "input_tokens": 7, - "output_tokens": 5, - "total_tokens": 12, - "output_tokens_details": {"reasoning_tokens": 2} - } - }) -} - -fn payload_has_function_call_output(payload: &Value) -> bool { - payload - .get("input") - .and_then(Value::as_array) - .map(|items| { - items - .iter() - .any(|item| item.get("type").and_then(Value::as_str) == Some("function_call_output")) - }) - .unwrap_or(false) -} - -fn responses_stream(payload: &Value, final_turn: bool) -> Bytes { - if final_turn { - let (token, _) = write_target(payload, "openai-responses"); - let token_json = json_compact(json!(token)); - return Bytes::from( - format!( - "event: response.reasoning_summary_text.delta\ndata: {{\"type\":\"response.reasoning_summary_text.delta\",\"delta\":\"ledger reasoning\"}}\n\n\ -event: response.output_text.delta\ndata: {{\"type\":\"response.output_text.delta\",\"delta\":{token_json}}}\n\n\ -event: response.output_text.done\ndata: {{\"type\":\"response.output_text.done\",\"text\":{token_json}}}\n\n\ -event: response.completed\ndata: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_capsem_mock\",\"status\":\"completed\",\"model\":\"gpt-5-nano\",\"usage\":{{\"input_tokens\":7,\"output_tokens\":5,\"total_tokens\":12,\"output_tokens_details\":{{\"reasoning_tokens\":2}}}}}}}}\n\n" - ), - ); - } - let (token, path) = write_target(payload, "openai-responses"); - let call_id = format!("call_{}", &token[..token.len().min(12)]); - let arguments = json_compact(json!({ - "cmd": shell_write_command(&token, &path), - "yield_time_ms": 1000, - "max_output_tokens": 2000 - })); - let arguments_json = json_compact(json!(arguments)); - Bytes::from(format!( - "event: response.output_item.added\ndata: {{\"type\":\"response.output_item.added\",\"item\":{{\"type\":\"reasoning\",\"id\":\"rs_capsem_mock\"}}}}\n\n\ -event: response.output_item.added\ndata: {{\"type\":\"response.output_item.added\",\"item\":{{\"type\":\"function_call\",\"id\":\"fc_capsem_mock\",\"call_id\":\"{call_id}\",\"name\":\"exec_command\",\"arguments\":{arguments_json}}}}}\n\n\ -event: response.function_call_arguments.delta\ndata: {{\"type\":\"response.function_call_arguments.delta\",\"delta\":{arguments_json}}}\n\n\ -event: response.function_call_arguments.done\ndata: {{\"type\":\"response.function_call_arguments.done\",\"arguments\":{arguments_json}}}\n\n\ -event: response.output_item.done\ndata: {{\"type\":\"response.output_item.done\",\"item\":{{\"type\":\"function_call\",\"id\":\"fc_capsem_mock\",\"status\":\"completed\",\"call_id\":\"{call_id}\",\"name\":\"exec_command\",\"arguments\":{arguments_json}}}}}\n\n\ -event: response.completed\ndata: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_capsem_mock\",\"status\":\"completed\",\"model\":\"gpt-5-nano\",\"usage\":{{\"input_tokens\":31,\"output_tokens\":17,\"total_tokens\":48,\"output_tokens_details\":{{\"reasoning_tokens\":2}}}}}}}}\n\n" - )) -} - /// The code-assist fixtures, carried in the binary rather than read at run time. /// /// These used to be read through `CARGO_MANIFEST_DIR`, which is an absolute diff --git a/crates/capsem-process/src/ipc.rs b/crates/capsem-process/src/ipc.rs index 44e07f7f7..503a4e478 100644 --- a/crates/capsem-process/src/ipc.rs +++ b/crates/capsem-process/src/ipc.rs @@ -684,14 +684,14 @@ pub(crate) async fn handle_ipc_connection( warn!(id, %error, "clone completion refused"); } } - ServiceToProcess::ReloadConfig { id } => { + ServiceToProcess::ReloadConfig { id, active_policy } => { info!( active_policy = %runtime_source.active_policy_path().display(), "Reloading runtime policy" ); // A policy that does not load leaves the previous one in // force; the service is told why instead of losing the socket. - let reply = match runtime_source.load() { + let reply = match runtime_source.load_bytes(&active_policy) { Ok(runtime_config) => { let digest = runtime_config.active_policy_digest.clone(); runtime_config.apply(&net_state, &mcp_runtime); @@ -802,7 +802,7 @@ pub(crate) async fn handle_ipc_connection( let mcp_builtin_binary = mcp_builtin_binary.clone(); let mcp_builtin_env = mcp_builtin_env.clone(); tokio::spawn(async move { - let runtime_config = match runtime_source.load() { + let runtime_config = match runtime_source.current() { Ok(config) => config, Err(e) => { capsem_core::try_send!( diff --git a/crates/capsem-process/src/ipc/tests.rs b/crates/capsem-process/src/ipc/tests.rs index a8dcf63e9..fe4fccb67 100644 --- a/crates/capsem-process/src/ipc/tests.rs +++ b/crates/capsem-process/src/ipc/tests.rs @@ -568,24 +568,23 @@ async fn negotiated_dispatcher_covers_stream_jobs_queries_and_lifecycle() { } if error == "read fixture failed" )); - std::fs::write( - temp.path().join("active_policy.toml"), - "[user_rules]\n[corp_rules]\n[network]\n[network.dns]\nupstreams = [\"127.0.0.1:5353\"]\n", - ) - .unwrap(); + let active_policy = + b"[user_rules]\n[corp_rules]\n[network]\n[network.dns]\nupstreams = [\"127.0.0.1:5353\"]\n".to_vec(); service_tx - .send(ServiceToProcess::ReloadConfig { id: 30 }) + .send(ServiceToProcess::ReloadConfig { + id: 30, + active_policy: active_policy.clone(), + }) .await .unwrap(); - let applied = capsem_core::net::policy_config::active_policy_digest( - &std::fs::read(temp.path().join("active_policy.toml")).unwrap(), - ); + let applied = capsem_core::net::policy_config::active_policy_digest(&active_policy); assert!(matches!( service_rx.recv().await.unwrap(), ProcessToService::ConfigReloadResult { id: 30, active_policy_digest: Some(digest), error: None } if digest == applied )); assert_eq!(dispatcher.mcp_runtime.proxy_policy.snapshot().digest(), applied); + std::fs::remove_file(dispatcher.runtime_source.active_policy_path()).unwrap(); service_tx .send(ServiceToProcess::McpListServers { id: 15 }) @@ -819,7 +818,10 @@ fn classify_container_pull_admission() { #[test] fn classify_reload_config() { assert_eq!( - classify_ipc_message(&ServiceToProcess::ReloadConfig { id: 1 }), + classify_ipc_message(&ServiceToProcess::ReloadConfig { + id: 1, + active_policy: Vec::new(), + }), IpcAction::Reload ); } diff --git a/crates/capsem-process/src/ipc/tests/classification.rs b/crates/capsem-process/src/ipc/tests/classification.rs index 841549779..fb149863e 100644 --- a/crates/capsem-process/src/ipc/tests/classification.rs +++ b/crates/capsem-process/src/ipc/tests/classification.rs @@ -36,7 +36,7 @@ pub(super) fn classify_ipc_message(msg: &ServiceToProcess) -> IpcAction { | ServiceToProcess::AbortPorts { .. } | ServiceToProcess::PlugCable { .. } | ServiceToProcess::UnplugCable { .. } => IpcAction::Unexpected, - ServiceToProcess::ReloadConfig { .. } => IpcAction::Reload, + ServiceToProcess::ReloadConfig { .. } | ServiceToProcess::InjectCredentials { .. } => IpcAction::Reload, ServiceToProcess::Shutdown | ServiceToProcess::Suspend => IpcAction::Lifecycle, } } diff --git a/crates/capsem-process/src/ipc/tests/reload.rs b/crates/capsem-process/src/ipc/tests/reload.rs index 44938617a..c880580e8 100644 --- a/crates/capsem-process/src/ipc/tests/reload.rs +++ b/crates/capsem-process/src/ipc/tests/reload.rs @@ -20,8 +20,13 @@ async fn failed_reload_answers_with_its_error_and_keeps_the_previous_policy() { .await .unwrap(); - std::fs::write(temp.path().join("active_policy.toml"), "id = [not toml").unwrap(); - service_tx.send(ServiceToProcess::ReloadConfig { id: 7 }).await.unwrap(); + service_tx + .send(ServiceToProcess::ReloadConfig { + id: 7, + active_policy: b"id = [not toml".to_vec(), + }) + .await + .unwrap(); match service_rx.recv().await.unwrap() { ProcessToService::ConfigReloadResult { id: 7, diff --git a/crates/capsem-process/src/main.rs b/crates/capsem-process/src/main.rs index 9cda5b130..2cd743bbe 100644 --- a/crates/capsem-process/src/main.rs +++ b/crates/capsem-process/src/main.rs @@ -5,11 +5,13 @@ mod ipc; mod job_store; mod mcp_runtime; mod metric_export; +mod owner; mod owner_sandbox; mod private_seats; mod proxy_mcp; mod runtime_config; mod terminal; +mod trace_hints; mod vsock; use anyhow::{Context, Result}; @@ -45,10 +47,12 @@ pub(crate) struct Shutdown { db: Option>, fs_monitor: Option, proxy_mcp: Option>, + proxy_trace_hints: Option>, } struct PreparedOwnerResources { seats: private_seats::Prepared, + workspace: capsem_foundation::unix::contained::ContainedDir, metric_service: Option, ipc_listener: std::os::unix::net::UnixListener, launched: std::fs::File, @@ -58,6 +62,29 @@ struct PreparedOwnerResources { mcp_servers: Vec, builtin_bin: Option, builtin_env: std::collections::HashMap, + ledger: PreparedLedger, +} + +struct PreparedLedger { + stream: std::os::unix::net::UnixStream, + commitment: std::os::unix::net::UnixStream, + grant: capsem_proto::ledger::LedgerChannelGrant, +} + +impl PreparedLedger { + fn from_grant( + (stream, commitment, grant): ( + std::os::unix::net::UnixStream, + std::os::unix::net::UnixStream, + capsem_proto::ledger::LedgerChannelGrant, + ), + ) -> Self { + Self { + stream, + commitment, + grant, + } + } } impl Shutdown { @@ -83,6 +110,10 @@ pub(crate) async fn drain_background_owners(shutdown: &Arc>) { proxy_mcp.abort(); let _ = proxy_mcp.await; } + if let Some(proxy_trace_hints) = owned.proxy_trace_hints.take() { + proxy_trace_hints.abort(); + let _ = proxy_trace_hints.await; + } if let Some(publisher) = owned.publisher.take() { publisher.shutdown().await; } @@ -259,6 +290,9 @@ fn main() -> Result<()> { pid: capsem_foundation::unix::process::parent_process_id().context("missing coordinator parent")?, uid: capsem_foundation::unix::process::current_uid(), }; + // Declared before VM/runtime resources, so normal teardown releases the + // generation lifetime fence last. SIGKILL still needs native exit proof. + let _spawn_owner = owner::enroll(&args)?; // Root span shared across the whole capsem-process run: every // subsequent log line inherits `vm_id` and `trace_id` as structured @@ -287,7 +321,7 @@ fn main() -> Result<()> { info!(id = %args.id, "capsem-sandbox-process starting"); let guest_dir = prepare_session_layout(&session_dir, args.scratch_disk_size_gb)?; - capsem_core::session::prepare_owner_state_dir(&session_dir)?; + capsem_core::container::publish::Publisher::prepare_session(&session_dir)?; // The image share is attached to every session, read-only at the device: // a device cannot be added after boot, and an image is pulled only once // the VM runs (its owner admits the pull). It stays empty until the @@ -405,6 +439,19 @@ fn main() -> Result<()> { let launched_path = args.uds_path.with_extension("launched"); let launched = prepare_sentinel(&launched_path)?; let ready = prepare_sentinel(&args.uds_path.with_extension("ready"))?; + // The grant response proves the supervised ledger worker has opened and + // initialized session.db. Acquire it before confinement so the denial + // attestation below always tests an existing ledger, then carry only the + // connected descriptors across the boundary. + let ledger = PreparedLedger::from_grant( + rt.block_on(upstream_grants.open_ledger()) + .context("acquire supervised session ledger")?, + ); + // Opening through the session root after confinement would require read + // authority over session.db and every other sibling. Resolve the + // guest-writable workspace before confinement, then carry only its + // no-follow directory descriptor into the VM owner. + let workspace = capsem_core::session::open_workspace(&session_dir).context("open workspace before confinement")?; let attestation = prepare_owner_sandbox_attestation(&session_dir)?; confine_owner(&args, &session_dir).context("install VM-owner confinement")?; @@ -434,6 +481,7 @@ fn main() -> Result<()> { let prepared_resources = PreparedOwnerResources { seats: prepared_seats, + workspace, metric_service, ipc_listener, launched, @@ -443,6 +491,7 @@ fn main() -> Result<()> { mcp_servers, builtin_bin, builtin_env, + ledger, }; // Emit boot timeline state transitions for process.log. @@ -560,6 +609,7 @@ async fn run_async_main_loop( ) -> Result<()> { let PreparedOwnerResources { seats: prepared_seats, + workspace, metric_service, ipc_listener, mut launched, @@ -569,13 +619,10 @@ async fn run_async_main_loop( mcp_servers, builtin_bin, builtin_env, + ledger, } = prepared; let terminal_output = Arc::new(capsem_core::TerminalOutputQueue::new()); - let (ledger_stream, commitment_stream, ledger_grant) = upstream_grants - .open_ledger() - .await - .context("acquire supervised session ledger")?; let ledger_path = session_dir.join("session.db"); // The config-owned queue covers the supported concurrent network burst. // A full queue makes every producer sleep in 5 ms steps on its reply path @@ -585,9 +632,9 @@ async fn run_async_main_loop( let db = Arc::new( tokio::task::spawn_blocking(move || { capsem_logger::DbWriter::from_ledger_channel( - ledger_stream, - commitment_stream, - ledger_grant, + ledger.stream, + ledger.commitment, + ledger.grant, &ledger_path, capsem_proto::LEDGER_PRODUCER_QUEUE_CAPACITY, ) @@ -620,10 +667,11 @@ async fn run_async_main_loop( let proxy_policy = capsem_core::net::proxy_engine::ProxyPolicyHandle::new(runtime_config.proxy_policy_snapshot()); let job_store = Arc::new(JobStore { publisher: Arc::new( - capsem_core::container::publish::Publisher::for_session( + capsem_core::container::publish::Publisher::for_prepared_session( &session_dir, runtime_config.network.router.clone(), )? + .with_listener_authority(upstream_grants.clone()) .with_security( args.id.clone(), args.vm_name.clone().unwrap_or_else(|| args.id.clone()), @@ -650,24 +698,15 @@ async fn run_async_main_loop( info!(restored, "restored published ports"); let model_trace_state = Arc::new(std::sync::Mutex::new(capsem_core::net::ai_traffic::TraceState::new())); - match capsem_core::session::open_workspace(&session_dir) - .map_err(anyhow::Error::from) - .and_then(|workspace| { - capsem_core::fs_monitor::FsMonitor::start( - workspace, - Arc::clone(&db), - Arc::clone(&security_rules), - Arc::clone(&model_trace_state), - ) - }) { - Ok(monitor) => { - info!("host file monitor started"); - shutdown.lock().await.fs_monitor = Some(monitor); - } - Err(e) => { - error!(error = %e, "failed to start host file monitor"); - } - } + let monitor = capsem_core::fs_monitor::FsMonitor::start( + workspace, + Arc::clone(&db), + Arc::clone(&security_rules), + Arc::clone(&model_trace_state), + ) + .context("start host file monitor")?; + info!("host file monitor started"); + shutdown.lock().await.fs_monitor = Some(monitor); let net_state = Arc::new(capsem_core::create_net_state_with_policy( &args.id, @@ -702,6 +741,19 @@ async fn run_async_main_loop( plugin_policy: Arc::clone(&plugin_policy), proxy_policy: proxy_policy.clone(), }); + let (proxy_trace_hints, owner_trace_hints) = std::os::unix::net::UnixStream::pair()?; + let owner_trace_state = Arc::clone(&model_trace_state); + let proxy_trace_hints_task = tokio::spawn(async move { + if let Err(error) = trace_hints::serve(owner_trace_hints, owner_trace_state).await { + tracing::warn!(%error, "proxy trace-hint capability stopped"); + } + }); + if let Err(error) = upstream_grants.attach_proxy_trace_hints(proxy_trace_hints.into()).await { + proxy_trace_hints_task.abort(); + let _ = proxy_trace_hints_task.await; + return Err(error.context("grant proxy trace-hint capability")); + } + shutdown.lock().await.proxy_trace_hints = Some(proxy_trace_hints_task); let (proxy_mcp, owner_mcp) = std::os::unix::net::UnixStream::pair()?; let proxy_mcp_task = tokio::spawn(async move { if let Err(error) = proxy_mcp::serve( diff --git a/crates/capsem-process/src/owner.rs b/crates/capsem-process/src/owner.rs index f7e51cd74..21bfc8526 100644 --- a/crates/capsem-process/src/owner.rs +++ b/crates/capsem-process/src/owner.rs @@ -1,5 +1,7 @@ -//! Admit only the original service-issued process generation, then bind the -//! VM owner's lifetime to that service before preparing any VM resources. +//! Admit only the original service-issued process generation and retain its +//! lifetime fence before preparing any VM resources. The process entrypoint +//! arms the parent watcher after confinement so that thread cannot retain +//! ambient authority. use crate::Args; use anyhow::{ensure, Context, Result}; use capsem_core::managed_sessions::VmBinding; @@ -7,16 +9,12 @@ use capsem_foundation::unix::contained::ContainedDir; use std::path::Path; pub(crate) fn enroll(args: &Args) -> Result { - let owner = admit( + admit( &args.id, args.spawn_generation.as_deref(), args.parent_pid, &args.session_dir, - )?; - // Recheck the actual parent while arming: it may have disappeared during - // metadata validation. The existing guard terminates even on SIGKILL/OOM. - capsem_guard::watch_parent_or_exit(args.parent_pid).context("VM service parent unavailable")?; - Ok(owner) + ) } fn admit( diff --git a/crates/capsem-process/src/owner_sandbox.rs b/crates/capsem-process/src/owner_sandbox.rs index d381994d0..61d5cd7b8 100644 --- a/crates/capsem-process/src/owner_sandbox.rs +++ b/crates/capsem-process/src/owner_sandbox.rs @@ -42,6 +42,19 @@ fn grant_owner_session_paths( Ok(policy) } +fn grant_owner_null( + policy: capsem_foundation::unix::worker_sandbox::Policy, +) -> Result { + use capsem_foundation::unix::worker_sandbox::Access; + + let null = Path::new("/dev/null"); + if null.exists() { + grant_owner_path(policy, null, Access::ReadWrite) + } else { + Ok(policy) + } +} + #[cfg(target_os = "linux")] pub(super) fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; @@ -50,16 +63,16 @@ pub(super) fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { policy = grant_owner_session_paths(policy, session_dir)?; policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; - policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; for path in args.kernel.iter().chain(args.initrd.iter()) { policy = grant_owner_path(policy, path, Access::ReadOnly)?; } for path in ["/dev/kvm", "/dev/vhost-vsock"] { let path = Path::new(path); if path.exists() { - policy = grant_owner_path(policy, path, Access::ReadWrite)?; + policy = grant_owner_path(policy, path, Access::ReadWriteDevice)?; } } + policy = grant_owner_null(policy)?; let executable = std::env::current_exe().context("locate VM-owner executable")?; let router = executable.with_file_name("capsem-router"); if router.exists() { @@ -93,7 +106,6 @@ pub(super) fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { policy = grant_owner_session_paths(policy, session_dir)?; policy = grant_owner_path(policy, &args.assets_dir, Access::ReadOnly)?; policy = grant_owner_path(policy, &args.rootfs, Access::ReadOnly)?; - policy = grant_owner_path(policy, &args.active_policy, Access::ReadOnly)?; for path in args.kernel.iter().chain(args.initrd.iter()) { policy = grant_owner_path(policy, path, Access::ReadOnly)?; } @@ -116,10 +128,7 @@ pub(super) fn confine_owner(args: &Args, session_dir: &Path) -> Result<()> { policy = grant_owner_path(policy, path, Access::ReadOnly)?; } } - let null = Path::new("/dev/null"); - if null.exists() { - policy = grant_owner_path(policy, null, Access::ReadWrite)?; - } + policy = grant_owner_null(policy)?; capsem_foundation::unix::worker_sandbox::confine(&policy)?; Ok(()) } @@ -222,3 +231,6 @@ pub(super) async fn attest_owner( std::fs::remove_file(&attestation.direct_path)?; Ok(()) } + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-process/src/owner_sandbox/tests.rs b/crates/capsem-process/src/owner_sandbox/tests.rs new file mode 100644 index 000000000..e23c88efb --- /dev/null +++ b/crates/capsem-process/src/owner_sandbox/tests.rs @@ -0,0 +1,80 @@ +use std::fs::File; +use std::path::{Path, PathBuf}; + +use capsem_foundation::unix::worker_sandbox::{Access, Policy, Role}; + +use super::*; + +fn touch(path: impl AsRef) -> PathBuf { + let path = path.as_ref(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + File::create(path).unwrap(); + path.to_path_buf() +} + +fn session_layout(root: &Path) { + std::fs::create_dir_all(capsem_core::guest_share_dir(root).join("workspace")).unwrap(); + std::fs::create_dir_all(root.join(capsem_core::session::OWNER_STATE_DIR)).unwrap(); + std::fs::create_dir_all(capsem_core::session::image_share_path(root)).unwrap(); + touch(capsem_core::session::system_overlay_image_path(root)); + touch(root.join("serial.log")); + touch(root.join("pty.log")); + touch(aggregator_log_path(root)); +} + +#[test] +fn owner_session_policy_accepts_only_materialized_canonical_paths() { + let directory = tempfile::tempdir().unwrap(); + session_layout(directory.path()); + + grant_owner_session_paths(Policy::new(Role::VmOwner), directory.path()).unwrap(); + + let missing = directory.path().join("missing"); + let error = grant_owner_path(Policy::new(Role::VmOwner), &missing, Access::ReadOnly).unwrap_err(); + assert!(format!("{error:#}").contains("resolve sandbox grant")); +} + +#[test] +fn owner_policy_grants_only_the_null_device_needed_for_router_stdio() { + let policy = grant_owner_null(Policy::new(Role::VmOwner)).unwrap(); + let null = Path::new("/dev/null").canonicalize().unwrap(); + assert!(policy + .paths() + .iter() + .any(|rule| rule.path() == null && rule.access() == Access::ReadWrite)); +} + +#[test] +fn owner_attestation_prepares_private_direct_and_brokered_targets() { + use std::os::unix::ffi::OsStrExt as _; + use std::os::unix::fs::PermissionsExt as _; + + let directory = tempfile::tempdir().unwrap(); + session_layout(directory.path()); + + let attestation = prepare_owner_sandbox_attestation(directory.path()).unwrap(); + + assert!(attestation + .direct_path + .starts_with(directory.path().join(capsem_core::session::OWNER_STATE_DIR))); + assert_eq!( + std::fs::metadata(&attestation.direct_path) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + attestation.guest_path, + capsem_core::guest_share_dir(directory.path()).join(std::ffi::OsStr::from_bytes(&attestation.guest_relative)) + ); + assert!(attestation.guest_path.exists()); + assert_eq!(attestation.ledger_path, directory.path().join("session.db")); + + drop(attestation.direct_file); + std::fs::remove_file(attestation.direct_path).unwrap(); + std::fs::remove_file(attestation.guest_path).unwrap(); +} diff --git a/crates/capsem-process/src/runtime_config.rs b/crates/capsem-process/src/runtime_config.rs index b0e8138ab..09f2a1729 100644 --- a/crates/capsem-process/src/runtime_config.rs +++ b/crates/capsem-process/src/runtime_config.rs @@ -5,11 +5,14 @@ use capsem_proto::mcp_contracts::McpServerDef; use std::collections::{BTreeMap, HashMap}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; -/// The session's active policy file, read at boot and on every reload. +/// The session's active policy file, read at boot. Reloads carry the service's +/// exact newly published bytes over authenticated IPC. #[derive(Debug, Clone)] pub(crate) struct RuntimePolicySource { active_policy_path: PathBuf, + current: Arc>>, } #[derive(Debug, Clone)] @@ -29,6 +32,7 @@ impl RuntimePolicySource { pub(crate) fn new(active_policy_path: impl Into) -> Self { Self { active_policy_path: active_policy_path.into(), + current: Arc::new(RwLock::new(None)), } } @@ -37,12 +41,31 @@ impl RuntimePolicySource { } pub(crate) fn load(&self) -> Result { - let content = std::fs::read_to_string(&self.active_policy_path) + let content = std::fs::read(&self.active_policy_path) .with_context(|| format!("read {}", self.active_policy_path.display()))?; + self.load_bytes(&content) + } + + pub(crate) fn load_bytes(&self, content: &[u8]) -> Result { + let content_text = + std::str::from_utf8(content).with_context(|| format!("decode {}", self.active_policy_path.display()))?; let active: ActivePolicyFile = - toml::from_str(&content).with_context(|| format!("parse {}", self.active_policy_path.display()))?; - let digest = capsem_core::net::policy_config::active_policy_digest(content.as_bytes()); - RuntimePolicyConfig::from_active(active, self.active_policy_path.clone(), digest) + toml::from_str(content_text).with_context(|| format!("parse {}", self.active_policy_path.display()))?; + let digest = capsem_core::net::policy_config::active_policy_digest(content); + let config = RuntimePolicyConfig::from_active(active, self.active_policy_path.clone(), digest)?; + *self.current.write().unwrap() = Some(config.clone()); + Ok(config) + } + + /// Return the last authenticated policy snapshot loaded at boot or over + /// service IPC. The confined owner cannot reopen the coordinator-owned + /// policy path after startup. + pub(crate) fn current(&self) -> Result { + self.current + .read() + .unwrap() + .clone() + .context("runtime policy has not been loaded") } } diff --git a/crates/capsem-process/src/tests.rs b/crates/capsem-process/src/tests.rs index 17dd70ec3..83e9013de 100644 --- a/crates/capsem-process/src/tests.rs +++ b/crates/capsem-process/src/tests.rs @@ -1,5 +1,7 @@ use super::*; +use capsem_proto::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration}; use clap::Parser; +use std::os::fd::AsRawFd as _; #[test] fn entry_closes_ambient_descriptors_before_runtime_initialization() { @@ -24,11 +26,27 @@ fn production_vm_owner_never_opens_the_session_database() { ); } +#[test] +fn prepared_ledger_retains_the_connected_channels_and_exact_grant() { + let (stream, _stream_peer) = std::os::unix::net::UnixStream::pair().unwrap(); + let (commitment, _commitment_peer) = std::os::unix::net::UnixStream::pair().unwrap(); + let stream_fd = stream.as_raw_fd(); + let commitment_fd = commitment.as_raw_fd(); + let grant = LedgerChannelGrant::new(LedgerGeneration::new([7; 16]), 41, LedgerClientRole::VmOwner).unwrap(); + + let prepared = PreparedLedger::from_grant((stream, commitment, grant)); + + assert_eq!(prepared.stream.as_raw_fd(), stream_fd); + assert_eq!(prepared.commitment.as_raw_fd(), commitment_fd); + assert_eq!(prepared.grant, grant); +} + #[test] fn confinement_grants_exact_runtime_paths_and_attests_ledger_denial() { let sandbox = include_str!("owner_sandbox.rs"); assert!(!sandbox.contains("grant_owner_path(policy, session_dir")); assert!(sandbox.contains("grant_owner_session_paths(policy, session_dir)")); + assert!(!sandbox.contains("&args.active_policy")); assert!(sandbox.contains("session_dir.join(\"session.db\")")); assert!(sandbox.contains("std::fs::File::open(&attestation.ledger_path)")); assert!(sandbox.contains("open session ledger storage")); @@ -44,8 +62,17 @@ fn suspend_checkpoint_stays_in_owner_state() { fn platform_confinement_and_attestation_precede_owner_readiness() { let source = include_str!("main.rs"); let entry = source.split_once("fn main() -> Result<()> {").unwrap().1; + let ledger = entry + .find("open_ledger()") + .expect("VM owner acquires its initialized ledger before confinement"); + let workspace = entry + .find("let workspace = capsem_core::session::open_workspace") + .expect("VM owner preopens the contained workspace before confinement"); let prepare = entry.find("prepare_owner_sandbox_attestation").unwrap(); let runtime = entry.find("Builder::new_current_thread()").unwrap(); + let enrollment = entry + .find("owner::enroll(&args)") + .expect("VM owner retains its durable generation fence before runtime resources"); let confine = entry .find("confine_owner(&args") .expect("all supported platforms install the VM-owner policy"); @@ -56,8 +83,30 @@ fn platform_confinement_and_attestation_precede_owner_readiness() { let boot = entry.find("prepared_vm.boot()").unwrap(); let run = entry.find("run_async_main_loop(").unwrap(); assert!(!entry.contains("Builder::new_multi_thread()")); - assert!(runtime < prepare && prepare < confine && confine < attest && attest < watcher); + assert!( + enrollment < runtime + && runtime < ledger + && ledger < workspace + && workspace < prepare + && prepare < confine + && confine < attest + && attest < watcher + ); assert!(confine < boot && boot < run); + assert!( + !include_str!("owner.rs").contains("watch_parent_or_exit"), + "generation enrollment must not create a pre-confinement watcher thread" + ); + + let async_loop = source.split_once("async fn run_async_main_loop(").unwrap().1; + assert!( + !async_loop.contains("open_workspace(&session_dir)"), + "the confined owner must use its preopened workspace descriptor" + ); + assert!( + async_loop.contains(".context(\"start host file monitor\")?"), + "a missing filesystem audit rail must fail VM startup" + ); } #[test] diff --git a/crates/capsem-process/src/trace_hints.rs b/crates/capsem-process/src/trace_hints.rs new file mode 100644 index 000000000..f7f38b49b --- /dev/null +++ b/crates/capsem-process/src/trace_hints.rs @@ -0,0 +1,39 @@ +//! Receiver for the proxy's one-way model/file correlation capability. + +use std::io; +use std::os::fd::AsFd as _; +use std::os::unix::net::UnixStream; +use std::sync::{Arc, Mutex}; + +use capsem_core::net::ai_traffic::TraceState; +use capsem_proto::proxy_trace_hints::{decode_proxy_trace_hint, PROXY_TRACE_HINT_ACK, PROXY_TRACE_HINT_FRAME_SIZE}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + +pub(crate) async fn serve(socket: UnixStream, trace_state: Arc>) -> io::Result<()> { + capsem_foundation::unix::fd::set_nonblocking(socket.as_fd(), true)?; + let mut socket = tokio::net::UnixStream::from_std(socket)?; + loop { + let mut frame = [0; PROXY_TRACE_HINT_FRAME_SIZE]; + match socket.read_exact(&mut frame).await { + Ok(_) => {} + Err(error) if error.kind() == io::ErrorKind::UnexpectedEof => return Ok(()), + Err(error) => return Err(error), + } + let hint = + decode_proxy_trace_hint(&frame).map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + if !trace_state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .register_file_hint(&hint.trace_id, &hint.relative_path) + { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "proxy trace hint names an invalid workspace path", + )); + } + socket.write_all(&PROXY_TRACE_HINT_ACK).await?; + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-process/src/trace_hints/tests.rs b/crates/capsem-process/src/trace_hints/tests.rs new file mode 100644 index 000000000..9a22a847c --- /dev/null +++ b/crates/capsem-process/src/trace_hints/tests.rs @@ -0,0 +1,51 @@ +use std::os::fd::AsFd as _; +use std::os::unix::net::UnixStream; +use std::sync::{Arc, Mutex}; + +use capsem_core::net::ai_traffic::TraceState; +use capsem_proto::proxy_trace_hints::{encode_proxy_trace_hint, ProxyTraceHint, PROXY_TRACE_HINT_ACK}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + +#[tokio::test] +async fn proxy_hint_updates_the_workspace_owners_trace_state() { + let (proxy, owner) = UnixStream::pair().unwrap(); + let state = Arc::new(Mutex::new(TraceState::new())); + let serving = tokio::spawn(super::serve(owner, Arc::clone(&state))); + capsem_foundation::unix::fd::set_nonblocking(proxy.as_fd(), true).unwrap(); + let mut proxy = tokio::net::UnixStream::from_std(proxy).unwrap(); + let frame = encode_proxy_trace_hint(&ProxyTraceHint { + trace_id: "2e934d32-1e3e-4d51-b816-e14cc662833f".into(), + relative_path: "reports/result.txt".into(), + }) + .unwrap(); + proxy.write_all(&frame).await.unwrap(); + let mut acknowledgement = [0; PROXY_TRACE_HINT_ACK.len()]; + proxy.read_exact(&mut acknowledgement).await.unwrap(); + assert_eq!(acknowledgement, PROXY_TRACE_HINT_ACK); + drop(proxy); + serving.await.unwrap().unwrap(); + + assert_eq!( + state.lock().unwrap().lookup_file_path("reports/result.txt").as_deref(), + Some("2e934d32-1e3e-4d51-b816-e14cc662833f") + ); +} + +#[tokio::test] +async fn escaping_proxy_hint_closes_the_capability() { + let (proxy, owner) = UnixStream::pair().unwrap(); + let state = Arc::new(Mutex::new(TraceState::new())); + let serving = tokio::spawn(super::serve(owner, state)); + capsem_foundation::unix::fd::set_nonblocking(proxy.as_fd(), true).unwrap(); + let mut proxy = tokio::net::UnixStream::from_std(proxy).unwrap(); + let frame = encode_proxy_trace_hint(&ProxyTraceHint { + trace_id: "0707070707070707".into(), + relative_path: "../escape.txt".into(), + }) + .unwrap(); + proxy.write_all(&frame).await.unwrap(); + assert_eq!( + serving.await.unwrap().unwrap_err().kind(), + std::io::ErrorKind::InvalidData + ); +} diff --git a/crates/capsem-proto/build.rs b/crates/capsem-proto/build.rs index a51ef1868..48b58509b 100644 --- a/crates/capsem-proto/build.rs +++ b/crates/capsem-proto/build.rs @@ -22,7 +22,14 @@ impl VisitMut for StripDocs { } fn main() { - let files = ["lib.rs", "ipc.rs", "handshake.rs", "router.rs", "exec_stream.rs"]; + let files = [ + "lib.rs", + "ipc.rs", + "handshake.rs", + "router.rs", + "exec_stream.rs", + "credential_injection.rs", + ]; write_hash("schema_hash.txt", &files); write_hash( "ledger_schema_hash.txt", diff --git a/crates/capsem-proto/src/ipc.rs b/crates/capsem-proto/src/ipc.rs index 9fd0c2f59..c2682b30b 100644 --- a/crates/capsem-proto/src/ipc.rs +++ b/crates/capsem-proto/src/ipc.rs @@ -91,10 +91,12 @@ pub enum ServiceToProcess { #[serde(default, skip_serializing_if = "Option::is_none")] mime_type: Option, }, - /// Request the process to reload its active policy from disk. Answered by - /// `ConfigReloadResult` naming the digest of the bytes it applied. + /// Deliver the exact active-policy bytes the service published. Answered + /// by `ConfigReloadResult` naming the digest of the bytes it applied. ReloadConfig { id: u64, + #[serde(with = "serde_bytes")] + active_policy: Vec, }, /// Seed host broker memory. This material must never cross the guest bridge. InjectCredentials { @@ -492,7 +494,7 @@ impl ServiceToProcess { | Self::LinkAttach { id, .. } | Self::LinkDetach { id, .. } | Self::AdmitContainerPull { id, .. } - | Self::ReloadConfig { id } + | Self::ReloadConfig { id, .. } | Self::InjectCredentials { id, .. } => Some(*id), _ => None, } diff --git a/crates/capsem-proto/src/ipc/tests.rs b/crates/capsem-proto/src/ipc/tests.rs index 25485c701..83d42b8a2 100644 --- a/crates/capsem-proto/src/ipc/tests.rs +++ b/crates/capsem-proto/src/ipc/tests.rs @@ -416,10 +416,16 @@ fn job_ids_are_distinct() { #[test] fn reload_config_roundtrip() { - let msg = ServiceToProcess::ReloadConfig { id: 3 }; + let msg = ServiceToProcess::ReloadConfig { + id: 3, + active_policy: b"[network]".to_vec(), + }; let bytes = serde_json::to_vec(&msg).unwrap(); let msg2: ServiceToProcess = serde_json::from_slice(&bytes).unwrap(); - assert!(matches!(msg2, ServiceToProcess::ReloadConfig { id: 3 })); + assert!(matches!( + msg2, + ServiceToProcess::ReloadConfig { id: 3, active_policy } if active_policy == b"[network]" + )); } // ----------------------------------------------------------------------- diff --git a/crates/capsem-proto/src/ipc/tests/correlation.rs b/crates/capsem-proto/src/ipc/tests/correlation.rs index d84489e64..18e46fd9c 100644 --- a/crates/capsem-proto/src/ipc/tests/correlation.rs +++ b/crates/capsem-proto/src/ipc/tests/correlation.rs @@ -41,7 +41,13 @@ fn replies_correlate_to_requests_by_id_and_broadcasts_answer_nothing() { Some(12), ), (ServiceToProcess::Ping, None), - (ServiceToProcess::ReloadConfig { id: 13 }, Some(13)), + ( + ServiceToProcess::ReloadConfig { + id: 13, + active_policy: Vec::new(), + }, + Some(13), + ), ( ServiceToProcess::CloneStateComplete { id: 14, diff --git a/crates/capsem-proto/src/ipc/tests/wire.rs b/crates/capsem-proto/src/ipc/tests/wire.rs index 01a6218a8..6001bf12f 100644 --- a/crates/capsem-proto/src/ipc/tests/wire.rs +++ b/crates/capsem-proto/src/ipc/tests/wire.rs @@ -54,7 +54,10 @@ fn service_to_process_variant_names_and_roundtrips_are_stable() { size: 1, mime_type: None, }, - ServiceToProcess::ReloadConfig { id: 11 }, + ServiceToProcess::ReloadConfig { + id: 11, + active_policy: b"[network]".to_vec(), + }, ServiceToProcess::StartTerminalStream, ServiceToProcess::StopTerminalStream, ServiceToProcess::Suspend, diff --git a/crates/capsem-proto/src/ledger.rs b/crates/capsem-proto/src/ledger.rs index 23759d230..cc9390818 100644 --- a/crates/capsem-proto/src/ledger.rs +++ b/crates/capsem-proto/src/ledger.rs @@ -11,8 +11,8 @@ use thiserror::Error; /// Version of the ledger handshake, request envelopes and logger-owned /// operation/reply enums. Bump it when any of those wire shapes changes. -/// Version 3 adds maintainer-only coherent snapshot authority. -pub const LEDGER_PROTOCOL_VERSION: u16 = 3; +/// Version 4 carries the DB-owned read-cache epoch with counter snapshots. +pub const LEDGER_PROTOCOL_VERSION: u16 = 4; /// Hash of ledger wire declarations, separate from the guest protocol hash. pub const LEDGER_SCHEMA_HASH: u64 = include!(concat!(env!("OUT_DIR"), "/ledger_schema_hash.txt")); /// Largest diagnostic admitted into a response. diff --git a/crates/capsem-proto/src/ledger_commitment.rs b/crates/capsem-proto/src/ledger_commitment.rs index 8af74157b..aa0a4ab6a 100644 --- a/crates/capsem-proto/src/ledger_commitment.rs +++ b/crates/capsem-proto/src/ledger_commitment.rs @@ -141,7 +141,7 @@ impl LedgerCommitment { } fn calculate_hash(&self) -> [u8; 32] { - let mut hash = blake3::Hasher::new(); + let mut hash = capsem_crypto::blake3::Hasher::new(); hash.update(b"capsem-ledger-commitment-v1\0"); hash.update(&self.generation.as_bytes()); hash.update(&self.client_id.to_be_bytes()); diff --git a/crates/capsem-proto/src/lib.rs b/crates/capsem-proto/src/lib.rs index 892570b19..e20683e73 100644 --- a/crates/capsem-proto/src/lib.rs +++ b/crates/capsem-proto/src/lib.rs @@ -1,8 +1,7 @@ //! Capsem protocol types for host/guest communication over vsock. //! //! Defines disjoint `HostToGuest` and `GuestToHost` message enums with -//! MessagePack framing. No platform-specific dependencies, so this crate -//! cross-compiles for both macOS host and aarch64-linux-musl guest. +//! MessagePack framing with no platform dependencies for macOS and aarch64-linux-musl. //! //! # Security invariant (RFC T14) //! @@ -33,6 +32,7 @@ pub mod proxy_mcp; pub mod proxy_metrics; pub mod proxy_policy; pub mod proxy_private_names; +pub mod proxy_trace_hints; pub mod repeated; pub mod router; mod sparse; @@ -89,7 +89,8 @@ pub const MAX_BOOT_FILES: usize = 64; /// Version 12 makes cloning a coordinator operation bracketed by the owner's /// guest freeze, so no destination path crosses into the VM owner. /// Version 13 fixes suspend to the owner's confined checkpoint location. -pub const PROTOCOL_VERSION: u16 = 13; +/// Version 14 adds host-only credential injection; version 15 carries active-policy bytes. +pub const PROTOCOL_VERSION: u16 = 15; /// Guest loopback port of the agent's DNS proxy (port 53 is redirected here). pub const GUEST_DNS_PROXY_PORT: u16 = 1053; diff --git a/crates/capsem-proto/src/proxy_control.rs b/crates/capsem-proto/src/proxy_control.rs index 66d45259c..31922589a 100644 --- a/crates/capsem-proto/src/proxy_control.rs +++ b/crates/capsem-proto/src/proxy_control.rs @@ -6,7 +6,7 @@ use crate::ledger::{LedgerChannelGrant, LedgerClientRole, LedgerGeneration, Ledg pub const PROXY_CONTROL_FRAME_SIZE: usize = 56; pub const PROXY_CONTROL_MAX_FDS: usize = 2; -pub const PROXY_CONTROL_VERSION: u16 = 5; +pub const PROXY_CONTROL_VERSION: u16 = 6; const MAGIC: [u8; 2] = *b"PX"; const VERSION_RANGE: std::ops::Range = 2..4; @@ -64,6 +64,7 @@ pub enum ProxyCapability { Mcp = 7, Telemetry = 8, Policy = 9, + TraceHints = 10, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -380,6 +381,7 @@ fn decode_capability(value: u8) -> Result { 7 => Ok(ProxyCapability::Mcp), 8 => Ok(ProxyCapability::Telemetry), 9 => Ok(ProxyCapability::Policy), + 10 => Ok(ProxyCapability::TraceHints), value => Err(ProxyControlError::InvalidCapability(value)), } } diff --git a/crates/capsem-proto/src/proxy_control/tests.rs b/crates/capsem-proto/src/proxy_control/tests.rs index ad4933a00..2bc4388dc 100644 --- a/crates/capsem-proto/src/proxy_control/tests.rs +++ b/crates/capsem-proto/src/proxy_control/tests.rs @@ -14,6 +14,7 @@ fn grants_round_trip_without_session_path_or_destination_authority() { ProxyCapability::Mcp, ProxyCapability::Telemetry, ProxyCapability::Policy, + ProxyCapability::TraceHints, ] { let grant = ProxyChannelGrant::new(GENERATION, 41, capability).unwrap(); let request = ProxyControlRequest::Attach(grant); diff --git a/crates/capsem-proto/src/proxy_trace_hints.rs b/crates/capsem-proto/src/proxy_trace_hints.rs new file mode 100644 index 000000000..6ac24fcd0 --- /dev/null +++ b/crates/capsem-proto/src/proxy_trace_hints.rs @@ -0,0 +1,122 @@ +//! Fixed records carrying model-to-filesystem correlation hints. +//! +//! The confined proxy may name only a trace and a workspace-relative path. +//! The VM owner validates and stores that hint; the channel grants no read, +//! filesystem, network, or ledger authority back to the proxy. + +use thiserror::Error; + +pub const MAX_TRACE_HINT_ID_BYTES: usize = 64; +pub const MAX_TRACE_HINT_PATH_BYTES: usize = 4096; +pub const PROXY_TRACE_HINT_FRAME_SIZE: usize = 2 + 1 + 1 + 2 + MAX_TRACE_HINT_ID_BYTES + MAX_TRACE_HINT_PATH_BYTES; +/// Owner acknowledgement. The proxy awaits this before completing the model +/// response, so a following workspace write cannot outrun attribution state. +pub const PROXY_TRACE_HINT_ACK: [u8; 4] = *b"TH\x01A"; + +const MAGIC: [u8; 2] = *b"TH"; +const VERSION: u8 = 1; +const TRACE_LENGTH_OFFSET: usize = 3; +const PATH_LENGTH_RANGE: std::ops::Range = 4..6; +const TRACE_RANGE: std::ops::Range = 6..6 + MAX_TRACE_HINT_ID_BYTES; +const PATH_RANGE: std::ops::Range = TRACE_RANGE.end..TRACE_RANGE.end + MAX_TRACE_HINT_PATH_BYTES; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ProxyTraceHint { + pub trace_id: String, + pub relative_path: String, +} + +pub fn encode_proxy_trace_hint( + hint: &ProxyTraceHint, +) -> Result<[u8; PROXY_TRACE_HINT_FRAME_SIZE], ProxyTraceHintError> { + validate_trace_id(&hint.trace_id)?; + let path = hint.relative_path.as_bytes(); + if path.is_empty() || path.len() > MAX_TRACE_HINT_PATH_BYTES { + return Err(ProxyTraceHintError::InvalidPath); + } + let mut frame = [0; PROXY_TRACE_HINT_FRAME_SIZE]; + frame[..2].copy_from_slice(&MAGIC); + frame[2] = VERSION; + frame[TRACE_LENGTH_OFFSET] = u8::try_from(hint.trace_id.len()).expect("trace length is bounded below u8::MAX"); + frame[PATH_LENGTH_RANGE].copy_from_slice(&(path.len() as u16).to_be_bytes()); + frame[TRACE_RANGE.start..TRACE_RANGE.start + hint.trace_id.len()].copy_from_slice(hint.trace_id.as_bytes()); + frame[PATH_RANGE.start..PATH_RANGE.start + path.len()].copy_from_slice(path); + Ok(frame) +} + +pub fn decode_proxy_trace_hint( + frame: &[u8; PROXY_TRACE_HINT_FRAME_SIZE], +) -> Result { + if frame[..2] != MAGIC { + return Err(ProxyTraceHintError::Magic); + } + if frame[2] != VERSION { + return Err(ProxyTraceHintError::Version(frame[2])); + } + let trace_len = usize::from(frame[TRACE_LENGTH_OFFSET]); + let path_len = usize::from(u16::from_be_bytes( + frame[PATH_LENGTH_RANGE].try_into().expect("fixed path length range"), + )); + if trace_len == 0 || trace_len > MAX_TRACE_HINT_ID_BYTES { + return Err(ProxyTraceHintError::InvalidTraceId); + } + if path_len == 0 || path_len > MAX_TRACE_HINT_PATH_BYTES { + return Err(ProxyTraceHintError::InvalidPath); + } + if frame[TRACE_RANGE.start + trace_len..TRACE_RANGE.end] + .iter() + .chain(frame[PATH_RANGE.start + path_len..PATH_RANGE.end].iter()) + .any(|byte| *byte != 0) + { + return Err(ProxyTraceHintError::ReservedBytes); + } + let trace_id = std::str::from_utf8(&frame[TRACE_RANGE.start..TRACE_RANGE.start + trace_len]) + .map_err(|_| ProxyTraceHintError::InvalidTraceId)? + .to_string(); + validate_trace_id(&trace_id)?; + let relative_path = std::str::from_utf8(&frame[PATH_RANGE.start..PATH_RANGE.start + path_len]) + .map_err(|_| ProxyTraceHintError::InvalidPath)? + .to_string(); + Ok(ProxyTraceHint { + trace_id, + relative_path, + }) +} + +fn validate_trace_id(trace_id: &str) -> Result<(), ProxyTraceHintError> { + let lowercase_hex = |byte: u8| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte); + let bytes = trace_id.as_bytes(); + let canonical = match bytes.len() { + 16 => bytes.iter().copied().all(lowercase_hex), + 36 => bytes.iter().copied().enumerate().all(|(index, byte)| { + if matches!(index, 8 | 13 | 18 | 23) { + byte == b'-' + } else { + lowercase_hex(byte) + } + }), + _ => false, + } && bytes.iter().any(|byte| *byte != b'0' && *byte != b'-'); + if canonical { + Ok(()) + } else { + Err(ProxyTraceHintError::InvalidTraceId) + } +} + +#[derive(Clone, Copy, Debug, Eq, Error, PartialEq)] +pub enum ProxyTraceHintError { + #[error("invalid proxy trace-hint magic")] + Magic, + #[error("unsupported proxy trace-hint version {0}")] + Version(u8), + #[error("trace id is not canonical or exceeds its bound")] + InvalidTraceId, + #[error("trace-hint path is empty, invalid UTF-8, or exceeds its bound")] + InvalidPath, + #[error("proxy trace-hint reserved bytes are nonzero")] + ReservedBytes, +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proto/src/proxy_trace_hints/tests.rs b/crates/capsem-proto/src/proxy_trace_hints/tests.rs new file mode 100644 index 000000000..5e48855e9 --- /dev/null +++ b/crates/capsem-proto/src/proxy_trace_hints/tests.rs @@ -0,0 +1,44 @@ +use super::*; + +#[test] +fn fixed_trace_hint_round_trips_with_zeroed_slack() { + let hint = ProxyTraceHint { + trace_id: "2e934d32-1e3e-4d51-b816-e14cc662833f".into(), + relative_path: "reports/result.txt".into(), + }; + let frame = encode_proxy_trace_hint(&hint).unwrap(); + assert_eq!(frame.len(), PROXY_TRACE_HINT_FRAME_SIZE); + assert_eq!(decode_proxy_trace_hint(&frame).unwrap(), hint); +} + +#[test] +fn trace_hint_rejects_oversize_and_reserved_data() { + for trace_id in [ + "not-a-trace", + "0000000000000000", + "2e934d321e3e-4d51-b816-e14cc662833f", + "2e934d32-1e3e-4d51-b816-e14cc662833G", + ] { + assert_eq!( + encode_proxy_trace_hint(&ProxyTraceHint { + trace_id: trace_id.into(), + relative_path: "result.txt".into(), + }), + Err(ProxyTraceHintError::InvalidTraceId) + ); + } + assert_eq!( + encode_proxy_trace_hint(&ProxyTraceHint { + trace_id: "0707070707070707".into(), + relative_path: "x".repeat(MAX_TRACE_HINT_PATH_BYTES + 1), + }), + Err(ProxyTraceHintError::InvalidPath) + ); + let mut frame = encode_proxy_trace_hint(&ProxyTraceHint { + trace_id: "0707070707070707".into(), + relative_path: "result.txt".into(), + }) + .unwrap(); + frame[PROXY_TRACE_HINT_FRAME_SIZE - 1] = 1; + assert_eq!(decode_proxy_trace_hint(&frame), Err(ProxyTraceHintError::ReservedBytes)); +} diff --git a/crates/capsem-proto/src/upstream_grant.rs b/crates/capsem-proto/src/upstream_grant.rs index e8bf3d0f7..031844da5 100644 --- a/crates/capsem-proto/src/upstream_grant.rs +++ b/crates/capsem-proto/src/upstream_grant.rs @@ -153,6 +153,7 @@ pub enum UpstreamGrantDenial { ConnectFailed, Revoked, InvalidResource, + BindFailed, } impl UpstreamGrantDenial { @@ -165,6 +166,7 @@ impl UpstreamGrantDenial { Self::ConnectFailed => 5, Self::Revoked => 6, Self::InvalidResource => 7, + Self::BindFailed => 8, } } @@ -177,6 +179,7 @@ impl UpstreamGrantDenial { 5 => Ok(Self::ConnectFailed), 6 => Ok(Self::Revoked), 7 => Ok(Self::InvalidResource), + 8 => Ok(Self::BindFailed), _ => bail!("invalid upstream denial reason {code}"), } } @@ -227,6 +230,13 @@ pub enum UpstreamGrantRequest { AttachProxyMcp { request_id: u64, }, + AttachProxyTraceHints { + request_id: u64, + }, + OpenLoopbackListener { + request_id: u64, + port: u16, + }, } /// Coordinator-to-worker record. Ledger grants carry two independent fds. @@ -265,13 +275,21 @@ pub enum UpstreamGrantResponse { ProxyMcpAdopted { request_id: u64, }, + ProxyTraceHintsAdopted { + request_id: u64, + }, + LoopbackListenerGranted { + request_id: u64, + grant_id: u64, + port: u16, + }, } impl UpstreamGrantRequest { /// Requests never carry descriptors from the untrusted worker. pub const fn expected_descriptor_count(&self) -> usize { match self { - Self::AttachProxyTraffic { .. } | Self::AttachProxyMcp { .. } => 1, + Self::AttachProxyTraffic { .. } | Self::AttachProxyMcp { .. } | Self::AttachProxyTraceHints { .. } => 1, _ => 0, } } @@ -282,14 +300,15 @@ impl UpstreamGrantResponse { /// the direct worker channel and the trusted checkpoint channel. pub const fn expected_descriptor_count(&self) -> usize { match self { - Self::DescriptorGranted { .. } => 1, + Self::DescriptorGranted { .. } | Self::LoopbackListenerGranted { .. } => 1, Self::LedgerGranted { .. } => 2, Self::TcpResolved { .. } | Self::Denied { .. } | Self::GuestModeSet { .. } | Self::GuestTimesSet { .. } | Self::ProxyTrafficAdopted { .. } - | Self::ProxyMcpAdopted { .. } => 0, + | Self::ProxyMcpAdopted { .. } + | Self::ProxyTraceHintsAdopted { .. } => 0, } } } @@ -391,6 +410,17 @@ pub fn encode_upstream_grant_request(request: &UpstreamGrantRequest) -> Result<[ frame[KIND_OFFSET] = ATTACH_PROXY_MCP; put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); } + UpstreamGrantRequest::AttachProxyTraceHints { request_id } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = ATTACH_PROXY_TRACE_HINTS; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + } + UpstreamGrantRequest::OpenLoopbackListener { request_id, port } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = OPEN_LOOPBACK_LISTENER; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u16(&mut frame, PORT_RANGE, *port); + } } Ok(frame) } @@ -521,6 +551,23 @@ pub fn decode_upstream_grant_request(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) -> require_empty_path(&relative_path)?; Ok(UpstreamGrantRequest::AttachProxyMcp { request_id }) } + ATTACH_PROXY_TRACE_HINTS => { + require_nonzero("request id", request_id)?; + if resource_id != 0 { + bail!("proxy trace-hint request carries a resource"); + } + require_empty_fields(port, detail, &name)?; + require_empty_path(&relative_path)?; + Ok(UpstreamGrantRequest::AttachProxyTraceHints { request_id }) + } + OPEN_LOOPBACK_LISTENER => { + require_nonzero("request id", request_id)?; + if resource_id != 0 || detail != 0 || !name.is_empty() { + bail!("loopback listener request carries unrelated fields"); + } + require_empty_path(&relative_path)?; + Ok(UpstreamGrantRequest::OpenLoopbackListener { request_id, port }) + } kind => bail!("invalid upstream request kind {kind}"), } } @@ -595,6 +642,26 @@ pub fn encode_upstream_grant_response(response: &UpstreamGrantResponse) -> Resul frame[KIND_OFFSET] = PROXY_MCP_ADOPTED; put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); } + UpstreamGrantResponse::ProxyTraceHintsAdopted { request_id } => { + require_nonzero("request id", *request_id)?; + frame[KIND_OFFSET] = PROXY_TRACE_HINTS_ADOPTED; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + } + UpstreamGrantResponse::LoopbackListenerGranted { + request_id, + grant_id, + port, + } => { + require_nonzero("request id", *request_id)?; + require_nonzero("grant id", *grant_id)?; + if *port == 0 { + bail!("granted loopback listener port cannot be zero"); + } + frame[KIND_OFFSET] = LOOPBACK_LISTENER_GRANTED; + put_u64(&mut frame, REQUEST_ID_RANGE, *request_id); + put_u64(&mut frame, RESOURCE_ID_RANGE, *grant_id); + put_u16(&mut frame, PORT_RANGE, *port); + } } Ok(frame) } @@ -612,7 +679,7 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - let relative_path = get_path(frame)?; require_empty_path(&relative_path)?; require_nonzero("request id", request_id)?; - if port != 0 { + if port != 0 && frame[KIND_OFFSET] != LOOPBACK_LISTENER_GRANTED { bail!("upstream response cannot carry a port"); } match frame[KIND_OFFSET] { @@ -693,6 +760,23 @@ pub fn decode_upstream_grant_response(frame: &[u8; UPSTREAM_GRANT_FRAME_SIZE]) - } Ok(UpstreamGrantResponse::ProxyMcpAdopted { request_id }) } + PROXY_TRACE_HINTS_ADOPTED => { + if resource_id != 0 || detail != 0 || !name.is_empty() || policy_digest.is_some() { + bail!("proxy trace-hint adoption carries unrelated fields"); + } + Ok(UpstreamGrantResponse::ProxyTraceHintsAdopted { request_id }) + } + LOOPBACK_LISTENER_GRANTED => { + require_nonzero("grant id", resource_id)?; + if port == 0 || detail != 0 || !name.is_empty() || policy_digest.is_some() { + bail!("loopback listener grant carries invalid fields"); + } + Ok(UpstreamGrantResponse::LoopbackListenerGranted { + request_id, + grant_id: resource_id, + port, + }) + } kind => bail!("invalid upstream response kind {kind}"), } } @@ -854,7 +938,7 @@ fn decode_hex_nibble(byte: u8) -> Result { fn validate_policy_digest(digest: &str) -> Result<()> { let Some(hash) = digest.strip_prefix("blake3:") else { - bail!("policy digest must use blake3"); + bail!("policy digest must have the blake3 scheme"); }; if hash.len() != 64 || !hash diff --git a/crates/capsem-proto/src/upstream_grant/tests.rs b/crates/capsem-proto/src/upstream_grant/tests.rs index 0148da472..218e91338 100644 --- a/crates/capsem-proto/src/upstream_grant/tests.rs +++ b/crates/capsem-proto/src/upstream_grant/tests.rs @@ -46,6 +46,15 @@ fn every_request_round_trips_exactly() { service: ProxyTrafficService::Dns, }, UpstreamGrantRequest::AttachProxyMcp { request_id: 8 }, + UpstreamGrantRequest::AttachProxyTraceHints { request_id: 9 }, + UpstreamGrantRequest::OpenLoopbackListener { + request_id: 10, + port: 0, + }, + UpstreamGrantRequest::OpenLoopbackListener { + request_id: 11, + port: 41234, + }, UpstreamGrantRequest::Adopted { grant_id: 51 }, UpstreamGrantRequest::Release { resource_id: 61 }, UpstreamGrantRequest::SetGuestMode { @@ -115,6 +124,12 @@ fn every_response_round_trips_exactly() { }, UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 9 }, UpstreamGrantResponse::ProxyMcpAdopted { request_id: 10 }, + UpstreamGrantResponse::ProxyTraceHintsAdopted { request_id: 11 }, + UpstreamGrantResponse::LoopbackListenerGranted { + request_id: 12, + grant_id: 17, + port: 41234, + }, ] { response_roundtrip(response); } @@ -224,6 +239,10 @@ fn descriptor_expectations_are_explicit() { UpstreamGrantRequest::AttachProxyMcp { request_id: 10 }.expected_descriptor_count(), 1 ); + assert_eq!( + UpstreamGrantRequest::AttachProxyTraceHints { request_id: 11 }.expected_descriptor_count(), + 1 + ); assert_eq!( UpstreamGrantResponse::ProxyTrafficAdopted { request_id: 9 }.expected_descriptor_count(), 0 @@ -232,6 +251,10 @@ fn descriptor_expectations_are_explicit() { UpstreamGrantResponse::ProxyMcpAdopted { request_id: 10 }.expected_descriptor_count(), 0 ); + assert_eq!( + UpstreamGrantResponse::ProxyTraceHintsAdopted { request_id: 11 }.expected_descriptor_count(), + 0 + ); assert_eq!( UpstreamGrantResponse::LedgerGranted { request_id: 2, @@ -250,6 +273,15 @@ fn descriptor_expectations_are_explicit() { .expected_descriptor_count(), 1 ); + assert_eq!( + UpstreamGrantResponse::LoopbackListenerGranted { + request_id: 12, + grant_id: 17, + port: 41234, + } + .expected_descriptor_count(), + 1 + ); assert_eq!( UpstreamGrantResponse::Denied { request_id: 1, diff --git a/crates/capsem-proxy/src/credential_client.rs b/crates/capsem-proxy/src/credential_client.rs index 969340aec..6c047299a 100644 --- a/crates/capsem-proxy/src/credential_client.rs +++ b/crates/capsem-proxy/src/credential_client.rs @@ -86,10 +86,11 @@ impl CredentialClient { .recv_timeout(OPERATION_TIMEOUT) .map_err(|_| "proxy credential operation timed out".to_string())? }; - if tokio::runtime::Handle::try_current().is_ok() { - tokio::task::block_in_place(receive) - } else { - receive() + match tokio::runtime::Handle::try_current() { + Ok(handle) if handle.runtime_flavor() == tokio::runtime::RuntimeFlavor::MultiThread => { + tokio::task::block_in_place(receive) + } + _ => receive(), } } } diff --git a/crates/capsem-proxy/src/credential_client/tests.rs b/crates/capsem-proxy/src/credential_client/tests.rs index ca7f2cb32..539ca753d 100644 --- a/crates/capsem-proxy/src/credential_client/tests.rs +++ b/crates/capsem-proxy/src/credential_client/tests.rs @@ -48,3 +48,44 @@ async fn remote_substitution_preserves_correlated_headers_and_reports_disconnect Some(capsem_proto::proxy_control::ProxyChannelCloseReason::Disconnected) ); } + +#[test] +fn remote_substitution_does_not_panic_on_the_proxy_current_thread_runtime() { + let (service, proxy) = UnixStream::pair().unwrap(); + let (client, _closed) = super::CredentialClient::start(proxy).unwrap(); + let serving = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + let (sender, receiver) = + ipc_channel::channel_from_std::(service).unwrap(); + let ProxyCredentialRequest::Substitute { + request_id, headers, .. + } = receiver.recv().await.unwrap() + else { + panic!("expected substitution request") + }; + sender + .send(ProxyCredentialResponse::Substituted { + request_id, + headers, + query: None, + credential_ref: None, + }) + .await + .unwrap(); + }); + }); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + client + .substitute_upstream("example.com", None, &mut http::HeaderMap::new(), None) + .unwrap(); + }); + serving.join().unwrap(); +} diff --git a/crates/capsem-proxy/src/main.rs b/crates/capsem-proxy/src/main.rs index afbb6abcb..62baccf61 100644 --- a/crates/capsem-proxy/src/main.rs +++ b/crates/capsem-proxy/src/main.rs @@ -32,6 +32,7 @@ mod credential_client; mod mcp_client; mod metric_client; mod private_names_client; +mod trace_hint_client; const CONTROL_QUEUE_CAPACITY: usize = 16; const CONTROL_SEND_TIMEOUT: Duration = Duration::from_secs(5); @@ -70,7 +71,6 @@ fn main() -> Result<()> { sink: capsem_foundation::telemetry::LogSink::Stderr, default_filter: "capsem_proxy=info,capsem_core=info,capsem_foundation=warn", })?; - let args = Args::parse(); let stdin = io::stdin(); let control = UnixStream::from(fd::duplicate(stdin.as_fd())?); let denied_file = std::env::current_exe().context("resolve proxy executable before confinement")?; @@ -90,6 +90,10 @@ fn attest_confinement(denied_file: &std::path::Path, parent_pid: u32) -> Result< bail!("proxy worker retained inherited environment state"); } require_denied(std::fs::read(denied_file), "read an ambient file")?; + // Linux seccomp denies socket(2) itself. Seatbelt denies endpoint bind, + // listen and connect operations but permits authority-free private + // socketpairs, which Tokio and worker-internal channels may use. + #[cfg(target_os = "linux")] require_denied(UnixStream::pair(), "create a Unix socket")?; require_denied( std::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)), @@ -308,6 +312,13 @@ async fn run_control( .lock() .unwrap_or_else(|error| error.into_inner()) .attach_metric_exporter(exporter); + } else if capability == ProxyCapability::TraceHints { + let client = trace_hint_client::TraceHintClient::start(UnixStream::from(descriptor)) + .context("open proxy trace-hint capability")?; + state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .attach_trace_hints(client); } else if capability == ProxyCapability::HttpTraffic { let runtime = state .lock() @@ -433,6 +444,7 @@ struct ProxyRuntimeState { http_config: Option>, dns_runtime: Option>, metric_exporter: Option, + trace_hints: Option>, } impl Default for ProxyRuntimeState { @@ -463,6 +475,7 @@ impl ProxyRuntimeState { http_config: None, dns_runtime: None, metric_exporter: None, + trace_hints: None, } } @@ -540,6 +553,11 @@ impl ProxyRuntimeState { self.metric_exporter = Some(exporter); } + fn attach_trace_hints(&mut self, trace_hints: Arc) { + self.trace_hints = Some(trace_hints); + self.http_config = None; + } + fn traffic_ready(&mut self, capability: ProxyCapability) -> Result { match capability { ProxyCapability::HttpTraffic => Ok(self.http_runtime()?.is_some()), @@ -568,7 +586,16 @@ impl ProxyRuntimeState { pricing: Arc::new(capsem_core::net::ai_traffic::pricing::PricingTable::load()), trace_state: Arc::new(Mutex::new(capsem_core::net::ai_traffic::TraceState::new())), }); - let pipeline = capsem_core::net::mitm_proxy::make_production_pipeline(Arc::clone(&telemetry)); + let pipeline = match &self.trace_hints { + Some(trace_hints) => capsem_core::net::mitm_proxy::make_production_pipeline_with_trace_hints( + Arc::clone(&telemetry), + Arc::clone(trace_hints), + ), + None if self.standalone_provider.is_some() => { + capsem_core::net::mitm_proxy::make_production_pipeline(Arc::clone(&telemetry)) + } + None => return Ok(None), + }; let upstream_grants: Arc = upstream_grants.clone(); let mcp_endpoint = match &self.mcp_client { Some((mcp_client, mcp_hello)) => Some(Arc::new( @@ -776,5 +803,19 @@ fn parse_generation(value: &str) -> std::result::Result Ok(generation) } +fn trace_id_from_generation(generation: ProxyGeneration) -> String { + let bytes = generation.as_bytes(); + let compact = if bytes[8..].iter().any(|byte| *byte != 0) { + &bytes[8..] + } else { + &bytes[..8] + }; + compact.iter().fold(String::with_capacity(16), |mut trace_id, byte| { + use std::fmt::Write as _; + write!(trace_id, "{byte:02x}").expect("write to String"); + trace_id + }) +} + #[cfg(test)] mod tests; diff --git a/crates/capsem-proxy/src/tests.rs b/crates/capsem-proxy/src/tests.rs index 3d03b2735..a38321c72 100644 --- a/crates/capsem-proxy/src/tests.rs +++ b/crates/capsem-proxy/src/tests.rs @@ -135,6 +135,7 @@ match = 'http.host == "worker.example"' mcp, capsem_proto::proxy_mcp::ProxyMcpHello::new(Vec::new(), 1, 1, 1, 1).unwrap(), ); + state.attach_trace_hints(Arc::new(EmptyTraceHints)); let config = state.http_config().unwrap().expect("complete capability set"); assert!(config.upstream_grants.is_some()); assert!(config.mcp_endpoint.is_some()); @@ -144,6 +145,18 @@ match = 'http.host == "worker.example"' ); } +struct EmptyTraceHints; + +impl capsem_core::net::ai_traffic::TraceHintSink for EmptyTraceHints { + fn register<'a>( + &'a self, + _trace_id: &'a str, + _relative_paths: &'a [String], + ) -> capsem_core::net::ai_traffic::TraceHintFuture<'a> { + Box::pin(async { Ok(()) }) + } +} + #[tokio::test] async fn standalone_http_runtime_pins_provider_without_mcp_capability() { let active_policy = br#" diff --git a/crates/capsem-proxy/src/trace_hint_client.rs b/crates/capsem-proxy/src/trace_hint_client.rs new file mode 100644 index 000000000..cbae45429 --- /dev/null +++ b/crates/capsem-proxy/src/trace_hint_client.rs @@ -0,0 +1,58 @@ +//! One-way model/file correlation hints for the workspace-owning process. + +use std::io; +use std::os::fd::AsFd as _; +use std::os::unix::net::UnixStream; +use std::sync::Arc; +use std::time::Duration; + +use capsem_core::net::ai_traffic::{TraceHintFuture, TraceHintSink}; +use capsem_proto::proxy_trace_hints::{encode_proxy_trace_hint, ProxyTraceHint, PROXY_TRACE_HINT_ACK}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + +const WRITE_TIMEOUT: Duration = Duration::from_secs(1); + +pub(super) struct TraceHintClient { + stream: tokio::sync::Mutex, +} + +impl TraceHintClient { + pub(super) fn start(stream: UnixStream) -> io::Result> { + capsem_foundation::unix::fd::set_nonblocking(stream.as_fd(), true)?; + Ok(Arc::new(Self { + stream: tokio::sync::Mutex::new(tokio::net::UnixStream::from_std(stream)?), + })) + } +} + +impl TraceHintSink for TraceHintClient { + fn register<'a>(&'a self, trace_id: &'a str, relative_paths: &'a [String]) -> TraceHintFuture<'a> { + Box::pin(async move { + let mut stream = self.stream.lock().await; + for relative_path in relative_paths { + let frame = encode_proxy_trace_hint(&ProxyTraceHint { + trace_id: trace_id.to_string(), + relative_path: relative_path.clone(), + }) + .map_err(|error| error.to_string())?; + tokio::time::timeout(WRITE_TIMEOUT, stream.write_all(&frame)) + .await + .map_err(|_| "proxy trace-hint write timed out".to_string())? + .map_err(|error| format!("write proxy trace hint: {error}"))?; + let mut acknowledgement = [0; PROXY_TRACE_HINT_ACK.len()]; + tokio::time::timeout(WRITE_TIMEOUT, stream.read_exact(&mut acknowledgement)) + .await + .map_err(|_| "proxy trace-hint acknowledgement timed out".to_string())? + .map_err(|error| format!("read proxy trace-hint acknowledgement: {error}"))?; + if acknowledgement != PROXY_TRACE_HINT_ACK { + return Err("proxy trace-hint owner returned an invalid acknowledgement".to_string()); + } + } + drop(stream); + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/capsem-proxy/src/trace_hint_client/tests.rs b/crates/capsem-proxy/src/trace_hint_client/tests.rs new file mode 100644 index 000000000..160ee2ea6 --- /dev/null +++ b/crates/capsem-proxy/src/trace_hint_client/tests.rs @@ -0,0 +1,28 @@ +use std::os::unix::net::UnixStream; + +use capsem_core::net::ai_traffic::TraceHintSink as _; +use capsem_proto::proxy_trace_hints::PROXY_TRACE_HINT_ACK; +use capsem_proto::proxy_trace_hints::{decode_proxy_trace_hint, PROXY_TRACE_HINT_FRAME_SIZE}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + +#[tokio::test] +async fn writes_fixed_bounded_trace_hints_to_the_owner() { + let (proxy, owner) = UnixStream::pair().unwrap(); + let client = super::TraceHintClient::start(proxy).unwrap(); + let reader = tokio::spawn(async move { + owner.set_nonblocking(true).unwrap(); + let mut owner = tokio::net::UnixStream::from_std(owner).unwrap(); + let mut frame = [0; PROXY_TRACE_HINT_FRAME_SIZE]; + owner.read_exact(&mut frame).await.unwrap(); + let hint = decode_proxy_trace_hint(&frame).unwrap(); + owner.write_all(&PROXY_TRACE_HINT_ACK).await.unwrap(); + hint + }); + + client + .register("2e934d32-1e3e-4d51-b816-e14cc662833f", &["reports/result.txt".into()]) + .await + .unwrap(); + let hint = reader.await.unwrap(); + assert_eq!(hint.relative_path, "reports/result.txt"); +} diff --git a/crates/capsem-proxy/tests/subprocess.rs b/crates/capsem-proxy/tests/subprocess.rs index 2978ba839..7fe1bae8a 100644 --- a/crates/capsem-proxy/tests/subprocess.rs +++ b/crates/capsem-proxy/tests/subprocess.rs @@ -1,4 +1,3 @@ -use std::io::Read; use std::os::fd::{AsRawFd, OwnedFd}; use std::os::unix::net::UnixStream; use std::process::{Child, Command, Stdio}; @@ -14,6 +13,7 @@ use capsem_proto::proxy_control::{ }; use capsem_proto::proxy_metrics::{ProxyMetricBrokerMessage, ProxyMetricRequest}; use capsem_proto::proxy_policy::{ProxyPolicyRequest, ProxyPolicyResponse}; +use tokio::io::AsyncReadExt; type ControlSender = DescriptorSender; type ControlReceiver = DescriptorReceiver; @@ -47,6 +47,10 @@ async fn event(receiver: &ControlReceiver) -> ProxyControlEvent { } fn spawn() -> (ChildGuard, ControlSender, ControlReceiver) { + spawn_with_stderr(Stdio::inherit()) +} + +fn spawn_with_stderr(stderr: Stdio) -> (ChildGuard, ControlSender, ControlReceiver) { let (coordinator, child_control) = UnixStream::pair().unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_capsem-proxy")) .arg("--parent-pid") @@ -56,7 +60,7 @@ fn spawn() -> (ChildGuard, ControlSender, ControlReceiver) { .env("CAPSEM_PROXY_SECRET", "must-not-survive") .stdin(Stdio::from(OwnedFd::from(child_control))) .stdout(Stdio::null()) - .stderr(Stdio::piped()) + .stderr(stderr) .spawn() .unwrap(); let requests = ControlSender::new(coordinator.try_clone().unwrap()).unwrap(); @@ -71,7 +75,7 @@ async fn worker_rejects_http_traffic_until_runtime_is_ready() { event(&events).await, ProxyControlEvent::Ready { generation: GENERATION } ); - let (mut peer, granted) = UnixStream::pair().unwrap(); + let (peer, granted) = UnixStream::pair().unwrap(); let grant = ProxyChannelGrant::new(GENERATION, 1, ProxyCapability::HttpTraffic).unwrap(); requests .send( @@ -89,8 +93,16 @@ async fn worker_rejects_http_traffic_until_runtime_is_ready() { reason: ProxyControlRejection::NotReady, } ); - peer.set_read_timeout(Some(Duration::from_secs(1))).unwrap(); - assert_eq!(peer.read(&mut [0; 1]).unwrap(), 0); + peer.set_nonblocking(true).unwrap(); + let mut peer = tokio::net::UnixStream::from_std(peer).unwrap(); + let mut byte = [0; 1]; + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), peer.read(&mut byte)) + .await + .expect("rejected proxy grant remained open") + .unwrap(), + 0 + ); requests .send( @@ -129,6 +141,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { } let mut peers = Vec::new(); + let mut metric_peer = None; for (index, capability) in [ ProxyCapability::Upstream, ProxyCapability::Credential, @@ -142,7 +155,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { let grant_id = index as u64 + 1; let (peer, granted) = UnixStream::pair().unwrap(); if capability == ProxyCapability::Telemetry { - let (messages, _requests) = + let (messages, requests) = capsem_foundation::ipc_channel::channel_from_std::(peer) .unwrap(); messages @@ -151,6 +164,9 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { }) .await .unwrap(); + // Match the service broker's lifetime. Closing both halves before + // granting the descriptor races the worker's first read on macOS. + metric_peer = Some((messages, requests)); } else { peers.push(peer); } @@ -204,6 +220,7 @@ async fn worker_confines_before_ready_and_adopts_scoped_descriptors() { ProxyControlEvent::Stopped { generation: GENERATION } ); drop(peers); + drop(metric_peer); let child = child.take(); let output = tokio::task::spawn_blocking(move || child.wait_with_output()) @@ -324,7 +341,7 @@ async fn ledger_grant_is_authenticated_before_adoption() { #[tokio::test] async fn stale_generation_terminates_worker_and_releases_grants() { - let (mut child, requests, events) = spawn(); + let (mut child, requests, events) = spawn_with_stderr(Stdio::piped()); assert_eq!( event(&events).await, ProxyControlEvent::Ready { generation: GENERATION } diff --git a/crates/capsem-router/README.md b/crates/capsem-router/README.md index 04d7a0f85..4cccbe25f 100644 --- a/crates/capsem-router/README.md +++ b/crates/capsem-router/README.md @@ -15,11 +15,13 @@ destinations, and never opens a socket of its own. ## Current descriptor and process ownership -The VM owner binds and accepts the loopback listener, connects the guest VSOCK -stream, and grants the two connected descriptors over a private Unix socket. -The companion receives no listener. Linux also denies accept syscalls; macOS -preserves existing FD authority, so closing inherited FDs and rejecting listener -grants are essential. Ten-byte, +The service binds an exact loopback listener and grants it over the VM owner's +generation-bound coordinator channel. The owner validates and accepts that +listener, connects the guest VSOCK stream, and grants only the two connected +descriptors over a private Unix socket. The service never accepts or reads the +published byte stream, and the companion receives no listener. Linux also +denies accept syscalls; macOS preserves existing FD authority, so closing +inherited FDs and rejecting listener grants are essential. Ten-byte, versioned headers carry grants, acknowledgements, closes, and aborts; only grants carry FDs. Close reports append 17 bytes for a typed reason and two delivered-byte counts. Cooperative cancellation preserves those counts during active transfer @@ -99,13 +101,15 @@ guest cancellation identity until cleanup. Host control I/O is asynchronous, with five-second write and started-frame deadlines and an owned reader. TCP descriptors are armed for abortive close before setup/handoff and again at child adoption, covering SIGKILL without relying on destructors. Normal Complete -clears that setting; other exits avoid an early FIN. The trusted owner immediately revokes its TCP endpoint using -linger-zero plus `disconnectx` on macOS or `connect(AF_UNSPEC)` on Linux; this -also revokes a malicious child's retained copies. The child retains no connect -authority. Guest cancellation resets the container TCP endpoint. Normal Complete -still drains both directions. Guest `PortClosed` reports keep their VSOCK stream -open after abnormal termination until `PortCloseAck`: the host revokes TCP before -acknowledging, so stream EOF cannot race an unintended FIN through the copier. +clears that setting; other exits avoid an early FIN. The trusted owner can +immediately revoke its TCP endpoint with `disconnectx` on macOS. The confined +Linux owner retains no `connect` syscall, including `connect(AF_UNSPEC)`, so +abnormal cleanup instead closes the owner and router copies while linger-zero +remains armed. The child retains no connect authority. Guest cancellation +resets the container TCP endpoint. Normal Complete still drains both +directions. Guest `PortClosed` reports keep their VSOCK stream open after +abnormal termination until `PortCloseAck`, and the host arms TCP reset before +acknowledging so stream EOF cannot turn cleanup into an unintended FIN. The control actor accepts reports only for the owning generation and flow lease, but acknowledges retired duplicates to release guest credits. Child cancellation retains its observer for final byte counts; a missing close ACK terminates the child. diff --git a/crates/capsem-service/src/active_policy.rs b/crates/capsem-service/src/active_policy.rs index 20052f3a3..6b05c273b 100644 --- a/crates/capsem-service/src/active_policy.rs +++ b/crates/capsem-service/src/active_policy.rs @@ -43,7 +43,8 @@ impl ServiceState { .map_err(anyhow::Error::msg) .context("compile serialized active policy")?, ); - // capsem-process reads this on every reload: publish it whole. + // Keep the durable audit copy whole; the same exact bytes are sent to + // capsem-process over its authenticated coordinator channel. capsem_foundation::unix::fs::atomic_write_private(&active_policy_path, serialized.as_bytes()) .with_context(|| format!("write {}", active_policy_path.display()))?; Ok(PublishedActivePolicy { @@ -76,7 +77,7 @@ impl ServiceState { } /// Deliver the current policy to every running VM: re-materialize each -/// session's active policy, then ask capsem-process to reload it. Mutation +/// session's active policy, then send those exact bytes to capsem-process. Mutation /// routes call this so an edit is enforced before the route returns. /// /// A VM counts as updated only when it reports applying the exact bytes this @@ -137,6 +138,7 @@ pub(crate) async fn push_policy_to_running_instances( let expected = &published.digest; let request = ServiceToProcess::ReloadConfig { id: state.next_job_id(), + active_policy: published.bytes.clone(), }; match send_ipc_command(state, uds_path, request, Some(5)).await { Ok(ProcessToService::ConfigReloadResult { @@ -159,11 +161,15 @@ pub(crate) async fn push_policy_to_running_instances( Err(error) => return Some(format!("{id}: {error}")), }; match proxy.apply_policy(published.bytes.clone()).await { - Ok(applied) if applied == *expected => publisher - .publish(published.broker_policy()) - .await - .err() - .map(|error| format!("{id}: {error}")), + Ok(applied) if applied == *expected => { + if let Err(error) = publisher.publish(published.broker_policy()).await { + return Some(format!("{id}: {error}")); + } + connection_routes::policy::acknowledge(state, id, *generation, expected) + .await + .err() + .map(|_| format!("{id}: applied credential policy ownership unavailable")) + } Ok(applied) => Some(format!( "{id}: proxy applied active policy {applied}, expected {expected}" )), diff --git a/crates/capsem-service/src/connection_routes/policy.rs b/crates/capsem-service/src/connection_routes/policy.rs index e708373de..d73fd2436 100644 --- a/crates/capsem-service/src/connection_routes/policy.rs +++ b/crates/capsem-service/src/connection_routes/policy.rs @@ -24,12 +24,12 @@ fn denied() -> CredentialError { api::CredentialErrorCode::PolicyDenied.into() } -fn read_policy(path: &StdPath) -> Result<(String, Arc), CredentialError> { +fn read_policy(path: &StdPath) -> Result<(String, Arc, Vec), CredentialError> { let bytes = capsem_foundation::unix::fs::read_regular_file_no_follow(path).map_err(|_| denied())?; let policy: ActivePolicyFile = toml::from_str(std::str::from_utf8(&bytes).map_err(|_| denied())?).map_err(|_| denied())?; policy.validate().map_err(|_| denied())?; - Ok((active_policy_digest(&bytes), Arc::new(policy))) + Ok((active_policy_digest(&bytes), Arc::new(policy), bytes)) } fn record(state: &ServiceState, binding: &VmBinding, policy: Arc) -> Result<(), CredentialError> { @@ -75,7 +75,7 @@ pub(crate) async fn acknowledge( .filter(|instance| instance.generation == generation) .map(|instance| instance.session_dir.join(ACTIVE_POLICY_DIR).join(ACTIVE_POLICY_FILE)) .ok_or(api::CredentialErrorCode::OwnerChanged)?; - let (digest, policy) = state + let (digest, policy, _) = state .off_worker(move |_| read_policy(&path)) .await .map_err(|_| CredentialError(api::CredentialErrorCode::StorageUnavailable, false))??; @@ -109,14 +109,20 @@ fn compute( async fn confirm(state: &Arc, context: &Context) -> Result { let path = context.policy_path.clone(); - let (expected, applied) = state - .off_worker(move |_| read_policy(&path)) + let (expected, applied, bytes, runtime) = state + .off_worker(move |_| { + let (digest, policy, bytes) = read_policy(&path)?; + let runtime = Arc::new(policy.compile_runtime().map_err(|_| denied())?); + Ok::<_, CredentialError>((digest, policy, bytes, runtime)) + }) .await .map_err(|_| CredentialError(api::CredentialErrorCode::StorageUnavailable, false))??; let response = send_ipc_command( + state, &context.uds_path, ServiceToProcess::ReloadConfig { id: state.next_job_id(), + active_policy: bytes.clone(), }, Some(5), ) @@ -127,6 +133,24 @@ async fn confirm(state: &Arc, context: &Context) -> Result Some(ProcessToService::ConfigReloadResult { - id: *id, - active_policy_digest: digest.clone(), - error: error.clone(), - }), + ServiceToProcess::ReloadConfig { id, active_policy } => { + assert_eq!(active_policy, &expected_policy); + Some(ProcessToService::ConfigReloadResult { + id: *id, + active_policy_digest: digest.clone(), + error: error.clone(), + }) + } _ => None, }; Box::pin(async move { reply }) diff --git a/crates/capsem-service/src/credential_routes.rs b/crates/capsem-service/src/credential_routes.rs index 36b872e43..09028d96e 100644 --- a/crates/capsem-service/src/credential_routes.rs +++ b/crates/capsem-service/src/credential_routes.rs @@ -61,7 +61,7 @@ pub(crate) async fn sync_memory(state: &Arc, socket: &StdPath) -> let command = ServiceToProcess::InjectCredentials { id, credentials }; let result = tokio::time::timeout( std::time::Duration::from_secs(5), - vm_files::send_ipc_command(socket, command, Some(5)), + vm_files::send_ipc_command(state, socket, command, Some(5)), ) .await; match result { diff --git a/crates/capsem-service/src/credential_routes/tests.rs b/crates/capsem-service/src/credential_routes/tests.rs index d09d8cb8f..5477ed3c1 100644 --- a/crates/capsem-service/src/credential_routes/tests.rs +++ b/crates/capsem-service/src/credential_routes/tests.rs @@ -68,6 +68,17 @@ async fn memory_handoff_uses_correlated_real_host_ipc_and_acknowledgement() { let dir = tempfile::tempdir().unwrap(); let socket = dir.path().join("owner.sock"); let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let state = make_test_state(); + state.instances.lock().unwrap().insert( + "vm".into(), + InstanceInfo { + id: "vm".into(), + name: "vm".into(), + uds_path: socket.clone(), + session_dir: dir.path().to_path_buf(), + ..crate::tests::test_instance() + }, + ); let reference = CredentialStore::global() .inject( CredentialProvider::Google, @@ -111,13 +122,11 @@ async fn memory_handoff_uses_correlated_real_host_ipc_and_acknowledgement() { }); // Readiness must include host credential admission, not just the guest's // sentinel. Exercise the real bootstrap wait against the live IPC peer. - std::fs::write(socket.with_extension("ready"), b"").unwrap(); - vm_files::wait_for_vm_ready(&socket, 1, Some(&make_test_state()), None) + std::fs::write(socket.with_extension("ready"), b"ready\n").unwrap(); + vm_files::wait_for_vm_ready(&socket, 1, Some(&state), Some("vm")) .await .unwrap(); owner.await.unwrap(); - assert!(sync_memory(&make_test_state(), &dir.path().join("absent.sock")) - .await - .is_err()); + assert!(sync_memory(&state, &dir.path().join("absent.sock")).await.is_err()); CredentialStore::global().clear_for_test(); } diff --git a/crates/capsem-service/src/instance/tests.rs b/crates/capsem-service/src/instance/tests.rs index 7eb0c32af..f70ece109 100644 --- a/crates/capsem-service/src/instance/tests.rs +++ b/crates/capsem-service/src/instance/tests.rs @@ -20,7 +20,13 @@ async fn stale_shutdown_binding_leaves_replacement_socket_and_ledger_intact() { let ledger = state .register_session_db_handle("replacement", ledger_dir.path()) .unwrap(); - let result = shutdown_vm_process(&state, "replacement", ShutdownMode::Discard, Some(uuid::Uuid::new_v4())).await; + let result = shutdown_vm_process( + &state, + "replacement", + ShutdownMode::Discard, + Some(capsem_crypto::uuid_v4()), + ) + .await; assert_eq!(result.unwrap_err().0, StatusCode::CONFLICT); assert_eq!( state.instances.lock().unwrap().get("replacement").unwrap().generation, diff --git a/crates/capsem-service/src/instance_reaper.rs b/crates/capsem-service/src/instance_reaper.rs index 3e3adca1f..a5ac7f22c 100644 --- a/crates/capsem-service/src/instance_reaper.rs +++ b/crates/capsem-service/src/instance_reaper.rs @@ -4,12 +4,10 @@ use super::*; mod retirement; pub(crate) use retirement::{Registration, Retirements}; -pub(super) enum ExitedGeneration { - Current(Option>), - Replaced, -} - -pub(super) fn kill_and_reap(mut child: tokio::process::Child) { +pub(super) fn kill_and_reap( + mut child: tokio::process::Child, + mut retirement: Registration, +) -> tokio::task::JoinHandle<()> { let _ = child.start_kill(); tokio::spawn(async move { if retirement.wait_child(&mut child).await.is_ok() { @@ -37,10 +35,32 @@ pub(super) fn spawn_exit_reaper( // task can be delayed across an exit/replacement or PID reuse. let generation = retirement.generation(); tokio::spawn(async move { - let registration = revoke_exited_generation(&child, &id, &state, generation).await; - let Ok(exit_status) = child.wait().await else { + let Ok((exit_status, current)) = retirement + .wait_child_before_reap(&mut child, || { + state + .connections + .fence_runtime(&id, generation, capsem_credentials::GrantStopPolicy::Preserve); + let instances = state.instances.lock().unwrap(); + let current = instances + .get(&id) + .is_none_or(|instance| instance.generation == generation); + if let Some(instance) = instances.get(&id).filter(|instance| instance.generation == generation) { + instance.authority.revoke(); + } + drop(instances); + if current { + state.remove_proxy_worker(&id, generation); + } + current + }) + .await + else { return; }; + if !current { + retirement.complete(); + return; + } info!(id, ?exit_status, "capsem-process exited, cleaning up"); // Even a delayed reaper must fence its exact old runtime. A resumed // replacement may already own the map, but never owns these handles. @@ -70,9 +90,12 @@ pub(super) fn spawn_exit_reaper( // child exited without an explicit service-side shutdown removing it. // A clean process exit is a graceful shutdown regardless of whether // the guest or service initiated it; anything else is a crash. - let removed = match registration { - ExitedGeneration::Current(removed) => removed, - ExitedGeneration::Replaced => { + let removed = { + let mut instances = state.instances.lock().unwrap(); + if instances + .get(&id) + .is_some_and(|instance| Some(instance.pid) != pid || instance.generation != generation) + { // A cold fallback can replace a failed restore while holding // the exclusive guard. Its registry, DB and sockets are not // owned by this delayed reaper. @@ -84,8 +107,11 @@ pub(super) fn spawn_exit_reaper( retirement.complete(); return; } + instances.remove(&id) }; if let Some(info) = &removed { + info.authority.revoke(); + state.remove_proxy_worker(&id, generation); // The VM is gone: a container setup for it has nothing left to do. let policy = connection_routes::lifecycle::stop_policy(info.persistent, true); if let Err(error) = state.connections.stop_runtime(&id, generation, policy).await { @@ -188,47 +214,3 @@ pub(super) fn spawn_exit_reaper( retirement.complete(); }) } - -pub(super) async fn revoke_exited_generation( - child: &tokio::process::Child, - id: &str, - state: &ServiceState, - generation: uuid::Uuid, -) -> ExitedGeneration { - let pid = child.id(); - if let Some(raw_pid) = pid { - match capsem_foundation::unix::process::ProcessId::try_from(raw_pid) { - Ok(pid) => { - match tokio::task::spawn_blocking(move || capsem_foundation::unix::process::wait_for_child_exit(pid)) - .await - { - Ok(Ok(())) => {} - Ok(Err(error)) => { - error!(id, ?pid, error = %error, "failed to retain child exit identity") - } - Err(error) => { - error!(id, ?pid, error = %error, "child exit observer failed") - } - } - } - Err(error) => error!(id, ?pid, error = %error, "child has invalid process identity"), - } - } else { - error!(id, "child process identity disappeared before exit observation"); - } - - let mut instances = state.instances.lock().unwrap(); - if instances - .get(id) - .is_some_and(|instance| Some(instance.pid) != pid || instance.generation != generation) - { - return ExitedGeneration::Replaced; - } - let removed = instances.remove(id); - drop(instances); - if let Some(instance) = removed.as_ref() { - instance.authority.revoke(); - state.remove_proxy_worker(id, generation); - } - ExitedGeneration::Current(removed.map(Box::new)) -} diff --git a/crates/capsem-service/src/instance_reaper/retirement.rs b/crates/capsem-service/src/instance_reaper/retirement.rs index e8bd8ed16..208a0eccc 100644 --- a/crates/capsem-service/src/instance_reaper/retirement.rs +++ b/crates/capsem-service/src/instance_reaper/retirement.rs @@ -168,13 +168,27 @@ impl Registration { &mut self, child: &mut tokio::process::Child, ) -> std::io::Result { - let status = loop { + self.wait_child_before_reap(child, || ()) + .await + .map(|(status, ())| status) + } + + pub(crate) async fn wait_child_before_reap( + &mut self, + child: &mut tokio::process::Child, + before_reap: impl FnOnce() -> T, + ) -> std::io::Result<(std::process::ExitStatus, T)> { + let pid = child + .id() + .ok_or_else(|| std::io::Error::other("original child identity unavailable"))?; + let pid = ProcessId::try_from(pid)?; + let mut observed = tokio::task::spawn_blocking(move || process::wait_for_child_exit(pid)); + let observation = loop { tokio::select! { - // An already exited child is reaped and published before any - // queued signal. Otherwise it remains unreaped while signaling: - // its PID cannot be reused until this task next polls wait(). + // Retain the original PID until generation authority is revoked, + // while continuing to service owned shutdown signals. biased; - status = child.wait() => break status, + result = &mut observed => break result.map_err(std::io::Error::other).and_then(|result| result), Some(request) = self.signals.recv() => { let outcome = child.id().ok_or_else(|| std::io::Error::other("original child identity unavailable")) .and_then(ProcessId::try_from) @@ -183,6 +197,9 @@ impl Registration { } } }; + observation?; + let revoked = before_reap(); + let status = child.wait().await; self.exited.send_replace(if status.is_ok() { Outcome::Complete } else { @@ -197,7 +214,7 @@ impl Registration { }; let _ = request.reply.send(outcome); } - status + status.map(|status| (status, revoked)) } pub(crate) fn complete(mut self) { self.completed = true; diff --git a/crates/capsem-service/src/instance_reaper/retirement/tests.rs b/crates/capsem-service/src/instance_reaper/retirement/tests.rs index d4c2ac8b3..702968f47 100644 --- a/crates/capsem-service/src/instance_reaper/retirement/tests.rs +++ b/crates/capsem-service/src/instance_reaper/retirement/tests.rs @@ -11,6 +11,34 @@ fn sleeper() -> tokio::process::Child { .unwrap() } +#[tokio::test] +async fn revocation_callback_runs_while_original_child_pid_is_reserved() { + let owner = Retirements::default(); + let generation = capsem_crypto::uuid_v4(); + let mut registration = owner.register("vm", generation).unwrap(); + let mut child = tokio::process::Command::new("sh") + .args(["-c", "exit 11"]) + .spawn() + .unwrap(); + let pid = ProcessId::try_from(child.id().unwrap()).unwrap(); + let (status, ()) = registration + .wait_child_before_reap(&mut child, || { + assert!( + process::child_has_exited(pid).unwrap(), + "revocation must precede native reap" + ); + assert!( + owner.is_live("vm", generation).unwrap(), + "exit is unpublished before revocation" + ); + }) + .await + .unwrap(); + assert_eq!(status.code(), Some(11)); + assert!(owner.wait_exit("vm", generation).await.unwrap()); + registration.complete(); +} + #[tokio::test] async fn owned_child_exit_precedes_full_retirement_and_stale_signals_do_not_retarget() { let owner = std::sync::Arc::new(Retirements::default()); diff --git a/crates/capsem-service/src/ledger_commitment.rs b/crates/capsem-service/src/ledger_commitment.rs index 498c598f8..f89614eb0 100644 --- a/crates/capsem-service/src/ledger_commitment.rs +++ b/crates/capsem-service/src/ledger_commitment.rs @@ -76,7 +76,7 @@ pub(crate) struct CommitmentAuthority { impl CommitmentAuthority { pub(crate) async fn open(root: &Path, session_id: &str) -> Result> { - let session_key = blake3::hash(session_id.as_bytes()).to_hex().to_string(); + let session_key = capsem_crypto::blake3::hash(session_id.as_bytes()).to_hex().to_string(); let path = root.join(session_key).join("checkpoints.log"); let loaded_path = path.clone(); let state = tokio::task::spawn_blocking(move || load(&loaded_path)) @@ -477,7 +477,7 @@ fn append(path: &Path, record: &CheckpointRecord) -> Result<()> { file.write_all(&CHECKPOINT_MAGIC)?; file.write_all(&(payload.len() as u32).to_be_bytes())?; file.write_all(&payload)?; - file.write_all(blake3::hash(&payload).as_bytes())?; + file.write_all(capsem_crypto::blake3::hash(&payload).as_bytes())?; file.sync_data()?; Ok(()) } @@ -519,7 +519,7 @@ fn read_records(path: &Path) -> Result> { } return Err(error.into()); } - if checksum != *blake3::hash(&payload).as_bytes() { + if checksum != *capsem_crypto::blake3::hash(&payload).as_bytes() { bail!("ledger commitment checkpoint checksum is corrupt"); } records.push(rmp_serde::from_slice(&payload).context("decode commitment checkpoint")?); diff --git a/crates/capsem-service/src/ledger_worker.rs b/crates/capsem-service/src/ledger_worker.rs index cbb68d198..ded74f82e 100644 --- a/crates/capsem-service/src/ledger_worker.rs +++ b/crates/capsem-service/src/ledger_worker.rs @@ -8,6 +8,7 @@ ) )] +use std::ffi::OsStr; use std::fmt::Write as _; use std::io; use std::os::fd::{AsRawFd as _, OwnedFd}; @@ -165,6 +166,56 @@ impl LedgerWorkers { result } + /// Stop every remaining worker before the service relinquishes its + /// checkpoint authority. Continue after one failed shutdown so no sibling + /// generation is left running merely because an earlier child was sick. + pub(crate) async fn shutdown_all(&self) { + let session_ids = self.slots.lock().await.keys().cloned().collect::>(); + for session_id in session_ids { + if let Err(error) = self.shutdown(&session_id).await { + tracing::warn!(session_id, %error, "failed to stop ledger worker during service shutdown"); + } + } + } + + /// Remove the trusted checkpoint state after the corresponding session + /// directory has been permanently deleted. A stopped persistent session + /// keeps its checkpoints and therefore continues to verify on restart. + pub(crate) async fn retire(&self, session_id: &str) -> Result<()> { + if self.slots.lock().await.contains_key(session_id) { + bail!("cannot retire checkpoints for a live ledger worker"); + } + let root = self.commitment_root.clone(); + let session_id = session_id.to_string(); + tokio::task::spawn_blocking(move || retire_checkpoints(&root, &session_id)) + .await + .context("join ledger checkpoint retirement")? + } + + /// Move trusted checkpoints with a failed session's forensic directory. + /// Failed IDs are never live worker slots, so retention can later remove + /// both artifacts without weakening the active session name. + pub(crate) fn preserve_quiesced(&self, session_id: &str, failed_id: &str) -> Result<()> { + let root = match capsem_foundation::unix::contained::ContainedDir::open_root(&self.commitment_root) { + Ok(root) => root, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + let source = checkpoint_key(session_id); + let destination = checkpoint_key(failed_id); + match root.rename_to(OsStr::new(&source), &root, OsStr::new(&destination)) { + Ok(()) => root.sync().map_err(Into::into), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } + } + + /// Remove checkpoints for a session that cannot have a live worker, such + /// as a retained failed-session directory being culled synchronously. + pub(crate) fn retire_quiesced(&self, session_id: &str) -> Result<()> { + retire_checkpoints(&self.commitment_root, session_id) + } + async fn spawn(&self, database: &Path, log_path: &Path) -> Result { let log = tokio::fs::OpenOptions::new() .create(true) @@ -187,6 +238,23 @@ impl LedgerWorkers { } } +fn checkpoint_key(session_id: &str) -> String { + capsem_crypto::blake3::hash(session_id.as_bytes()).to_hex().to_string() +} + +fn retire_checkpoints(root: &Path, session_id: &str) -> Result<()> { + let root = match capsem_foundation::unix::contained::ContainedDir::open_root(root) { + Ok(root) => root, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + match root.remove_tree(OsStr::new(&checkpoint_key(session_id))) { + Ok(()) => root.sync().map_err(Into::into), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + #[cfg(not(test))] async fn verify_commitments( worker: &LedgerWorker, @@ -630,7 +698,7 @@ fn fail_pending(requests: &mut mpsc::Receiver, reason: &str) { } fn fresh_generation() -> LedgerGeneration { - LedgerGeneration::new(*uuid::Uuid::new_v4().as_bytes()) + LedgerGeneration::new(*capsem_crypto::uuid_v4().as_bytes()) } fn generation_hex(generation: LedgerGeneration) -> String { diff --git a/crates/capsem-service/src/ledger_worker/tests.rs b/crates/capsem-service/src/ledger_worker/tests.rs index 0a366d560..1febe925e 100644 --- a/crates/capsem-service/src/ledger_worker/tests.rs +++ b/crates/capsem-service/src/ledger_worker/tests.rs @@ -333,6 +333,60 @@ async fn stopping_one_session_leaves_an_unrelated_session_generation_alive() { drop(client_b); drop(second_b); + workers.shutdown_all().await; + assert_eq!(workers.generation("b").await, None); +} + +#[tokio::test] +async fn preserving_a_failed_session_moves_its_checkpoint_authority() { + let directory = tempfile::tempdir().unwrap(); + let binary = fake_binary(directory.path(), "normal"); + let commitment_root = directory.path().join("commitments"); + let workers = Arc::new(LedgerWorkers::new(binary, commitment_root.clone())); + let (database, log) = session_paths(directory.path(), "session-a"); + let client = workers + .acquire("a", &database, &log, LedgerClientRole::Reader) + .await + .unwrap(); + drop(client); + workers.shutdown("a").await.unwrap(); + + workers.preserve_quiesced("a", "a-failed-1").unwrap(); + assert!(!commitment_root.join(checkpoint_key("a")).exists()); + assert!(commitment_root.join(checkpoint_key("a-failed-1")).exists()); + workers.retire_quiesced("a-failed-1").unwrap(); + assert!(!commitment_root.join(checkpoint_key("a-failed-1")).exists()); +} + +#[tokio::test] +async fn retiring_a_stopped_session_removes_only_its_trusted_checkpoints() { + let directory = tempfile::tempdir().unwrap(); + let binary = fake_binary(directory.path(), "normal"); + let commitment_root = directory.path().join("commitments"); + let workers = Arc::new(LedgerWorkers::new(binary, commitment_root.clone())); + let (database_a, log_a) = session_paths(directory.path(), "session-a"); + let (database_b, log_b) = session_paths(directory.path(), "session-b"); + let client_a = workers + .acquire("a", &database_a, &log_a, LedgerClientRole::Reader) + .await + .unwrap(); + let client_b = workers + .acquire("b", &database_b, &log_b, LedgerClientRole::Reader) + .await + .unwrap(); + let key_a = capsem_crypto::blake3::hash(b"a").to_hex().to_string(); + let key_b = capsem_crypto::blake3::hash(b"b").to_hex().to_string(); + assert!(commitment_root.join(&key_a).exists()); + assert!(commitment_root.join(&key_b).exists()); + + assert!(workers.retire("a").await.is_err()); + drop(client_a); + workers.shutdown("a").await.unwrap(); + workers.retire("a").await.unwrap(); + assert!(!commitment_root.join(key_a).exists()); + assert!(commitment_root.join(key_b).exists()); + + drop(client_b); workers.shutdown("b").await.unwrap(); } diff --git a/crates/capsem-service/src/main.rs b/crates/capsem-service/src/main.rs index b22675f5a..528706163 100644 --- a/crates/capsem-service/src/main.rs +++ b/crates/capsem-service/src/main.rs @@ -46,6 +46,7 @@ mod owner_connection; mod owner_handoff; mod policy_mutation; use policy_mutation::{apply_policy_mutation, MutationRoute, PolicyMutation}; +mod connection_routes; mod credential_routes; mod ledger_commitment; mod ledger_worker; diff --git a/crates/capsem-service/src/managed_routes/tests.rs b/crates/capsem-service/src/managed_routes/tests.rs index d38e4acdf..59130541c 100644 --- a/crates/capsem-service/src/managed_routes/tests.rs +++ b/crates/capsem-service/src/managed_routes/tests.rs @@ -11,7 +11,7 @@ async fn failed_managed_http_create_remains_owned_until_awaited_close_and_cannot let state = crate::tests::make_test_state(); let app = build_service_router(Arc::clone(&state)); let token = "af".repeat(32); - // The actual service adapter has no installed runtime assets or process in this + // The actual service adapter has no installed VM assets or process binary in this // fixture. Its failure must retire under ownership rather than fall back // to ordinary provisioning or lose the close handle. let create = json!({"managed": {"request_id": REQUEST, "ownership_token": token}}); diff --git a/crates/capsem-service/src/managed_sessions.rs b/crates/capsem-service/src/managed_sessions.rs index 6f6fd7e43..c269a039c 100644 --- a/crates/capsem-service/src/managed_sessions.rs +++ b/crates/capsem-service/src/managed_sessions.rs @@ -316,6 +316,7 @@ impl ManagedLifecycle { async fn remove_session(&self, binding: VmBinding, require_identity: bool) -> Result<()> { let state = Arc::clone(&self.state); let target = binding; + let id = target.id().to_string(); self.state .off_worker(move |_| { let directory = state.run_dir.join("sessions").join(target.id()); @@ -334,7 +335,8 @@ impl ManagedLifecycle { Ok(()) }) .await - .map_err(app_error)? + .map_err(app_error)??; + self.state.ledger_workers.retire(&id).await } } diff --git a/crates/capsem-service/src/owner_connection/tests.rs b/crates/capsem-service/src/owner_connection/tests.rs index 85012e34a..aa5eab26f 100644 --- a/crates/capsem-service/src/owner_connection/tests.rs +++ b/crates/capsem-service/src/owner_connection/tests.rs @@ -14,7 +14,7 @@ fn fixture() -> (Arc, OwnerConnection) { async fn replaced_generation_is_rejected_before_connect() { let (state, grant) = fixture(); let listener = tokio::net::UnixListener::bind(&grant.uds_path).unwrap(); - state.instances.lock().unwrap().get_mut("owner").unwrap().generation = uuid::Uuid::new_v4(); + state.instances.lock().unwrap().get_mut("owner").unwrap().generation = capsem_crypto::uuid_v4(); assert!(grant.open(&state, "capsem-service", false).await.is_err()); assert!( tokio::time::timeout(std::time::Duration::from_millis(20), listener.accept()) @@ -42,7 +42,7 @@ async fn generation_changed_during_hello_cannot_receive_a_request() { .unwrap() .get_mut("owner") .unwrap() - .generation = uuid::Uuid::new_v4(); + .generation = capsem_crypto::uuid_v4(); // The initiator may observe the replaced generation and close // before completing the hello. Either outcome must leave the // rejected connection with no request authority. @@ -69,7 +69,7 @@ fn shutdown_grant_accepts_only_the_claimed_generation_or_an_empty_slot() { let mut instance = state.instances.lock().unwrap().remove("owner").unwrap(); assert!(grant.validate(&state, false).is_err()); assert!(grant.validate(&state, true).is_ok()); - instance.generation = uuid::Uuid::new_v4(); + instance.generation = capsem_crypto::uuid_v4(); state.instances.lock().unwrap().insert("owner".into(), instance); assert!(grant.validate(&state, true).is_err()); } diff --git a/crates/capsem-service/src/proxy_worker.rs b/crates/capsem-service/src/proxy_worker.rs index 19ecf7688..50397d8f3 100644 --- a/crates/capsem-service/src/proxy_worker.rs +++ b/crates/capsem-service/src/proxy_worker.rs @@ -748,7 +748,7 @@ fn fail_pending(requests: &mut mpsc::Receiver, reason: &str) { } fn fresh_generation() -> ProxyGeneration { - ProxyGeneration::new(*uuid::Uuid::new_v4().as_bytes()) + ProxyGeneration::new(*capsem_crypto::uuid_v4().as_bytes()) } fn generation_hex(generation: ProxyGeneration) -> String { @@ -907,4 +907,4 @@ impl ServiceState { } #[cfg(test)] -mod tests; +pub(crate) mod tests; diff --git a/crates/capsem-service/src/router_runtime.rs b/crates/capsem-service/src/router_runtime.rs index 0e39149fb..f58f2a5e5 100644 --- a/crates/capsem-service/src/router_runtime.rs +++ b/crates/capsem-service/src/router_runtime.rs @@ -64,6 +64,8 @@ pub(super) fn build_service_router(state: Arc) -> Router { .route("/proxies", post(standalone_proxy::handle_create)) .route("/proxies/{id}/heartbeat", post(standalone_proxy::handle_heartbeat)) .route("/proxies/{id}/stop", post(standalone_proxy::handle_stop)) + .route("/vms/managed/{request_id}/claim", post(managed_routes::claim)) + .route("/vms/managed/{request_id}/close", post(managed_routes::close)) .route("/vms/list", get(handle_list)) .route("/vms/{id}/info", get(handle_info)) .route("/vms/{id}/status", get(handle_vm_status)) diff --git a/crates/capsem-service/src/router_runtime/exposures/tests.rs b/crates/capsem-service/src/router_runtime/exposures/tests.rs index bfe0e15ff..601dea307 100644 --- a/crates/capsem-service/src/router_runtime/exposures/tests.rs +++ b/crates/capsem-service/src/router_runtime/exposures/tests.rs @@ -390,7 +390,7 @@ async fn preview_admission_rejects_a_reply_from_a_replaced_owner_generation() { let ServiceToProcess::AdmitPreviewConnection { id, .. } = message else { panic!("unexpected {message:?}") }; - owner_state.instances.lock().unwrap().get_mut("box").unwrap().generation = uuid::Uuid::new_v4(); + owner_state.instances.lock().unwrap().get_mut("box").unwrap().generation = capsem_crypto::uuid_v4(); let reply = ProcessToService::PreviewConnectionAdmitted { id: *id, handoff_socket: expected.to_string_lossy().into_owned(), diff --git a/crates/capsem-service/src/router_runtime/streams/tests/flow.rs b/crates/capsem-service/src/router_runtime/streams/tests/flow.rs index eb2f151fc..843122a27 100644 --- a/crates/capsem-service/src/router_runtime/streams/tests/flow.rs +++ b/crates/capsem-service/src/router_runtime/streams/tests/flow.rs @@ -128,7 +128,7 @@ async fn replaced_generation_output_is_rejected_and_the_old_exec_is_cancelled() let ServiceToProcess::ExecStream { id, .. } = rx.recv().await.unwrap() else { panic!("expected ExecStream") }; - state.instances.lock().unwrap().get_mut("box").unwrap().generation = uuid::Uuid::new_v4(); + state.instances.lock().unwrap().get_mut("box").unwrap().generation = capsem_crypto::uuid_v4(); tx.send(ProcessToService::ExecOutput { id, channel: capsem_proto::ExecOutputChannel::Stdout, diff --git a/crates/capsem-service/src/service_runtime.rs b/crates/capsem-service/src/service_runtime.rs index 7c347a5a7..0638f123c 100644 --- a/crates/capsem-service/src/service_runtime.rs +++ b/crates/capsem-service/src/service_runtime.rs @@ -468,8 +468,14 @@ async fn start_and_serve(args: Args, run_dir: PathBuf) -> Result<()> { } } info!("service shutting down, stopping VM processes and draining replies"); - kill_all_vm_processes(&shutdown_state); + let deleted_ephemeral = kill_all_vm_processes(&shutdown_state).await; shutdown_state.stop_all_standalone_proxies().await; + shutdown_state.ledger_workers.shutdown_all().await; + for id in deleted_ephemeral { + if let Err(error) = shutdown_state.ledger_workers.retire(&id).await { + warn!(id, %error, "failed to retire ephemeral checkpoints during service shutdown"); + } + } }) .await; @@ -559,7 +565,7 @@ pub(super) fn reap_orphan_capsem_processes(run_dir: &std::path::Path) -> Result< /// NOT orphan running guests. Without this, each service shutdown leaked one /// `capsem-process` per live VM, which in turn held Apple VZ memory -- making /// long test runs increasingly slow until boots timed out. -pub(super) async fn kill_all_vm_processes(state: &Arc) { +pub(super) async fn kill_all_vm_processes(state: &Arc) -> Vec { state.lifecycle.begin_shutdown(); state.lifecycle.wait_for_launches().await; let targets: Vec<_> = state @@ -569,17 +575,20 @@ pub(super) async fn kill_all_vm_processes(state: &Arc) { .values() .map(|instance| (instance.id.clone(), instance.generation)) .collect(); + let mut deleted_ephemeral = Vec::new(); for (id, generation) in targets { match vm_lifecycle::shutdown_vm_process(state, &id, ShutdownMode::ServiceStop, Some(generation)).await { Ok(Some((directory, false, _))) => { - if let Err(error) = state + match state .off_worker(move |state| state.delete_session_dir(&directory)) .await .and_then(|result| { result.map_err(|error| AppError(StatusCode::INTERNAL_SERVER_ERROR, error.to_string())) - }) - { - tracing::warn!(id,error = %error.1,"service shutdown session cleanup failed"); + }) { + Ok(()) => deleted_ephemeral.push(id), + Err(error) => { + tracing::warn!(id,error = %error.1,"service shutdown session cleanup failed"); + } } } Ok(_) => {} @@ -590,6 +599,7 @@ pub(super) async fn kill_all_vm_processes(state: &Arc) { ), } } + deleted_ephemeral } pub(super) async fn shutdown_signal() { diff --git a/crates/capsem-service/src/session_cleanup.rs b/crates/capsem-service/src/session_cleanup.rs index ff65a1daa..a6165d27a 100644 --- a/crates/capsem-service/src/session_cleanup.rs +++ b/crates/capsem-service/src/session_cleanup.rs @@ -71,24 +71,30 @@ pub(super) async fn finalize_one_shot_session( } let session_dir = shutdown_result .map(|(path, _, _)| path) - .unwrap_or_else(|| state.run_dir.join("sessions").join(id)); - if !session_dir.exists() { - return Ok(None); + .unwrap_or_else(|| state.run_dir.join("sessions").join(&id)); + if session_dir.exists() { + let delete_state = Arc::clone(&state); + tokio::task::spawn_blocking(move || delete_state.delete_session_dir(&session_dir)) + .await + .map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("one-shot cleanup task failed: {error}"), + ) + })? + .map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("one-shot cleanup failed: {error:#}"), + ) + })?; } - tokio::task::spawn_blocking(move || state.delete_session_dir(&session_dir)) - .await - .map_err(|error| { - AppError( - StatusCode::INTERNAL_SERVER_ERROR, - format!("one-shot cleanup task failed: {error}"), - ) - })? - .map_err(|error| { - AppError( - StatusCode::INTERNAL_SERVER_ERROR, - format!("one-shot cleanup failed: {error:#}"), - ) - })?; + state.ledger_workers.retire(&id).await.map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("one-shot checkpoint cleanup failed: {error:#}"), + ) + })?; Ok(None) } diff --git a/crates/capsem-service/src/session_cleanup/tests.rs b/crates/capsem-service/src/session_cleanup/tests.rs index 62b50fdef..b6646e915 100644 --- a/crates/capsem-service/src/session_cleanup/tests.rs +++ b/crates/capsem-service/src/session_cleanup/tests.rs @@ -139,7 +139,11 @@ async fn one_shot_completion_deletes_success_and_preserves_ipc_failure() { let clean_id = "code-run-clean"; let clean_dir = state.run_dir.join("sessions").join(clean_id); + let commitments = state.run_dir.join("ledger-commitments"); + let clean_checkpoints = commitments.join(capsem_crypto::blake3::hash(clean_id.as_bytes()).to_hex().to_string()); std::fs::create_dir_all(&clean_dir).unwrap(); + std::fs::create_dir_all(&clean_checkpoints).unwrap(); + std::fs::write(clean_checkpoints.join("checkpoints.log"), b"trusted").unwrap(); std::fs::write(clean_dir.join("process.log"), b"command completed").unwrap(); finalize_one_shot_session( Arc::clone(&state), @@ -150,10 +154,29 @@ async fn one_shot_completion_deletes_success_and_preserves_ipc_failure() { .await .unwrap(); assert!(!clean_dir.exists(), "successful one-shot state must be deleted"); + assert!( + !clean_checkpoints.exists(), + "successful one-shot checkpoints must be retired" + ); + + let already_deleted_id = "code-run-already-deleted"; + let already_deleted_checkpoints = commitments.join( + capsem_crypto::blake3::hash(already_deleted_id.as_bytes()) + .to_hex() + .to_string(), + ); + std::fs::create_dir_all(&already_deleted_checkpoints).unwrap(); + finalize_one_shot_session(Arc::clone(&state), already_deleted_id.to_string(), None, false) + .await + .unwrap(); + assert!(!already_deleted_checkpoints.exists()); let failed_id = "code-run-ipc"; let failed_dir = state.run_dir.join("sessions").join(failed_id); + let failed_checkpoints = commitments.join(capsem_crypto::blake3::hash(failed_id.as_bytes()).to_hex().to_string()); std::fs::create_dir_all(&failed_dir).unwrap(); + std::fs::create_dir_all(&failed_checkpoints).unwrap(); + std::fs::write(failed_checkpoints.join("checkpoints.log"), b"trusted").unwrap(); std::fs::write(failed_dir.join("process.log"), b"exec IPC closed").unwrap(); finalize_one_shot_session( Arc::clone(&state), @@ -166,6 +189,17 @@ async fn one_shot_completion_deletes_success_and_preserves_ipc_failure() { assert!(!failed_dir.exists()); let preserved = find_failed_session_dir(&state.run_dir, failed_id) .expect("one-shot exec IPC loss must retain the process evidence"); + let preserved_id = preserved.file_name().unwrap().to_str().unwrap(); + let preserved_checkpoints = commitments.join( + capsem_crypto::blake3::hash(preserved_id.as_bytes()) + .to_hex() + .to_string(), + ); + assert!(!failed_checkpoints.exists()); + assert_eq!( + std::fs::read(preserved_checkpoints.join("checkpoints.log")).unwrap(), + b"trusted" + ); assert_eq!( capsem_foundation::telemetry::read_log_tail(&preserved.join("process.log"), usize::MAX).unwrap(), "exec IPC closed" diff --git a/crates/capsem-service/src/session_db_handles.rs b/crates/capsem-service/src/session_db_handles.rs index 66cf40157..baa030487 100644 --- a/crates/capsem-service/src/session_db_handles.rs +++ b/crates/capsem-service/src/session_db_handles.rs @@ -21,7 +21,7 @@ struct RemoteLedger { session_dir: PathBuf, workers: Arc, client: tokio::sync::Mutex, - counters: Mutex>>, + observed_remote_epoch: Mutex>, read_cache_epoch: AtomicU64, } @@ -40,7 +40,7 @@ impl SessionLedger { session_dir, workers, client: tokio::sync::Mutex::new(client), - counters: Mutex::new(None), + observed_remote_epoch: Mutex::new(None), read_cache_epoch: AtomicU64::new(0), }), #[cfg(test)] @@ -66,10 +66,7 @@ impl SessionLedger { if let Some(db) = &self.embedded { return db.ready().await; } - self.remote_state() - .call(|client| async move { client.counters().await }) - .await - .map(|_| ()) + self.remote_state().refresh_counters().await.map(|_| ()) } pub(crate) async fn query(&self, query: LedgerQuery) -> Result, String> { @@ -90,17 +87,7 @@ impl SessionLedger { if let Some(db) = &self.embedded { return db.ledger_counters().await; } - let current = self - .remote_state() - .call(|client| async move { client.counters().await }) - .await?; - let mut previous = self.remote_state().counters.lock().unwrap(); - if previous.as_deref() != Some(current.as_ref()) { - *previous = Some(Arc::clone(¤t)); - self.remote_state().read_cache_epoch.fetch_add(1, Ordering::AcqRel); - } - drop(previous); - Ok(current) + self.remote_state().refresh_counters().await } #[cfg(test)] @@ -160,6 +147,22 @@ impl SessionLedger { } impl RemoteLedger { + async fn refresh_counters(&self) -> Result, String> { + let (current, remote_epoch) = self + .call(|client| async move { + let counters = client.counters().await?; + Ok((counters, client.read_cache_epoch())) + }) + .await?; + let mut observed = self.observed_remote_epoch.lock().unwrap(); + if *observed != Some(remote_epoch) { + *observed = Some(remote_epoch); + self.read_cache_epoch.fetch_add(1, Ordering::AcqRel); + } + drop(observed); + Ok(current) + } + async fn call(&self, operation: F) -> Result where F: Fn(capsem_logger::ledger_client::LedgerClient) -> Fut, @@ -175,6 +178,7 @@ impl RemoteLedger { .reconnect() .await .map_err(|error| format!("ledger operation failed ({first_error}); reconnect failed: {error}"))?; + *self.observed_remote_epoch.lock().unwrap() = None; *client = replacement.clone(); drop(client); operation(replacement).await diff --git a/crates/capsem-service/src/session_housekeeping.rs b/crates/capsem-service/src/session_housekeeping.rs index abf754f35..88171da01 100644 --- a/crates/capsem-service/src/session_housekeeping.rs +++ b/crates/capsem-service/src/session_housekeeping.rs @@ -45,9 +45,8 @@ impl ServiceState { /// every such path. /// /// If the rename fails (EEXIST, permission, different filesystem, - /// etc.) we `warn!` with the specific error and fall back to - /// `remove_dir_all` so disk isn't leaked when the filesystem is - /// already unhappy. + /// etc.) we `warn!` with the specific error and fall back to the same + /// contained deletion used by every permanent session cleanup. pub(crate) fn preserve_failed_session_dir(&self, session_dir: &std::path::Path, id: &str) -> Option { let failed_id = format!("{}-failed-{}", id, capsem_core::session::generate_session_id(),); let failed_dir = self.run_dir.join("sessions").join(&failed_id); @@ -57,6 +56,18 @@ impl ServiceState { .and_then(|()| std::fs::rename(session_dir, &failed_dir)); match moved { Ok(()) => { + if let Err(error) = self.ledger_workers.preserve_quiesced(id, &failed_id) { + warn!(id, failed_id, %error, "failed to preserve trusted ledger checkpoints; discarding unauthenticated evidence"); + if let Err(delete_error) = self.delete_session_dir(&failed_dir) { + warn!(id, failed_id, %delete_error, "failed to discard unauthenticated session evidence"); + } + for checkpoint_id in [id, failed_id.as_str()] { + if let Err(retire_error) = self.ledger_workers.retire_quiesced(checkpoint_id) { + warn!(checkpoint_id, %retire_error, "failed to retire unpreserved ledger checkpoints"); + } + } + return None; + } info!( id, path = %failed_dir.display(), @@ -78,7 +89,7 @@ impl ServiceState { error = %e, "failed to preserve session dir for post-mortem -- logs lost; removing to reclaim disk" ); - if let Err(e) = std::fs::remove_dir_all(session_dir) { + if let Err(e) = self.delete_session_dir(session_dir) { warn!( id, path = %session_dir.display(), @@ -86,6 +97,9 @@ impl ServiceState { "also failed to remove session dir -- orphaned on disk" ); } + if let Err(error) = self.ledger_workers.retire_quiesced(id) { + warn!(id, %error, "failed to retire checkpoints after lost session evidence"); + } None } } @@ -161,9 +175,16 @@ impl ServiceState { let mut culled = 0; for (path, _) in failed_dirs.iter().take(to_delete) { info!(path = %path.display(), "culling old failed session dir"); - match std::fs::remove_dir_all(path) { - Ok(()) => culled += 1, - Err(e) => warn!(path = %path.display(), error = %e, "cull remove_dir_all failed"), + match self.delete_session_dir(path) { + Ok(()) => { + if let Some(id) = path.file_name().and_then(|name| name.to_str()) { + if let Err(error) = self.ledger_workers.retire_quiesced(id) { + warn!(id, %error, "failed to retire culled session checkpoints"); + } + } + culled += 1; + } + Err(e) => warn!(path = %path.display(), error = %e, "cull failed session delete failed"), } } Ok(culled) diff --git a/crates/capsem-service/src/standalone_proxy.rs b/crates/capsem-service/src/standalone_proxy.rs index 65be13c4f..a2c338134 100644 --- a/crates/capsem-service/src/standalone_proxy.rs +++ b/crates/capsem-service/src/standalone_proxy.rs @@ -52,7 +52,7 @@ impl ServiceState { )); } - let session_id = format!("proxy-{}", uuid::Uuid::new_v4().simple()); + let session_id = format!("proxy-{}", capsem_crypto::uuid_v4().simple()); let session_dir = self.run_dir.join("sessions").join(&session_id); capsem_foundation::unix::fs::ensure_private_dir(&session_dir) .map_err(|error| internal(format!("create standalone proxy session: {error}")))?; @@ -166,7 +166,7 @@ impl ServiceState { } let broker_task = broker.start(authority.grant()); let accept_task = spawn_accept_loop(listener, worker.clone(), authority.grant()); - let lease_token = uuid::Uuid::new_v4().simple().to_string(); + let lease_token = capsem_crypto::uuid_v4().simple().to_string(); Ok(( StandaloneProxy { lease_token, diff --git a/crates/capsem-service/src/standalone_proxy/tests.rs b/crates/capsem-service/src/standalone_proxy/tests.rs index 66d630552..4b5afca0c 100644 --- a/crates/capsem-service/src/standalone_proxy/tests.rs +++ b/crates/capsem-service/src/standalone_proxy/tests.rs @@ -3,6 +3,22 @@ use serde_json::json; use super::*; +async fn running_proxy(state: &ServiceState, session_id: &str, lease_token: &str) -> StandaloneProxy { + let session_dir = state.run_dir.join("sessions").join(session_id); + std::fs::create_dir_all(&session_dir).unwrap(); + StandaloneProxy { + lease_token: lease_token.to_string(), + expires_at: Instant::now() + LEASE_TTL, + worker: crate::proxy_worker::tests::fake("normal").await.unwrap(), + authority: WorkerAuthority::default(), + accept_task: tokio::spawn(async {}), + broker_task: tokio::spawn(async {}), + session_dir, + provider_id: "openai".to_string(), + upstream_policy: crate::upstream_broker::test_policy_publisher(), + } +} + #[tokio::test] async fn create_rejects_a_non_ip_bind_before_starting_any_worker() { let state = crate::tests::make_test_state(); @@ -39,6 +55,122 @@ async fn create_rejects_an_empty_provider_before_starting_any_worker() { assert!(state.standalone_proxies.lock().await.is_empty()); } +#[tokio::test] +async fn create_cleans_its_session_when_provider_or_worker_startup_fails() { + for provider in ["missing-provider", "openai"] { + let state = crate::tests::make_test_state(); + std::fs::create_dir_all(state.run_dir.join("sessions")).unwrap(); + let app = crate::router_runtime::build_service_router(Arc::clone(&state)); + + let (status, body) = crate::tests::route_request( + app, + Method::POST, + "/proxies", + Some(json!({"provider": provider, "bind": "127.0.0.1", "port": 0})), + ) + .await; + + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR, "{body}"); + assert!( + body["error"].as_str().unwrap().contains("start standalone proxy"), + "{body}" + ); + assert!(state.standalone_proxies.lock().await.is_empty()); + let sessions = state.run_dir.join("sessions"); + assert!( + !sessions.exists() || std::fs::read_dir(sessions).unwrap().next().is_none(), + "failed startup retained its private session" + ); + } +} + +#[tokio::test] +async fn unknown_proxy_lease_routes_fail_without_mutating_registry() { + let state = crate::tests::make_test_state(); + let app = crate::router_runtime::build_service_router(Arc::clone(&state)); + for (method, path) in [ + (Method::POST, "/proxies/missing/heartbeat"), + (Method::POST, "/proxies/missing/stop"), + ] { + let (status, body) = + crate::tests::route_request(app.clone(), method, path, Some(json!({"lease_token": "unknown"}))).await; + assert_eq!(status, StatusCode::NOT_FOUND, "{body}"); + assert_eq!(body["error"], "proxy session not found"); + } + assert!(state.standalone_proxies.lock().await.is_empty()); +} + +#[tokio::test] +async fn empty_proxy_registry_refresh_and_shutdown_are_noops() { + let state = crate::tests::make_test_state(); + + assert_eq!(refresh_policies(&state).await.unwrap(), (0, Vec::new())); + state.stop_all_standalone_proxies().await; + assert!(state.standalone_proxies.lock().await.is_empty()); +} + +#[tokio::test] +async fn heartbeat_and_stop_require_the_exact_lease_and_cleanup_the_worker() { + let state = crate::tests::make_test_state(); + let id = "proxy-live"; + let proxy = running_proxy(&state, id, "secret").await; + state.standalone_proxies.lock().await.insert(id.to_string(), proxy); + let app = crate::router_runtime::build_service_router(Arc::clone(&state)); + + let (status, body) = crate::tests::route_request( + app.clone(), + Method::POST, + &format!("/proxies/{id}/heartbeat"), + Some(json!({"lease_token": "wrong"})), + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN, "{body}"); + + let (status, body) = crate::tests::route_request( + app.clone(), + Method::POST, + &format!("/proxies/{id}/heartbeat"), + Some(json!({"lease_token": "secret"})), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body["session_id"], id); + + let (status, body) = crate::tests::route_request( + app, + Method::POST, + &format!("/proxies/{id}/stop"), + Some(json!({"lease_token": "secret"})), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!(body, json!({"session_id": id, "stopped": true})); + assert!(state.standalone_proxies.lock().await.is_empty()); + assert!(!state.run_dir.join("sessions").join(id).exists()); +} + +#[tokio::test] +async fn expired_lease_revokes_and_cleans_up_its_worker_automatically() { + let state = crate::tests::make_test_state(); + let id = "proxy-expired"; + let mut proxy = running_proxy(&state, id, "secret").await; + proxy.expires_at = Instant::now(); + state.standalone_proxies.lock().await.insert(id.to_string(), proxy); + + spawn_lease_monitor(Arc::clone(&state), id.to_string()); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if state.standalone_proxies.lock().await.is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("expired lease was not removed"); + assert!(!state.run_dir.join("sessions").join(id).exists()); +} + #[test] fn wire_expiry_is_bounded_by_the_lease_ttl() { let before = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis() as u64; diff --git a/crates/capsem-service/src/switches.rs b/crates/capsem-service/src/switches.rs index ab6a7c8ab..9af3de678 100644 --- a/crates/capsem-service/src/switches.rs +++ b/crates/capsem-service/src/switches.rs @@ -451,7 +451,7 @@ pub(crate) async fn plug(state: &Arc, network: Uuid, vm_id: &str) drop(registry); (generation, address) }; - let connection = Uuid::new_v4(); + let connection = capsem_crypto::uuid_v4(); match handshake(state, network, vm_id, address, generation).await { Ok((host, port, keepalive)) => { let mut registry = state.networks.lock().await; diff --git a/crates/capsem-service/src/tests.rs b/crates/capsem-service/src/tests.rs index 081e347bd..bbf14ee13 100644 --- a/crates/capsem-service/src/tests.rs +++ b/crates/capsem-service/src/tests.rs @@ -45,6 +45,8 @@ mod asset_status; mod asset_wait; mod instance_reaper; mod policy_push; +#[path = "tests/session_housekeeping.rs"] +mod session_housekeeping_tests; pub(crate) static SETTINGS_ENV_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); @@ -192,7 +194,7 @@ pub(super) fn make_asset_state(assets_dir: PathBuf) -> Arc { /// The fields every fake instance shares; a test names only what it varies. pub(crate) fn test_instance() -> InstanceInfo { InstanceInfo { - generation: uuid::Uuid::new_v4(), + generation: capsem_crypto::uuid_v4(), authority: Default::default(), upstream_policy: upstream_broker::test_policy_publisher(), id: String::new(), @@ -236,7 +238,8 @@ pub(crate) fn spawn_fake_process( std::fs::write(uds_path.with_extension("ready"), b"ready").unwrap(); tokio::spawn(async move { let mut messages = Vec::new(); - for received in 0..expected { + while messages.len() < expected { + let received = messages.len(); // A route that stops talking to its VM used to hang the whole // binary here: tests serialized on SETTINGS_ENV_LOCK stalled // behind the one waiting forever. Fail with what was missing. @@ -258,6 +261,12 @@ pub(crate) fn spawn_fake_process( capsem_foundation::ipc_channel::Receiver, ) = capsem_foundation::ipc_channel::channel_from_std(std_stream).unwrap(); let message = rx.recv().await.unwrap(); + if let ServiceToProcess::InjectCredentials { id, .. } = &message { + tx.send(ProcessToService::CredentialsInjected { id: *id, error: None }) + .await + .unwrap(); + continue; + } if let Some(reply) = handler(&message).await { tx.send(reply).await.unwrap(); } @@ -338,24 +347,17 @@ pub(crate) fn spawn_fake_fork_owner( } /// A fake process that answers ping, and reloads by reporting the digest of -/// the active policy it finds in its session, as capsem-process does. +/// the exact active policy the service delivered, as capsem-process does. pub(crate) fn spawn_fake_process_reload_ack( uds_path: &StdPath, expected: usize, ) -> tokio::task::JoinHandle> { - let active_policy = uds_path - .parent() - .unwrap() - .join(ACTIVE_POLICY_DIR) - .join(ACTIVE_POLICY_FILE); - spawn_fake_process(uds_path, expected, move |message| { + spawn_fake_process(uds_path, expected, |message| { let reply = match message { ServiceToProcess::Ping => Some(ProcessToService::Pong), - ServiceToProcess::ReloadConfig { id } => Some(ProcessToService::ConfigReloadResult { + ServiceToProcess::ReloadConfig { id, active_policy } => Some(ProcessToService::ConfigReloadResult { id: *id, - active_policy_digest: Some(capsem_core::net::policy_config::active_policy_digest( - &std::fs::read(&active_policy).unwrap(), - )), + active_policy_digest: Some(capsem_core::net::policy_config::active_policy_digest(active_policy)), error: None, }), _ => None, diff --git a/crates/capsem-service/src/tests/assets_registry.rs b/crates/capsem-service/src/tests/assets_registry.rs index 26720e8c5..16f751c3c 100644 --- a/crates/capsem-service/src/tests/assets_registry.rs +++ b/crates/capsem-service/src/tests/assets_registry.rs @@ -597,22 +597,22 @@ fn drain_dead_instances_releases_mutex_before_returning() { "mutex still held after drain_dead_instances returned" ); } - pub(crate) fn make_state_in(test_root: PathBuf) -> Arc { let run_dir = test_root.join("run"); std::fs::create_dir_all(run_dir.join("sessions")).unwrap(); let mut state = make_test_state_owned(); - state.persistent_registry = SharedRegistry::new( - PersistentRegistry::load(run_dir.join("persistent_registry.json")).expect("registry loads"), - ); - state.networks = tokio::sync::Mutex::new(capsem_core::net::network_registry::NetworkRegistry::new(PathBuf::from( - "/nonexistent/networks", - ))); + let registry = PersistentRegistry::load(run_dir.join("persistent_registry.json")).unwrap(); + state.persistent_registry = SharedRegistry::new(registry); + let networks = capsem_core::net::network_registry::NetworkRegistry::new("/nonexistent/networks".into()); + state.networks = tokio::sync::Mutex::new(networks); state.service_socket = PathBuf::from("/nonexistent/service.sock"); state.asset_status_path = asset_status_path_for_run_dir(&run_dir); state.host_ledger = test_host_ledger(&run_dir); + state.ledger_workers = Arc::new(ledger_worker::LedgerWorkers::new( + "/no".into(), + run_dir.join("ledger-commitments"), + )); state.run_dir = run_dir; - // The caller owns `test_root`; the fixture's own temporary root goes. state._test_tempdir = None; Arc::new(state) } diff --git a/crates/capsem-service/src/tests/files_api.rs b/crates/capsem-service/src/tests/files_api.rs index c2855757f..2d2b49e9e 100644 --- a/crates/capsem-service/src/tests/files_api.rs +++ b/crates/capsem-service/src/tests/files_api.rs @@ -83,7 +83,7 @@ pub(super) fn setup_vm_with_workspace_and_uds( state.instances.lock().unwrap().insert( vm_id.into(), InstanceInfo { - generation: uuid::Uuid::new_v4(), + generation: capsem_crypto::uuid_v4(), authority: Default::default(), upstream_policy: upstream_broker::test_policy_publisher(), id: vm_id.into(), diff --git a/crates/capsem-service/src/tests/instance_reaper.rs b/crates/capsem-service/src/tests/instance_reaper.rs index 36585f09a..732dd397d 100644 --- a/crates/capsem-service/src/tests/instance_reaper.rs +++ b/crates/capsem-service/src/tests/instance_reaper.rs @@ -1,33 +1,128 @@ use super::*; #[tokio::test] -async fn exited_generation_is_revoked_while_its_pid_is_still_reserved() { +async fn vm_shutdown_targets_owned_child_even_when_cached_pid_names_an_unrelated_process() { let (state, _dir) = make_test_state_with_tempdir(); - let id = "unreaped-owner"; + let id = "owned-signal-target"; let session = state.run_dir.join("sessions").join(id); std::fs::create_dir_all(&session).unwrap(); - let mut child = tokio::process::Command::new("sh") - .args(["-c", "sleep 0.05; exit 11"]) + let child = tokio::process::Command::new("sleep") + .arg("30") + .kill_on_drop(true) .spawn() .unwrap(); - let pid = child.id().unwrap(); - insert_fake_instance_with_session_dir(&state, id, pid, session); - let generation = state.instances.lock().unwrap()[id].generation; - let grant = state.instances.lock().unwrap()[id].authority.grant(); + let mut unrelated = tokio::process::Command::new("sleep") + .arg("30") + .kill_on_drop(true) + .spawn() + .unwrap(); + insert_fake_instance_with_session_dir(&state, id, unrelated.id().unwrap(), session.clone()); + let generation = state.instances.lock().unwrap().get(id).unwrap().generation; + let registration = state.retirements.register(id, generation).unwrap(); + let reaper = crate::instance_reaper::spawn_exit_reaper( + child, + id.into(), + id.into(), + state.clone(), + state.instance_socket_path(id).unwrap(), + session, + registration, + ); + let result = tokio::time::timeout( + std::time::Duration::from_secs(3), + shutdown_vm_process(&state, id, ShutdownMode::Discard, Some(generation)), + ) + .await + .unwrap() + .unwrap(); + assert!(result.is_some()); + assert!( + unrelated.try_wait().unwrap().is_none(), + "shutdown signaled a cached/recycled PID" + ); + reaper.await.unwrap(); + assert!(state.retirements.wait(id, generation).await.unwrap()); + unrelated.kill().await.unwrap(); +} - let revoked = crate::instance_reaper::revoke_exited_generation(&child, id, &state, generation).await; +#[tokio::test] +async fn vm_shutdown_without_original_child_authority_cannot_acknowledge_exit() { + let (state, _dir) = make_test_state_with_tempdir(); + let id = "missing-child-authority"; + let session = state.run_dir.join("sessions").join(id); + std::fs::create_dir_all(&session).unwrap(); + let mut unrelated = tokio::process::Command::new("sleep") + .arg("30") + .kill_on_drop(true) + .spawn() + .unwrap(); + insert_fake_instance_with_session_dir(&state, id, unrelated.id().unwrap(), session.clone()); + let generation = state.instances.lock().unwrap().get(id).unwrap().generation; + let result = shutdown_vm_process(&state, id, ShutdownMode::Discard, Some(generation)).await; + assert!(matches!(result, Err(AppError(StatusCode::SERVICE_UNAVAILABLE, _)))); + assert!(unrelated.try_wait().unwrap().is_none()); + assert!(session.exists(), "unknown exit must not remove owned session bytes"); + assert!( + state.instances.lock().unwrap().contains_key(id), + "unknown owner cannot claim the instance" + ); + unrelated.kill().await.unwrap(); +} - assert!(!state.instances.lock().unwrap().contains_key(id)); - tokio::time::timeout(std::time::Duration::from_millis(100), grant.revoked()) +#[tokio::test] +async fn service_shutdown_uses_original_child_owner_before_removing_session_bytes() { + let (state, _dir) = make_test_state_with_tempdir(); + let launch = state.lifecycle.admit().unwrap(); + let id = "service-owned-shutdown"; + let session = state.run_dir.join("sessions").join(id); + std::fs::create_dir_all(&session).unwrap(); + std::fs::write(session.join("marker"), b"original").unwrap(); + let child = tokio::process::Command::new("sleep") + .arg("30") + .kill_on_drop(true) + .spawn() + .unwrap(); + let mut unrelated = tokio::process::Command::new("sleep") + .arg("30") + .kill_on_drop(true) + .spawn() + .unwrap(); + let shutdown = { + let state = state.clone(); + tokio::spawn(async move { kill_all_vm_processes(&state).await }) + }; + capsem_foundation::poll::poll_until( + capsem_foundation::poll::PollOpts::new("shutdown-seals-admission", std::time::Duration::from_secs(1)), + || std::future::ready(state.lifecycle.admit().is_err().then_some(())), + ) + .await + .unwrap(); + assert!(!shutdown.is_finished(), "shutdown missed the admitted original worker"); + insert_fake_instance_with_session_dir(&state, id, unrelated.id().unwrap(), session.clone()); + let generation = state.instances.lock().unwrap().get(id).unwrap().generation; + let registration = state.retirements.register(id, generation).unwrap(); + let reaper = crate::instance_reaper::spawn_exit_reaper( + child, + id.into(), + id.into(), + state.clone(), + state.instance_socket_path(id).unwrap(), + session.clone(), + registration, + ); + drop(launch); + tokio::time::timeout(std::time::Duration::from_secs(4), shutdown) .await - .expect("exit observation revokes grants before the removed instance is dropped"); - let pid = capsem_foundation::unix::process::ProcessId::try_from(pid).unwrap(); + .unwrap() + .unwrap(); assert!( - capsem_foundation::unix::process::child_has_exited(pid).unwrap(), - "generation revocation must happen before the child is reaped" + unrelated.try_wait().unwrap().is_none(), + "service stop targeted unrelated cached PID" ); - drop(revoked); - assert_eq!(child.wait().await.unwrap().code(), Some(11)); + reaper.await.unwrap(); + assert!(state.retirements.wait(id, generation).await.unwrap()); + assert!(!session.exists()); + unrelated.kill().await.unwrap(); } #[tokio::test] @@ -185,24 +280,6 @@ fn child_reapers_start_after_instance_registration() { } } -#[test] -fn child_exit_authority_is_revoked_before_reaping() { - let source = include_str!("../instance_reaper.rs") - .split_once("pub(super) fn spawn_exit_reaper") - .unwrap() - .1; - let revoke = source - .find("revoke_exited_generation(&child") - .expect("the reaper observes exit and revokes the registered generation"); - let reap = source - .find("child.wait().await") - .expect("the reaper eventually consumes the child exit status"); - assert!( - revoke < reap, - "the kernel must still reserve the worker PID when its authority is revoked" - ); -} - /// There is one child reaper. The resume path had its own, which removed the /// instance and nothing else: no session-index stop, no checkpoint or defunct /// bookkeeping in the persistent registry, no crash evidence. A resumed diff --git a/crates/capsem-service/src/tests/ipc_command.rs b/crates/capsem-service/src/tests/ipc_command.rs index 292b27857..f482904b2 100644 --- a/crates/capsem-service/src/tests/ipc_command.rs +++ b/crates/capsem-service/src/tests/ipc_command.rs @@ -126,7 +126,7 @@ async fn replaced_generation_cannot_supply_an_accepted_command_reply() { .unwrap() .get_mut("replaced") .unwrap() - .generation = uuid::Uuid::new_v4(); + .generation = capsem_crypto::uuid_v4(); let reply = ProcessToService::ExecResult { id: *id, stdout: b"old generation".to_vec(), diff --git a/crates/capsem-service/src/tests/policy_push.rs b/crates/capsem-service/src/tests/policy_push.rs index 4776223e2..dd2f8b2b9 100644 --- a/crates/capsem-service/src/tests/policy_push.rs +++ b/crates/capsem-service/src/tests/policy_push.rs @@ -195,7 +195,7 @@ async fn a_reload_acknowledging_another_active_policy_fails_the_push() { let (state, _, _env) = state_with_running_vms(&dir, &["stale-ack-vm"]); let process = spawn_fake_process(&uds_path(&state, "stale-ack-vm"), 1, |message| { let reply = match message { - ServiceToProcess::ReloadConfig { id } => Some(ProcessToService::ConfigReloadResult { + ServiceToProcess::ReloadConfig { id, .. } => Some(ProcessToService::ConfigReloadResult { id: *id, active_policy_digest: Some("blake3:some-other-policy".to_string()), error: None, @@ -225,15 +225,15 @@ async fn a_reload_ack_from_a_replaced_generation_cannot_update_its_broker() { let dir = tempfile::tempdir().unwrap(); let (state, sessions, _env) = state_with_running_vms(&dir, &["replaced-vm"]); let session_dir = sessions[0].clone(); - let active_path = session_dir.join("vm/active_policy.toml"); let process = spawn_fake_process(&uds_path(&state, "replaced-vm"), 1, { let state = Arc::clone(&state); move |message| { - let ServiceToProcess::ReloadConfig { id } = *message else { + let ServiceToProcess::ReloadConfig { id, active_policy } = message else { return Box::pin(async { None }); }; + let id = *id; + let digest = capsem_core::net::policy_config::active_policy_digest(active_policy); insert_fake_instance_with_session_dir(&state, "replaced-vm", std::process::id(), session_dir.clone()); - let digest = capsem_core::net::policy_config::active_policy_digest(&std::fs::read(&active_path).unwrap()); Box::pin(async move { Some(ProcessToService::ConfigReloadResult { id, @@ -258,9 +258,9 @@ async fn a_reload_ack_from_a_replaced_generation_cannot_update_its_broker() { /// Start edit A, hold its VM acknowledgement, then start edit B. Release A /// once B is either waiting for A or has already rewritten A's session. The -/// fake VM answers each reload with the digest of the active policy it finds -/// when it answers, as capsem-process does. Returns the session's active -/// policy after both edits. +/// fake VM answers each reload with the digest of the active policy bytes in +/// that request, as capsem-process does. Returns the session's active policy +/// after both edits. async fn race_edit_b_against_held_edit_a( edit_b: impl FnOnce(Arc) -> tokio::task::JoinHandle>, ) -> ActivePolicyFile { @@ -272,13 +272,13 @@ async fn race_edit_b_against_held_edit_a( let (release, released) = tokio::sync::oneshot::channel::<()>(); let released = Arc::new(std::sync::Mutex::new(Some(released))); let process = spawn_fake_process(&uds_path(&state, "race-vm"), 2, { - let (held, active_path) = (Arc::clone(&held), active_path.clone()); + let held = Arc::clone(&held); move |message| { - let ServiceToProcess::ReloadConfig { id } = *message else { + let ServiceToProcess::ReloadConfig { id, active_policy } = message else { return Box::pin(async { None }); }; - let (held, released, active_path) = - (Arc::clone(&held), released.lock().unwrap().take(), active_path.clone()); + let (id, active_policy) = (*id, active_policy.clone()); + let (held, released) = (Arc::clone(&held), released.lock().unwrap().take()); Box::pin(async move { if let Some(released) = released { held.notify_one(); @@ -286,9 +286,7 @@ async fn race_edit_b_against_held_edit_a( } Some(ProcessToService::ConfigReloadResult { id, - active_policy_digest: Some(capsem_core::net::policy_config::active_policy_digest( - &std::fs::read(&active_path).unwrap(), - )), + active_policy_digest: Some(capsem_core::net::policy_config::active_policy_digest(&active_policy)), error: None, }) }) diff --git a/crates/capsem-service/src/tests/session_housekeeping.rs b/crates/capsem-service/src/tests/session_housekeeping.rs index d3142ab97..f786cbbca 100644 --- a/crates/capsem-service/src/tests/session_housekeeping.rs +++ b/crates/capsem-service/src/tests/session_housekeeping.rs @@ -60,6 +60,31 @@ fn preserve_renames_session_dir_and_keeps_logs() { ); } +#[test] +fn preserve_discards_evidence_when_its_checkpoint_authority_cannot_move() { + let dir = tempfile::tempdir().unwrap(); + let state = make_state_in(dir.path().to_path_buf()); + let session_dir = state.run_dir.join("sessions").join("vm-broken-authority"); + std::fs::create_dir_all(&session_dir).unwrap(); + std::fs::write(session_dir.join("session.db"), b"untrusted without its checkpoints").unwrap(); + std::fs::write(state.run_dir.join("ledger-commitments"), b"not a directory").unwrap(); + + assert!( + state + .preserve_failed_session_dir(&session_dir, "vm-broken-authority") + .is_none(), + "evidence without its checkpoint authority must not be exposed" + ); + assert!(!session_dir.exists()); + assert!(std::fs::read_dir(state.run_dir.join("sessions")) + .unwrap() + .flatten() + .all(|entry| !entry + .file_name() + .to_string_lossy() + .starts_with("vm-broken-authority-failed-"))); +} + #[test] fn cull_keeps_newest_and_prunes_oldest() { let dir = tempfile::tempdir().unwrap(); @@ -275,6 +300,12 @@ async fn a_registered_session_handle_reads_bodies_after_the_process_trims_them() owner.flush().await.expect("flush"); } }; + let encoded = |payload| { + capsem_proto::forensic::SecurityForensicEvent::from_json(payload, "model.call") + .unwrap() + .encode() + .unwrap() + }; write_body("0000000000ab", r#"{"old":1}"#).await; // A block stays open across flushes; a retention that keeps everything // closes it, so the next body starts the second block this test trims to. @@ -291,22 +322,20 @@ async fn a_registered_session_handle_reads_bodies_after_the_process_trims_them() .expect("register the session handle"); assert_eq!( handle - .read_body( - "0000000000cd", - "security_rule_events", - capsem_logger::BodyDirection::Payload - ) + .read_bodies("0000000000cd") .await .expect("read a body") + .into_iter() + .find(|body| body.direction == capsem_logger::BodyDirection::Payload) .expect("the body is archived") .bytes, - br#"{"new":2}"# + encoded(r#"{"new":2}"#) ); // The VM's process trims on its way out. The handle above is still // registered: this is the window the reaper leaves open. let cutoff = { - let raw = handle + let raw = owner .query("SELECT sealed_at FROM body_blocks ORDER BY block_offset", &[]) .await .expect("read the block seal times"); @@ -317,28 +346,22 @@ async fn a_registered_session_handle_reads_bodies_after_the_process_trims_them() assert_eq!( handle - .read_body( - "0000000000cd", - "security_rule_events", - capsem_logger::BodyDirection::Payload - ) + .read_bodies("0000000000cd") .await .expect("a still-registered handle must follow the archive, not fail on it") + .into_iter() + .find(|body| body.direction == capsem_logger::BodyDirection::Payload) .expect("the surviving body is still archived") .bytes, - br#"{"new":2}"#, + encoded(r#"{"new":2}"#), "the same read returns the same bytes after the file was replaced under it" ); assert!( handle - .read_body( - "0000000000ab", - "security_rule_events", - capsem_logger::BodyDirection::Payload - ) + .read_bodies("0000000000ab") .await .expect("read the dropped body") - .is_none(), + .is_empty(), "and a dropped body is absent rather than stale bytes from the old inode" ); diff --git a/crates/capsem-service/src/tests/settings_files.rs b/crates/capsem-service/src/tests/settings_files.rs index 3576934e4..eb2a16d31 100644 --- a/crates/capsem-service/src/tests/settings_files.rs +++ b/crates/capsem-service/src/tests/settings_files.rs @@ -187,7 +187,7 @@ fn resolve_rejects_symlink_escape() { state.instances.lock().unwrap().insert( "test-vm".into(), InstanceInfo { - generation: uuid::Uuid::new_v4(), + generation: capsem_crypto::uuid_v4(), authority: Default::default(), upstream_policy: upstream_broker::test_policy_publisher(), id: "test-vm".into(), @@ -222,7 +222,7 @@ fn resolve_valid_path_inside_workspace() { state.instances.lock().unwrap().insert( "test-vm".into(), InstanceInfo { - generation: uuid::Uuid::new_v4(), + generation: capsem_crypto::uuid_v4(), authority: Default::default(), upstream_policy: upstream_broker::test_policy_publisher(), id: "test-vm".into(), diff --git a/crates/capsem-service/src/upstream_broker.rs b/crates/capsem-service/src/upstream_broker.rs index 07a9e0e6a..17637a214 100644 --- a/crates/capsem-service/src/upstream_broker.rs +++ b/crates/capsem-service/src/upstream_broker.rs @@ -75,6 +75,11 @@ struct ActiveGrant { adopted: bool, } +struct ActiveListenerGrant { + descriptor: Option, + adopted: bool, +} + enum LedgerSource { #[cfg(test)] Ready(LedgerClient), @@ -113,12 +118,13 @@ impl LedgerSource { enum AwaitingAdoption { Upstream(u64), Ledger(u64), + Listener(u64), } impl AwaitingAdoption { const fn grant_id(self) -> u64 { match self { - Self::Upstream(grant_id) | Self::Ledger(grant_id) => grant_id, + Self::Upstream(grant_id) | Self::Ledger(grant_id) | Self::Listener(grant_id) => grant_id, } } } @@ -259,6 +265,7 @@ async fn run( }); let mut selections = HashMap::new(); let mut active = HashMap::new(); + let mut listeners = HashMap::new(); let mut next_resource_id = 1_u64; let mut awaiting_adoption = None; let ledger_configured = ledger.is_some(); @@ -571,6 +578,18 @@ async fn run( awaiting_adoption = None; continue; } + if awaiting_adoption == Some(AwaitingAdoption::Listener(grant_id)) { + let Some(grant) = listeners.get_mut(&grant_id) else { + break Err(format!("listener grant {grant_id} disappeared before adoption")); + }; + if grant.adopted { + break Err(format!("listener grant {grant_id} was already adopted")); + } + grant.adopted = true; + drop(grant.descriptor.take()); + awaiting_adoption = None; + continue; + } let Some(grant) = active.get_mut(&grant_id) else { break Err(format!("adopted unknown descriptor grant {grant_id}")); }; @@ -582,13 +601,18 @@ async fn run( } UpstreamGrantRequest::Release { resource_id } => { if selections.remove(&resource_id).is_none() { - let Some(grant) = active.remove(&resource_id) else { + if let Some(grant) = active.remove(&resource_id) { + if !grant.adopted { + break Err(format!("released unadopted descriptor grant {resource_id}")); + } + revoke_grant(resource_id, grant)?; + } else if let Some(grant) = listeners.remove(&resource_id) { + if !grant.adopted { + break Err(format!("released unadopted listener grant {resource_id}")); + } + } else { break Err(format!("released unknown upstream resource {resource_id}")); - }; - if !grant.adopted { - break Err(format!("released unadopted descriptor grant {resource_id}")); } - revoke_grant(resource_id, grant)?; } } UpstreamGrantRequest::SetGuestMode { @@ -702,6 +726,34 @@ async fn run( } } } + UpstreamGrantRequest::AttachProxyTraceHints { request_id } => { + let Some(proxy) = proxy.as_ref() else { + send_denied(&responses, request_id, UpstreamGrantDenial::NotConfigured).await?; + continue; + }; + let descriptor = frame + .fds + .into_iter() + .next() + .expect("one validated trace-hint descriptor"); + match proxy + .grant(capsem_proto::proxy_control::ProxyCapability::TraceHints, descriptor) + .await + { + Ok(()) => { + send_response( + &responses, + &UpstreamGrantResponse::ProxyTraceHintsAdopted { request_id }, + None, + ) + .await?; + } + Err(error) => { + warn!(%error, "proxy worker refused trace-hint descriptor"); + send_denied(&responses, request_id, UpstreamGrantDenial::Revoked).await?; + } + } + } } } } diff --git a/crates/capsem-service/src/upstream_broker/tests.rs b/crates/capsem-service/src/upstream_broker/tests.rs index 30671eb6f..aa84aa97f 100644 --- a/crates/capsem-service/src/upstream_broker/tests.rs +++ b/crates/capsem-service/src/upstream_broker/tests.rs @@ -22,6 +22,67 @@ struct TestClient { responses: WireReceiver, } +#[tokio::test] +async fn loopback_listener_grant_is_bound_by_the_coordinator_and_usable_by_the_owner() { + let (client, _publisher, authority, task) = start_broker(test_policy("policy-a", None, vec![])); + let (response, mut fds) = client + .request(&UpstreamGrantRequest::OpenLoopbackListener { request_id: 1, port: 0 }) + .await; + let (grant_id, granted_port) = match response { + UpstreamGrantResponse::LoopbackListenerGranted { + request_id: 1, + grant_id, + port, + } => (grant_id, port), + response => panic!("unexpected listener grant response: {response:?}"), + }; + client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; + let listener = std::net::TcpListener::from(fds.pop().unwrap()); + assert!(fds.is_empty()); + let address = listener.local_addr().unwrap(); + assert_eq!(address.ip(), std::net::Ipv4Addr::LOCALHOST); + assert_eq!(address.port(), granted_port); + let client_stream = std::net::TcpStream::connect(address).unwrap(); + let (accepted, peer) = listener.accept().unwrap(); + assert_eq!(accepted.local_addr().unwrap(), address); + assert_eq!(peer, client_stream.local_addr().unwrap()); + drop(accepted); + drop(client_stream); + drop(listener); + client + .send(&UpstreamGrantRequest::Release { resource_id: grant_id }) + .await; + let (replacement, descriptors) = client + .request(&UpstreamGrantRequest::OpenLoopbackListener { + request_id: 2, + port: granted_port, + }) + .await; + let replacement_id = match replacement { + UpstreamGrantResponse::LoopbackListenerGranted { + request_id: 2, + grant_id, + port, + } => { + assert_eq!(port, granted_port); + grant_id + } + response => panic!("released listener port was not reusable: {response:?}"), + }; + client + .send(&UpstreamGrantRequest::Adopted { + grant_id: replacement_id, + }) + .await; + drop(descriptors); + client + .send(&UpstreamGrantRequest::Release { + resource_id: replacement_id, + }) + .await; + stop_broker(authority, task).await; +} + impl TestClient { fn new(socket: UnixStream) -> Self { Self { @@ -91,6 +152,18 @@ async fn typed_guest_traffic_is_adopted_by_the_registered_proxy_generation() { .await; assert_eq!(response, UpstreamGrantResponse::ProxyMcpAdopted { request_id: 2 }); + let (trace_hints, _trace_hints_peer) = UnixStream::pair().unwrap(); + let response = client + .request_with_descriptor( + &UpstreamGrantRequest::AttachProxyTraceHints { request_id: 3 }, + &trace_hints, + ) + .await; + assert_eq!( + response, + UpstreamGrantResponse::ProxyTraceHintsAdopted { request_id: 3 } + ); + authority.revoke(); assert!(task.await.unwrap().unwrap_err().contains("revoked")); } @@ -112,7 +185,7 @@ fn test_policy(digest: &str, tcp_override: Option, dns_upstreams: Ve } fn policy_digest(label: &str) -> String { - format!("blake3:{}", blake3::hash(label.as_bytes()).to_hex()) + format!("blake3:{}", capsem_crypto::blake3::hash(label.as_bytes()).to_hex()) } #[tokio::test] @@ -850,10 +923,13 @@ async fn active_descriptor_capacity_is_bounded_and_recovers_after_release() { UpstreamGrantRequest::OpenDns { request_id, upstream_index: 0, - }) - .await; - let UpstreamGrantResponse::DescriptorGranted { grant_id, .. } = response else { - panic!("expected descriptor grant, got {response:?}"); + } + }; + let (response, mut fds) = client.request(&request).await; + let grant_id = match response { + UpstreamGrantResponse::DescriptorGranted { grant_id, .. } + | UpstreamGrantResponse::LoopbackListenerGranted { grant_id, .. } => grant_id, + response => panic!("expected descriptor grant, got {response:?}"), }; client.send(&UpstreamGrantRequest::Adopted { grant_id }).await; grant_ids.push(grant_id); diff --git a/crates/capsem-service/src/vm_files.rs b/crates/capsem-service/src/vm_files.rs index a11a509ad..61c5eb357 100644 --- a/crates/capsem-service/src/vm_files.rs +++ b/crates/capsem-service/src/vm_files.rs @@ -627,9 +627,16 @@ pub(super) fn classify_attempt_decision(outcome: ProvisionAttemptOutcome, id: &s async fn failed_process_log_tail(state: &Arc, id: &str) -> String { let vm_id = id.to_string(); state - .off_worker(move |state| match find_failed_session_dir(&state.run_dir, &vm_id) { - Some(dir) => read_process_log_tail(&dir, 20), - None => "(no preserved log found)".to_string(), + .off_worker(move |state| { + if let Some(entry) = find_persistent_entry_by_route_id(&state, &vm_id) { + return entry + .last_error + .unwrap_or_else(|| read_process_log_tail(&entry.session_dir, 20)); + } + match find_failed_session_dir(&state.run_dir, &vm_id) { + Some(dir) => read_process_log_tail(&dir, 20), + None => "(no preserved log found)".to_string(), + } }) .await .unwrap_or_else(|error| format!("(log read failed: {})", error.1)) @@ -742,16 +749,9 @@ pub(super) async fn provision_attempt( launch::LaunchWait::Launched => ProvisionAttemptOutcome::Launched { uds_path }, launch::LaunchWait::TimedOut => ProvisionAttemptOutcome::StillBootingTimedOut { uds_path }, launch::LaunchWait::Crashed => { - // Crash before ready. Prefer the persistent entry's - // cached last_error (already computed by the child-exit - // handler) to avoid re-reading the log; fall back to - // find_failed_session_dir for ephemeral VMs whose dir was - // renamed to `-failed-*`. - let cached = find_persistent_entry_by_route_id(state, id).and_then(|e| e.last_error); - let tail = match cached { - Some(tail) => tail, - None => failed_process_log_tail(state, id).await, - }; + // The instance may disappear before its persistent last_error is + // cached. Read its log during that window; ephemeral evidence lives in `-failed-*`. + let tail = failed_process_log_tail(state, id).await; if is_launchd_cleanup_transient(&tail) { warn!( id, @@ -1019,83 +1019,6 @@ pub(super) async fn handle_stats_summary( Ok(Json(ledger_routes::activity::stats_summary(&counters))) } -/// Wait until a VM signals readiness via a `.ready` sentinel file. -/// The capsem-process creates this file once the guest handshake completes. -/// -/// If `state` and `id` are provided, also checks on every poll iteration that -/// the VM is still in the instance registry. The resume_sandbox / spawn child- -/// exit handlers remove the instance when capsem-process dies; observing that -/// removal lets us fail fast (within ~50ms) instead of polling the dead -/// sentinel for the full timeout. Without this, a capsem-process that crashes -/// or exits during boot/restore would hang the API for `timeout_secs` (was -/// reproducibly 30s under heavy suspend/resume churn). -#[tracing::instrument(skip_all, fields(timeout_secs))] -pub(super) async fn wait_for_vm_ready( - uds_path: &std::path::Path, - timeout_secs: u64, - state: Option<&Arc>, - id: Option<&str>, -) -> Result<(), String> { - let ready_span = tracing::debug_span!( - target: "capsem.launch", - capsem_foundation::telemetry::LAUNCH_VSOCK_READY_SPAN, - status = tracing::field::Empty, - ); - let ready_path = uds_path.with_extension("ready"); - // Override the PollOpts::new defaults (50ms / 500ms): VM ready-time is - // sub-second in the common case and the sentinel check is a single stat, - // so 500ms max_delay overshoots readiness by ~500ms and blows the - // exec_ready / boot_ready latency gates. Peer callers (service-connect, - // gateway-ready) wait for remote processes with seconds-scale startup - // where 500ms is appropriate; this poll is different. - let opts = vm_ready_poll_opts(timeout_secs); - let died: Arc = Arc::new(std::sync::atomic::AtomicBool::new(false)); - let res = capsem_foundation::poll::poll_until(opts, || { - let ready = ready_path.clone(); - let state = state.cloned(); - let id = id.map(|s| s.to_string()); - let died = Arc::clone(&died); - async move { - if launch::stamped(&ready, "ready") { - return Some(()); - } - if let (Some(st), Some(name)) = (state.as_ref(), id.as_ref()) { - if !st.instances.lock().unwrap().contains_key(name) { - died.store(true, std::sync::atomic::Ordering::Release); - // Returning Some short-circuits the poll loop; the - // outer caller distinguishes via `died`. - return Some(()); - } - } - None - } - }) - .instrument(ready_span.clone()) - .await; - if died.load(std::sync::atomic::Ordering::Acquire) { - ready_span.record("status", "error"); - return Err("capsem-process exited before signalling ready".into()); - } - match res { - Ok(()) => { - ready_span.record("status", "ok"); - Ok(()) - } - Err(error) => { - ready_span.record("status", "error"); - Err(format!("{error}")) - } - } -} - -pub(super) fn vm_ready_poll_opts(timeout_secs: u64) -> capsem_foundation::poll::PollOpts { - capsem_foundation::poll::PollOpts { - initial_delay: std::time::Duration::from_millis(5), - max_delay: std::time::Duration::from_millis(50), - ..capsem_foundation::poll::PollOpts::new("vm-ready", std::time::Duration::from_secs(timeout_secs)) - } -} - pub(super) fn running_uds_path(state: &ServiceState, id: &str) -> Result { let instances = state.instances.lock().unwrap(); let path = instances diff --git a/crates/capsem-service/src/vm_files/fork.rs b/crates/capsem-service/src/vm_files/fork.rs index dfa611402..d956ba191 100644 --- a/crates/capsem-service/src/vm_files/fork.rs +++ b/crates/capsem-service/src/vm_files/fork.rs @@ -278,7 +278,7 @@ async fn clone_ledger( let (stream, commitment, grant) = channel.into_parts(); debug_assert!(commitment.is_none()); let client = capsem_logger::ledger_client::LedgerClient::connect(stream, grant, database).await?; - let snapshot = client.snapshot(*uuid::Uuid::new_v4().as_bytes()).await?; + let snapshot = client.snapshot(*capsem_crypto::uuid_v4().as_bytes()).await?; let snapshot_copy = snapshot.clone(); let destination = destination.to_path_buf(); let copied = tokio::task::spawn_blocking(move || { diff --git a/crates/capsem-service/src/vm_files/provision/tests.rs b/crates/capsem-service/src/vm_files/provision/tests.rs index beef023cf..a0eaea15a 100644 --- a/crates/capsem-service/src/vm_files/provision/tests.rs +++ b/crates/capsem-service/src/vm_files/provision/tests.rs @@ -71,6 +71,21 @@ impl ImageSource for RefusingImages { } } +#[tokio::test] +async fn persistent_boot_crash_reads_the_log_before_the_reaper_caches_its_tail() { + let state = Arc::new(crate::tests::make_test_state_owned()); + let session_dir = state.run_dir.join("persistent").join("named-box"); + std::fs::create_dir_all(&session_dir).unwrap(); + std::fs::write(session_dir.join("process.log"), b"ledger confinement failed\n").unwrap(); + let mut entry = crate::tests::test_persistent_entry("named-box", session_dir); + entry.id = "box".into(); + state.persistent_registry.lock().unwrap().register(entry).unwrap(); + + let tail = failed_process_log_tail(&state, "box").await; + + assert_eq!(tail, "ledger confinement failed"); +} + /// A create whose container fails after the VM is registered used to answer /// 500 and leave the VM running: the caller never learned its id, and a named /// VM kept its name, so the retry got 409. The failed create is discarded -- diff --git a/crates/capsem-service/src/vm_files/readiness.rs b/crates/capsem-service/src/vm_files/readiness.rs index 00a51995d..7f8748bce 100644 --- a/crates/capsem-service/src/vm_files/readiness.rs +++ b/crates/capsem-service/src/vm_files/readiness.rs @@ -26,7 +26,7 @@ pub(crate) async fn wait_for_vm_ready( let id = id.map(|s| s.to_string()); let died = Arc::clone(&died); async move { - if ready.exists() { + if super::launch::stamped(&ready, "ready") { return Some(()); } if let (Some(st), Some(name)) = (state.as_ref(), id.as_ref()) { diff --git a/crates/capsem-service/src/vm_lifecycle.rs b/crates/capsem-service/src/vm_lifecycle.rs index c0ad6ee02..df7f4b02c 100644 --- a/crates/capsem-service/src/vm_lifecycle.rs +++ b/crates/capsem-service/src/vm_lifecycle.rs @@ -3,8 +3,10 @@ use super::*; mod provision; mod resume; mod resume_process; +mod run; #[cfg(test)] pub(crate) use resume_process::configure_resume_storage; +pub(crate) use run::handle_run; mod session_dirs; pub(crate) use resume::handle_resume; #[cfg(test)] @@ -352,8 +354,8 @@ pub(super) async fn shutdown_vm_process( }; if let Ok((tx, _)) = channel { capsem_core::try_send!("ipc_graceful_shutdown", tx.send(ServiceToProcess::Shutdown).await); - } else if pid > 0 { - process_control::send_or_log(pid, process_control::Signal::Terminate, "vm-ipc-shutdown-fallback"); + } else { + signal_vm_process(state, id, generation, process_control::Signal::Terminate).await?; } } else { // Destructive delete has no state to flush. SIGKILL also prevents the @@ -442,7 +444,7 @@ pub(super) async fn handle_suspend( } .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e))?; let uds_path = owner.uds_path.clone(); - let pid = owner.pid(); + let generation = owner.generation(); let (tx, rx) = owner .open(&state, "capsem-service", false) .await @@ -506,11 +508,16 @@ pub(super) async fn handle_suspend( // released its VZ checkpoint and VirtioFS share before resume can spawn a // replacement. The helper bounds natural exit, then SIGKILLs and waits for // reaping instead of guessing that a fixed post-kill sleep was sufficient. - wait_for_process_exit(pid, std::time::Duration::from_millis(500)).await; + wait_for_process_exit(&state, &id, generation, std::time::Duration::from_millis(500)).await?; owner .validate(&state, true) .map_err(|e| AppError(StatusCode::BAD_GATEWAY, e))?; + state + .connections + .stop_runtime(&id, generation, capsem_credentials::GrantStopPolicy::Preserve) + .await?; + tracing::warn!(id, "handle_suspend (success) removing instance"); state.evict_instance(&id, owner.generation()); state.unregister_session_db_handle(&id); @@ -558,6 +565,12 @@ pub(super) async fn handle_stop( format!("ephemeral session cleanup failed: {error:#}"), ) })?; + state.ledger_workers.retire(&id).await.map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("ephemeral checkpoint cleanup failed: {error:#}"), + ) + })?; } Ok(Json(api::StopResponse { success: true, @@ -615,6 +628,12 @@ pub(super) async fn handle_delete( format!("delete session state failed: {error:#}"), ) })?; + state.ledger_workers.retire(&id).await.map_err(|error| { + AppError( + StatusCode::INTERNAL_SERVER_ERROR, + format!("delete session checkpoints failed: {error:#}"), + ) + })?; // Unregister from persistent registry only after filesystem deletion // succeeds. An unsafe or failed delete therefore remains discoverable @@ -851,134 +870,3 @@ pub(super) async fn handle_purge( ephemeral_purged, })) } - -/// One-shot exec: provision a temp VM, run a command, return output, destroy VM. -pub(super) async fn handle_run( - State(state): State>, - Json(payload): Json, -) -> Result, AppError> { - let timeout_secs = - capsem_api::exec_timeout_secs(payload.timeout_secs).map_err(|e| AppError(StatusCode::BAD_REQUEST, e))?; - let _launch = state - .lifecycle - .admit() - .map_err(|e| AppError(StatusCode::CONFLICT, e.to_string()))?; - if let Some(reason) = state.off_worker(|state| vm_asset_block_reason(&state)).await? { - return Err(AppError(StatusCode::PRECONDITION_FAILED, reason)); - } - - let id = { - let existing = state.off_worker(|state| existing_session_names(&state)).await?; - generate_session_name(existing.iter().map(|s| s.as_str())) - }; - - let resources = resolve_vm_resources(payload.ram_mb, payload.cpus); - let ram_mb = resources.ram_mb; - let cpus = resources.cpus; - let scratch_disk_size_gb = resources.scratch_disk_size_gb; - - // 1. Provision ephemeral VM. `provision_sandbox` is synchronous and - // does heavy I/O (APFS clonefile, rootfs.img fsync, child spawn); - // offload to the blocking pool, matching `handle_provision` -- the - // tokio::process::Command::spawn inside still works because - // spawn_blocking preserves the runtime handle via thread-locals. - let state_clone = Arc::clone(&state); - let id_clone = id.clone(); - let version = state.current_version.clone(); - let env = payload.env.clone(); - { - let _vz_guard = state.lifecycle.vz.read().await; - let _vz_host_guard = acquire_vz_host_lock(startup::VzHostLockMode::Shared).await?; - let provision_result = tokio::task::spawn_blocking(move || { - state_clone.provision_sandbox(ProvisionOptions { - id: &id_clone, - name: &id_clone, - ram_mb, - cpus, - scratch_disk_size_gb, - version_override: Some(version), - persistent: false, - env, - from: None, - description: None, - }) - }) - .await - .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, format!("provision task: {e}")))?; - provision_result.map_err(|e| provision_failure(&e))?; - // The blocking worker has published its child. Executing the user's - // command is no longer launch admission work for service shutdown. - drop(_launch); - - // 3. Wait for VM socket to appear while still holding the VZ - // lifecycle rail. The child does its Apple VZ start/restore before it - // writes the ready sentinel; releasing earlier reintroduces the - // sibling-service overlap this lock exists to prevent. - let uds_path = state - .instance_socket_path(&id) - .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - if let Err(e) = wait_for_vm_ready(&uds_path, 30, Some(&state), Some(&id)).await { - drop(_vz_host_guard); - drop(_vz_guard); - // Wait for the child to actually exit before renaming. Rename on - // an open-for-write dir is safe (fds survive) but any path-based - // reopens the child might do during shutdown (log rotation, db - // reopen) would ENOENT -- so we let it finish flushing first. - // shutdown_vm_process now blocks until exit (5s budget, SIGKILL - // fallback) and cleans the UDS socket inline. Graceful because - // preserve_failed_session_dir inspects session logs that capsem-process - // is still flushing. - let shutdown_result = shutdown_vm_process(&state, &id, ShutdownMode::Retain, None).await?; - preserve_failed_run_shutdown_result(Arc::clone(&state), id.clone(), shutdown_result).await?; - return Err(AppError(StatusCode::INTERNAL_SERVER_ERROR, e)); - } - } - - let uds_path = state - .instance_socket_path(&id) - .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; - - // 2. Execute command. - let job_id = state.next_job_id(); - let exec_result = send_ipc_command( - &state, - &uds_path, - ServiceToProcess::Exec { - id: job_id, - command: payload.command, - // A one-shot run boots a VM with no workload. - target: capsem_proto::ipc::ExecTarget::Vm, - }, - Some(timeout_secs), - ) - .await; - - // 3. Tear down VM process and build response. shutdown_vm_process - // blocks until the process is actually gone -- the leak detector needs - // that guarantee. Route handlers must not mine session.db before returning; - // durable telemetry is recovered by the ledger rails. - let shutdown_result = shutdown_vm_process(&state, &id, ShutdownMode::Retain, None).await?; - let failed = !matches!(&exec_result, Ok(ProcessToService::ExecResult { .. })); - finalize_one_shot_session(Arc::clone(&state), id.clone(), shutdown_result, failed).await?; - - let response = match exec_result { - Ok(ProcessToService::ExecResult { - stdout, - stderr, - exit_code, - truncated, - .. - }) => Ok(Json(ExecResponse { - stdout: ExecOutput::from_bytes(stdout), - stderr: ExecOutput::from_bytes(stderr), - exit_code, - truncated, - })), - Ok(_) => Err(AppError( - StatusCode::INTERNAL_SERVER_ERROR, - "unexpected IPC response".into(), - )), - Err(e) => Err(AppError(StatusCode::INTERNAL_SERVER_ERROR, format!("exec failed: {e}"))), - }; - response -} diff --git a/crates/capsem-service/src/vm_lifecycle/provision.rs b/crates/capsem-service/src/vm_lifecycle/provision.rs index ead9bd43e..5c21221b4 100644 --- a/crates/capsem-service/src/vm_lifecycle/provision.rs +++ b/crates/capsem-service/src/vm_lifecycle/provision.rs @@ -299,29 +299,29 @@ impl ServiceState { ) { Ok(proxy) => proxy, Err(error) => { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error.context("start confined proxy worker")); } }; if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_ledger(id, &session_dir, &proxy)) { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error); } if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_credentials(&proxy)) { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error); } if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_private_names(id, &proxy)) { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error); } if let Err(error) = tokio::runtime::Handle::current().block_on(proxy.grant( capsem_proto::proxy_control::ProxyCapability::Upstream, proxy_upstream.into(), )) { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error.context("grant proxy upstream broker")); } let upstream_broker = upstream_broker.with_proxy(proxy.clone()); @@ -402,14 +402,14 @@ impl ServiceState { drop(instances); if let Err(error) = self.register_proxy_worker(id, generation, proxy.clone()) { self.evict_instance(id, generation); - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error.context("register proxy worker")); } if let Err(error) = tokio::runtime::Handle::current().block_on( crate::service_runtime::telemetry_export::grant_proxy_metric_broker(self, id, generation, &proxy), ) { self.evict_instance(id, generation); - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error); } let _upstream_broker = upstream_broker.start(upstream_grant); diff --git a/crates/capsem-service/src/vm_lifecycle/resume.rs b/crates/capsem-service/src/vm_lifecycle/resume.rs index 03ca32932..401ff70ca 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume.rs @@ -104,18 +104,18 @@ pub(crate) async fn handle_resume( /// Tear down a warm-restore process that failed to reach ready while the /// caller already holds the save/restore locks. -pub(crate) async fn stop_failed_restore_process_under_lock(state: &ServiceState, id: &str) { - let Some((uds_path, pid, generation)) = ({ +pub(crate) async fn stop_failed_restore_process_under_lock(state: &ServiceState, id: &str) -> Result<(), AppError> { + let Some((uds_path, generation)) = ({ let instances = state.instances.lock().unwrap(); - instances.get(id).map(|i| (i.uds_path.clone(), i.pid, i.generation)) + instances.get(id).map(|i| (i.uds_path.clone(), i.generation)) }) else { return Ok(()); }; signal_vm_process(state, id, generation, process_control::Signal::Terminate).await?; - tracing::warn!(id, pid, "removing failed warm restore before cold fallback"); - state.evict_instance(id, generation); + tracing::warn!(id, "removing failed warm restore before cold fallback"); + claim_shutdown_instance(state, id, generation); state.unregister_session_db_handle(id); wait_for_process_exit(state, id, generation, std::time::Duration::from_secs(1)).await?; state diff --git a/crates/capsem-service/src/vm_lifecycle/resume_process.rs b/crates/capsem-service/src/vm_lifecycle/resume_process.rs index 39e17b1b7..12e4f05eb 100644 --- a/crates/capsem-service/src/vm_lifecycle/resume_process.rs +++ b/crates/capsem-service/src/vm_lifecycle/resume_process.rs @@ -233,29 +233,29 @@ impl ServiceState { ) { Ok(proxy) => proxy, Err(error) => { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error.context("start confined proxy worker")); } }; if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_ledger(&vm_id, &entry.session_dir, &proxy)) { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error); } if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_credentials(&proxy)) { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error); } if let Err(error) = tokio::runtime::Handle::current().block_on(self.grant_proxy_private_names(&vm_id, &proxy)) { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error); } if let Err(error) = tokio::runtime::Handle::current().block_on(proxy.grant( capsem_proto::proxy_control::ProxyCapability::Upstream, proxy_upstream.into(), )) { - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error.context("grant proxy upstream broker")); } let upstream_broker = upstream_broker.with_proxy(proxy.clone()); @@ -296,14 +296,14 @@ impl ServiceState { drop(instances); if let Err(error) = self.register_proxy_worker(&vm_id, generation, proxy.clone()) { self.evict_instance(&vm_id, generation); - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error.context("register proxy worker")); } if let Err(error) = tokio::runtime::Handle::current().block_on( crate::service_runtime::telemetry_export::grant_proxy_metric_broker(self, &vm_id, generation, &proxy), ) { self.evict_instance(&vm_id, generation); - instance_reaper::kill_and_reap(child); + let _reaper = instance_reaper::kill_and_reap(child, retirement); return Err(error); } let _upstream_broker = upstream_broker.start(upstream_grant); diff --git a/crates/capsem-service/src/vm_lifecycle/run.rs b/crates/capsem-service/src/vm_lifecycle/run.rs new file mode 100644 index 000000000..df76dd6c5 --- /dev/null +++ b/crates/capsem-service/src/vm_lifecycle/run.rs @@ -0,0 +1,134 @@ +//! One-shot VM execution and its owned cleanup. + +use super::*; + +/// One-shot exec: provision a temp VM, run a command, return output, destroy VM. +pub(crate) async fn handle_run( + State(state): State>, + Json(payload): Json, +) -> Result, AppError> { + let timeout_secs = + capsem_api::exec_timeout_secs(payload.timeout_secs).map_err(|e| AppError(StatusCode::BAD_REQUEST, e))?; + let _launch = state + .lifecycle + .admit() + .map_err(|e| AppError(StatusCode::CONFLICT, e.to_string()))?; + if let Some(reason) = state.off_worker(|state| vm_asset_block_reason(&state)).await? { + return Err(AppError(StatusCode::PRECONDITION_FAILED, reason)); + } + + let id = { + let existing = state.off_worker(|state| existing_session_names(&state)).await?; + generate_session_name(existing.iter().map(|s| s.as_str())) + }; + + let resources = resolve_vm_resources(payload.ram_mb, payload.cpus); + let ram_mb = resources.ram_mb; + let cpus = resources.cpus; + let scratch_disk_size_gb = resources.scratch_disk_size_gb; + + // 1. Provision ephemeral VM. `provision_sandbox` is synchronous and + // does heavy I/O (APFS clonefile, rootfs.img fsync, child spawn); + // offload to the blocking pool, matching `handle_provision` -- the + // tokio::process::Command::spawn inside still works because + // spawn_blocking preserves the runtime handle via thread-locals. + let state_clone = Arc::clone(&state); + let id_clone = id.clone(); + let version = state.current_version.clone(); + let env = payload.env.clone(); + { + let _vz_guard = state.lifecycle.vz.read().await; + let _vz_host_guard = acquire_vz_host_lock(startup::VzHostLockMode::Shared).await?; + let provision_result = tokio::task::spawn_blocking(move || { + state_clone.provision_sandbox(ProvisionOptions { + id: &id_clone, + name: &id_clone, + ram_mb, + cpus, + scratch_disk_size_gb, + version_override: Some(version), + persistent: false, + env, + from: None, + description: None, + }) + }) + .await + .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, format!("provision task: {e}")))?; + provision_result.map_err(|e| provision_failure(&e))?; + // The blocking worker has published its child. Executing the user's + // command is no longer launch admission work for service shutdown. + drop(_launch); + + // 3. Wait for VM socket to appear while still holding the VZ + // lifecycle rail. The child does its Apple VZ start/restore before it + // writes the ready sentinel; releasing earlier reintroduces the + // sibling-service overlap this lock exists to prevent. + let uds_path = state + .instance_socket_path(&id) + .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + if let Err(e) = wait_for_vm_ready(&uds_path, 30, Some(&state), Some(&id)).await { + drop(_vz_host_guard); + drop(_vz_guard); + // Wait for the child to actually exit before renaming. Rename on + // an open-for-write dir is safe (fds survive) but any path-based + // reopens the child might do during shutdown (log rotation, db + // reopen) would ENOENT -- so we let it finish flushing first. + // shutdown_vm_process now blocks until exit (5s budget, SIGKILL + // fallback) and cleans the UDS socket inline. Graceful because + // preserve_failed_session_dir inspects session logs that capsem-process + // is still flushing. + let shutdown_result = shutdown_vm_process(&state, &id, ShutdownMode::Retain, None).await?; + preserve_failed_run_shutdown_result(Arc::clone(&state), id.clone(), shutdown_result).await?; + return Err(AppError(StatusCode::INTERNAL_SERVER_ERROR, e)); + } + } + + let uds_path = state + .instance_socket_path(&id) + .map_err(|e| AppError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + + // 2. Execute command. + let job_id = state.next_job_id(); + let exec_result = send_ipc_command( + &state, + &uds_path, + ServiceToProcess::Exec { + id: job_id, + command: payload.command, + // A one-shot run boots a VM with no workload. + target: capsem_proto::ipc::ExecTarget::Vm, + }, + Some(timeout_secs), + ) + .await; + + // 3. Tear down VM process and build response. shutdown_vm_process + // blocks until the process is actually gone -- the leak detector needs + // that guarantee. Route handlers must not mine session.db before returning; + // durable telemetry is recovered by the ledger rails. + let shutdown_result = shutdown_vm_process(&state, &id, ShutdownMode::Retain, None).await?; + let failed = !matches!(&exec_result, Ok(ProcessToService::ExecResult { .. })); + finalize_one_shot_session(Arc::clone(&state), id.clone(), shutdown_result, failed).await?; + + let response = match exec_result { + Ok(ProcessToService::ExecResult { + stdout, + stderr, + exit_code, + truncated, + .. + }) => Ok(Json(ExecResponse { + stdout: ExecOutput::from_bytes(stdout), + stderr: ExecOutput::from_bytes(stderr), + exit_code, + truncated, + })), + Ok(_) => Err(AppError( + StatusCode::INTERNAL_SERVER_ERROR, + "unexpected IPC response".into(), + )), + Err(e) => Err(AppError(StatusCode::INTERNAL_SERVER_ERROR, format!("exec failed: {e}"))), + }; + response +} diff --git a/crates/capsem-service/src/vm_lifecycle/session_dirs.rs b/crates/capsem-service/src/vm_lifecycle/session_dirs.rs index 095db1d92..b1cdc9db0 100644 --- a/crates/capsem-service/src/vm_lifecycle/session_dirs.rs +++ b/crates/capsem-service/src/vm_lifecycle/session_dirs.rs @@ -79,6 +79,13 @@ pub(crate) fn settle_persistent_session_dir(state: &ServiceState, name: &str, ex /// reports is data, not authority, and used to go straight to /// `remove_dir_all`. pub(super) async fn remove_purged_session_dir(state: &Arc, vm_id: &str, session_dir: PathBuf) -> bool { + let _deletion_guard = match connection_routes::lifecycle::deletion(state, vm_id).await { + Ok(guard) => guard, + Err(error) => { + warn!(vm_id, error = %error.1, "purge credential retirement remains pending"); + return false; + } + }; state.unregister_session_db_handle(vm_id); if let Err(error) = state.ledger_workers.shutdown(vm_id).await { warn!(vm_id, error = %error, "purge could not stop the session ledger worker"); @@ -88,7 +95,10 @@ pub(super) async fn remove_purged_session_dir(state: &Arc, vm_id: let dir = session_dir.clone(); let outcome = tokio::task::spawn_blocking(move || delete_state.delete_session_dir(&dir)).await; let error = match outcome { - Ok(Ok(())) => return true, + Ok(Ok(())) => match state.ledger_workers.retire(vm_id).await { + Ok(()) => return true, + Err(error) => format!("purge checkpoint cleanup failed: {error:#}"), + }, Ok(Err(error)) => error.to_string(), Err(join_error) => format!("purge delete task failed: {join_error}"), }; diff --git a/mcp/typescript/tests/image-tools.test.ts b/mcp/typescript/tests/image-tools.test.ts index 48feee3cb..983184103 100644 --- a/mcp/typescript/tests/image-tools.test.ts +++ b/mcp/typescript/tests/image-tools.test.ts @@ -99,7 +99,7 @@ it('rejects malformed image catalog responses without claiming cache readiness', expect(result.isError).toBe(true); expect(result.structuredContent).toEqual({error: {kind: 'internal'}}); expect(requests).toHaveLength(1); - }, {body: {images: [{...catalog.images[0], cached: 'available'}]}}); + }, {body: {images: [{...catalog.images[0], cached: 'not-a-cache-state'}]}}); }); it('keeps gateway bearer ownership separate for each MCP server', async () => { diff --git a/mcp/typescript/tests/packed-package.test.ts b/mcp/typescript/tests/packed-package.test.ts index 3fc0575a8..e09803533 100644 --- a/mcp/typescript/tests/packed-package.test.ts +++ b/mcp/typescript/tests/packed-package.test.ts @@ -1,9 +1,9 @@ import {execFileSync} from 'node:child_process'; -import {copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync} from 'node:fs'; -import {createHash} from 'node:crypto'; +import {copyFileSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync} from 'node:fs'; +import {sha256} from '../../../sdk/typescript/tools/crypto.mjs'; import {createServer as createHttpServer} from 'node:http'; import type {AddressInfo} from 'node:net'; -import {tmpdir} from 'node:os'; +import {homedir, tmpdir} from 'node:os'; import {dirname, join} from 'node:path'; import {fileURLToPath} from 'node:url'; import {Client} from '@modelcontextprotocol/sdk/client/index.js'; @@ -46,8 +46,7 @@ describe('packed-package', () => { retained.push({path, sha256: digest(path)}); copyFileSync(path, join(destination, `capsem-${owner}-${version}.tgz`)); } else execFileSync('pnpm', ['pack', '--config.ignore-scripts=true', '--pack-destination', destination], { - cwd: source, stdio: 'pipe', timeout: 15_000, - env: {...process.env, HOME: fixture}, + cwd: source, stdio: 'pipe', timeout: 15_000, env: {...process.env, HOME: fixture}, }); const name = readdirSync(destination).find(entry => entry.endsWith('.tgz')); if (!name) throw new Error('pnpm pack did not create a tarball'); @@ -57,12 +56,10 @@ describe('packed-package', () => { execFileSync('tar', ['-xzf', archive, '-C', destination], {timeout: 5000}); } writeFileSync(join(fixture, 'package.json'), JSON.stringify({name: 'capsem-mcp-clean-consumer', private: true})); - const npmCache = process.env.NPM_CONFIG_CACHE - ?? (process.env.HOME && existsSync(process.env.HOME) ? join(process.env.HOME, '.npm') : undefined); execFileSync('npm', ['install', '--offline', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', ...archives], { cwd: fixture, stdio: 'pipe', timeout: 60_000, - env: {HOME: fixture, PATH: process.env.PATH ?? '', - ...(npmCache ? {NPM_CONFIG_CACHE: npmCache} : {})}, + env: {PATH: process.env.PATH ?? '', HOME: fixture, + NPM_CONFIG_CACHE: process.env.NPM_CONFIG_CACHE ?? join(homedir(), '.npm')}, }); const dependencies = join(fixture, 'node_modules'); for (const [owner, source] of Object.entries(sources)) { diff --git a/mcp/typescript/tools/prewarm-package.mjs b/mcp/typescript/tools/prewarm-package.mjs index 52badc874..ca436c864 100644 --- a/mcp/typescript/tools/prewarm-package.mjs +++ b/mcp/typescript/tools/prewarm-package.mjs @@ -1,15 +1,14 @@ // Warm npm's complete runtime dependency graph before offline acceptance. import assert from 'node:assert/strict'; import {execFileSync} from 'node:child_process'; -import {existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; -import {tmpdir} from 'node:os'; +import {mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync} from 'node:fs'; +import {homedir, tmpdir} from 'node:os'; import {join} from 'node:path'; import process from 'node:process'; import {fileURLToPath} from 'node:url'; const project = fileURLToPath(new URL('../', import.meta.url)); const fixture = mkdtempSync(join(tmpdir(), 'capsem-mcp-prewarm-')); -const home = process.env.HOME && existsSync(process.env.HOME) ? process.env.HOME : fixture; try { const archives = []; for (const [owner, source] of [['sdk', join(project, '../../sdk/typescript')], ['mcp', project]]) { @@ -17,7 +16,7 @@ try { const output = join(fixture, owner); mkdirSync(output); execFileSync('pnpm', ['pack', '--config.ignore-scripts=true', '--pack-destination', output], { - cwd: source, stdio: 'pipe', timeout: 15_000, env: {...process.env, HOME: home}, + cwd: source, stdio: 'pipe', timeout: 15_000, env: {...process.env, HOME: fixture}, }); const names = readdirSync(output).filter(name => name.endsWith('.tgz')); assert.equal(names.length, 1); @@ -27,8 +26,8 @@ try { writeFileSync(join(fixture, 'package.json'), JSON.stringify({name: 'capsem-mcp-prewarm', private: true})); execFileSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', ...archives], { cwd: fixture, stdio: 'inherit', timeout: 60_000, - env: {HOME: home, PATH: process.env.PATH ?? '', - ...(process.env.NPM_CONFIG_CACHE ? {NPM_CONFIG_CACHE: process.env.NPM_CONFIG_CACHE} : {})}, + env: {PATH: process.env.PATH ?? '', HOME: fixture, + NPM_CONFIG_CACHE: process.env.NPM_CONFIG_CACHE ?? join(homedir(), '.npm')}, }); } finally { rmSync(fixture, {recursive: true, force: true}); diff --git a/sdk/python/capsem/_operations/__init__.py b/sdk/python/capsem/_operations/__init__.py index 70a7df2da..3c66e8823 100644 --- a/sdk/python/capsem/_operations/__init__.py +++ b/sdk/python/capsem/_operations/__init__.py @@ -12,6 +12,9 @@ from .close_managed_session import close_managed_session as close_managed_session from .create_network import create_network as create_network from .create_proxy import create_proxy as create_proxy +from .create_session_credential_grant import ( + create_session_credential_grant as create_session_credential_grant, +) from .create_vm import create_vm as create_vm from .create_vm_exposure import create_vm_exposure as create_vm_exposure from .create_vm_preview_session import ( @@ -50,6 +53,8 @@ from .get_vm_status import get_vm_status as get_vm_status from .get_vm_timeline import get_vm_timeline as get_vm_timeline from .heartbeat_proxy import heartbeat_proxy as heartbeat_proxy +from .inject_credential import inject_credential as inject_credential +from .list_connections import list_connections as list_connections from .list_images import list_images as list_images from .list_mcp_servers import list_mcp_servers as list_mcp_servers from .list_mcp_tools import list_mcp_tools as list_mcp_tools diff --git a/sdk/python/capsem/models/__init__.py b/sdk/python/capsem/models/__init__.py index 6e1edbaf2..4664fcd47 100644 --- a/sdk/python/capsem/models/__init__.py +++ b/sdk/python/capsem/models/__init__.py @@ -34,6 +34,10 @@ from .create_network_request import CreateNetworkRequest as CreateNetworkRequest from .create_proxy_request import CreateProxyRequest as CreateProxyRequest from .create_proxy_response import CreateProxyResponse as CreateProxyResponse +from .create_session_grant_request import ( + CreateSessionGrantRequest as CreateSessionGrantRequest, +) +from .credential_error_code import CredentialErrorCode as CredentialErrorCode from .credential_event import CredentialEvent as CredentialEvent from .credential_event_type import CredentialEventType as CredentialEventType from .credential_inject_provider import ( diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 999040e31..c131cb88a 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -13,13 +13,14 @@ build-backend = "hatchling.build" packages = ["capsem"] [dependency-groups] +package-build = ["hatchling==1.29.0"] dev = [ + { include-group = "package-build" }, "pytest>=9", "pytest-cov>=6", "ruff>=0.15.16", "ty>=0.0.46", "build>=1.2", - "hatchling==1.29.0", "editables==0.5", "openai==2.54.0", ] diff --git a/sdk/python/tests/image_package_acceptance.py b/sdk/python/tests/image_package_acceptance.py index ee86f28fb..106a9a10e 100644 --- a/sdk/python/tests/image_package_acceptance.py +++ b/sdk/python/tests/image_package_acceptance.py @@ -11,6 +11,7 @@ import sys import tarfile import threading +import time import tomllib import zipfile from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index 203e249f5..7f4742b8f 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -208,6 +208,9 @@ dev = [ { name = "ruff" }, { name = "ty" }, ] +package-build = [ + { name = "hatchling" }, +] [package.metadata] requires-dist = [ @@ -227,6 +230,7 @@ dev = [ { name = "ruff", specifier = ">=0.15.16" }, { name = "ty", specifier = ">=0.0.46" }, ] +package-build = [{ name = "hatchling", specifier = "==1.29.0" }] [[package]] name = "certifi" diff --git a/sdk/rust/src/operations/generated/mod.rs b/sdk/rust/src/operations/generated/mod.rs index e504b4daf..e0baf982d 100644 --- a/sdk/rust/src/operations/generated/mod.rs +++ b/sdk/rust/src/operations/generated/mod.rs @@ -15,6 +15,8 @@ mod create_network; pub use create_network::{create_network, CreateNetworkParams}; mod create_proxy; pub use create_proxy::{create_proxy, CreateProxyParams}; +mod create_session_credential_grant; +pub use create_session_credential_grant::{create_session_credential_grant, CreateSessionCredentialGrantParams}; mod create_vm; pub use create_vm::{create_vm, CreateVmParams}; mod create_vm_exposure; @@ -83,6 +85,10 @@ mod get_vm_timeline; pub use get_vm_timeline::{get_vm_timeline, GetVmTimelineParams}; mod heartbeat_proxy; pub use heartbeat_proxy::{heartbeat_proxy, HeartbeatProxyParams}; +mod inject_credential; +pub use inject_credential::{inject_credential, InjectCredentialParams}; +mod list_connections; +pub use list_connections::{list_connections, ListConnectionsParams}; mod list_images; pub use list_images::{list_images, ListImagesParams}; mod list_mcp_servers; diff --git a/sdk/rust/src/operations/tests.rs b/sdk/rust/src/operations/tests.rs index 2a9609924..391fb7cfc 100644 --- a/sdk/rust/src/operations/tests.rs +++ b/sdk/rust/src/operations/tests.rs @@ -89,6 +89,9 @@ operation!(get_panics, "getPanics", GetPanicsParams); operation!(get_triage, "getTriage", GetTriageParams); operation!(get_update_status, "getUpdateStatus"); operation!(update_hypervisor, "updateHypervisor", UpdateHypervisorParams); +operation!(create_proxy, "createProxy", CreateProxyParams); +operation!(heartbeat_proxy, "heartbeatProxy", HeartbeatProxyParams); +operation!(stop_proxy, "stopProxy", StopProxyParams); operation!(create_network, "createNetwork", CreateNetworkParams); operation!(list_networks, "listNetworks"); operation!(get_network, "getNetwork", GetNetworkParams); diff --git a/sdk/specification/openapi.json b/sdk/specification/openapi.json index f0e05c624..3dded0f62 100644 --- a/sdk/specification/openapi.json +++ b/sdk/specification/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "Capsem Gateway", - "version": "3.1.0" + "version": "3.3.0" }, "paths": { "/assets/status": { @@ -4303,6 +4303,67 @@ } } }, + "CreateSessionGrantRequest": { + "type": "object", + "required": [ + "request_id", + "expected_owner_generation", + "connection_id", + "capability_ids", + "expires_at" + ], + "properties": { + "capability_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "connection_id": { + "type": "string" + }, + "expected_owner_generation": { + "type": "string", + "format": "uuid" + }, + "expires_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "request_id": { + "type": "string", + "format": "uuid" + } + }, + "additionalProperties": false + }, + "CredentialErrorCode": { + "type": "string", + "enum": [ + "configuration_required", + "unsupported_capability", + "unknown_scope_profile", + "policy_denied", + "scope_denied", + "connection_unavailable", + "reauth_required", + "storage_unavailable", + "storage_commit_failed", + "transaction_expired", + "transaction_cancelled", + "invalid_callback", + "provider_rejected", + "provider_timeout", + "owner_changed", + "session_unavailable", + "grant_unavailable", + "request_conflict", + "revocation_pending" + ] + }, "CredentialEvent": { "type": "object", "required": [ diff --git a/sdk/typescript/src/models/index.ts b/sdk/typescript/src/models/index.ts index 20098d946..534c54f09 100644 --- a/sdk/typescript/src/models/index.ts +++ b/sdk/typescript/src/models/index.ts @@ -28,6 +28,8 @@ export { ContainerSurfaceKind } from "./ContainerSurfaceKind.js"; export type { CreateNetworkRequest } from "./CreateNetworkRequest.js"; export type { CreateProxyRequest } from "./CreateProxyRequest.js"; export type { CreateProxyResponse } from "./CreateProxyResponse.js"; +export type { CreateSessionGrantRequest } from "./CreateSessionGrantRequest.js"; +export { CredentialErrorCode } from "./CredentialErrorCode.js"; export type { CredentialEvent } from "./CredentialEvent.js"; export { CredentialEventType } from "./CredentialEventType.js"; export { CredentialInjectProvider } from "./CredentialInjectProvider.js"; diff --git a/sdk/typescript/src/operations/index.ts b/sdk/typescript/src/operations/index.ts index 356526635..c7094bf63 100644 --- a/sdk/typescript/src/operations/index.ts +++ b/sdk/typescript/src/operations/index.ts @@ -8,6 +8,7 @@ export {claimManagedSession} from "./claimManagedSession.js"; export {closeManagedSession} from "./closeManagedSession.js"; export {createNetwork} from "./createNetwork.js"; export {createProxy} from "./createProxy.js"; +export {createSessionCredentialGrant} from "./createSessionCredentialGrant.js"; export {createVm} from "./createVm.js"; export {createVmExposure} from "./createVmExposure.js"; export {createVmPreviewSession} from "./createVmPreviewSession.js"; @@ -42,6 +43,8 @@ export {getVmStatsSummary} from "./getVmStatsSummary.js"; export {getVmStatus} from "./getVmStatus.js"; export {getVmTimeline} from "./getVmTimeline.js"; export {heartbeatProxy} from "./heartbeatProxy.js"; +export {injectCredential} from "./injectCredential.js"; +export {listConnections} from "./listConnections.js"; export {listImages} from "./listImages.js"; export {listMcpServers} from "./listMcpServers.js"; export {listMcpTools} from "./listMcpTools.js"; diff --git a/sdk/typescript/src/validation/index.ts b/sdk/typescript/src/validation/index.ts index 4f065e895..ad925e526 100644 --- a/sdk/typescript/src/validation/index.ts +++ b/sdk/typescript/src/validation/index.ts @@ -28,6 +28,8 @@ export {ContainerSurfaceKindSchema} from "./ContainerSurfaceKind.js"; export {CreateNetworkRequestSchema} from "./CreateNetworkRequest.js"; export {CreateProxyRequestSchema} from "./CreateProxyRequest.js"; export {CreateProxyResponseSchema} from "./CreateProxyResponse.js"; +export {CreateSessionGrantRequestSchema} from "./CreateSessionGrantRequest.js"; +export {CredentialErrorCodeSchema} from "./CredentialErrorCode.js"; export {CredentialEventSchema} from "./CredentialEvent.js"; export {CredentialEventTypeSchema} from "./CredentialEventType.js"; export {CredentialInjectProviderSchema} from "./CredentialInjectProvider.js"; diff --git a/sdk/typescript/tests/package-install.test.ts b/sdk/typescript/tests/package-install.test.ts index fcae7077d..29a85d4d2 100644 --- a/sdk/typescript/tests/package-install.test.ts +++ b/sdk/typescript/tests/package-install.test.ts @@ -11,7 +11,7 @@ interface Report {ok: boolean; sha256: string; version: string; httpPaths: strin const source = fileURLToPath(new URL('../', import.meta.url)); const root = resolve(source, '../..'); -const sha = (path: string): string => createHash('sha256').update(readFileSync(path)).digest('hex'); +const sha = (path: string): string => sha256(readFileSync(path)); const usableHome = (): string => process.env.HOME && existsSync(process.env.HOME) ? process.env.HOME : tmpdir(); const npmCache = (): string => process.env.NPM_CONFIG_CACHE ?? join(usableHome(), '.npm'); diff --git a/sdk/typescript/tools/prewarm-package.mjs b/sdk/typescript/tools/prewarm-package.mjs index 88860475f..537ee43b7 100644 --- a/sdk/typescript/tools/prewarm-package.mjs +++ b/sdk/typescript/tools/prewarm-package.mjs @@ -1,8 +1,9 @@ // Fetch dependencies before the clean-consumer acceptance enters the sandbox. import assert from 'node:assert/strict'; import {execFileSync} from 'node:child_process'; -import {existsSync, readFileSync} from 'node:fs'; -import {tmpdir} from 'node:os'; +import {mkdtempSync, readFileSync, rmSync} from 'node:fs'; +import {homedir, tmpdir} from 'node:os'; +import {join} from 'node:path'; import process from 'node:process'; /** @type {unknown} */ @@ -10,12 +11,16 @@ const parsed = JSON.parse(readFileSync(new URL('../package.json', import.meta.ur const manifest = /** @type {{dependencies:Record,devDependencies:Record}} */ (parsed); const compiler = manifest.devDependencies.typescript; assert.ok(compiler); -const home = process.env.HOME && existsSync(process.env.HOME) ? process.env.HOME : tmpdir(); -for (const [name, version] of Object.entries({...manifest.dependencies, typescript: compiler})) { - execFileSync('npm', ['cache', 'add', `${name}@${version}`], { - stdio: 'inherit', timeout: 30_000, - env: {HOME: home, PATH: process.env.PATH ?? '', - ...(process.env.NPM_CONFIG_CACHE ? {NPM_CONFIG_CACHE: process.env.NPM_CONFIG_CACHE} : {}), - ...(process.env.NPM_CONFIG_OFFLINE ? {NPM_CONFIG_OFFLINE: process.env.NPM_CONFIG_OFFLINE} : {})}, - }); +const cache = process.env.NPM_CONFIG_CACHE ?? join(homedir(), '.npm'); +const home = mkdtempSync(join(tmpdir(), 'capsem-sdk-prewarm-home-')); +try { + for (const [name, version] of Object.entries({...manifest.dependencies, typescript: compiler})) { + execFileSync('npm', ['cache', 'add', `${name}@${version}`], { + stdio: 'inherit', timeout: 30_000, + env: {PATH: process.env.PATH ?? '', HOME: home, NPM_CONFIG_CACHE: cache, + ...(process.env.NPM_CONFIG_OFFLINE ? {NPM_CONFIG_OFFLINE: process.env.NPM_CONFIG_OFFLINE} : {})}, + }); + } +} finally { + rmSync(home, {recursive: true, force: true}); } diff --git a/skills/dev-capsem/SKILL.md b/skills/dev-capsem/SKILL.md index b671f196d..8bfa20ba3 100644 --- a/skills/dev-capsem/SKILL.md +++ b/skills/dev-capsem/SKILL.md @@ -30,7 +30,7 @@ then lead with a recommendation. The full contract is | `capsem-foundation` | Dependency-light host primitives shared across product crates. | `paths.rs`, `uds.rs`, `poll.rs`, `telemetry.rs`, `log_layer.rs`, `ipc_handshake.rs` | | `capsem-cache` | Bounded observable cache mechanics; callers own freshness policy. | `bounded.rs`, `snapshot.rs`, `telemetry.rs` | | `capsem-archive` | Append-only, block-compressed body archive for one session ledger. | `format.rs`, `writer.rs`, `reader.rs` | -| `capsem-crypto` | Central owner of hashing, randomness, TLS providers, signatures, and secret handling. | `lib.rs`, `random.rs`, `tls.rs`, `verify.rs` | +| `capsem-crypto` | Accelerated hashes, integrity verification, randomness, TLS keys, JWT verification, signatures, and secret handling. | `verify.rs`, `random.rs`, `tls.rs`, `jwt.rs`, `secrets.rs` | | `capsem-ledger` | Confined session-ledger worker and archive codec executor. | `main.rs`, `codec.rs` | | `capsem-telemetry` | Single owner of every `metrics`-facade metric name, kind, unit, and description. Emitters import names from here. | `lib.rs` (`MetricSpec`, `describe_all`), `db.rs`, `dns.rs`, `mitm.rs`, `security.rs`, `virtio_blk.rs` | | `capsem-assets` | VM asset lifecycle and manifest compatibility. | `asset_manager.rs`, `manifest_compat.rs` | diff --git a/skills/dev-testing/references/test-matrix.md b/skills/dev-testing/references/test-matrix.md index 8af9b37a4..12c457640 100644 --- a/skills/dev-testing/references/test-matrix.md +++ b/skills/dev-testing/references/test-matrix.md @@ -15,7 +15,7 @@ this table, enforced by `tests/citadel/test_rust_workspace_documentation.py`. | `capsem-foundation` | Host primitives | Yes | Compile/no-run | Clippy | Yes | | `capsem-cache` | Bounded observable cache primitives | Yes | Compile/no-run | Clippy | Yes | | `capsem-archive` | Session body archive | Yes | Compile/no-run | Clippy | Yes | -| `capsem-crypto` | Cryptographic providers and primitives | Yes | Compile/no-run | Clippy | Yes | +| `capsem-crypto` | Cryptographic primitives and integrity verification; speed measured by `capsem-bench` | Yes | Compile/no-run | Clippy | Yes | | `capsem-ledger` | Confined ledger and archive codec worker | Yes | Subprocess | Clippy | Yes | | `capsem-telemetry` | Metric name/unit/description catalog | Yes | Compile/no-run | Clippy | Yes | | `capsem-assets` | Asset lifecycle | Yes | Compile/no-run | Clippy | Yes | diff --git a/skills/site-architecture/references/crate-and-privilege-model.md b/skills/site-architecture/references/crate-and-privilege-model.md index 18e85cebe..a98247a13 100644 --- a/skills/site-architecture/references/crate-and-privilege-model.md +++ b/skills/site-architecture/references/crate-and-privilege-model.md @@ -17,8 +17,10 @@ Sharing alone is not a reason to put code in `capsem-core`. - **`capsem-archive`**: block-compressed body archive for session ledgers. Pure Rust (`miniz_oxide`); SQLite keeps the index, this crate keeps the bytes, and every block is blake3-verified before a body is returned. -- **`capsem-crypto`**: central owner of hashing, randomness, TLS providers, - signatures, constant-time verification, and secret handling. +- **`capsem-crypto`**: first-party Rust cryptographic operations: accelerated + hashes and bounded integrity verification, randomness, TLS key/provider + selection, JWT verification, signatures, constant-time verification, and + secret handling. Speed measurements belong in `capsem-bench`. - **`capsem-ledger`**: confined session-ledger worker and the sole runtime executor of body archive codecs. - **`capsem-telemetry`**: the one owner of every `metrics`-facade metric name, @@ -37,11 +39,12 @@ Sharing alone is not a reason to put code in `capsem-core`. - **`capsem-logger`**: session DB schema, queries, storage, and async writer. - **`capsem-guard`**: parent-watch and singleton-flock lifecycle primitives. - **`capsem-service`**: daemon HTTP/UDS API and VM-process orchestration. -- **`capsem-process`**: per-VM boot, vsock, IPC, and job runtime; still holds - only the paths and descriptors granted for its session and generation. +- **`capsem-process`**: low-privilege per-VM boot, vsock, IPC, and job runtime; + it holds only the paths and descriptors granted for its session and generation. - **`capsem-proxy`**: confined HTTP/DNS worker for VM and standalone proxy traffic; upstream, ledger, credential, and MCP authority arrives by grant. -- **`capsem`**: CLI client; HTTP/UDS to the service, including shell streams. +- **`capsem`**: CLI client; HTTP/UDS to the service, including streamed shell + and terminal operations. It never dials a per-VM process socket. - **`capsem-tui`**: terminal control UI over the gateway API. - **`capsem-admin`**: runtime image build plus asset, release, and config validation. - **`@capsem/mcp`** (`mcp/typescript`): separately installed host MCP server; @@ -73,6 +76,20 @@ credential, MCP, and ledger channels. `capsem-ledger` receives the ledger files and typed producer channels; durable service-owned commitment checkpoints anchor the sequence outside that worker. +1. **Minimal environment**: service uses `env_clear()` before spawn, then passes only `HOME`, `PATH`, `USER`, `TMPDIR`, `RUST_LOG`. API keys and tokens from the user's shell never reach the process. +2. **Socket permissions 0600**: Per-VM owner sockets (`{id}.sock` IPC, `{id}-handoff.sock`) are chmod 0600 after bind; no client dials them -- terminals and attach go through the service `/vms/{id}/stream` route. Only the owning user can connect. +3. **Session directory 0700**: created by the service via `create_virtiofs_session`. Contains workspace/, system/, serial.log (0600), session.db. +4. **No guest-triggered process exit**: control channel read errors cause `break` (loop exit), not `process::exit()`. Guest cannot DoS the host process. +5. **Gateway auth layer**: external access goes through capsem-gateway (Bearer token, rate limiting, localhost CORS). Per-VM sockets are not exposed to the network. +6. **Rootfs read-only**: runtime rootfs asset mounted read-only. Guest binaries deployed chmod 555. +7. **Guest binary security**: all injected binaries are read-only. Guest cannot modify its own agent. +8. **VirtioFS boundary**: `session_dir/guest/` is the guest-writable share and + contains the workspace. The writable overlay image stays in host-only + `session_dir/system/`, alongside the host-only ledger, logs and checkpoint. + The separate `session_dir/image/` share contains immutable image blobs and + is read-only at the device. The workspace compatibility symlink points into + `guest/`; the system image must never be reachable through that share. + Socket permissions remain 0600 and session directories 0700 as defense in depth. The guest sees only `session_dir/guest/` through VirtioFS; the writable system overlay, serial log, ledger, and worker state remain outside the share. diff --git a/tests/capsem-build-chain/test_process_reads_only_active_policy.py b/tests/capsem-build-chain/test_process_reads_only_active_policy.py index 7d4a22361..42842b369 100644 --- a/tests/capsem-build-chain/test_process_reads_only_active_policy.py +++ b/tests/capsem-build-chain/test_process_reads_only_active_policy.py @@ -1,8 +1,9 @@ -"""capsem-process reads its runtime policy from the session's active policy file. +"""capsem-process consumes only the service's published active policy. The service merges built-in defaults, settings.toml and corp config into -`vm/active_policy.toml`; the per-VM process never re-reads those sources, or a -reload could apply policy the service never published. +`vm/active_policy.toml`; the per-VM process boots from that file and receives +the same exact published bytes for reload. It never reads the source files, or +a reload could apply policy the service never published. """ from pathlib import Path @@ -23,6 +24,8 @@ def test_capsem_process_runtime_does_not_load_settings_or_corp_files() -> None: text = path.read_text() for needle, reason in forbidden.items(): if needle in text: - offenders.append(f"{path.relative_to(PROJECT_ROOT)} contains {needle!r}: {reason}") + offenders.append( + f"{path.relative_to(PROJECT_ROOT)} contains {needle!r}: {reason}" + ) assert not offenders, "\n".join(offenders) diff --git a/tests/capsem-release/test_binary_lane_gate.py b/tests/capsem-release/test_binary_lane_gate.py index 0c9017fa9..9ede3a627 100644 --- a/tests/capsem-release/test_binary_lane_gate.py +++ b/tests/capsem-release/test_binary_lane_gate.py @@ -48,7 +48,7 @@ def test_every_packaged_executable_has_hashes_and_sbom_ref() -> None: assert "pub fn verify_package_contents_match_binary_inventory" in source assert 'format!("{:x}", Sha256::digest(&file.bytes))' in source assert "use capsem_crypto::{blake3::hash, Sha256};" in source - assert "hash(&file.bytes)" in source + assert "hash(&file.bytes).to_hex().to_string()" in source assert "sbom_component_refs" in source assert "missing SBOM component reference" in source @@ -66,10 +66,7 @@ def test_sha1_only_spdx_is_rejected() -> None: tests = ADMIN_CHANNEL_BUILD_TESTS.read_text(encoding="utf-8") assert "fn validate_host_spdx_sbom_bytes" in render - assert ( - "let blake3 = capsem_crypto::blake3::hash(&bytes).to_hex().to_string();" - in package_inspection - ) + assert "let blake3 = capsem_crypto::blake3::hash(&bytes).to_hex().to_string();" in package_inspection assert "blake3: file.blake3.clone()" in render assert "channel manifest host binary {} has malformed blake3" in validation assert 'algorithm.eq_ignore_ascii_case("SHA256")' in render diff --git a/tests/capsem-release/test_release_test_composition.py b/tests/capsem-release/test_release_test_composition.py index 71f96e0d9..d87775910 100644 --- a/tests/capsem-release/test_release_test_composition.py +++ b/tests/capsem-release/test_release_test_composition.py @@ -270,6 +270,26 @@ def test_every_ci_job_provisions_the_tools_its_own_steps_invoke() -> None: assert not missing, "CI jobs missing tool provisioning:\n" + "\n".join(missing) +def test_macos_ci_builds_python_sdk_distributions_before_testing_them() -> None: + """The package acceptance tests consume the wheel and sdist from the + configured build output; a fresh runner has neither until this job builds + them.""" + workflow = yaml.safe_load((PROJECT_ROOT / ".github/workflows/ci.yaml").read_text()) + steps = workflow["jobs"]["test"]["steps"] + names = [step.get("name") for step in steps] + build = names.index("Build Python SDK distributions") + tested = names.index("Python SDK tests with coverage") + assert build < tested + + command = steps[build]["run"] + settings = CONFIG.sdk_python + assert f"uv sync --project {settings.project} --frozen --no-install-project" in command + assert ( + f"uv run --project {settings.project} --frozen --no-sync python -m build " + f"--no-isolation --outdir {settings.build_output} {settings.project}" + ) in command + + def _source_digest_module(): script = PROJECT_ROOT / "build_system" / "scripts" / "build" / "source-state-digest.py" spec = importlib.util.spec_from_file_location("source_state_digest", script) diff --git a/tests/capsem-sdk/test_mcp_cli_parity.py b/tests/capsem-sdk/test_mcp_cli_parity.py index b65882921..e96b6e509 100644 --- a/tests/capsem-sdk/test_mcp_cli_parity.py +++ b/tests/capsem-sdk/test_mcp_cli_parity.py @@ -115,6 +115,7 @@ "stop": "stop the background service daemon", "support-bundle": "host-side bug-report bundler; no service round-trip, not an AI concept", "cp": "host/session file copy convenience; MCP uses capsem_read_file/capsem_write_file", + "proxy": "long-lived local HTTP proxy process; MCP tools use the service gateway", "version": "human CLI build metadata; MCP status reports typed gateway state", "assets status": "runtime asset diagnostics; MCP status includes aggregate asset health", "assets ensure": "local runtime asset repair; host MCP owns no asset download authority", diff --git a/tests/capsem-service/test_companion_lifecycle.py b/tests/capsem-service/test_companion_lifecycle.py index 7840bb149..be68a9fdf 100644 --- a/tests/capsem-service/test_companion_lifecycle.py +++ b/tests/capsem-service/test_companion_lifecycle.py @@ -24,6 +24,7 @@ import os import shutil import signal +import socket import subprocess import sys import tempfile @@ -48,6 +49,7 @@ # Parent-watch poll interval is 500ms in capsem-guard; give a generous factor # for loaded CI while still catching real regressions. WATCH_DEADLINE_SECS = 5.0 +_GATEWAY_TEST_SOCKETS: dict[int, tuple[socket.socket, socket.socket]] = {} def _sign(): @@ -89,12 +91,28 @@ def _spawn_under_parent( This matches production: the service is a long-lived parent process that spawns companions; the test shell stands in for the service. """ + grant_peer = None + grant_stdin = None + service_listener = None + if binary == GATEWAY_BIN: + assert env_extra is not None + grant_stdin, grant_peer = socket.socketpair() + service_listener = socket.socket(socket.AF_UNIX) + service_path = Path(env_extra["CAPSEM_RUN_DIR"]) / f"service-{child_pid_file.stem}.sock" + service_listener.bind(str(service_path)) + service_listener.listen() + extra_args = [ + *extra_args, + "--service-grant-stdin", + "--uds-path", + str(service_path), + ] args_q = " ".join(f'"{a}"' for a in extra_args) # Parent shell: prints its own PID, forks the companion with --parent-pid $$, # writes the companion's pid, then sleeps forever to stay as parent. script = ( f'echo "$$" > "{parent_pid_file}"\n' - f'"{binary}" --parent-pid "$$" {args_q} &\n' + f'"{binary}" --parent-pid "$$" {args_q} <&0 &\n' f'echo "$!" > "{child_pid_file}"\n' f"exec sleep 600\n" ) @@ -103,12 +121,20 @@ def _spawn_under_parent( env.setdefault("CAPSEM_TRAY_HEADLESS", "1") if env_extra: env.update(env_extra) - return subprocess.Popen( - ["bash", "-c", script], - env=env, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) + try: + parent = subprocess.Popen( + ["bash", "-c", script], + env=env, + stdin=grant_stdin.fileno() if grant_stdin is not None else None, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if grant_peer is not None and service_listener is not None: + _GATEWAY_TEST_SOCKETS[parent.pid] = (grant_peer, service_listener) + return parent + finally: + if grant_stdin is not None: + grant_stdin.close() def _read_pid(path: Path, deadline: float) -> int: @@ -124,6 +150,11 @@ def _read_pid(path: Path, deadline: float) -> int: raise TimeoutError(f"pid file {path} never populated") +def _close_gateway_test_sockets(parent: subprocess.Popen) -> None: + for test_socket in _GATEWAY_TEST_SOCKETS.pop(parent.pid, ()): + test_socket.close() + + def _wait_exit(proc: subprocess.Popen, deadline: float) -> int | None: """Poll for exit; returns exit code or None if still alive at deadline.""" end = time.time() + deadline @@ -243,9 +274,11 @@ def test_tray_second_spawn_exits(self): finally: second_parent.kill() second_parent.wait(timeout=5) + _close_gateway_test_sockets(second_parent) finally: first_parent.kill() first_parent.wait(timeout=5) + _close_gateway_test_sockets(first_parent) def test_tray_hammer_20_parallel_yields_one_live(self): """Spawn 20 trays concurrently against one lock path. Exactly one @@ -329,9 +362,11 @@ def test_gateway_second_spawn_exits(self): finally: second_parent.kill() second_parent.wait(timeout=5) + _close_gateway_test_sockets(second_parent) finally: first_parent.kill() first_parent.wait(timeout=5) + _close_gateway_test_sockets(first_parent) class TestCompanionDiesWithParent: @@ -371,6 +406,7 @@ def test_tray_exits_when_parent_sigkilled(self): parent.kill() with contextlib.suppress(subprocess.TimeoutExpired): parent.wait(timeout=5) + _close_gateway_test_sockets(parent) def test_gateway_exits_when_parent_sigkilled(self): _sign() @@ -402,6 +438,7 @@ def test_gateway_exits_when_parent_sigkilled(self): parent.kill() with contextlib.suppress(subprocess.TimeoutExpired): parent.wait(timeout=5) + _close_gateway_test_sockets(parent) class TestServiceSigkillReapsAllCompanions: diff --git a/tests/capsem-service/test_dbwriter_contract.py b/tests/capsem-service/test_dbwriter_contract.py index f50b24624..5d42c212f 100644 --- a/tests/capsem-service/test_dbwriter_contract.py +++ b/tests/capsem-service/test_dbwriter_contract.py @@ -34,8 +34,9 @@ def test_dbwriter_source_boundaries_are_single_rail() -> None: # Workspace snapshots were retired (#228): no status query reaches the process. assert "SnapshotStatus" not in service_prod - assert "capsem_logger::DbWriter::open(" in process_prod - assert '&session_dir.join("session.db")' in process_prod + assert "capsem_logger::DbWriter::from_ledger_channel(" in process_prod + assert "DbWriter::open(" not in process_prod + assert 'let ledger_path = session_dir.join("session.db");' in process_prod assert "Arc" in process_vsock assert "rusqlite::Connection" not in process_vsock assert "write_many" not in process_vsock diff --git a/tests/capsem-service/test_settings_security_routes.py b/tests/capsem-service/test_settings_security_routes.py index 56d30c228..6db6e672d 100644 --- a/tests/capsem-service/test_settings_security_routes.py +++ b/tests/capsem-service/test_settings_security_routes.py @@ -10,6 +10,8 @@ import json from typing import Any +from helpers.service import ServiceInstance + SERVER = "local" @@ -49,16 +51,22 @@ def _seed_mcp_tool_cache(service_env: Any) -> None: ) -def test_settings_security_routes_expose_single_contract(client: Any, service_env: Any) -> None: - _seed_mcp_tool_cache(service_env) - refresh = client.post(f"/mcp/servers/{SERVER}/refresh") - assert refresh["success"] is True - assert refresh["server_id"] == SERVER - - plugins = client.get("/plugins/list") - mcp_info = client.get("/mcp/info") - mcp_default = client.get("/mcp/default/info") - mcp_tools = client.get(f"/mcp/servers/{SERVER}/tools/list") +def test_settings_security_routes_expose_single_contract() -> None: + service = ServiceInstance() + _seed_mcp_tool_cache(service) + service.start() + try: + client = service.client() + refresh = client.post(f"/mcp/servers/{SERVER}/refresh") + assert refresh["success"] is True + assert refresh["server_id"] == SERVER + + plugins = client.get("/plugins/list") + mcp_info = client.get("/mcp/info") + mcp_default = client.get("/mcp/default/info") + mcp_tools = client.get(f"/mcp/servers/{SERVER}/tools/list") + finally: + service.stop() assert set(plugins) == {"plugins"} assert {plugin["id"] for plugin in plugins["plugins"]} == { diff --git a/tests/capsem-service/test_svc_startup_failure_logged.py b/tests/capsem-service/test_svc_startup_failure_logged.py index c951584f2..cf1c3f2eb 100644 --- a/tests/capsem-service/test_svc_startup_failure_logged.py +++ b/tests/capsem-service/test_svc_startup_failure_logged.py @@ -1,10 +1,8 @@ """A service that fails to start says why in its own log. -The gate and the installer launch capsem-service detached and, when it dies -before listening, point at its log. A startup error returned from -`run_service` went only to stderr, which nobody reads: a dev build refusing -a malformed installed ledger left a service log that ended at "sqlite mmap -telemetry recorded" with no cause anywhere. +The gate and installer point to the service log when startup fails before +listening. A corrupt host ledger must record its cause there, rather than +leaving it only on the detached process's stderr. """ from __future__ import annotations @@ -23,11 +21,9 @@ def test_a_startup_error_is_written_to_the_service_log() -> None: home_dir, run_dir = make_service_home_run_dirs() sessions = home_dir / "sessions" sessions.mkdir() - # A malformed current ledger must fail loudly. A pre-v4 ledger is kept - # aside and replaced, so it cannot exercise the startup failure path. - with closing(sqlite3.connect(sessions / "host.db")) as broken: - broken.execute("CREATE TABLE archive_state (id INTEGER PRIMARY KEY)") - broken.commit() + # Pre-v4 ledgers are archived and replaced. A corrupt SQLite file still + # fails startup, so it exercises the error-log contract directly. + (sessions / "host.db").write_bytes(b"corrupt host ledger") sign_binary(PROCESS_BINARY) sign_binary(SERVICE_BINARY) env = { @@ -59,6 +55,6 @@ def test_a_startup_error_is_written_to_the_service_log() -> None: logs = sorted(run_dir.glob("service*.log")) assert logs, f"no service log in {run_dir}" text = "".join(path.read_text() for path in logs) - assert "archive_state must contain exactly one row" in text, ( + assert "file is not a database" in text, ( "the cause of a failed start must reach the service log, not only stderr" ) diff --git a/tests/citadel/cargo_lock_identity.toml b/tests/citadel/cargo_lock_identity.toml index b1b9d44f4..681c59260 100644 --- a/tests/citadel/cargo_lock_identity.toml +++ b/tests/citadel/cargo_lock_identity.toml @@ -1,11 +1,3 @@ -# The intentionally selected Cargo.lock for this tree, recorded on purpose. -# -# A lock change is the most expensive input in the repository: it changes the -# identity of the guest Rust builder image, so every guest agent is recompiled -# inside Docker, the initrd is repacked, and every host binary rebuilds. Four -# measurement runs in one evening paid that cost because unrelated commits -# each moved the lock. `test_cargo_lock_changes_are_deliberate` fails until -# this record matches the lock again, so the cost is chosen, reviewed and -# batched rather than discovered twenty minutes into a gate. -sha256 = "bb76b3833746aa2a26714484474ceb59e004bd467097d6b9645c8415403ae7d4" -reason = "The unified observable cache adds quick_cache 0.7.0. The 0.7 crypto ownership retrofit centralizes first-party hashes, entropy, TLS keys, JWT verification and secret primitives; BLAKE2b/BLAKE2s SIMD 1.0.5 support normal-harness comparisons. Managed-session APIs add zeroizing request capabilities, and Rust managed scopes add SDK edges to existing capsem-crypto random/secrets, futures, humantime and tokio-util rt. Central runtime-limit generation adds capsem-proto's direct toml edge, and exact benchmark reporting adds capsem-bench's direct capsem-proto edge. rustls 0.23.46 fixes RUSTSEC-2026-0346, RUSTSEC-2026-0347, RUSTSEC-2026-0348 and RUSTSEC-2026-0349. The host and sealed guest cohort will be rebuilt and qualified together." +# Deliberately combined dependency graph for #206 on 0.7. +sha256 = "00898cc7fd2bdb704d7f6fd494d0ef3ca677f91597731d25c63e09658213edf2" +reason = "The unified observable cache adds quick_cache 0.7.0. The 0.7 crypto ownership retrofit centralizes first-party hashes, entropy, TLS keys, JWT verification and secret primitives; BLAKE2b/BLAKE2s SIMD 1.0.5 support normal-harness comparisons. Managed-session APIs add zeroizing request capabilities, and Rust managed scopes add SDK edges to existing capsem-crypto random/secrets, futures, humantime and tokio-util rt. Central runtime-limit generation adds capsem-proto's direct toml edge, and exact benchmark reporting adds capsem-bench's direct capsem-proto edge. rustls 0.23.46 fixes RUSTSEC-2026-0346, RUSTSEC-2026-0347, RUSTSEC-2026-0348 and RUSTSEC-2026-0349. The host and sealed guest cohort will be rebuilt and qualified together. Rebase onto shared 0.7 retains the approved time parsing and safe Sigma edges, while preserving the confined worker codecs, shared cache and patched rustls dependencies." diff --git a/tests/citadel/fixture_ownership.toml b/tests/citadel/fixture_ownership.toml index 3200d21e2..9f5fa935c 100644 --- a/tests/citadel/fixture_ownership.toml +++ b/tests/citadel/fixture_ownership.toml @@ -93,7 +93,7 @@ symbols = [ # session through today's writer: # cargo test -p capsem-logger --test roundtrip -- --ignored regenerate_session_fixture # then update these digests in the same commit. -sha256 = "ec0f21a3de5d1a03d376455b17d31730fd66022b4080968fd1de4011302ad598" +sha256 = "02f4a0bcd631f0745f2346fec758eaecca1fdf7bbc8c7d435b7fc399e2ecc25a" regenerator = "crates/capsem-logger/tests/roundtrip/fixture_regen.rs" # The ledger's bodies live in the selected generation beside it. The index in diff --git a/tests/citadel/python_boundary_debt.toml b/tests/citadel/python_boundary_debt.toml index da53eef21..89e2df28c 100644 --- a/tests/citadel/python_boundary_debt.toml +++ b/tests/citadel/python_boundary_debt.toml @@ -20,6 +20,6 @@ old_import_sha256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b785 # images holds the official image publication scripts, run through the # build_system project (`uv run --project build_system`) and type-checked as a # [lint].python_roots entry. -# integrations carries pure Inspect text transforms; the same frozen project -# owns their source-contract tests, strict type checks and coverage. +# integrations/inspect-ai carries pure Inspect text transforms; the same frozen +# project owns their source-contract tests, strict type checks and coverage. python_source_roots = [".claude", "benchmarks", "build_system", "guest", "images", "integrations", "sdk", "tests"] diff --git a/tests/citadel/repository_surface_ownership.toml b/tests/citadel/repository_surface_ownership.toml index 31115870c..fa0349c28 100644 --- a/tests/citadel/repository_surface_ownership.toml +++ b/tests/citadel/repository_surface_ownership.toml @@ -196,18 +196,6 @@ coverage = "not-applicable: image recipes and publication scripts are proved by debt = "none: images/ is a new root" reason = "The official OCI images and their publication scripts; only .github/workflows/images.yaml builds and publishes them." -[[surface]] -id = "external-integrations" -state = "retained" -current = ["integrations/"] -targets = ["integrations/"] -lint = ["python.ruff", "python.ty.strict", "fast.audit.source-syntax", "fast.citadel"] -tests = ["integrations/inspect-ai/tests/containers/", "tests/citadel/test_repository_surface_ownership.py"] -ci = ["fast-gate", "test-linux", "test", "test-install", "docs-build", "site-build", "release-site-build", "pr-gate"] -coverage = "codecov unit flag; carried pure transforms join the Python coverage rail" -debt = "none: carried sources pass strict types and source ceilings; full Inspect adapter/package acceptance remains in #342" -reason = "Already-carried external integration source has one checker/test/CI owner without claiming an installed or published adapter." - [[surface]] id = "developer-skills" state = "retained" diff --git a/tests/citadel/test_bench_route_coverage.py b/tests/citadel/test_bench_route_coverage.py index 757f758c8..fd513d8d5 100644 --- a/tests/citadel/test_bench_route_coverage.py +++ b/tests/citadel/test_bench_route_coverage.py @@ -167,9 +167,10 @@ def test_the_debt_is_a_ratchet_that_can_only_shrink() -> None: coverage = _config() unmeasured = len(coverage["unmeasured"]) # The reviewed provider and lifecycle mutations add six stateful paths. - # Catalog/grant reads are measured; record the actual remaining 75 rather - # than retaining headroom that could hide future classification debt. - assert unmeasured <= 75, ( + # The combined #206 surface adds three supervised proxy lease paths; + # managed claim/close now have successful tombstone measurements. These + # changes leave exactly 76 stateful paths with named acceptance owners. + assert unmeasured <= 76, ( f"{unmeasured} routes are unmeasured, more than when this guard was " "written. A new route should be measured, not added to the debt." ) diff --git a/tests/citadel/test_exposure_forwarding_boundary.py b/tests/citadel/test_exposure_forwarding_boundary.py index 2d357a755..780fcde61 100644 --- a/tests/citadel/test_exposure_forwarding_boundary.py +++ b/tests/citadel/test_exposure_forwarding_boundary.py @@ -4,8 +4,8 @@ service and gateway: a reverse proxy that read and wrote workload bytes in processes holding credentials, the session ledger and every VM's lifecycle. The shipped design keeps the privilege split of the network router: the -service decides and sends bounded IPC, the VM owner binds the loopback -listener and connects the authorized guest target, and only the +service decides and mints an exact loopback listener descriptor, the VM owner +admits that listener and connects the authorized guest target, and only the environment-cleared, sandbox-confirmed `capsem-router` copies bytes between the two connected descriptors it is handed. """ @@ -26,7 +26,7 @@ capsem-gateway may accept a loopback browser socket and read bounded control material needed to authenticate it, but hands that same descriptor to the VM owner without opening the guest destination or carrying workload bytes. The VM -owner binds loopback TCP exposures, connects the authorized guest target, and +owner receives loopback TCP exposures, connects the authorized guest target, and grants connected descriptor pairs to a router spawned with a cleared environment, "/" as its directory, no stdout, and a confirmed sandbox. Do not add a service or gateway reverse proxy, and do not give the router privileged @@ -110,14 +110,20 @@ def test_the_gateway_tunnel_is_the_stream_upgrade_to_the_service_socket() -> Non assert "SWITCHING_PROTOCOLS" in tunnel and "hyper::upgrade::on" in tunnel, RATIONALE -def test_vm_owner_binds_loopback_and_grants_connected_pairs() -> None: +def test_coordinator_mints_loopback_listener_and_owner_grants_connected_pairs() -> None: publisher = ( PROJECT_ROOT / "crates/capsem-core/src/container/publish.rs" ).read_text(encoding="utf-8") + coordinator = ( + PROJECT_ROOT / "crates/capsem-service/src/upstream_broker.rs" + ).read_text(encoding="utf-8") broker = ( PROJECT_ROOT / "crates/capsem-core/src/container/publish/broker.rs" ).read_text(encoding="utf-8") - assert "TcpListener::bind((Ipv4Addr::LOCALHOST, host_port))" in publisher, RATIONALE + assert "TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port))" in coordinator, RATIONALE + assert ".accept(" not in coordinator and not BYTE_COPY.search(coordinator), RATIONALE + assert re.search(r"listener_authority\s*\.open\(host_port\)", publisher), RATIONALE + assert "TcpListener::bind" not in publisher, RATIONALE grant = " ".join(broker.split()) assert ".grant( flow.source.as_fd(), destination.as_fd()," in grant, RATIONALE diff --git a/tests/citadel/test_generated_output_boundary.py b/tests/citadel/test_generated_output_boundary.py index 1323558eb..d44bcc90d 100644 --- a/tests/citadel/test_generated_output_boundary.py +++ b/tests/citadel/test_generated_output_boundary.py @@ -525,12 +525,9 @@ def test_test_evidence_and_coverage_use_canonical_target_roots() -> None: "root erased the diagnostics of the failure that triggered teardown." ) assert workspace_evidence.parent == Path(evidence), RATIONALE - # Surface owners choose coverage inputs; this boundary owns every report - # destination, including when another source surface joins the unit rail. - assert [ - flag for flag in config["suites"]["pytest"]["coverage_flags"] - if flag.startswith("--cov-report=") - ] == [ + assert config["suites"]["pytest"]["coverage_flags"] == [ + "--cov=build_system/builder", + "--cov=integrations/inspect-ai/inspect_capsem", f"--cov-report=xml:{coverage}/python/codecov.xml", ], RATIONALE assert nextest["store"]["dir"] == f"{coverage}/nextest", RATIONALE diff --git a/tests/citadel/test_hot_build_contract.py b/tests/citadel/test_hot_build_contract.py index 76c38db59..1a9458fce 100644 --- a/tests/citadel/test_hot_build_contract.py +++ b/tests/citadel/test_hot_build_contract.py @@ -33,6 +33,8 @@ "ryu", "quick_cache", "rusqlite", + "libsqlite3-sys", + "capsem-logger", } RATIONALE = """\ diff --git a/tests/citadel/test_repository_surface_ownership.py b/tests/citadel/test_repository_surface_ownership.py index e75d95899..aacbfb835 100644 --- a/tests/citadel/test_repository_surface_ownership.py +++ b/tests/citadel/test_repository_surface_ownership.py @@ -45,7 +45,7 @@ "crates/", "guest/", "images/", - "integrations/", + "integrations/inspect-ai/", "mcp/", "sdk/", "skills/", diff --git a/tests/citadel/test_sdk_ci.py b/tests/citadel/test_sdk_ci.py index 1e622ee80..d77657369 100644 --- a/tests/citadel/test_sdk_ci.py +++ b/tests/citadel/test_sdk_ci.py @@ -64,7 +64,14 @@ def _assert_mcp_owner(fast: dict[str, Any], ci: dict[str, Any], coverage: dict[s sdk = next(index for index, step in enumerate(steps) if step.get("working-directory") == "sdk/typescript") mcp = next(index for index, step in enumerate(steps) if step.get("working-directory") == MCP) assert sdk < mcp, "the MCP build links the SDK package the SDK step builds; " + RATIONALE - assert "pnpm test " in steps[mcp]["run"], RATIONALE + assert steps[mcp]["run"].splitlines() == [ + "pnpm install --frozen-lockfile", + "pnpm run prewarm:packed", + ( + "pnpm test --reporter=default --reporter=junit " + "--outputFile=../../cache/target/coverage/junit/mcp-typescript.xml" + ), + ], RATIONALE upload = next(step for step in steps if step.get("with", {}).get("flags") == "mcp-server") assert upload["with"]["files"] == MCP_LCOV, RATIONALE vitest = (ROOT / MCP / "vitest.config.ts").read_text() diff --git a/tests/citadel/test_sdk_rust_quality.py b/tests/citadel/test_sdk_rust_quality.py index 03804321b..a7aedcca9 100644 --- a/tests/citadel/test_sdk_rust_quality.py +++ b/tests/citadel/test_sdk_rust_quality.py @@ -27,10 +27,11 @@ def _assert_owners(manifest: dict, floors: dict, ci: dict, coverage: dict) -> No assert all(manifest.get("lib", {}).get(key, True) for key in ("test", "doctest")), SDK_RATIONALE assert floors.get("capsem-sdk", 0) >= 90, SDK_RATIONALE local = {name for name in manifest["dependencies"] if name.startswith("capsem")} - assert local == {"capsem-api", "capsem-crypto"}, ( - "SDK dependencies are limited to the public API and owned secret primitives; " - "they must not grant service/core or local discovery access" - ) + assert local == {"capsem-api", "capsem-crypto"}, SDK_CRYPTO_RATIONALE + crypto = manifest["dependencies"]["capsem-crypto"] + assert crypto.get("path") == "../../crates/capsem-crypto", SDK_CRYPTO_RATIONALE + assert crypto.get("default-features") is False, SDK_CRYPTO_RATIONALE + assert set(crypto.get("features", [])) == {"random", "secrets"}, SDK_CRYPTO_RATIONALE steps = ci["jobs"]["test"]["steps"] commands = next(step["run"] for step in steps if step.get("name") == "Unit tests with coverage") for command in commands.splitlines(): diff --git a/tests/citadel/test_sdk_typescript_ci.py b/tests/citadel/test_sdk_typescript_ci.py index 073736431..77a33f02e 100644 --- a/tests/citadel/test_sdk_typescript_ci.py +++ b/tests/citadel/test_sdk_typescript_ci.py @@ -17,10 +17,14 @@ def _assert_owners(fast: dict, ci: dict, coverage: dict) -> None: assert prewarm < sealed, RATIONALE steps = ci["jobs"]["test"]["steps"] test = next(step for step in steps if step.get("name") == "TypeScript SDK tests with coverage") - assert test["run"].splitlines() == ["pnpm install --frozen-lockfile", ( - "pnpm test --reporter=default --reporter=junit " - "--outputFile=../../cache/target/coverage/junit/typescript-sdk.xml" - )], RATIONALE + assert test["run"].splitlines() == [ + "pnpm install --frozen-lockfile", + "pnpm run prewarm:package", + ( + "pnpm test --reporter=default --reporter=junit " + "--outputFile=../../cache/target/coverage/junit/typescript-sdk.xml" + ), + ], RATIONALE upload = next(step for step in steps if step.get("with", {}).get("flags") == "typescript-sdk") assert upload["with"]["files"] == "cache/target/coverage/typescript-sdk/lcov.info", RATIONALE assert upload["uses"].startswith("codecov/codecov-action@"), RATIONALE diff --git a/tests/citadel/unix_boundary_debt.toml b/tests/citadel/unix_boundary_debt.toml index f90cb4496..f98f0620d 100644 --- a/tests/citadel/unix_boundary_debt.toml +++ b/tests/citadel/unix_boundary_debt.toml @@ -31,7 +31,7 @@ "crates/capsem-core/src/hypervisor/kvm/virtio_console.rs" = "b0a92055b6ed9190b992f25a16b92b2aa0242b3c4be245d474410b0b2fa6cdeb" "crates/capsem-core/src/hypervisor/kvm/virtio_fs/mod.rs" = "f3ce99ec68343009ffcab5ba74802fc3527361ee646109157d9ad55f443976be" "crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_dir.rs" = "1c2723207e04d0be3594c7ba21fdc13d6ff761e6f6d4dbff85ab2fc06da3f4c4" -"crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs" = "b5f5c223d1f514cb692d8e6604227f687087f481b07ed84b42b1a20b7a693be4" +"crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_file.rs" = "9f99917b8f9e273f2de49df5c515f41e154cb5ba2284c7b8e74f5dd80caf4a20" "crates/capsem-core/src/hypervisor/kvm/virtio_fs/ops_meta.rs" = "b933aa796bce77d6bf0face797a38163ce4c473dacc149c23c5baa3390166a4c" "crates/capsem-core/src/hypervisor/kvm/virtio_mmio.rs" = "f9797a518c8d6273151bc775f9a176a72f3b0a62e09cd4cd4fb636477b58627b" "crates/capsem-core/src/hypervisor/kvm/virtio_vsock.rs" = "e8497a3dd53669d1e463c69a098214ece6257555c2a11d10a3d5642027e0a56a" @@ -152,3 +152,19 @@ reason = "macOS Seatbelt error release paired with sandbox_init_with_parameters" [foundation.raw_calls."crates/capsem-foundation/src/unix/worker_sandbox/macos.rs::sandbox_init_with_parameters"] fingerprint = "4d6b9f0844b92206d15c663c6a82d1619a9a7117d06167eb037c779af1fc9f44" reason = "macOS parameterized Seatbelt confinement for worker-specific path grants; nix has no wrapper" + +[foundation.raw_calls."crates/capsem-foundation/src/unix/process/pinned.rs::syscall"] +fingerprint = "e7b0fa754e405a0d00ed0808bf38b8c6bedae2df72a82154e0bbd61d42990c93" +reason = "Linux pidfd_open and pidfd_send_signal retain original process identity across exit and PID reuse; pinned nix 0.29 provides no wrapper for either syscall" + +[foundation.raw_calls."crates/capsem-foundation/src/unix/process/pinned_macos.rs::proc_pidinfo"] +fingerprint = "33ed8bd6a7cea2e0bef25a259110c71112d2e9e0bcebaef0297bf64e85b8af63" +reason = "Darwin combined native BSD status and immutable process identity including zombies; nix has no libproc query for this fixed ABI" + +[foundation.raw_calls."crates/capsem-foundation/src/unix/process/pinned_macos.rs::dlsym"] +fingerprint = "806c2d3062055d3c1bf7a97779715cbadc6d136ffae2fa2c59141886d0222b3e" +reason = "Resolve Darwin's PID-version guarded signal API only when present; macOS 14 lacks it and must retain exit observation without raw PID fallback; nix has no libproc symbol wrapper" + +[foundation.raw_calls."crates/capsem-foundation/src/unix/process/scope.rs::sysctlbyname"] +fingerprint = "017ea7709880405f8f0ef6262209e68cb7250c7c308852bdd237dbe936d36771" +reason = "Read Darwin's bounded kernel boot-session UUID so durable process witnesses cannot cross reboot; nix has no Darwin named sysctl wrapper" diff --git a/tests/fixtures/oci/README.md b/tests/fixtures/oci/README.md index 073476fd6..cb8a291fc 100644 --- a/tests/fixtures/oci/README.md +++ b/tests/fixtures/oci/README.md @@ -173,6 +173,20 @@ the broker's hermetic OAuth endpoint -- stays a `"target": "vm"` exec. amd64 emulator here) and add its `linux/amd64` digest beside the arm64 one. Until then the x86_64 lanes fail at the prefetch, naming the missing pin. +### Building and pinning the reference image (maintainers) + +The official `images/dev` image follows the same immutable-layout workflow. +Run `capsem-gate reference-image` on each native architecture, qualify the +printed layout with `capsem-gate image-qualify --image dev --layout +`, and update `[functional.reference_image] digests` only after +that native qualification passes. Publish the exact layout bytes to +`ghcr.io/google/capsem/dev` with `oras cp`; rebuilding or using `docker push` +can produce a different manifest digest. Attach the OBOM with +`images/ci/rootfs.py --image dev:`, make the package public, and verify an +anonymous manifest fetch by digest before relying on the pin in CI. The pinned +layout is also mounted read-only into installed-package proofs through +`CAPSEM_REFERENCE_IMAGE_LAYOUT`. + ## Artifact identities and evidence Supported ARM64 proof: **23 passed in 123.47s** at diff --git a/tests/fixtures/session/test.db b/tests/fixtures/session/test.db index a748383cb7ce6f850ea892e92eb2d77290bab5a5..b11a39f6cd8536c5ce4f006f7e01e14bfbca8752 100644 GIT binary patch delta 6136 zcmcgw3v^V~xjz3sk9}q)GbisMnItol$t0O1lL-Vw9tjDEybJ^)6;KEXi4X~SO#;}+ zV{E0oDuEGx9twySdM~#YNoCsAAm?lMg=cNGG+yWOxeC;XAAJ9$)j)xEj(3+tQO zs~hTEh;!_d3pp6-d3*-~0`!ko7HI-1f@(^%bB|KN>j2mg}|4Ied6!_zNG zmEY3jPNo;v)KW8b0yUGyP&2-ano*_H48N0_qC#qNa;Qnkpe8Ps8momGMc=f{WNJaX zOs%j&yA1uvBo(^*#r=m39_zm1T;_l^-OGmc0<7+CE}?&`!qp>wVrG1az>o7Kyo2U9 z^Z$R#qiDoBb9_Fti#|Rp?OzhR1yzXUYhZVz2UtGq&{$|9O3; zC58-sShu&8hwHW!fjPW!%ScedC%0sRDSUa$$#BsQf4FO#J-m7L3&6W4!VG3!w^-zM zj|X-P4(}Ha^yO9@pUzy}gJu2?`qr?}blt-JWs<8~+Q(hc)g9CGi3aPsue=Ab$ySXC zvQ#J7*2($O1*uNESKKG=Bg?mn7A3}DX2@p=KiyRdTKIHVBN<$=eo(kw*A$3fwdA4wm!K#~`+*=syr3Ln3EBpDk{&k- z1?$U`O$+0>eV7|U=~$BmiTI@l;&EVrnw~3#7GY)5DK>(8R`4pCh zyXn~+;<4O5$^)T67}qH$VTL};ytKft3wVtF)q*nv;0F!Y1fY;!acQBPj;ngaM7$V) zB=f2QE4TZ6upt2}yx>Nb3kewV!kWI2#I4!z=|83l2A=-w%voO=6P2W0W7=zK8|&Mf zn;WZjqk}h|Vo}V6Eq(0kDct@?(HxoJgG;xC*QwWcM-J9TkefUWO?CB)8yao+W)|cV zQiNt2I=zs!y9ipct+q!68x#oel+A2?B<9nY&KSY6%Hm)LNQx)hPO(%d&=!T=u4+?r z2DcBP72^!Y??_>Bkfl5JE;XIotH`6{@KOhPbYdz?ur70Q`*;3gB>Proq*Z zTMQ-~>SQIigGzYN$;wQP{kVNF!D=;N?TuxM$%OThLrrZv=kRsjLt(v41K{$9GdHejWNQlkrN2%+cdu@wB_>V3mXAK`JhFuz}z@uoEnm^5c5h zKr(zxiTvqvaF~=I>#25aV!6fD)*q%sY1;qK^ zCb7HBt7BZ;?sP(UE^5il4%wKK%r^HCFnc0Rc2*9zW0&#Qcd4xtrj--L+&aZl(38Rv z!F!!u7Pc@OSNF02zIv9>eY2PO`t}e>j_n*Pf_xltj*T_xGV|T54>S=sTEA7aM($qD zX767v#db?Ozto(zY=H}{f;yI;;14RFC`%O(TD6dtXxn2OYzq)`CY#q-3%T9vg^e?D z{W+Enci_*@v2A_fiK`Qt1;=Kvgoyn^w(T|}+vz9UNjcoulE)L#eV*vN?>xm+)p-_B z3yd_KWDIUUPc+ZEK!fz8;v46gTg|^|M*Ibuu~DBvPDsRu&eP6kUtne;_xuYi52cGV zwEqJ0s6K8lq1UDuD}Q`}B|v|C{ydRzWsjH?{bgZ$!-D!?d(DCt-%V=L6=)#xu-?>1S9b znN@a%72sQEm=o8WVQwhWJJRuYXPAcfoTc9X(R%}!dX^1Sl(vdrKs8ho;^RPkEU6UN-!I(>V-3Vl-!F}2QVCT!7%1eKIB}Mg zficxmj{aoEr)EioFdSc-C6&S5ShiG7#m5V&jxDa12B^chebOKZm0-~&9a}q8~;^--#EiX=eHfarJjVuOD>v!+{qfGyWh( z#M`0RIO9!mi=MhihtsyDMma@A*EJengD;-3Hw5e3>HIMFE2yj^ zTNf$u%Gj&6CXC_s2^4XIuw@CIQQkf*wGs*Lc~#o1j^_4pG}7M~F&&Yb_+7*X>cZCP zQUV@7OfLS@5y^xVJ)#MpJ0i{3);LBPfjH9$#P<)=#9>FJ1emDv;-`txSgQ%nzv;1_-& zREFKI0jzckKGQDGQ70QIIL@dcvpeMJP>IVsO`aBl8p+rZJ}IA?09J* zO}GaO7RhbdD+W)9)>CT-MGc?V9BjXuB);-EE%Ndrc_j0E;}TZXk9U5smL1w zJBqKueMm~i_YX<0rbKaG(9vEK)w8eMFh#mwlKu@0w3{32Zazcc+*c&ut;*a}uSjE! zb(-oIeQllp`a9O~9G1GG*+wL3qa3rlOKur&HBS}n!b(9~t9fh(Y~xHHnSv%w-K7rU zNBB_Xr-~vkmQF~i;*(+u+riS{$Dj!-tv%Mdn9pL?#-v&HS%#bUnrqFq=V+8Ji0Z#Ep0$K6na4SBp6nvGiES9!cpnoX3)LDCOpJzqEK3x8vn}9)L!@ z#C711Jle&y0`7pBXzr8;!F>Iola5^lw5j=>vI`!-wViS?JcI{3LZxkFd(VK_UFb1`I@j@DMW6oZA-uf0ma`uY;tCr{({sU_GmOG?7n zrE&oGERh|!@3<6)TAS=AvX!*SWq78QdZTIJ*)};3zK6%#Q|UVJ?$7eK9nU~;?cRI9GV4xLt#v8tVR|2M*JZh|0)pXo;&SwH)a zXt{PxhqJS|hwdolnWXYU$kv39XYqBg05iNis5TnaplnoiKlAcR0`sq4-b~mvW%Ide z1MQm@H5pO-Nj9&7d3bjYpAU2O71Hr|4*#C=AYJavzs0z}sY)`m$7m09$x3WQ=~Di8 z-GLriij7ER{4YB;bV<4K@(MwFLMyNxwB7Y}zN#twB%h?bsQBa+B-ArxG308;*9y5G zet>^=6LsPOd?p03Ilza(qqsM~rLd+@IwZn1n-Hx^~`--Gzi!CcibrxR9^ zA;6F1auq9b`MqlWHJ3(j>R!y{@nn4TCXSv}{4AHJ=%n=Vd}!0-DQ#0H9l&S$ct2w0 wLLcGQYFxhk*2kT@FR^RMAtJ@9ZFj)R$fCQ|Cl&ZnB(Xxh$l!;&E7hI<2LyM5ZU6uP delta 5527 zcmd5=dstNGnVS#Xlha! zQ?J}J!8aGRnoY=d-85RdjA_%>wq|z|8)K7pw|TaSB>M>Q*^QWHgKbS@zcYgj3cLU8 zv-|9M9=`d```zB}{r%p{p|v%obuhWx6w##$KTSRMrEzNc14ZJIZ^@(b7xKT$!}6!{ zNAmCG5BwJM3lbG*aDm^q*$P7c>CF!jO`DSwjmgP?k$(BX0m<3oba~6`YtT?dUd8R- z1#|yP+eBVp-R!JkBlVRH<;~6=c*D*w(b(o#tr|0Xtp2COjhgm&5RDcKJO};{#s0f@ z>R+X)`+~oT99wCR<&Qg1+o1W zvjqLIkwbd;ZvU8RZ#*37-|h4 zxKE;V+#YAk96}$d7`|o(o^S;zGwqm|Dt!d z0ciX^m$KL~o=wX^?H|~b1UmnRn|@D~C0HArHMP#>@@jWuW4*VLvB*mgN zBnGE%n>tjUJyoVYxSoo}>}=?Kz*||>;G8;Rip2j>S#0uz4(9YAcNPBDHn^)8u|3Ym zcRF3wPWRQw`gLm<~(!u z!byv&8|oGR!pJ|8D1UyLlrk=;ueoRUre=4|&T3}0W_JTZI&a)CS>iOWDmACiX=yBX z?_yfav76_x;oVOtooctcxu)J#>2)_V{|xIpvHs3A{f@F(X;Y;4=n;wsnEfcz_r^PV zd+7bKdRL9ryu7$0!X^ z<_Q?=<|!7Lbgl7~_Z|=mg1)taXw1j}{~Kkg!D#Z*fI*^$e%dOSp{rka@{clf_K&>| zh96#rtW14}O0ROLND-iTMJVEbCm+_2@egu?+(z;?m{c#T9Qqz5(y?b}9JaMVCa!FQ zMEs{V$dG1<#%0OiOT?e+X$+d$A&QN>)(*3kzJyoWK!;BP=ycxtMyJPH+1O-liN&Yd zVJ^;V2NPaw2P^q+Ld9CnM~yhz4ml8yH?E5@cwS94xTXW= zArhZ*6Agad0Ww7QW`YA|DXS8CcSB;NCq0CFeU05*Ut`@f6Tjz!xeyziftNc$*E<5q zq#+WYl_3M7@w^NLn*Fm3qS0!F!;P3ozydRI8G++>hbG+3!I#tJrkUZ~)R7?v)6xqT z{56GHFaw7uJb%|d$MI4%(FB%CaAUe8wZstKje*hJ5RZur17=0#is)>E6pc@e5{*R+ zksJk)Xv_wP0i%LQB02%?W5=EV$Yi!-mrKBF0Jc5@G!1lvcD>M{@m?v$I3eL+=ODmVkIz8PYPi=?Q~)7_9mghO%}kMNt|{!5)m8RB$>D^ znCSyMq>VGP71F_siThv)q+&!DgL3gch{8krAd`4Z_;Du#G({OGmS&5_JcjXO*X#qU(z6MP)VL}+m~?Tt3u3jFL#q>l#_!&a=l4N8JL3=gAZ%ylItLVLr(?Xt zNVNXPoF>3ZvA+RbwcF)Y5{MGl&~yMC9J?Eyd6cY}?&4s>e>nmLFO!PCCQ_kP5h*gc zq!M&I%7^3_?E{+MYUm2ve-6qF}e$1UVcy?BKy!GClN-7K%hsi(T9D*xHG%mtIEseu1w}}CAu}Mp3e5d!VZz@PMo^`Q>->zUj!u&59PgfA#%rNT?d^=k- zrlvy2BHVL>B(ezi{0;Kr-8T+=@dbHfT9U zvL&zIB*|dM4{kEoe?}3^{u8mK99)?Z40xT}gWeN9Io!*}hCh+JW8kZSU;ddykm+bN z$$y$`_)Z0JKo+iZvGbxUNvgWVI?nx&kuNloD7>i{`Kv}^?Ykiw`UZ)i?<6T;e|P%p zmA*pABuSN^(w))?T9@WM=|1sG)dePCk1sV(G;YWN-v)#fem3T%@oIc(gf-eU-lgfZ zFJ)ZGffx7S(`mc|itub2fBG)4f`#i@2z=klXG|Nln0qA6<)0@-xk}-=oL|0w<`SIo zBQ6UHmEGb4-}(`^l5oC+#W*^^ErSBQ9N^XVUhooi@DiR&<7Le7Gf$oSNuV@=E=S>RHlG z@l$aX?I&*%9c+ZvESK6rOH0Q;i59KJBPBu+9$6vS@MejyMp`2pH!lX?HtZ=8EZDF@ zm<^lpQi)(y`cf#Q{NoD2syXO=Ab9sK4sWdxKGinv^ag8w_K6%AT`Bx6NbZK3P&huf zTCg(AK3pS|#kicifBUl`(hVHIW#^Up~uMdK>Q zsBAnn#3$nC!@P@G>EX}#Q}U6$CBabKn;LXzj2vb)v*{Wi1?v=^#$enqGw<$7vKZU1 z@#|=%=P~6Ae&ibe{i(>G1!NA95>S86KMPQdC$96Wpfm)>nGxQA$9fpBBRzZ;+>hpG z_`mDB!QbJW;~(c&G%H%WQj_DR6HTt=r3rQFf4pW5cpI40`4OB5cxG(67e4q zJ9IzSN!lY?Q9hu1Q#O ztb9!rsR_1Wo=EedI^@kyQmVt(M4A8(;yWVE!di*WK%GQ0VJqfJ)DD%QI6O8Mtp@8hD z#mo^giSwx{FoB~v@Mx&$baU(m<#>Xl^PxUebY53P-@-(mu7kP>lNs<0j_MVgN8rZ_ z0qO;65dIh8v6FlKakYj8{Dt>;q?YWi#8ZT3!(*Y+@j9UyeXT<4cq%Cv=w^xr=t=sQ zt_2bN)2bNu$)xpau4-!4gKDo@C!Lg1#53ak^tZG@_*&=|68JXw8H;NRRWHj8`nMSm zn>e4m6VDHcOR;TOOh(hNm@Nuk(OB#N+&V1IhrRgCkeGx&7#3}s{m>lzXUX9lO#V!K zb;7p-gP(~PfiCk*{Bs3cH8fePy{VFdsZ9Sk{y;;!pb7J})B|q(p_V=dE}X5SZfL|# z9j%obLP=B|`h1uXK{v5`?}%XY9#vK);z$JD!jLSvF782R1dYV^uZxM$tdy#G%zGFK zGVoU;;tcF45VY7bBAQ?a9vu-Ky}x9HKC@iV;U`SU`FLk<8(oC=Lx|@QA^ckj<$J%R ziOgvtfpNwaV!a(0H;{_4bPj(KzCJPIHc&HEp^4BOe4nu0D_pE<$t9s3 zOlp;iykBf$MKV5K%W4FMr(QEr(uYBI!2czaEu7E13lCov9k34rS4A(hpnXWp#jl4L uA=VCwYV;3@w!SR!3?p9FfLUr;2nPeZwn)b)90<&OMEW-ZPxn toolbar = read("lib/components/shell/Toolbar.svelte") assert "Sessions" in dashboard + assert "import { sessionActionError }" in dashboard assert "sessionActionError(error)" in dashboard assert "Failed to create session" in session_errors assert "Session {vmId} ledger" in stats diff --git a/tests/helpers/body_archive.py b/tests/helpers/body_archive.py index 65a86567c..cf9989cc4 100644 --- a/tests/helpers/body_archive.py +++ b/tests/helpers/body_archive.py @@ -2,7 +2,7 @@ Bodies left SQLite for a block archive beside it: ``event_body_blobs`` says which block a body is in and where inside that block it starts, and the block -itself is a deflated span of raw bytes. A black-box test that wants to know +itself is a compressed span of raw bytes. A black-box test that wants to know what the ledger actually stored has to read it the way the product does, so this is the reader, in the few lines the format needs. @@ -20,14 +20,15 @@ file header (80 bytes): magic "CAPSEMBL" u16 version=3 u16 flags=0 u32 header_len=80 archive_id[16] generation_id[16] blake3(bytes[0:48])[32] - block header (8 bytes): magic "BLK2" u8 codec (1 = raw deflate) u8 flags u16 pad + block header (8 bytes): magic "BLK2" u8 codec (1 = raw deflate, 2 = zstd) + u8 flags u16 pad segment (repeated): magic "SGMT" u8 flags (bit 0 FINAL) u8[3] pad u32 raw_start u32 raw_len u32 comp_len - blake3(segment raw)[32] deflate bytes + blake3(segment raw)[32] compressed bytes -A block is one deflate stream cut at sync-flush points into segments; one -inflater runs across them, and the last segment of a closed block ends the -stream. A block that is still being written simply has no FINAL segment yet. +A block is one codec stream cut at flush points into segments; one decoder +runs across them, and the last segment of a closed block ends the stream. A +block that is still being written simply has no FINAL segment yet. Every read verifies each segment's blake3 against its header and the body's own blake3 against the index row that named it, exactly as the product reader @@ -47,6 +48,8 @@ from pathlib import Path from typing import Any, BinaryIO, Self +import pyzstd + FILE_MAGIC = b"CAPSEMBL" FILE_VERSION = 3 # The ledger's own format (`archive_state.format_version`). It moved to 4 when @@ -56,6 +59,7 @@ BLOCK_MAGIC = b"BLK2" BLOCK_HEADER_BYTES = 8 CODEC_DEFLATE = 1 +CODEC_ZSTD = 2 SEGMENT_MAGIC = b"SGMT" SEGMENT_HEADER_BYTES = 52 SEGMENT_FINAL = 0x01 @@ -131,11 +135,18 @@ def generation_path_for_db(db_path: Path | str) -> Path: class _Cursor: """One block inflated up to the end of some segment, as the product keeps it.""" - def __init__(self, block_offset: int, disk_len: int, raw_len: int) -> None: + def __init__( + self, block_offset: int, disk_len: int, raw_len: int, codec: int + ) -> None: self.block_offset = block_offset self.block_end = block_offset + disk_len self.raw_limit = raw_len - self.inflater = zlib.decompressobj(-15) + self.codec = codec + self.inflater: Any = ( + zlib.decompressobj(-15) + if codec == CODEC_DEFLATE + else pyzstd.ZstdDecompressor() + ) self.raw = bytearray() self.next_segment_at = block_offset + BLOCK_HEADER_BYTES self.finished = False @@ -198,10 +209,17 @@ def read(self, event_id: str, source_table: str, direction: str) -> bytes | None if row is None: return None if row[7] != self._expected_generation_id: - raise AssertionError(f"{self.db_path} selected another archive generation while it was being read") + raise AssertionError( + f"{self.db_path} selected another archive generation while it was being read" + ) self.committed_end = max(self.committed_end, int(row[6])) block_offset, body_offset, body_len, body_hash, disk_len, raw_len = ( - int(row[0]), int(row[1]), int(row[2]), row[3], int(row[4]), int(row[5]) + int(row[0]), + int(row[1]), + int(row[2]), + row[3], + int(row[4]), + int(row[5]), ) end = body_offset + body_len try: @@ -232,7 +250,9 @@ def security_payload( raise AssertionError(f"{source_table} does not archive a security payload") body = self.read(event_id, source_table, "payload") if body is None: - raise AssertionError(f"{source_table} has no archived payload for {event_id}") + raise AssertionError( + f"{source_table} has no archived payload for {event_id}" + ) return forensic_payload(body) def _verify_file_header(self) -> None: @@ -249,7 +269,9 @@ def _verify_file_header(self) -> None: or _blake3(header[:48]) != header[48:80].hex() or self.generation_path.stat().st_size < self.committed_end ): - raise AssertionError(f"{self.generation_path} is not the selected v3 archive generation") + raise AssertionError( + f"{self.generation_path} is not the selected v3 archive generation" + ) def _span( self, block_offset: int, start: int, end: int, disk_len: int, raw_len: int @@ -288,13 +310,18 @@ def _start_block(self, block_offset: int, disk_len: int, raw_len: int) -> _Curso file.seek(block_offset) header = file.read(BLOCK_HEADER_BYTES) if len(header) != BLOCK_HEADER_BYTES or header[:4] != BLOCK_MAGIC: - raise AssertionError(f"no block at offset {block_offset} of {self._archive}") - if header[4] != CODEC_DEFLATE or header[5:] != b"\x00\x00\x00": + raise AssertionError( + f"no block at offset {block_offset} of {self._archive}" + ) + if ( + header[4] not in (CODEC_DEFLATE, CODEC_ZSTD) + or header[5:] != b"\x00\x00\x00" + ): raise AssertionError( f"block at {block_offset} of {self.generation_path} uses codec {header[4]}, " "which this reader does not know" ) - return _Cursor(block_offset, disk_len, raw_len) + return _Cursor(block_offset, disk_len, raw_len, header[4]) def _inflate_segment(self, file: BinaryIO, cursor: _Cursor) -> None: at = cursor.next_segment_at @@ -302,7 +329,9 @@ def _inflate_segment(self, file: BinaryIO, cursor: _Cursor) -> None: header = file.read(SEGMENT_HEADER_BYTES) where = f"segment at {at} of {self.generation_path}" if len(header) != SEGMENT_HEADER_BYTES: - raise AssertionError(f"{where} is truncated: the block was never written this far") + raise AssertionError( + f"{where} is truncated: the block was never written this far" + ) flags = header[4] raw_start = int.from_bytes(header[8:12], "little") raw_len = int.from_bytes(header[12:16], "little") @@ -333,7 +362,11 @@ def _inflate_segment(self, file: BinaryIO, cursor: _Cursor) -> None: compressed = file.read(comp_len) if len(compressed) != comp_len: raise AssertionError(f"{where} is truncated") - if not final and not compressed.endswith(SYNC_FLUSH_TAIL): + if ( + cursor.codec == CODEC_DEFLATE + and not final + and not compressed.endswith(SYNC_FLUSH_TAIL) + ): raise AssertionError(f"{where} does not end on a sync flush") # One inflater across the block's segments, bounded by the length the # header declared plus one byte, so an overlong segment is seen rather @@ -342,15 +375,21 @@ def _inflate_segment(self, file: BinaryIO, cursor: _Cursor) -> None: inflater = cursor.inflater try: raw = inflater.decompress(compressed, raw_len + 1) - except zlib.error as error: + except (zlib.error, pyzstd.ZstdError) as error: raise AssertionError(f"{where} did not inflate: {error}") from error + if cursor.codec == CODEC_DEFLATE: + stream_invalid = bool(inflater.unconsumed_tail) + else: + stream_invalid = inflater.needs_input != (not final) if ( len(raw) != raw_len - or inflater.unconsumed_tail + or stream_invalid or inflater.eof != final or (final and inflater.unused_data) ): - raise AssertionError(f"{where} did not inflate to the {raw_len} bytes it declares") + raise AssertionError( + f"{where} did not inflate to the {raw_len} bytes it declares" + ) expected_hash = header[20:SEGMENT_HEADER_BYTES].hex() if _blake3(raw) != expected_hash: raise AssertionError( @@ -360,10 +399,17 @@ def _inflate_segment(self, file: BinaryIO, cursor: _Cursor) -> None: cursor.raw += raw cursor.next_segment_at = at + SEGMENT_HEADER_BYTES + comp_len cursor.finished = final - if cursor.next_segment_at == cursor.block_end and len(cursor.raw) != cursor.raw_limit: - raise AssertionError(f"{where} ends at {len(cursor.raw)} raw bytes, not {cursor.raw_limit}") + if ( + cursor.next_segment_at == cursor.block_end + and len(cursor.raw) != cursor.raw_limit + ): + raise AssertionError( + f"{where} ends at {len(cursor.raw)} raw bytes, not {cursor.raw_limit}" + ) if final and cursor.next_segment_at != cursor.block_end: - raise AssertionError(f"{where} is final before the recorded block extent ends") + raise AssertionError( + f"{where} is final before the recorded block extent ends" + ) def archived_bodies(db_path: Path | str) -> list[tuple[str, str, str, bytes]]: @@ -374,7 +420,9 @@ def archived_bodies(db_path: Path | str) -> list[tuple[str, str, str, bytes]]: bodies when they moved to the archive, and security payloads when those followed; it has to walk this too, or it passes by not looking. """ - with contextlib.closing(sqlite3.connect(f"file:{Path(db_path)}?mode=ro", uri=True)) as conn: + with contextlib.closing( + sqlite3.connect(f"file:{Path(db_path)}?mode=ro", uri=True) + ) as conn: has_index = conn.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'event_body_blobs'" ).fetchone() @@ -393,7 +441,9 @@ def archived_bodies(db_path: Path | str) -> list[tuple[str, str, str, bytes]]: for source_table, event_id, direction in keys: body = archive.read(event_id, source_table, direction) if body is None: - raise AssertionError(f"{source_table}/{direction} of {event_id} vanished mid-scan") + raise AssertionError( + f"{source_table}/{direction} of {event_id} vanished mid-scan" + ) bodies.append((source_table, event_id, direction, body)) return bodies @@ -500,7 +550,7 @@ def forensic_payload(body: bytes) -> dict[str, Any]: """ payload = _messagepack().decode(body) if not isinstance(payload, dict): - raise AssertionError(f"security payload is not a map: {type(payload).__name__}") + raise TypeError(f"security payload is not a map: {type(payload).__name__}") for name in FORENSIC_LIST_FIELDS: payload.setdefault(name, []) for name in FORENSIC_OPTION_FIELDS: @@ -516,7 +566,11 @@ def _security_payload_from_response( continue assert not body["truncated"] and not body["truncated_for_transport"], body content = str(body["content"]) - raw = base64.b64decode(content) if body["encoding"] == "base64" else content.encode() + raw = ( + base64.b64decode(content) + if body["encoding"] == "base64" + else content.encode() + ) return forensic_payload(raw) raise AssertionError( f"{source_table} has no served payload for {event_id}: {served}" diff --git a/tests/helpers/image_session.py b/tests/helpers/image_session.py index 697bd534e..d677ff216 100644 --- a/tests/helpers/image_session.py +++ b/tests/helpers/image_session.py @@ -106,7 +106,7 @@ def session_of( with layout_registry(workdir, layout, image_name) as (reference, certificate, _): catalog = ( f"{reference.split('/', 1)[0]}/library/catalog:stable" - if (layout / "rootfs-v2").is_dir() + if (layout / "rootfs-v2").is_dir() or (layout / "rootfs").is_dir() else None ) grant_exact(service.home_dir, reference, catalog=catalog, certificate=certificate) diff --git a/tests/ironbank/kingslanding/test_publish.py b/tests/ironbank/kingslanding/test_publish.py index d26110605..071a2c024 100644 --- a/tests/ironbank/kingslanding/test_publish.py +++ b/tests/ironbank/kingslanding/test_publish.py @@ -251,9 +251,8 @@ def test_port_collision_does_not_replace_a_listener(service, tmp_path): timeout=30, check=False, ) - assert ( - result.returncode != 0 and b"bind publication listener" in result.stderr - ), result.stderr + assert result.returncode != 0, result.stderr + assert b"loopback listener grant denied: BindFailed" in result.stderr, result.stderr assert service.client().get("/vms/list")["sandboxes"] == [] with socket.create_connection(("127.0.0.1", port), timeout=2): accepted, _ = listener.accept() diff --git a/tests/ironbank/model_ledger.py b/tests/ironbank/model_ledger.py index dfa89bac3..3db4551df 100644 --- a/tests/ironbank/model_ledger.py +++ b/tests/ironbank/model_ledger.py @@ -403,7 +403,7 @@ def assert_two_turn_model_ledger_exchange( } assert turn.call_response in (trace_tool_responses[0]["content_preview"] or "") - file_row = _assert_created_file_row( + file_row = _assert_written_file_row( conn, trace_id=trace_id, file_path=turn.file_path, @@ -896,7 +896,7 @@ def _assert_tool_output_file( ] -def _assert_created_file_row( +def _assert_written_file_row( conn: sqlite3.Connection, *, trace_id: str, @@ -912,20 +912,30 @@ def _assert_created_file_row( """ SELECT * FROM fs_events - WHERE action = 'created' + WHERE action IN ('created', 'modified') ORDER BY id """ ).fetchall() - if any(row["trace_id"] == trace_id and row["name"] == path for row in rows): + if any( + row["trace_id"] == trace_id + and row["name"] == path + and row["size"] == len(file_content.encode()) + for row in rows + ): break time.sleep(0.25) - matches = [row for row in rows if row["trace_id"] == trace_id and row["name"] == path] + matches = [ + row + for row in rows + if row["trace_id"] == trace_id + and row["name"] == path + and row["size"] == len(file_content.encode()) + ] assert len(matches) == 1, [dict(row) for row in rows] row = matches[0] _assert_event_id(row["event_id"]) assert row["path"] == path, dict(row) assert row["directory"] == ".", dict(row) - assert row["size"] == len(file_content.encode()), dict(row) assert row["credential_ref"] is None, dict(row) return row diff --git a/tests/ironbank/test_doctor_ledger.py b/tests/ironbank/test_doctor_ledger.py index 803032cdd..9237dc7db 100644 --- a/tests/ironbank/test_doctor_ledger.py +++ b/tests/ironbank/test_doctor_ledger.py @@ -303,6 +303,10 @@ def test_capsem_doctor_pays_protocol_and_security_ledger_debt(): assert mcp_default["action"] in {"allow", "ask", "block", "disable"} assert mcp_default["source"] in {"corp", "settings", "default"} + mcp_refresh = client.post("/mcp/servers/local/refresh", timeout=30) + assert mcp_refresh["success"] is True + assert mcp_refresh["instances"] >= 1 + mcp_servers = client.get("/mcp/servers/list", timeout=30) assert isinstance(mcp_servers, list) assert mcp_servers @@ -764,11 +768,18 @@ def test_runtime_plugin_action_matrix_pays_file_import_ledger_debt(): assert read_status == 200 assert read_body.decode() == EICAR_TEXT - # DbWriter.write().await accepts into the process-owned buffer. Stop - # the persistent VM to drain and checkpoint that writer before opening - # session.db directly; polling the live file is not a flush barrier. - stopped = client.post(f"/vms/{vm_id}/stop", {}, timeout=60) - assert stopped == {"success": True, "persistent": True}, stopped + plugins = client.get("/plugins/list", timeout=30) + by_id = {plugin["id"]: plugin for plugin in plugins["plugins"]} + assert by_id["dummy_pre_eicar"]["runtime"]["enabled"] is False + assert by_id["dummy_post_allow"]["runtime"]["enabled"] is True + assert by_id["dummy_post_allow"]["runtime"]["execution_count"] == 0 + # The info route aggregates the plugin's runtime across sessions. + dummy_post_detail = client.get("/plugins/dummy_post_allow/info", timeout=30) + assert dummy_post_detail["runtime"]["enabled"] is True + assert dummy_post_detail["runtime"]["execution_count"] >= 1 + # Stop awaits the owner and its DB flush before inspecting disk rows. + stopped = client.post(f"/vms/{vm_id}/stop", timeout=60) + assert stopped is not None conn = ledgers.enter_context(contextlib.closing(_connect_session_db(service, client, vm_id))) security_rows = conn.execute( diff --git a/tests/ironbank/test_mcp_settings_ledger.py b/tests/ironbank/test_mcp_settings_ledger.py index 4c8892217..166d39757 100644 --- a/tests/ironbank/test_mcp_settings_ledger.py +++ b/tests/ironbank/test_mcp_settings_ledger.py @@ -131,6 +131,11 @@ def test_mcp_call_pays_full_ledger_blackbox(): assert isinstance(vm_id, str) assert created.get("name") == session_id assert wait_exec_ready(client, vm_id, timeout=EXEC_READY_TIMEOUT) + assert client.post("/mcp/servers/local/refresh", timeout=30) == { + "success": True, + "server_id": "local", + "instances": 1, + } with packed_npm_mcp(service.tmp_dir) as mcp: listed = mcp.request("tools/list") diff --git a/tests/ironbank/test_model_client_ledger_contract.py b/tests/ironbank/test_model_client_ledger_contract.py index 1d452ada5..f06073fd3 100644 --- a/tests/ironbank/test_model_client_ledger_contract.py +++ b/tests/ironbank/test_model_client_ledger_contract.py @@ -566,21 +566,14 @@ def test_openai_two_tool_calls_have_exact_item_cardinality( "SELECT * FROM tool_calls WHERE tool_name = 'exec_command' ORDER BY id" ).fetchall() tool_responses = conn.execute("SELECT * FROM tool_responses ORDER BY id").fetchall() - expected_filenames = {item["filename"] for item in result["results"]} - file_rows = _eventually( - lambda: conn.execute( - """ - SELECT * - FROM fs_events - WHERE action = 'created' - ORDER BY id - """ - ).fetchall(), - lambda rows: ( - expected_filenames <= {row["name"] for row in rows if row["name"] is not None} - ), - timeout_s=15, - ) + file_rows = conn.execute( + """ + SELECT * + FROM fs_events + WHERE action IN ('created', 'modified') + ORDER BY id + """ + ).fetchall() net_rows = conn.execute( """ SELECT * @@ -699,16 +692,17 @@ def test_openai_two_tool_calls_have_exact_item_cardinality( assert len(trace_tool_responses) == 1, [dict(row) for row in tool_responses] assert trace_tool_responses[0]["call_id"] == expected["call_id"] assert expected["call_response"] in (trace_tool_responses[0]["content_preview"] or "") - created = [ + written = [ row for row in file_rows - if row["trace_id"] == trace_id and row["name"] == expected["filename"] + if row["trace_id"] == trace_id + and row["name"] == expected["filename"] + and row["size"] == len((expected["nonce"] + "\n").encode()) ] - assert len(created) == 1, [dict(row) for row in file_rows] - assert created[0]["size"] == len((expected["nonce"] + "\n").encode()) - assert created[0]["directory"] == ".", dict(created[0]) - assert created[0]["credential_ref"] is None, dict(created[0]) - file_event_ids.append(created[0]["event_id"]) + assert len(written) == 1, [dict(row) for row in file_rows] + assert written[0]["directory"] == ".", dict(written[0]) + assert written[0]["credential_ref"] is None, dict(written[0]) + file_event_ids.append(written[0]["event_id"]) event_ids = [row["event_id"] for row in [*model_calls, *net_rows, dns]] event_ids.extend(file_event_ids) @@ -826,7 +820,7 @@ def test_openai_two_tool_calls_have_exact_item_cardinality( assert route_row["source"] == db_row["origin"] assert route_row["credential_ref"] == db_row["credential_ref"] - created_by_event = {row["event_id"]: row for row in file_rows if row["event_id"]} + written_by_event = {row["event_id"]: row for row in file_rows if row["event_id"]} for event_id in file_event_ids: assert event_id in route_file_by_event, { "missing_file_event_id": event_id, @@ -838,7 +832,7 @@ def test_openai_two_tool_calls_have_exact_item_cardinality( ), } route_row = route_file_by_event[event_id] - db_row = created_by_event[event_id] + db_row = written_by_event[event_id] assert route_row["action"] == db_row["action"] assert route_row["path"] == db_row["path"] assert route_row["size"] == db_row["size"] diff --git a/tests/ironbank/test_model_sdk_ledger.py b/tests/ironbank/test_model_sdk_ledger.py index c04a3cc8c..6e74712a6 100644 --- a/tests/ironbank/test_model_sdk_ledger.py +++ b/tests/ironbank/test_model_sdk_ledger.py @@ -2533,7 +2533,12 @@ def test_codex_cli_poem_path_pays_full_ledger_debt_blackbox(): "SELECT * FROM fs_events WHERE path = ? ORDER BY id", (filename,), ).fetchall(), - lambda rows: any(row["action"] in {"created", "modified"} for row in rows), + lambda rows: any( + row["action"] in {"created", "modified"} + and row["size"] == len((nonce + "\n").encode()) + and row["trace_id"] == tool_row["trace_id"] + for row in rows + ), ) assert all(row["credential_ref"] is None for row in file_rows) created_file_rows = [ diff --git a/tests/ironbank/test_standalone_proxy.py b/tests/ironbank/test_standalone_proxy.py index 32a490c55..e81e553cf 100644 --- a/tests/ironbank/test_standalone_proxy.py +++ b/tests/ironbank/test_standalone_proxy.py @@ -278,6 +278,8 @@ def test_standalone_proxy_serves_official_sdks_without_a_vm(): proxy_proc.send_signal(signal.SIGTERM) assert proxy_proc.wait(timeout=15) == 0 + assert proxy_proc.stdout is not None + proxy_proc.stdout.close() proxy_proc = None deadline = time.monotonic() + 5 while session_dir.exists() and time.monotonic() < deadline: @@ -291,6 +293,8 @@ def test_standalone_proxy_serves_official_sdks_without_a_vm(): if proxy_proc is not None: proxy_proc.kill() proxy_proc.wait(timeout=5) + if proxy_proc.stdout is not None: + proxy_proc.stdout.close() service.stop() stop_process(mock_proc) if previous_corp is None: diff --git a/tests/test_benchmark_retention_contract.py b/tests/test_benchmark_retention_contract.py index bbd53b8bd..2a186c961 100644 --- a/tests/test_benchmark_retention_contract.py +++ b/tests/test_benchmark_retention_contract.py @@ -205,7 +205,7 @@ def test_lifecycle_startup_metric_spans_create_and_ready_boundary() -> None: def test_latest_benchmark_evidence_ignores_untracked_results(tmp_path: Path) -> None: subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) - evidence = tmp_path / PRUNE.BENCHMARKS.relative_to(PROJECT_ROOT) / "fork" + evidence = tmp_path / PRUNE.BENCHMARKS.relative_to(PROJECT_ROOT) / BenchmarkCategory.FORK.value evidence.mkdir(parents=True) (evidence / "tracked.json").write_text( json.dumps({"timestamp": 1, "identity": "tracked"}), encoding="utf-8" diff --git a/tests/test_benchmark_route_collector.py b/tests/test_benchmark_route_collector.py index ae1796d49..c4fadf383 100644 --- a/tests/test_benchmark_route_collector.py +++ b/tests/test_benchmark_route_collector.py @@ -24,9 +24,9 @@ def test_every_measured_route_resolves_to_a_concrete_fixture_request() -> None: request_path = collector["request_path"] unresolved = { - route: request_path(route) + route: request_path(route, "live-owner") for route in collector["measured_routes"]() - if "{" in request_path(route) or "}" in request_path(route) + if "{" in request_path(route, "live-owner") or "}" in request_path(route, "live-owner") } assert not unresolved, ( diff --git a/tests/test_benchmark_routes.py b/tests/test_benchmark_routes.py index 223eb373d..c01b66f0b 100644 --- a/tests/test_benchmark_routes.py +++ b/tests/test_benchmark_routes.py @@ -73,6 +73,8 @@ def test_fixture_diagnostics_do_not_corrupt_collector_json( monkeypatch.setattr(collector, "ServiceInstance", lambda: service) monkeypatch.setattr(collector, "GatewayInstance", lambda _: gateway) monkeypatch.setattr(collector, "seed_stopped_workspace", lambda *_: None) + monkeypatch.setattr(collector, "clone_benchmark_assets", lambda *_: tmp_path / "benchmark-assets") + original_process = collector.psutil.Process monkeypatch.setattr(collector.psutil, "Process", lambda _: None) monkeypatch.setattr(collector, "measured_routes", lambda: ["/status"]) monkeypatch.setattr( @@ -80,12 +82,21 @@ def test_fixture_diagnostics_do_not_corrupt_collector_json( lambda target, *_: ([0.1], dict.fromkeys(target.cpu_processes, 0.0)), ) monkeypatch.setattr(collector, "sample_cold_route", lambda *_: [0.2]) + monkeypatch.setattr( + collector, "sample_concurrent_route", + lambda target, *_: ([0.3], dict.fromkeys(target.cpu_processes, 0.0), 10.0), + ) monkeypatch.setattr(sys, "argv", ["routes", "--samples", "1"]) - assert collector.main() == 0 + try: + assert collector.main() == 0 + finally: + # psutil is shared with the process-leak fixture, which tears down + # before monkeypatch's automatic undo. + monkeypatch.setattr(collector.psutil, "Process", original_process) captured = capsys.readouterr() payload = json.loads(captured.out) - assert len(payload["metrics"]) == 7 + assert len(payload["metrics"]) == 14 assert "SERVICE LOG:" in captured.err assert "GATEWAY LOG:" in captured.err assert stopped == ["gateway", "service"] diff --git a/tests/test_body_archive_reader.py b/tests/test_body_archive_reader.py index 9b05a3918..db431e562 100644 --- a/tests/test_body_archive_reader.py +++ b/tests/test_body_archive_reader.py @@ -43,16 +43,25 @@ def _ledger(tmp_path: Path) -> Path: bodies.mkdir(mode=0o700) for generation in FIXTURE_DB.with_suffix(".bodies").iterdir(): shutil.copyfile(generation, bodies / generation.name) - os.close(os.open(db.with_name(db.name + "-archive.lock"), os.O_CREAT | os.O_WRONLY, 0o600)) + os.close( + os.open( + db.with_name(db.name + "-archive.lock"), os.O_CREAT | os.O_WRONLY, 0o600 + ) + ) return db def _committed_end(db: Path, value: int | None = None) -> int: with closing(sqlite3.connect(db)) as conn: if value is not None: - conn.execute("UPDATE archive_state SET committed_end = ? WHERE singleton = 1", (value,)) + conn.execute( + "UPDATE archive_state SET committed_end = ? WHERE singleton = 1", + (value,), + ) conn.commit() - return conn.execute("SELECT committed_end FROM archive_state WHERE singleton = 1").fetchone()[0] + return conn.execute( + "SELECT committed_end FROM archive_state WHERE singleton = 1" + ).fetchone()[0] def test_a_body_committed_after_the_reader_opened_is_read(tmp_path: Path) -> None: @@ -60,7 +69,10 @@ def test_a_body_committed_after_the_reader_opened_is_read(tmp_path: Path) -> Non of the block and the index row naming it, then the reader asks for it. The row and the committed end come from one snapshot, so the read succeeds.""" db = _ledger(tmp_path) - expected = {(table, event_id, direction): body for table, event_id, direction, body in archived_bodies(db)} + expected = { + (table, event_id, direction): body + for table, event_id, direction, body in archived_bodies(db) + } committed = _committed_end(db) _committed_end(db, FILE_HEADER_BYTES) # what the reader saw when it opened with SessionArchive(db) as archive: diff --git a/web/app/src/lib/__tests__/session-language-contract.test.ts b/web/app/src/lib/__tests__/session-language-contract.test.ts index e9e4c2135..b1df8cee9 100644 --- a/web/app/src/lib/__tests__/session-language-contract.test.ts +++ b/web/app/src/lib/__tests__/session-language-contract.test.ts @@ -6,10 +6,6 @@ const dashboard = readFileSync( new URL('../components/shell/NewTabPage.svelte', import.meta.url), 'utf8', ); -const sessionActionErrors = readFileSync( - new URL('../models/session-action-error.ts', import.meta.url), - 'utf8', -); const appShell = readFileSync( new URL('../components/shell/App.svelte', import.meta.url), 'utf8', @@ -50,8 +46,8 @@ describe('user-facing session language contract', () => { expect(dashboard).toContain('Sessions'); expect(dashboard).toContain('Loading sessions'); expect(dashboard).toContain('No sessions'); - expect(dashboard).toContain('sessionActionError(error)'); - expect(sessionActionErrors).toContain('Failed to create session'); + expect(dashboard).toContain('actionError = sessionActionError(error)'); + expect(sessionActionError(null)).toBe('Failed to create session'); expect(dashboard).not.toContain('>' + legacyVmPlural + '<'); expect(dashboard).not.toContain('Customize ' + 'VM'); expect(dashboard).not.toContain('Loading ' + legacyVmPlural); diff --git a/web/docs/src/content/docs/architecture/host-isolation.md b/web/docs/src/content/docs/architecture/host-isolation.md index f87ce1f2c..51d750447 100644 --- a/web/docs/src/content/docs/architecture/host-isolation.md +++ b/web/docs/src/content/docs/architecture/host-isolation.md @@ -50,6 +50,7 @@ flowchart TB Owner -->|"virtio and VSOCK"| VM Owner -->|"connected stream pairs"| Relay Proxy -->|"MCP requests after policy"| MCPBridge + Proxy -->|"bounded trace hints"| Owner Owner -->|"owns"| MCPBridge MCPBridge --> Aggregator Owner -->|"one cable per membership"| Switch @@ -80,7 +81,7 @@ authority. |---|---|---| | Control | client API, service-to-owner IPC, worker grant frames, policy reload | Typed and bounded messages; fresh process generation required | | Private data plane | VSOCK, private-network frames, published-port byte streams | Connected descriptors for one session, network membership, or flow | -| Proxied traffic | HTTP, DNS, model API, framed guest MCP | A descriptor granted to the session proxy plus separate upstream, policy, credential, ledger, and telemetry capabilities | +| Proxied traffic | HTTP, DNS, model API, framed guest MCP | A descriptor granted to the session proxy plus separate upstream, policy, credential, ledger, telemetry, and trace-hint capabilities | A worker does not receive a path and then reopen the resource. The coordinator opens or accepts the resource, creates a connected channel, attaches the file @@ -92,10 +93,10 @@ grants fail closed. | Process | Permitted after readiness | Denied or absent | |---|---|---| -| `capsem-service` | Global lifecycle, settings and corp policy, credential store, MCP discovery catalog, session registry, resource creation and capability grants; producer ordering and external commitment checkpoints | It is part of the trusted computing base; clients reach it through the local UDS or authenticated gateway routes | +| `capsem-service` | Global lifecycle, settings and corp policy, credential store, MCP discovery catalog, session registry, resource creation and capability grants, including exact loopback publication listeners; producer ordering and external commitment checkpoints | It is part of the trusted computing base; clients reach it through the local UDS or authenticated gateway routes | | `capsem-gateway` | Accept on listeners bound before confinement; use coordinator-granted service/owner channels; update its private readiness files | Ambient file reads, path-based UDS connects, outbound TCP, new binds, process execution | -| `capsem-process` | One VM and its exact session runtime paths; read exact boot assets and active policy; use coordinator-granted upstream and ledger channels; execute only the installed router helper | Other sessions, the global MCP discovery catalog, direct `session.db` access, ambient outbound connects/binds, unrelated host files and processes | -| `capsem-proxy` | Consume granted HTTP, DNS, MCP, upstream, credential, ledger, metrics, private-name, and policy channels | All filesystem access, arbitrary dial/bind/listen and control-socket access, process execution, signals to the coordinator | +| `capsem-process` | One VM and its exact session runtime paths; read exact boot assets and initial active policy; receive policy reload bytes and use upstream, loopback-listener, and ledger capabilities granted by the coordinator; execute only the installed router helper | Other sessions, the global MCP discovery catalog, direct `session.db` access, ambient outbound connects/binds, unrelated host files and processes | +| `capsem-proxy` | Consume granted HTTP, DNS, MCP, upstream, credential, ledger, metrics, private-name, policy, and one-way trace-hint channels | All filesystem access, arbitrary dial/bind/listen and control-socket access, process execution, signals to the coordinator | | `capsem-ledger` | Read and write one session directory; serve authenticated operations over granted connected channels | Other paths, ambient network and control-socket access, process execution, signals to the coordinator | | `capsem-router --expose` | Copy bytes between connected host TCP and guest VSOCK descriptors | Listener ownership, destination selection, filesystem, arbitrary network, VM control | | `capsem-router --network` | Switch frames among the descriptor-backed cables of one named network | Uplink, listener, outbound socket, another network's members, policy decisions | @@ -112,8 +113,9 @@ On macOS, workers install Seatbelt profiles. Proxy and ledger workers use `deny default`; the proxy has no path parameters, and the ledger receives one read-write session directory. Gateway and VM-owner profiles deny filesystem, network, execution, and cross-sandbox signal operations, then add their exact -path grants. Only the gateway may accept inbound traffic on a listener it -already owns. +path grants. The gateway accepts only on listeners it already owns. A VM owner +accepts only on an exact loopback listener granted over its generation-bound +coordinator channel; it cannot create or bind another network socket. On Linux, workers support Landlock ABI 6 and newer. ABI 8 adds synchronized restriction of existing threads. On ABI 6 and 7, startup remains @@ -121,19 +123,42 @@ single-threaded until Landlock is installed; every later runtime, hypervisor, database, monitor, logging, and parent-watch thread inherits that domain. A worker refuses startup if an older ABI already has a sibling thread. Seccomp is installed with thread synchronization after the Landlock ruleset. Landlock -limits filesystem scope; seccomp denies direct connect, bind, listen, process -inspection, namespace and mount operations, kernel attack surfaces, and -permission or ownership changes. Proxy and ledger workers also cannot create -sockets or send signals. Existing connected descriptors remain usable because -they are the capability. +limits filesystem scope; seccomp denies direct socket creation, connect, bind, +listen, process inspection, namespace and mount operations, kernel attack +surfaces, and permission or ownership changes. On macOS, Seatbelt denies +external bind, listen and connect operations but permits private unnamed Unix +socketpairs, which carry no filesystem or network destination authority. Proxy +and ledger workers cannot send signals on either platform. Existing connected +descriptors remain usable because they are the capability. + +A writable Landlock path does not carry device-ioctl authority. The VM owner +receives that authority only for the exact `/dev/kvm` and `/dev/vhost-vsock` +device nodes it needs. A regular file is rejected if code tries to grant it the +device class, so widening an ordinary writable path cannot also widen KVM or +VSOCK control. The VM owner prepares only resources that cannot be acquired after the sandbox: its inherited coordinator channel, exact IPC listeners, readiness and -log files, metric channel, and Linux AF_VSOCK listeners. It then installs and -self-attests confinement before starting the parent watcher, hypervisor, -Tokio or blocking worker threads, ledger client, filesystem monitor, metric -exporter, or any VM worker. KVM consumes the prepared VSOCK descriptors after -confinement; it does not regain `socket` or `bind` authority. +log files, metric channel, Linux AF_VSOCK listeners, `/dev/null` for the router's +closed output streams, and saved publication state. Publication state is +materialized and migrated before confinement; the +confined publisher consumes that prepared state without reopening the session +root. Updates create private files at their final mode and need no later +permission change. When that publisher opens or restores a port, the +coordinator binds the exact `127.0.0.1` listener and passes its descriptor to +the owner. The owner validates the descriptor address before admission and +acceptance. Host TCP streams are armed for reset on close before handoff; on +Linux, abnormal cleanup closes the owner and confined-router copies with that +setting still armed instead of restoring a direct `connect` syscall. The +service never accepts a publication connection or reads its workload bytes. +Before confinement, the owner asks the coordinator for its ledger +capability. The grant arrives only after `capsem-ledger` has initialized +`session.db`; the owner keeps the connected descriptors, installs its sandbox, +and verifies that opening that existing database by path is denied. It then +starts the parent watcher, hypervisor, ledger client, filesystem monitor, +metric exporter, and VM workers. KVM opens only its explicitly granted device +nodes and consumes the prepared VSOCK descriptors after confinement; it does +not regain general device-ioctl, `socket`, or `bind` authority. Both platforms clear inherited environment state where secrets could otherwise leak and close descriptors that were not deliberately preserved. The sandbox @@ -182,6 +207,7 @@ flowchart LR Engine --> Ledger["session ledger capability"] Engine --> Metrics["metric capability"] Engine --> MCP["scoped MCP capability"] + Engine --> Hints["one-way trace hints
to VM owner"] ``` The engine owns request normalization, policy evaluation, preprocessing and diff --git a/web/docs/src/content/docs/architecture/session-telemetry.md b/web/docs/src/content/docs/architecture/session-telemetry.md index e25b20c1f..4e67620a1 100644 --- a/web/docs/src/content/docs/architecture/session-telemetry.md +++ b/web/docs/src/content/docs/architecture/session-telemetry.md @@ -9,8 +9,8 @@ Every Capsem VM and standalone proxy session gets its own session ledger: a SQLite database (`session.db`) that records network requests, DNS queries, AI model calls, MCP tool invocations, exec activity, kernel audit events, file changes, security rule matches, and credential substitutions, and beside it a -body archive (`session.bodies`) that holds the full request, response, tool, -exec and security payloads those rows describe. The two files live in the +body archive directory (`session.bodies/`) that holds the full request, response, tool, +exec and security payloads those rows describe. The database and archive directory live in the session directory and follow its lifecycle; retained and forked VMs keep both for forensic review, and one without the other is not a ledger. @@ -687,10 +687,10 @@ graph LR ORDER -->|"sync after ledger flush"| CHECKPOINTS ``` -The ledger worker owns both files; its writer thread owns the SQLite +The ledger worker owns the database and archive directory; its writer thread owns the SQLite connection and archive writer. A body is fed to the open block's compressor as its event is written; each disk flush appends what the block -produced since the last one to `session.bodies` as a segment, syncs it, and +produced since the last one to the selected generation file as a segment, syncs it, and only then commits the body's index row and the block's grown extent in `body_blocks`. @@ -736,8 +736,17 @@ removes any externally anchored identity. ## Body archive -`session.bodies` is append-only: a 16-byte file header, then blocks. A block -is one streaming codec instance: an 8-byte header (magic and codec), then segments. +`session.bodies/` contains generation files named `g-.cbl`, +where the generation ID is 32 hexadecimal digits. The database's singleton +`archive_state` row selects the archive and generation identities, +committed byte extent, and revision. Readers never select a file by its age. +The ledger format is version 4; the generation byte format is version 3. + +Each generation is append-only: an 80-byte header followed by blocks. +The header contains `CAPSEMBL`, version and flags, its length, 16-byte archive +and generation identities, and a 32-byte BLAKE3 hash of the first 48 bytes. +A block uses one streaming codec instance: an 8-byte header identifies its codec, +followed by segments. Confined workers encode zstd; legacy deflate remains readable. Each disk flush sync-flushes the stream -- byte-aligned, dictionary kept -- and appends what it produced as a segment behind a 52-byte header (`raw_start`, `raw_len`, `comp_len` and the blake3 of the segment's raw bytes). The block @@ -808,7 +817,7 @@ seek within it without Capsem code. It is streamed, not buffered. - At service start, failed-session directories older than the period are deleted. - When a persistent VM stops, its `capsem-ledger` worker drops the blocks sealed - before the cutoff, compacts `session.bodies`, and rewrites the index rows. + before the cutoff, compacts the selected archive generation, and rewrites the index rows atomically with its generation state. Commitment rows and event metadata remain intact. - Ephemeral sessions are deleted whole and are not trimmed. @@ -1062,7 +1071,7 @@ projection. | File | Purpose | |------|---------| | `capsem-logger/src/schema.rs` | Table DDL and pragmas | -| `capsem-archive/src/format.rs` | The `session.bodies` byte format | +| `capsem-archive/src/format.rs` | The archive generation byte format | | `capsem-ledger/src/codec.rs` | Confined zstd codec implementation and legacy deflate registration | | `capsem-service/src/ledger_commitment.rs` | Trusted global ordering, external checkpoints, and startup verification | | `capsem-archive/src/warc.rs` | WARC record writer | From d581703a0f3bfbb32b94dd9b36e6041e7edcce9b Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 21:01:13 +0000 Subject: [PATCH 459/464] fix(logger): reject incomplete schema at flush boundaries --- CHANGELOG.md | 3 + .../capsem-logger/src/schema/memory_sync.rs | 30 +++---- .../src/schema/tests/integrity.rs | 87 +++++++++++++++++++ crates/capsem-logger/src/writer/worker.rs | 15 ++-- 4 files changed, 113 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ca26f432..32f87c5f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -282,6 +282,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Ledger flushes reject missing tables or ID columns even when empty, keeping + pending rows available for retry instead of acknowledging broken schema. + - The guest DNS listener sizes its UDP receive queue from the supported client, session and EDNS datagram limits, preventing kernel receive drops during a 512-client dual-stack burst. diff --git a/crates/capsem-logger/src/schema/memory_sync.rs b/crates/capsem-logger/src/schema/memory_sync.rs index 0b5605b95..134211b13 100644 --- a/crates/capsem-logger/src/schema/memory_sync.rs +++ b/crates/capsem-logger/src/schema/memory_sync.rs @@ -50,6 +50,11 @@ const DISK_ONLY_TABLES: &[&str] = &[ "network_schema", ]; +fn required_hot_table(table: &str) -> rusqlite::Result<&'static str> { + canonical_hot_table(table) + .ok_or_else(|| rusqlite::Error::InvalidParameterName(format!("unknown ledger table {table}"))) +} + pub(crate) fn is_disk_only_table(name: &str) -> bool { DISK_ONLY_TABLES.contains(&name) } @@ -114,12 +119,10 @@ pub(crate) fn seed_memory_sequences<'a>( tables: impl IntoIterator, ) -> rusqlite::Result<()> { for table in tables { - if is_disk_only_table(table) - || !table_exists(conn, "main", table)? - || !table_exists(conn, MEMORY_SCHEMA, table)? - { + if is_disk_only_table(table) { continue; } + let table = required_hot_table(table)?; let next: i64 = conn.query_row( &format!( "SELECT MAX( @@ -157,12 +160,7 @@ pub(crate) fn initial_memory_flush_watermarks<'a>( if is_disk_only_table(table) { continue; } - if !table_exists(conn, "main", table)? { - continue; - } - let Some(table) = canonical_hot_table(table) else { - continue; - }; + let table = required_hot_table(table)?; let max_id = max_table_id(conn, "main", table)?; watermarks.insert(table, max_id); } @@ -193,12 +191,12 @@ pub fn flush_memory_tables_to_disk<'a>( if is_disk_only_table(table) { continue; } - if !table_exists(conn, "main", table)? || !table_exists(conn, MEMORY_SCHEMA, table)? { - continue; - } - let Some(table) = canonical_hot_table(table) else { - continue; - }; + let table = required_hot_table(table)?; + // Execute the cached probe so a schema-cookie change reparses it. + // Even an empty memory table must not hide a missing disk table/id. + let _: i64 = conn + .prepare_cached(&format!("SELECT COUNT(id) FROM main.{table} WHERE 0"))? + .query_row([], |row| row.get(0))?; let last_flushed_id = *watermarks.get(table).unwrap_or(&0); let max_memory_id = max_table_id(conn, MEMORY_SCHEMA, table)?; if max_memory_id > last_flushed_id { diff --git a/crates/capsem-logger/src/schema/tests/integrity.rs b/crates/capsem-logger/src/schema/tests/integrity.rs index 49feec8e4..90f021d3a 100644 --- a/crates/capsem-logger/src/schema/tests/integrity.rs +++ b/crates/capsem-logger/src/schema/tests/integrity.rs @@ -1,5 +1,92 @@ use super::*; +fn memory_integrity_database(name: &str) -> Connection { + let conn = Connection::open_in_memory().unwrap(); + create_tables(&conn).unwrap(); + create_memory_tables(&conn, &memory_uri_for_name(name)).unwrap(); + conn +} + +#[test] +fn memory_sync_empty_flush_refuses_missing_tables_and_id_columns() { + for (index, corruption) in [ + "DROP TABLE main.net_events", + "DROP TABLE mem.net_events", + "ALTER TABLE main.net_events RENAME COLUMN id TO lost_id", + "ALTER TABLE mem.net_events RENAME COLUMN id TO lost_id", + ] + .iter() + .enumerate() + { + let conn = memory_integrity_database(&format!("memory-sync-empty-{index}")); + let watermarks = MemoryFlushWatermarks::new(); + assert!(flush_memory_tables_to_disk(&conn, ["net_events"], &watermarks) + .unwrap() + .is_empty()); + conn.execute_batch(corruption).unwrap(); + assert!( + flush_memory_tables_to_disk(&conn, ["net_events"], &watermarks).is_err(), + "a warmed empty flush acknowledged broken schema: {corruption}" + ); + } +} + +#[test] +fn memory_sync_missing_disk_table_preserves_pending_rows_until_explicit_repair() { + let conn = memory_integrity_database("memory-sync-repair"); + let watermarks = initial_memory_flush_watermarks(&conn, ["net_events"]).unwrap(); + conn.execute_batch( + "INSERT INTO mem.net_events(event_id,timestamp,domain,decision) + VALUES('012345abcdef','t','fixture','allowed'); DROP TABLE main.net_events", + ) + .unwrap(); + assert!(flush_memory_tables_to_disk(&conn, ["net_events"], &watermarks).is_err()); + let pending: i64 = conn + .query_row("SELECT COUNT(*) FROM mem.net_events", [], |row| row.get(0)) + .unwrap(); + assert_eq!(pending, 1); + assert_eq!(watermarks["net_events"], 0); + create_tables(&conn).unwrap(); // Explicit fixture repair, never a production fallback. + let advanced = flush_memory_tables_to_disk(&conn, ["net_events"], &watermarks).unwrap(); + assert_eq!(advanced["net_events"], 1); + let event: String = conn + .query_row("SELECT event_id FROM main.net_events", [], |row| row.get(0)) + .unwrap(); + assert_eq!(event, "012345abcdef"); + let pending: i64 = conn + .query_row("SELECT COUNT(*) FROM mem.net_events", [], |row| row.get(0)) + .unwrap(); + assert_eq!(pending, 0); +} + +#[test] +fn memory_sync_sequence_seed_refuses_missing_tables() { + for schema in ["main", "mem"] { + let conn = memory_integrity_database(&format!("memory-sync-seed-{schema}")); + conn.execute_batch(&format!("DROP TABLE {schema}.net_events")).unwrap(); + assert!( + seed_memory_sequences(&conn, ["net_events"]).is_err(), + "missing {schema}.net_events was ignored" + ); + } +} + +#[test] +fn memory_sync_initial_watermark_refuses_missing_disk_table() { + let conn = memory_integrity_database("memory-sync-watermark"); + conn.execute_batch("DROP TABLE main.net_events").unwrap(); + assert!(initial_memory_flush_watermarks(&conn, ["net_events"]).is_err()); +} + +#[test] +fn memory_sync_refuses_unknown_hot_table_names() { + let conn = memory_integrity_database("memory-sync-unknown"); + let watermarks = MemoryFlushWatermarks::new(); + assert!(seed_memory_sequences(&conn, ["unknown_ledger"]).is_err()); + assert!(initial_memory_flush_watermarks(&conn, ["unknown_ledger"]).is_err()); + assert!(flush_memory_tables_to_disk(&conn, ["unknown_ledger"], &watermarks).is_err()); +} + fn seed_integrity_rows(conn: &Connection) { let hash = format!("blake3:{}", "0123456789abcdef".repeat(4)); conn.execute_batch( diff --git a/crates/capsem-logger/src/writer/worker.rs b/crates/capsem-logger/src/writer/worker.rs index 0693866ce..ebe1cb7a9 100644 --- a/crates/capsem-logger/src/writer/worker.rs +++ b/crates/capsem-logger/src/writer/worker.rs @@ -15,12 +15,15 @@ pub(super) fn writer_loop( capacity: batch_capacity, interval, } = schedule; - let mut flush_watermarks = - schema::with_memory_schema_lock(|| schema::initial_memory_flush_watermarks(&conn, schema::hot_ledger_tables())) - .unwrap_or_else(|error| { - warn!(error = %error, "db initial memory flush watermark load failed"); - schema::MemoryFlushWatermarks::new() - }); + let mut flush_watermarks = match schema::with_memory_schema_lock(|| { + schema::initial_memory_flush_watermarks(&conn, schema::hot_ledger_tables()) + }) { + Ok(watermarks) => watermarks, + Err(error) => { + warn!(error = %error, "db initial memory flush watermark load failed"); + return; + } + }; // Rows at or below the open-time watermark were written by an earlier // writer; exec ids restart with the process, so a completion must not // reach one of those. From 8e1efc2344c151144dbc88bf5d958a1117fb00c1 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 21:10:16 +0000 Subject: [PATCH 460/464] refactor(logger): validate empty tables without aggregate SQL --- crates/capsem-logger/src/schema/memory_sync.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/capsem-logger/src/schema/memory_sync.rs b/crates/capsem-logger/src/schema/memory_sync.rs index 134211b13..6655e32d9 100644 --- a/crates/capsem-logger/src/schema/memory_sync.rs +++ b/crates/capsem-logger/src/schema/memory_sync.rs @@ -194,9 +194,10 @@ pub fn flush_memory_tables_to_disk<'a>( let table = required_hot_table(table)?; // Execute the cached probe so a schema-cookie change reparses it. // Even an empty memory table must not hide a missing disk table/id. - let _: i64 = conn - .prepare_cached(&format!("SELECT COUNT(id) FROM main.{table} WHERE 0"))? - .query_row([], |row| row.get(0))?; + let _: Option = conn + .prepare_cached(&format!("SELECT id FROM main.{table} WHERE 0"))? + .query_row([], |row| row.get(0)) + .optional()?; let last_flushed_id = *watermarks.get(table).unwrap_or(&0); let max_memory_id = max_table_id(conn, MEMORY_SCHEMA, table)?; if max_memory_id > last_flushed_id { From f8ab01adcb513411f0f7f61b0d3a621da41d8c0e Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 21:10:39 +0000 Subject: [PATCH 461/464] chore(qualification): reconcile merged source inventories and lint --- build_system/tests/scripts/test_rust_affected.py | 1 - .../tests/sdkgen/test_python_operations.py | 3 ++- build_system/tests/test_test_ownership.py | 2 -- config/dependency-inventory.json | 16 +++++++++++++--- sdk/python/tests/test_connections.py | 4 ++-- skills/dev-gate/SKILL.md | 2 +- tests/citadel/test_bench_route_coverage.py | 3 +-- tests/citadel/test_hot_build_contract.py | 5 +---- tests/citadel/test_sdk_rust_quality.py | 2 +- tests/test_benchmark_routes.py | 2 +- 10 files changed, 22 insertions(+), 18 deletions(-) diff --git a/build_system/tests/scripts/test_rust_affected.py b/build_system/tests/scripts/test_rust_affected.py index b1625f12b..063cfff09 100644 --- a/build_system/tests/scripts/test_rust_affected.py +++ b/build_system/tests/scripts/test_rust_affected.py @@ -123,7 +123,6 @@ def test_real_workspace_inventory_matches_cargo_members() -> None: "capsem-core", "capsem-crypto", "capsem-credentials", - "capsem-crypto", "capsem-foundation", "capsem-gateway", "capsem-guard", diff --git a/build_system/tests/sdkgen/test_python_operations.py b/build_system/tests/sdkgen/test_python_operations.py index df9d890ac..d9da35740 100644 --- a/build_system/tests/sdkgen/test_python_operations.py +++ b/build_system/tests/sdkgen/test_python_operations.py @@ -13,13 +13,14 @@ import pytest from aiohttp import web -from build_system.tests.sdkgen.test_python import _sample as sample from capsem_builder.sdkgen.operations import Route, is_binary_media_type, read_operations from capsem_builder.sdkgen.python import module_name, render_models, type_name from capsem_builder.sdkgen.python_operations import render_operations from capsem_builder.sdkgen.schema import read_schemas from pydantic import TypeAdapter, ValidationError +from build_system.tests.sdkgen.test_python import _sample as sample + ROOT = Path(__file__).resolve().parents[3] SPEC = ROOT / "sdk/specification/openapi.json" SCHEMAS = read_schemas(SPEC) diff --git a/build_system/tests/test_test_ownership.py b/build_system/tests/test_test_ownership.py index dc9a2ad44..a7be32e7d 100644 --- a/build_system/tests/test_test_ownership.py +++ b/build_system/tests/test_test_ownership.py @@ -107,7 +107,6 @@ "build_system/tests/image/test_guest_binary_source_contract.py", "build_system/tests/image/test_kernel_config_honored.py", "build_system/tests/image/test_kernel_defconfig.py", - "build_system/tests/image/test_kernel_fuse_io.py", "build_system/tests/image/test_runtime_packages.py", "build_system/tests/packaging/test_deb_platform_support.py", "build_system/tests/packaging/test_linux_packaging_boundary.py", @@ -142,7 +141,6 @@ "build_system/tests/sdkgen/test_typescript_operations.py", "build_system/tests/sdkgen/test_typescript_validation.py", "build_system/tests/sdkgen/test_write_only.py", - "build_system/tests/test_crypto.py", "build_system/tests/conftest.py", "build_system/tests/test_bootstrap.py", "build_system/tests/test_ownership.toml", diff --git a/config/dependency-inventory.json b/config/dependency-inventory.json index 5961ab15f..ff900d41a 100644 --- a/config/dependency-inventory.json +++ b/config/dependency-inventory.json @@ -860,6 +860,7 @@ "dependencies": [ "anyhow", "capsem-foundation", + "capsem-proto", "clap", "futures", "reqwest", @@ -960,20 +961,24 @@ "dependencies": [ "base64", "bytes", + "capsem-api", "capsem-crypto", "capsem-foundation", "capsem-proto", "futures", "http-body-util", + "humantime", "hyper", "hyper-util", "reqwest", "serde", "serde_json", "tempfile", + "time", "tokio", "tracing", - "url" + "url", + "uuid" ] }, "capsem-crypto": { @@ -1212,7 +1217,8 @@ "syn", "tempfile", "thiserror", - "tokio" + "tokio", + "toml" ] }, "capsem-proxy": { @@ -1257,11 +1263,15 @@ "dependencies": [ "axum", "capsem-api", + "capsem-crypto", + "futures", + "humantime", "reqwest", "serde", "serde_json", "thiserror", - "tokio" + "tokio", + "tokio-util" ] }, "capsem-service": { diff --git a/sdk/python/tests/test_connections.py b/sdk/python/tests/test_connections.py index fd6125bfb..fc96c185b 100644 --- a/sdk/python/tests/test_connections.py +++ b/sdk/python/tests/test_connections.py @@ -112,7 +112,7 @@ async def run(): def test_grant_creation_refuses_substituted_authority(field, value): async def run(): async with gateway(response={**grant(), field: value}) as (url, received), Hypervisor(url, "gateway-token") as hv: - with pytest.raises(ValueError, match="grant.*changed"): + with pytest.raises(ValueError, match=r"grant.*changed"): await hv.vm(id=SESSION).credential_grants.create(grant_request()) assert len(received) == 1 asyncio.run(run()) @@ -152,7 +152,7 @@ async def run(): request = grant_request() async with gateway(response={**grant(), "capability_ids": ["other"]}, on_request=lambda: setattr(request, "capability_ids", ["other"])) as (url, received), Hypervisor(url, "gateway-token") as hv: - with pytest.raises(ValueError, match="grant.*changed"): + with pytest.raises(ValueError, match=r"grant.*changed"): await hv.vm(id=SESSION).credential_grants.create(request) assert received[0][2]["capability_ids"] == ["agy"] assert request.capability_ids == ["other"] diff --git a/skills/dev-gate/SKILL.md b/skills/dev-gate/SKILL.md index ddb10a90a..6e296b11a 100644 --- a/skills/dev-gate/SKILL.md +++ b/skills/dev-gate/SKILL.md @@ -144,7 +144,7 @@ before invoking its sandboxed modules. Native supervisor restart uses its existing glow-up owner because supervisor setup must precede inherited privilege confinement. Linux then runs the SDK worker as the original user with `no_new_privs` in a loopback-only namespace; -see `/dev-testing`'s `references/local-ci-parity.md`. Never run sudo after +see `/dev-testing`'s `../dev-testing/references/local-ci-parity.md`. Never run sudo after that restriction or disable it to make setup work. ## Declaring a command diff --git a/tests/citadel/test_bench_route_coverage.py b/tests/citadel/test_bench_route_coverage.py index fd513d8d5..2533bcea7 100644 --- a/tests/citadel/test_bench_route_coverage.py +++ b/tests/citadel/test_bench_route_coverage.py @@ -17,9 +17,8 @@ from __future__ import annotations -from pathlib import Path - import tomllib +from pathlib import Path PROJECT_ROOT = Path(__file__).parents[2] SERVICE = PROJECT_ROOT / "crates" / "capsem-service" / "src" / "router_runtime.rs" diff --git a/tests/citadel/test_hot_build_contract.py b/tests/citadel/test_hot_build_contract.py index 1a9458fce..f5eefeb7c 100644 --- a/tests/citadel/test_hot_build_contract.py +++ b/tests/citadel/test_hot_build_contract.py @@ -7,9 +7,8 @@ from __future__ import annotations -from pathlib import Path - import tomllib +from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parents[2] @@ -33,8 +32,6 @@ "ryu", "quick_cache", "rusqlite", - "libsqlite3-sys", - "capsem-logger", } RATIONALE = """\ diff --git a/tests/citadel/test_sdk_rust_quality.py b/tests/citadel/test_sdk_rust_quality.py index a7aedcca9..0ba3a5ac2 100644 --- a/tests/citadel/test_sdk_rust_quality.py +++ b/tests/citadel/test_sdk_rust_quality.py @@ -3,11 +3,11 @@ from __future__ import annotations import re +import tomllib from copy import deepcopy from types import SimpleNamespace import pytest -import tomllib from capsem_builder.gate import sdkchecks from capsem_builder.gate.plan import Plan from citadel.test_sdk_ci import _documents diff --git a/tests/test_benchmark_routes.py b/tests/test_benchmark_routes.py index c01b66f0b..d96ba9aed 100644 --- a/tests/test_benchmark_routes.py +++ b/tests/test_benchmark_routes.py @@ -2,13 +2,13 @@ import json import sys +import tomllib from importlib.machinery import SourceFileLoader from importlib.util import module_from_spec, spec_from_loader from pathlib import Path from types import SimpleNamespace import pytest -import tomllib ROOT = Path(__file__).resolve().parents[1] config = tomllib.loads((ROOT / "config/gate.toml").read_text()) From c4bd494ea37e04276a8878acaafdcb04461e3841 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 21:10:40 +0000 Subject: [PATCH 462/464] fix(gate): bind restart proof to runtime and reuse SDK preparation --- build_system/builder/gate/actions.py | 4 +-- build_system/builder/gate/managedrestart.py | 12 +++++++-- build_system/builder/gate/module_glowup.py | 8 ++++-- build_system/tests/gate/test_gate_actions.py | 10 ++++++++ .../tests/gate/test_gate_functional_sdks.py | 25 ++++++++++++++++++- config/gate.toml | 8 ------ 6 files changed, 52 insertions(+), 15 deletions(-) diff --git a/build_system/builder/gate/actions.py b/build_system/builder/gate/actions.py index ccb80a065..73fc51af3 100644 --- a/build_system/builder/gate/actions.py +++ b/build_system/builder/gate/actions.py @@ -101,7 +101,7 @@ def render(self) -> str: """ rendered = str(self._command) if self._command.cwd is not None: - rendered = f"(in {self._command.cwd.name}) {rendered}" + rendered = f"(in {self._command.cwd}) {rendered}" if self._outside_sandbox: rendered = f"{rendered} [outside kernel sandbox]" return rendered @@ -167,7 +167,7 @@ def render(self) -> str: ) ) if self._root is not None: - rendered = f"(in {self._root.name}) {rendered}" + rendered = f"(in {self._root}) {rendered}" if self._outside_sandbox: rendered = f"{rendered} [outside kernel sandbox]" return rendered diff --git a/build_system/builder/gate/managedrestart.py b/build_system/builder/gate/managedrestart.py index abf8bf9ef..9bb0d22d9 100644 --- a/build_system/builder/gate/managedrestart.py +++ b/build_system/builder/gate/managedrestart.py @@ -11,24 +11,32 @@ from . import host, pytestsuite, sdkchecks, toolchain from .actions import Run from .config import GateConfig +from .content import RuntimeContent from .execution import Kind, Needs, Speed, Step, step from .plan import Plan from .pythonenv import uv_run -def fragment(plan: Plan, config: GateConfig, *, after: tuple[Step, ...]) -> Step: +def fragment( + plan: Plan, config: GateConfig, *, after: tuple[Step, ...], content: RuntimeContent +) -> Step: """Run the managed-restart suite once the SDK it drives is usable.""" settings = config.suites.pytest suite = pytestsuite.Suite( label="pytest.managed-restart", paths=(settings.managed_restart,), + assets_dir=str(content.assets), ) bundle = sdkchecks.typescript_bundle(config) if bundle.label in plan.labels: bundled = plan.step_named(bundle.label) else: node = toolchain.node(config, (config.sdk_typescript.project,)) - installed = plan.step_named(node.label) if node.label in plan.labels else plan.shared(node) + installed = ( + plan.step_named("fast.toolchain.node") + if "fast.toolchain.node" in plan.labels + else plan.step_named(node.label) if node.label in plan.labels else plan.shared(node) + ) bundled = plan.shared(bundle, after=(installed,)) argv = suite.argv(config) if not host.on_macos(): diff --git a/build_system/builder/gate/module_glowup.py b/build_system/builder/gate/module_glowup.py index 41a6a80fe..f53805e69 100644 --- a/build_system/builder/gate/module_glowup.py +++ b/build_system/builder/gate/module_glowup.py @@ -81,7 +81,7 @@ def glowup( phase = plan.phase("glowup") if qualification.pulled: content = staged or RuntimeContent.standalone(config) - native = managedrestart.fragment(plan, config, after=after) + native = managedrestart.fragment(plan, config, after=after, content=content) return pulled_package(phase, config, qualification, (*after, native), content) content = local_content or RuntimeContent.standalone(config) return _build_and_prove( @@ -282,7 +282,11 @@ def _build_and_prove( after=previous, ), ) - previous = (managedrestart.fragment(plan, config, after=(*previous, *filter(None, (runtime,)))),) + previous = ( + managedrestart.fragment( + plan, config, after=(*previous, *filter(None, (runtime,))), content=content + ), + ) sbom = phase.add(hostpackage.sbom_step(config), after=previous) exact_install_image = installplan.fragment(plan, config) diff --git a/build_system/tests/gate/test_gate_actions.py b/build_system/tests/gate/test_gate_actions.py index 11bb9b142..e54ae760f 100644 --- a/build_system/tests/gate/test_gate_actions.py +++ b/build_system/tests/gate/test_gate_actions.py @@ -73,6 +73,16 @@ def test_render_shows_the_environment_an_action_adds() -> None: assert action.render() == "CAPSEM_TEST_MODULE=fast just _test-candidate-run" +def test_render_distinguishes_projects_with_the_same_directory_name() -> None: + """SDK and MCP builds are separate work even though both end in typescript.""" + sdk = Run(["pnpm", "run", "build"], cwd=PROJECT_ROOT / "sdk/typescript") + mcp = Run(["pnpm", "run", "build"], cwd=PROJECT_ROOT / "mcp/typescript") + + assert sdk.render() != mcp.render() + assert str(PROJECT_ROOT / "sdk/typescript") in sdk.render() + assert str(PROJECT_ROOT / "mcp/typescript") in mcp.render() + + # --------------------------------------------------------------------------- # Performing # --------------------------------------------------------------------------- diff --git a/build_system/tests/gate/test_gate_functional_sdks.py b/build_system/tests/gate/test_gate_functional_sdks.py index dd17a0c65..cd826b78b 100644 --- a/build_system/tests/gate/test_gate_functional_sdks.py +++ b/build_system/tests/gate/test_gate_functional_sdks.py @@ -11,6 +11,9 @@ from capsem_builder.gate import config as gate_config from capsem_builder.gate import host +from capsem_builder.gate.content import RuntimeContent +from capsem_builder.gate.module_glowup import glowup +from capsem_builder.gate.plan import Plan from capsem_builder.gate.qualification import BinaryQualification, RuntimeQualification from helpers.gate import PROJECT_ROOT, gate_plan @@ -134,6 +137,7 @@ def test_native_supervisor_restart_is_owned_by_glowup_on_both_platforms() -> Non label = "glowup.pytest.managed-restart" rendered = plan.step_named(label).actions[0].render() assert managed in rendered and "[outside kernel sandbox]" in rendered + assert f"{config.environment.assets_dir}={RuntimeContent.built(config).assets} " in rendered assert "sdk.python.sync" in _ancestors(plan, label) assert "sdk.typescript.bundle" in _ancestors(plan, label) if host.on_macos(): @@ -150,4 +154,23 @@ def test_pulled_package_lanes_keep_native_restart_and_sdk_prerequisites() -> Non label = "glowup.pytest.managed-restart" assert {"sdk.python.sync", "sdk.typescript.bundle"} <= _ancestors(plan, label) assert label in _ancestors(plan, "glowup.package") - assert config.suites.pytest.managed_restart in plan.step_named(label).actions[0].render() + rendered = plan.step_named(label).actions[0].render() + assert config.suites.pytest.managed_restart in rendered + assert f"{config.environment.assets_dir}={RuntimeContent.standalone(config).assets} " in rendered + + +def test_pulled_restart_uses_the_handed_runtime_assets() -> None: + config = gate_config.load(PROJECT_ROOT) + content = RuntimeContent.staged(config, PROJECT_ROOT / "cache/target/selected-cohort") + for owner in (BinaryQualification, RuntimeQualification): + qualification = owner(input_dir="/staged", package="/candidate.deb", bin_dir="/candidate-bin") + plan = Plan("selected-cohort") + glowup(plan, config, qualification=qualification, staged=content) + rendered = plan.step_named("glowup.pytest.managed-restart").actions[0].render() + assert f"{config.environment.assets_dir}={content.assets} " in rendered + + +def test_candidate_restart_reuses_the_prepared_sdk_dependencies() -> None: + plan = gate_plan("candidate") + assert "fast.toolchain.node" in _ancestors(plan, "sdk.typescript.bundle") + assert "toolchain.node" not in plan.labels diff --git a/config/gate.toml b/config/gate.toml index 971a55775..eda108853 100644 --- a/config/gate.toml +++ b/config/gate.toml @@ -2884,14 +2884,6 @@ capsem-admin through Cargo and replace workspace outputs, then signs the exact post-generation bytes used by the VM and package lanes. """ -[[boundary.repeated_actions]] -subject = "(in typescript) CI=true pnpm" -reason = """ -The SDK and npm MCP are distinct package roots that share the basename -`typescript`; the plan renderer shortens both working directories to that -basename, so their separate dependency fetch and install actions render alike. -""" - # --------------------------------------------------------------------------- # The execution log # --------------------------------------------------------------------------- From cdb8154913905cfa94d28b97e1f236a8f398abc3 Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 21:40:44 +0000 Subject: [PATCH 463/464] test(api): align proxy leases with the 0.7 contract version CI 38638215911 fails the same proxy OpenAPI assertion on macOS and both Linux architectures: the approved document contract advanced to 3.3.0 while this test still expects 3.1.0. Preserve all proxy path checks and use the current explicit version. --- crates/capsem-api/src/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/capsem-api/src/tests.rs b/crates/capsem-api/src/tests.rs index ed5cd7e77..de13cfb88 100644 --- a/crates/capsem-api/src/tests.rs +++ b/crates/capsem-api/src/tests.rs @@ -261,7 +261,7 @@ fn openapi_describes_standalone_proxy_leases() { "heartbeatProxy" ); assert_eq!(paths["/proxies/{id}/stop"]["post"]["operationId"], "stopProxy"); - assert_eq!(crate::CONTRACT_VERSION, "3.1.0"); + assert_eq!(crate::CONTRACT_VERSION, "3.3.0"); } #[test] From 558361c1f5aae07cdb8fdb364b1a6ed4235f38dd Mon Sep 17 00:00:00 2001 From: Elie Bursztein Date: Wed, 14 Oct 2026 21:40:46 +0000 Subject: [PATCH 464/464] test(gate): compare execution identity and declared phases CI 38638215911 static fails seven Citadel cases after full-directory command rendering and shared prerequisite reuse. Inspect SDK installation argv, cwd and env rather than basename labels; strip only plan-declared phases for counterpart coverage. Keep graph-order assertions and add missing-SDK-install and unrelated-domain adversaries. --- ...test_local_gate_covers_the_release_lane.py | 44 +++++++++++++------ tests/citadel/test_sdk_ui_dependency.py | 30 ++++++++++++- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/tests/citadel/test_local_gate_covers_the_release_lane.py b/tests/citadel/test_local_gate_covers_the_release_lane.py index 4cb2f7f4a..e4362514f 100644 --- a/tests/citadel/test_local_gate_covers_the_release_lane.py +++ b/tests/citadel/test_local_gate_covers_the_release_lane.py @@ -28,6 +28,8 @@ from pathlib import Path from capsem_builder.gate import config as gate_config +from capsem_builder.gate.execution import step as plan_step +from capsem_builder.gate.plan import Plan from capsem_builder.gate.qualification import from_environment from helpers.gate import built_command @@ -39,11 +41,11 @@ REHEARSAL = "rehearsal" -def _labels(name: str, arguments: tuple, qualification) -> tuple[str, ...]: - return tuple(built_command(ROOT, name, arguments, qualification)._describe().labels) +def _plan(name: str, arguments: tuple, qualification) -> Plan: + return built_command(ROOT, name, arguments, qualification)._describe() -def _plans() -> tuple[tuple[str, ...], tuple[str, ...]]: +def _plans() -> tuple[Plan, Plan]: """The local gate's steps, and the binary release lane's.""" config = gate_config.load(ROOT) settings = config.modules @@ -55,27 +57,30 @@ def _plans() -> tuple[tuple[str, ...], tuple[str, ...]]: settings.release_bin_dir: str(STAGED / "cache/target/cargo/debug"), }, ) - local = _labels("candidate", (), from_environment(config, {})) - pulled = _labels("qualify-binaries", (("workspace_root", STAGED),), release) + local = _plan("candidate", (), from_environment(config, {})) + pulled = _plan("qualify-binaries", (("workspace_root", STAGED),), release) return local, pulled +def _owner(plan: Plan, label: str) -> str: + stage = plan.stage_of(label) + return label.removeprefix(f"{stage}.") if stage else label + + +def _uncovered(local: Plan, pulled: Plan) -> list[str]: + covered = set(local.labels) | {_owner(local, label) for label in local.labels} + return [label for label in pulled.labels if _owner(pulled, label) not in covered] + + def test_every_release_step_has_a_local_counterpart() -> None: """The whole point: a green `just test` has to mean something about CI.""" local, pulled = _plans() - covered = set(local) - # Matched by what the step does rather than by which phase does it. A # release lane makes the generated settings in its functional phase because # that is the only phase it has; the local gate makes them once in the fast # phase and hands the step on. Same work, same command, different label -- # and a guard that insisted on the phase would report that as a blind spot. - suffixes = {label.split(".", 1)[-1] for label in covered} - uncovered = [ - label - for label in pulled - if label not in covered and label.split(".", 1)[-1] not in suffixes - ] + uncovered = _uncovered(local, pulled) assert not uncovered, ( "these steps run only in a binary release, so the only way to find a " @@ -83,6 +88,17 @@ def test_every_release_step_has_a_local_counterpart() -> None: ) +def test_shared_owner_matches_its_phase_without_erasing_the_domain() -> None: + local, pulled = Plan("local"), Plan("pulled") + local.phase("fast").add(plan_step("toolchain.node")) + local.phase("fast").add(plan_step("sdk.typescript.build")) + pulled.shared(plan_step("toolchain.node")) + pulled.shared(plan_step("fast.sdk.typescript.build")) + assert _uncovered(local, pulled) == [] + pulled.shared(plan_step("network.node")) + assert _uncovered(local, pulled) == ["network.node"] + + def test_the_rehearsal_covers_exactly_the_steps_that_differ() -> None: """A rehearsal step with nothing to rehearse is a step proving itself. @@ -184,7 +200,7 @@ def test_a_release_lane_does_not_rehearse_itself() -> None: fabricated beside it. """ _, pulled = _plans() - doubled = [label for label in pulled if label.startswith(f"{REHEARSAL}.")] + doubled = [label for label in pulled.labels if label.startswith(f"{REHEARSAL}.")] assert not doubled, f"the release lane rehearses itself at {doubled}" diff --git a/tests/citadel/test_sdk_ui_dependency.py b/tests/citadel/test_sdk_ui_dependency.py index f4534afe7..e2d1217f0 100644 --- a/tests/citadel/test_sdk_ui_dependency.py +++ b/tests/citadel/test_sdk_ui_dependency.py @@ -5,10 +5,12 @@ import argparse import json from copy import deepcopy +from dataclasses import replace from types import SimpleNamespace import pytest import yaml +from capsem_builder.gate.actions import Run from capsem_builder.gate.devloop import DevCommand from capsem_builder.gate.hostpackage import BuildUiCommand from citadel.test_sdk_quality import CONFIG, ROOT @@ -35,8 +37,13 @@ def _ancestors(plan, label: str) -> set[str]: def _assert_order(plan, node: str, build: str, consumers: tuple[str, ...]) -> None: assert node in _ancestors(plan, build), RATIONALE assert all(build in _ancestors(plan, consumer) for consumer in consumers), RATIONALE - rendered = "\n".join(plan.step_named(node).render()) - assert "(in typescript) CI=true pnpm install --offline --frozen-lockfile" in rendered, RATIONALE + assert any( + isinstance(action, Run) + and action._command.cwd == CONFIG.path(CONFIG.sdk_typescript.project) + and action._command.argv == CONFIG.toolchain.node_install + and action._command.env == dict(CONFIG.toolchain.node_env) + for action in plan.step_named(node).actions + ), RATIONALE @pytest.mark.parametrize(("command", "prefix", "consumers"), [ @@ -120,6 +127,25 @@ def test_missing_sdk_predecessors_are_rejected(removed: str) -> None: ("fast.web.frontend-build", "fast.web.frontend-verify")) +def test_mcp_install_cannot_stand_in_for_the_sdk_install() -> None: + plan = gate_plan("test-fast") + node = plan.step_named("fast.toolchain.node") + broken_node = replace(node, actions=tuple( + action for action in node.actions + if not (isinstance(action, Run) + and action._command.cwd == CONFIG.path(CONFIG.sdk_typescript.project) + and action._command.argv == CONFIG.toolchain.node_install) + )) + assert len(broken_node.actions) == len(node.actions) - 1 + broken = SimpleNamespace( + step_named=lambda label: broken_node if label == node.label else plan.step_named(label), + after_of=plan.after_of, + ) + with pytest.raises(AssertionError, match="compiled SDK"): + _assert_order(broken, node.label, "fast.sdk.typescript.build", + ("fast.web.frontend-build", "fast.web.frontend-verify")) + + def _assert_ci(workflow: dict) -> None: for job in workflow["jobs"].values(): steps = job.get("steps", [])