From 37d1373e3efcdefcc88a6a5a7f29f84a8ad3cd0c Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 3 Nov 2026 17:10:54 +0530 Subject: [PATCH 1/3] fix(supervisor): report isolation backend Attach refusals to the gateway When an isolation backend refused Attach, the supervisor exited without telling the gateway. The configuration stayed `ConfigurationReady: True (ConfigurationAccepted)`, and a revision the supervisor had written at startup stayed Pending with no error. The supervisor now reports the refusal: the gateway records the accepted configuration as rejected, and a revision this start wrote ends Failed with the backend's message. Signed-off-by: Shiju --- crates/openshell-core/src/grpc_client.rs | 66 ++- crates/openshell-sdk/README.md | 8 + .../src/grpc/mutation_replay/ordinary.rs | 2 + crates/openshell-server/src/grpc/policy.rs | 217 +++++++++- crates/openshell-server/src/storage_proto.rs | 4 +- .../openshell-supervisor/src/backend_setup.rs | 11 +- .../src/backend_setup/tests.rs | 45 ++ .../src/config_runtime.rs | 28 ++ .../src/legacy_startup_regression_tests.rs | 81 +++- crates/openshell-supervisor/src/lib.rs | 383 +++++++++++++++++- docs/extensibility/isolation-backends.mdx | 15 + proto/openshell.proto | 11 + .../v1/internal/converter/coverage_test.go | 5 + .../v1/internal/converter/setting.go | 11 +- .../v1/internal/converter/setting_test.go | 13 +- sdk/go/openshell/v1/types/setting.go | 3 + sdk/go/proto/openshellv1/openshell.pb.go | 41 +- sdk/typescript/README.md | 4 + 18 files changed, 884 insertions(+), 64 deletions(-) diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 64268a7866..e1c571319b 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -1287,8 +1287,8 @@ async fn sync_policy_with_client( sandbox: &str, policy: &ProtoSandboxPolicy, workspace: &str, -) -> Result<()> { - client +) -> Result { + let response = client .update_config(UpdateConfigRequest { sandbox: sandbox.to_string(), workspace_scope: Some(crate::proto::workspace_selector(workspace)), @@ -1299,7 +1299,7 @@ async fn sync_policy_with_client( .map_err(grpc_status_error) .wrap_err("failed to sync policy to server")?; - Ok(()) + Ok(response.into_inner()) } /// Discover and sync policy using a single gRPC connection. @@ -1344,7 +1344,17 @@ pub async fn sync_policy( ) -> Result<()> { debug!(endpoint = %gateway.url(), sandbox = %sandbox, "Syncing enriched policy to gateway"); let mut client = connect(gateway).await?; - sync_policy_with_client(&mut client, sandbox, policy, workspace).await + sync_policy_with_client(&mut client, sandbox, policy, workspace) + .await + .map(|_| ()) +} + +/// The authoritative snapshot and the acknowledged revision this write created. +/// A deduplicated or unknown write outcome never establishes startup ownership. +#[derive(Debug)] +pub struct PolicySyncResult { + pub snapshot: SettingsPollResult, + pub created_policy_version: Option, } /// Sync an enriched policy and return the authoritative revision snapshot. @@ -1358,20 +1368,23 @@ pub async fn sync_policy_and_fetch_snapshot( sandbox: &str, policy: &ProtoSandboxPolicy, workspace: &str, -) -> Result { +) -> Result { let status = ClientSpanStatus::current(); let mut client = connect(gateway).await?; - sync_policy_with_client(&mut client, sandbox, policy, workspace).await?; - status.finish(fetch_settings_snapshot_with_client(&mut client, sandbox, Some(workspace)).await) + let write = sync_policy_with_client(&mut client, sandbox, policy, workspace).await?; + let snapshot = status + .finish(fetch_settings_snapshot_with_client(&mut client, sandbox, Some(workspace)).await)?; + Ok(PolicySyncResult { + snapshot, + created_policy_version: write + .created_policy_revision + .then_some(write.version) + .filter(|version| *version > 0), + }) } /// Report an exact runtime configuration generation. Pending registration uses /// the snapshot's instance fence; retain that snapshot across registration retries. -#[tracing::instrument( - name = "supervisor.gateway.report_sandbox_configuration", - skip_all, - fields(otel.kind = "client", otel.status_code = tracing::field::Empty) -)] pub async fn report_sandbox_configuration( gateway: &GatewayEndpoint, sandbox_id: &str, @@ -1380,10 +1393,9 @@ pub async fn report_sandbox_configuration( state: crate::proto::ConfigurationAdmissionState, error: &str, ) -> Result<()> { - let status = ClientSpanStatus::current(); - let mut client = connect(gateway).await?; - client - .report_sandbox_configuration(crate::proto::ReportSandboxConfigurationRequest { + report_sandbox_configuration_request( + gateway, + crate::proto::ReportSandboxConfigurationRequest { sandbox_id: sandbox_id.to_string(), expected_instance_id: snapshot.map_or_else(String::new, |snapshot| { snapshot.configuration_instance_id.clone() @@ -1399,7 +1411,27 @@ pub async fn report_sandbox_configuration( .map_or(0, |snapshot| snapshot.provider_env_revision), error: error.to_string(), }), - }) + refused_before_start: false, + }, + ) + .await +} + +/// Send a configuration report the caller built itself, such as a refusal of +/// the generation this instance already accepted. +#[tracing::instrument( + name = "supervisor.gateway.report_sandbox_configuration", + skip_all, + fields(otel.kind = "client", otel.status_code = tracing::field::Empty) +)] +pub async fn report_sandbox_configuration_request( + gateway: &GatewayEndpoint, + request: crate::proto::ReportSandboxConfigurationRequest, +) -> Result<()> { + let status = ClientSpanStatus::current(); + let mut client = connect(gateway).await?; + client + .report_sandbox_configuration(request) .await .map_err(grpc_status_error)?; status.finish(Ok(())) diff --git a/crates/openshell-sdk/README.md b/crates/openshell-sdk/README.md index 001b2e9249..ac878f6383 100644 --- a/crates/openshell-sdk/README.md +++ b/crates/openshell-sdk/README.md @@ -212,3 +212,11 @@ These helpers use the raw client's authentication slot. They do not perform OIDC - Async-only. Tonic is async-native; callers needing a blocking call can wrap with their own runtime. - The curated surface will grow as more RPCs graduate from `raw`. + +## Next minor API migration + +This source-breaking API change targets the next minor release and must not be backported into the `0.1.x` patch series. + +Raw `ReportSandboxConfigurationRequest` adds `refused_before_start`, and `UpdateConfigResponse` adds `created_policy_revision`. Complete generated Rust literals must initialize both fields to `false` or use `..Default::default()`. Complete raw TypeScript messages need the corresponding boolean defaults; generated message builders supply them. + +Legacy wire payloads decode both flags as false. A false or unacknowledged creation receipt does not establish startup ownership. Upgrade gateway replicas and supervisors together before relying on this metadata; older writers can discard it. diff --git a/crates/openshell-server/src/grpc/mutation_replay/ordinary.rs b/crates/openshell-server/src/grpc/mutation_replay/ordinary.rs index a88c6abb9d..6f069cfa7c 100644 --- a/crates/openshell-server/src/grpc/mutation_replay/ordinary.rs +++ b/crates/openshell-server/src/grpc/mutation_replay/ordinary.rs @@ -940,6 +940,8 @@ mutation!( let _: Sandbox = live(store, &id).await?; } Ok(UpdateConfigResponse { + // Legacy durable replay outcomes do not attest which write created a policy. + created_policy_revision: false, version, policy_hash, settings_revision, diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 504c339690..5d3d1e918a 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -2670,6 +2670,7 @@ fn update_config_response( settings_revision, deleted, annotations, + created_policy_revision: false, }) } @@ -4853,13 +4854,15 @@ async fn handle_update_config_inner( spawn_pending_chunk_refresh(state, &workspace, &sandbox); emit_full_policy_update_success(sandbox_caller, next_version); - Ok(update_config_response( + let mut response = update_config_response( u32::try_from(next_version).unwrap_or(0), hash, 0, false, response_annotations, - )) + ); + response.get_mut().created_policy_revision = true; + Ok(response) } // --------------------------------------------------------------------------- @@ -5025,6 +5028,26 @@ fn configuration_failure_diagnostic(error: &Status) -> &'static str { } } +/// Both admissions name the same instance and configuration generation. +fn same_generation( + current: &openshell_core::proto::SandboxConfigurationAdmission, + reported: &openshell_core::proto::SandboxConfigurationAdmission, +) -> bool { + ( + ¤t.instance_id, + current.policy_version, + ¤t.policy_hash, + current.config_revision, + current.provider_env_revision, + ) == ( + &reported.instance_id, + reported.policy_version, + &reported.policy_hash, + reported.config_revision, + reported.provider_env_revision, + ) +} + fn configuration_generation_matches( admission: &openshell_core::proto::SandboxConfigurationAdmission, config: &GetSandboxConfigResponse, @@ -5062,6 +5085,12 @@ pub(super) async fn handle_report_sandbox_configuration( if reported == ConfigurationAdmissionState::Unspecified { return Err(Status::invalid_argument("admission state is required")); } + let refused_before_start = request.get_ref().refused_before_start; + if refused_before_start && reported != ConfigurationAdmissionState::Rejected { + return Err(Status::invalid_argument( + "refused_before_start requires a rejected admission", + )); + } let Some(_mutation_guard) = state .compute .sandbox_mutation_guard_by_id(&sandbox_id) @@ -5146,10 +5175,16 @@ pub(super) async fn handle_report_sandbox_configuration( if let Some(current) = current && current.state == i32::from(ConfigurationAdmissionState::Accepted) { - // A rejected desired update does not invalidate an accepted runtime. - let error = admission.error; - admission = current.clone(); - admission.error = error; + if refused_before_start && same_generation(current, &admission) { + // The runtime refused what it accepted, so the workload never + // started on it. Runtime text stays out of public status. + admission.error = "The isolation backend refused this configuration before the workload started; create the sandbox again with a policy the backend accepts".to_string(); + } else { + // A rejected desired update does not invalidate an accepted runtime. + let error = admission.error; + admission = current.clone(); + admission.error = error; + } } } else { admission.error.clear(); @@ -8836,6 +8871,7 @@ mod tests { sandbox_id: sandbox_id.to_string(), admission: Some(admission), expected_instance_id: String::new(), + refused_before_start: false, }), sandbox_id, ) @@ -8973,6 +9009,134 @@ mod tests { } } + #[tokio::test] + async fn configuration_admission_records_refusal_of_accepted_generation() { + use openshell_core::proto::{ + ConfigurationAdmissionState as Admission, ReportSandboxConfigurationRequest, + SandboxConfigurationAdmission, + }; + let state = test_server_state().await; + let sandbox_id = "sb-refused"; + state + .store + .put_message(&test_sandbox( + sandbox_id, + "refused", + openshell_policy::restrictive_default_policy(), + Vec::new(), + )) + .await + .unwrap(); + let instance_id = uuid::Uuid::new_v4().to_string(); + let report = |admission, refused_before_start| { + with_sandbox( + Request::new(ReportSandboxConfigurationRequest { + sandbox_id: sandbox_id.to_string(), + admission: Some(admission), + expected_instance_id: String::new(), + refused_before_start, + }), + sandbox_id, + ) + }; + handle_report_sandbox_configuration( + &state, + report( + SandboxConfigurationAdmission { + instance_id: instance_id.clone(), + state: Admission::Pending.into(), + ..Default::default() + }, + false, + ), + ) + .await + .unwrap(); + let config = handle_get_sandbox_config( + &state, + with_sandbox( + Request::new(GetSandboxConfigRequest { + name: "refused".to_string(), + workspace_scope: Some(openshell_core::proto::workspace_selector( + "default".to_string(), + )), + }), + sandbox_id, + ), + ) + .await + .unwrap() + .into_inner(); + let accepted = SandboxConfigurationAdmission { + instance_id, + state: Admission::Accepted.into(), + policy_version: config.version, + policy_hash: config.policy_hash, + config_revision: config.config_revision, + provider_env_revision: config.provider_env_revision, + error: String::new(), + }; + handle_report_sandbox_configuration(&state, report(accepted.clone(), false)) + .await + .unwrap(); + let readiness = || async { + let persisted = state + .store + .get_message::(sandbox_id) + .await + .unwrap() + .unwrap(); + let status = persisted.status.unwrap(); + let ready = status + .conditions + .into_iter() + .find(|condition| condition.r#type == "ConfigurationReady") + .unwrap(); + (status.configuration_admission.unwrap().state, ready) + }; + + // A runtime that fails to apply the generation it runs, such as a + // reconcile of unreachable middleware, keeps the accepted state. + let mut refused = accepted; + refused.state = Admission::Rejected.into(); + refused.error = "backend text is not shown".to_string(); + handle_report_sandbox_configuration(&state, report(refused.clone(), false)) + .await + .unwrap(); + let (admission_state, ready) = readiness().await; + assert_eq!(admission_state, i32::from(Admission::Accepted)); + assert_eq!(ready.status, "True"); + + let mut flagged_acceptance = refused.clone(); + flagged_acceptance.state = Admission::Accepted.into(); + assert_eq!( + handle_report_sandbox_configuration(&state, report(flagged_acceptance, true)) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut other_generation = refused.clone(); + other_generation.policy_hash.push('x'); + assert!(same_generation(&refused, &refused)); + assert!(!same_generation(&refused, &other_generation)); + + handle_report_sandbox_configuration(&state, report(refused, true)) + .await + .unwrap(); + let (admission_state, ready) = readiness().await; + assert_eq!(admission_state, i32::from(Admission::Rejected)); + assert_eq!( + (ready.status.as_str(), ready.reason.as_str()), + ("False", "ConfigurationInvalid") + ); + assert!( + ready.message.contains("isolation backend refused"), + "{ready:?}" + ); + assert!(!ready.message.contains("backend text"), "{ready:?}"); + } + #[test] fn configuration_diagnostic_is_bounded_and_removes_control_characters() { let diagnostic = bounded_configuration_diagnostic(&format!("rule\n{}", "é".repeat(1000))); @@ -9014,6 +9178,7 @@ mod tests { state: Admission::Pending.into(), ..Default::default() }), + refused_before_start: false, }), sandbox_id, ), @@ -10022,6 +10187,45 @@ mod tests { } } + #[tokio::test] + async fn policy_write_receipt_distinguishes_created_and_deduplicated_revisions() { + let state = test_server_state().await; + let policy = mcp_policy_with_versions(&["2025-11-25"]); + state + .store + .put_message(&test_sandbox( + "write-receipt", + "write-receipt", + policy.clone(), + vec![], + )) + .await + .unwrap(); + let mut narrowed = policy.clone(); + narrowed.network_policies.clear(); + for (candidate, expected_version, created) in [ + (policy.clone(), 1, true), + (policy, 1, false), + (narrowed.clone(), 2, true), + (narrowed, 2, false), + ] { + let response = handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + sandbox: "write-receipt".into(), + workspace_scope: Some(openshell_core::proto::workspace_selector("default")), + policy: Some(candidate), + ..Default::default() + })), + ) + .await + .unwrap() + .into_inner(); + assert_eq!(response.version, expected_version); + assert_eq!(response.created_policy_revision, created); + } + } + #[tokio::test] async fn policy_record_identity_sandbox_deduplicates_defaulted_mcp_history() { let state = test_server_state().await; @@ -10072,6 +10276,7 @@ mod tests { .into_inner(); assert_eq!(response.version, 1, "{case}"); + assert!(!response.created_policy_revision, "{case}"); assert_eq!(response.policy_hash, canonical_hash, "{case}"); let revisions = state .store diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 5f7d87555b..eb6760fabf 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -177,8 +177,10 @@ mod tests { // selections absent; storage.v1 remains frozen. Public field 3 retains its // image wire identity, now with explicit presence. Compute configuration // updates use a separate capability/service and add one classified RPC. + // ReportSandboxConfigurationRequest field 4 adds a public-only refusal + // flag. Older supervisors omit it; durable schemas remain unchanged. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "33e32bb6cb2251b3e75c6425555502e6c2fd675b640094437ef06856a46866e3"; + "71c803759900cef4bfaafc4a2bbc45521adf90bf1216eea56aebd7614036a8b8"; const DURABLE_SCHEMA_SHA256: &str = "cb15f8f79a855064a43fe8c6cd908bf07eca6604e2bf8ab7d2de0ef33e97da4e"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = diff --git a/crates/openshell-supervisor/src/backend_setup.rs b/crates/openshell-supervisor/src/backend_setup.rs index a0cd52ea87..ca8be97e38 100644 --- a/crates/openshell-supervisor/src/backend_setup.rs +++ b/crates/openshell-supervisor/src/backend_setup.rs @@ -85,6 +85,10 @@ pub trait PreparedBackend: Send + Sync { ) -> std::result::Result; } +/// Diagnostic code of an Attach error that is a backend refusal, as opposed to +/// a transient failure the next start can retry. +pub(crate) const ATTACH_REFUSED: &str = "openshell::backend::attach_refused"; + /// Created only after name and launch identity checks. Consuming attachment /// prevents reusing one prepared startup to build or attach a second client. pub struct SelectedBackend { @@ -200,7 +204,12 @@ impl SelectedBackend { }, ) .await - .map_err(|error| miette::miette!(error.to_string()))?; + .map_err(|error| match error { + BackendError::Denied(_) => { + miette::miette!(code = ATTACH_REFUSED, "{error}") + } + error => miette::miette!("{error}"), + })?; Ok((bound, built.proxy_listener)) } } diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 18564d13ec..9602c7d68d 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -117,6 +117,7 @@ struct Observed { deny_confirmation: AtomicBool, require_networking: AtomicBool, active: AtomicBool, + attach_error: Mutex>, starts: AtomicUsize, releases: AtomicUsize, suspendable: AtomicBool, @@ -308,6 +309,10 @@ impl PreparedBackend for TestLaunch { Ok(match request { Request::Attach { .. } => { observed.record("attach"); + let attach_error = observed.attach_error.lock().unwrap().take(); + if let Some(error) = attach_error { + return Err(error); + } if observed.active.swap(true, Ordering::SeqCst) { return Err(BackendError::Denied("already attached".into())); } @@ -583,6 +588,46 @@ async fn attachment_receives_live_supervisor_services() { assert!(setup.observed.services.lock().unwrap().upgrade().is_none()); } +#[tokio::test] +async fn attach_refusal_is_marked_as_a_refusal() { + let setup = TestSetup::new(); + setup.observed.active.store(true, Ordering::SeqCst); + let error = setup + .select() + .attach(setup.services(), policy(), agent()) + .await + .err() + .expect("an already attached boundary refuses attachment"); + assert_eq!( + error.code().map(|code| code.to_string()).as_deref(), + Some(ATTACH_REFUSED) + ); + assert!(error.to_string().contains("already attached")); +} + +#[tokio::test] +async fn attach_transport_failures_are_not_policy_refusals() { + for failure in [ + BackendError::Attach("mediation transport failed".into()), + BackendError::Unavailable("backend unavailable".into()), + ] { + let setup = TestSetup::new(); + *setup.observed.attach_error.lock().unwrap() = Some(failure); + let error = setup + .select() + .attach(setup.services(), policy(), agent()) + .await + .err() + .expect("backend transport failure must fail startup"); + assert_ne!( + error.code().map(|code| code.to_string()).as_deref(), + Some(ATTACH_REFUSED), + "infrastructure failures must not be reported as policy denials" + ); + assert_eq!(setup.observed.starts.load(Ordering::SeqCst), 0); + } +} + #[tokio::test] async fn restored_boundary_attaches_with_retained_processes_and_starts_agent() { let setup = TestSetup::new(); diff --git a/crates/openshell-supervisor/src/config_runtime.rs b/crates/openshell-supervisor/src/config_runtime.rs index ecda1a7194..7b45d29d0e 100644 --- a/crates/openshell-supervisor/src/config_runtime.rs +++ b/crates/openshell-supervisor/src/config_runtime.rs @@ -2196,6 +2196,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&settings)), has_last_valid_policy: true, + written_at_startup: false, }, connector, ); @@ -2317,6 +2318,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&settings)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -2364,6 +2366,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -2464,6 +2467,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&settings)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -2522,6 +2526,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -2584,6 +2589,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -2676,6 +2682,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -2845,6 +2852,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(loaded_revision), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -2916,6 +2924,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(loaded_revision), has_last_valid_policy: true, + written_at_startup: false, }, middleware_connector, ); @@ -2993,6 +3002,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(loaded_revision), has_last_valid_policy: true, + written_at_startup: false, }, connector, ); @@ -3060,6 +3070,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(loaded_revision), has_last_valid_policy: true, + written_at_startup: false, }, connector, ); @@ -3137,6 +3148,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(loaded_v1.clone()), has_last_valid_policy: true, + written_at_startup: false, }, Some(1), ) @@ -3153,6 +3165,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(empty_loaded), has_last_valid_policy: true, + written_at_startup: false, }, Some(1), ) @@ -3164,6 +3177,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(loaded_v1.clone()), has_last_valid_policy: true, + written_at_startup: false, }, Some(1), ) @@ -3176,6 +3190,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(loaded_v2), has_last_valid_policy: true, + written_at_startup: false, }, Some(2), ) @@ -3202,6 +3217,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(loaded_revision), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3323,6 +3339,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3406,6 +3423,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3480,6 +3498,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3609,6 +3628,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3713,6 +3733,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3752,6 +3773,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3807,6 +3829,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3859,6 +3882,7 @@ mod tests { revision: has_last_valid .then(|| LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: has_last_valid, + written_at_startup: false, }, default_middleware_connector(), ); @@ -3983,6 +4007,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: None, has_last_valid_policy: false, + written_at_startup: false, }, default_middleware_connector(), ); @@ -4028,6 +4053,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -4067,6 +4093,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); @@ -4160,6 +4187,7 @@ mod tests { LoadedPolicyOrigin::Gateway { revision: Some(LoadedPolicyRevision::from_snapshot(&initial)), has_last_valid_policy: true, + written_at_startup: false, }, default_middleware_connector(), ); diff --git a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs index 7b2b60859f..9d4a28694d 100644 --- a/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs +++ b/crates/openshell-supervisor/src/legacy_startup_regression_tests.rs @@ -15,11 +15,18 @@ use std::sync::atomic::Ordering; use tonic::codegen::tokio_stream::{StreamExt as _, wrappers::TcpListenerStream}; use tonic::codegen::{Body, BoxFuture, BoxStream, Bytes, Service, StdError, http}; +#[derive(Clone)] +struct PolicySyncCase { + write: UpdateConfigResponse, + snapshot: GetSandboxConfigResponse, +} + #[derive(Clone, Default)] struct LegacyReadyProbe { ready_event: Arc, premature_ready: Arc, discovered: Arc, + policy_sync: Arc>>, } #[derive(Clone)] @@ -132,10 +139,15 @@ where .await } "/openshell.v1.OpenShell/GetSandboxConfig" => { - let mut policy = openshell_policy::restrictive_default_policy(); - enrich_proto_baseline_paths(&mut policy); - legacy_unary::( - request, + let scripted = probe + .policy_sync + .lock() + .unwrap() + .as_ref() + .map(|case| case.snapshot.clone()); + let snapshot = scripted.unwrap_or_else(|| { + let mut policy = openshell_policy::restrictive_default_policy(); + enrich_proto_baseline_paths(&mut policy); GetSandboxConfigResponse { policy: Some(policy), version: 1, @@ -147,9 +159,9 @@ where policy_source: openshell_core::proto::PolicySource::Sandbox.into(), policy_validation_failure_mode: "fail_closed".into(), ..Default::default() - }, - ) - .await + } + }); + legacy_unary::(request, snapshot).await } "/openshell.v1.OpenShell/GetSandboxProviderEnvironment" => { legacy_unary::( @@ -173,15 +185,15 @@ where .await } "/openshell.v1.OpenShell/UpdateConfig" => { - legacy_unary::( - request, - UpdateConfigResponse { + let response = probe.policy_sync.lock().unwrap().as_ref().map_or_else( + || UpdateConfigResponse { version: 1, policy_hash: "legacy-policy".into(), ..Default::default() }, - ) - .await + |case| case.write.clone(), + ); + legacy_unary::(request, response).await } _ => http::Response::builder() .status(200) @@ -344,3 +356,48 @@ async fn config_push_regression_legacy_gateway_never_observes_ready_before_workl .unwrap_err(); assert!(error.to_string().contains("test boundary remains held")); } + +#[tokio::test] +async fn policy_sync_preserves_write_receipt_separately_from_refetched_policy() { + for (created, snapshot_version) in [(true, 2), (true, 3), (false, 2)] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let probe = LegacyReadyProbe::default(); + *probe.policy_sync.lock().unwrap() = Some(PolicySyncCase { + write: UpdateConfigResponse { + version: 2, + created_policy_revision: created, + ..Default::default() + }, + snapshot: GetSandboxConfigResponse { + version: snapshot_version, + policy_source: openshell_core::proto::PolicySource::Sandbox.into(), + policy_validation_failure_mode: "fail_closed".into(), + workspace: "default".into(), + ..Default::default() + }, + }); + let server = tokio::spawn( + tonic::transport::Server::builder() + .add_service(LegacyGateway { probe }) + .serve_with_incoming(TcpListenerStream::new(listener)), + ); + let gateway = openshell_core::grpc_client::GatewayEndpoint::new( + endpoint, + openshell_core::grpc_client::GatewayCredentials::from_gateway_token("test-token") + .unwrap(), + ); + let synced = openshell_core::grpc_client::sync_policy_and_fetch_snapshot( + &gateway, + "sandbox", + &openshell_policy::restrictive_default_policy(), + "default", + ) + .await + .expect("sync the policy through the production RPC helper"); + assert_eq!(synced.snapshot.version, snapshot_version); + assert_eq!(synced.created_policy_version, created.then_some(2)); + server.abort(); + let _ = server.await; + } +} diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index 398c4bef77..ae412b2000 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -1317,7 +1317,7 @@ pub async fn run_sandbox_instance( // supervisor-owned handles shared with the backend. let admitted_backend_name = selected_backend.backend_name().to_string(); let ca_file_paths = Arc::new(std::sync::Mutex::new(None)); - let (bound, proxy_listener) = selected_backend + let attached = selected_backend .attach( backend_setup::BackendServices { ca_file_paths: ca_file_paths.clone(), @@ -1337,7 +1337,22 @@ pub async fn run_sandbox_instance( }, ) .instrument(tracing::info_span!(parent: &startup, "supervisor.boundary.attach")) - .await?; + .await; + let (bound, proxy_listener) = match attached { + Ok(attached) => attached, + Err(error) => { + if let (Some(sandbox_id), Some(gateway)) = (sandbox_id.as_deref(), gateway.as_ref()) { + Box::pin(report_attach_refusal( + gateway, + sandbox_id, + &loaded_policy_origin, + &error, + )) + .await; + } + return Err(error); + } + }; info!(backend = %admitted_backend_name, "Isolation boundary attached"); let remote_boundary = (bound, admitted_backend_name, ca_file_paths); @@ -2615,7 +2630,7 @@ trait StartupGateway: Send + Sync { sandbox: &str, policy: &openshell_core::proto::SandboxPolicy, workspace: &str, - ) -> impl Future> + Send; + ) -> impl Future> + Send; fn report( &self, id: &str, @@ -2656,7 +2671,7 @@ impl StartupGateway for RemoteStartupGateway { sandbox: &str, policy: &openshell_core::proto::SandboxPolicy, workspace: &str, - ) -> Result { + ) -> Result { openshell_core::grpc_client::sync_policy_and_fetch_snapshot( self.gateway()?, sandbox, @@ -2923,6 +2938,7 @@ async fn load_policy_with_gateway( .await?; let discovery = image_discovery.unwrap_or_else(discover_image_policy); let mut reconciliation_attempts = 0u32; + let mut startup_created_policy_version = None; let mut rejection_log = StartupRejectionLog::default(); loop { if reconciliation_attempts == 5 { @@ -2984,7 +3000,12 @@ async fn load_policy_with_gateway( }) .await; snapshot = match synced { - Ok(synced) => synced, + Ok(synced) => { + if let Some(version) = synced.created_policy_version { + startup_created_policy_version = Some(version); + } + synced.snapshot + } Err(error) => { // The gateway stored nothing, so report the rejection // against the snapshot this upload was built from and @@ -3044,7 +3065,12 @@ async fn load_policy_with_gateway( }) .await; let canonical = match synced { - Ok(canonical) => canonical, + Ok(canonical) => { + if let Some(version) = canonical.created_policy_version { + startup_created_policy_version = Some(version); + } + canonical.snapshot + } Err(error) => { // The stored policy is unchanged, so report the // rejection against the snapshot it was read from and @@ -3262,6 +3288,9 @@ async fn load_policy_with_gateway( LoadedPolicyOrigin::Gateway { revision: loaded_policy_revision, has_last_valid_policy, + written_at_startup: snapshot.policy_source + == openshell_core::proto::PolicySource::Sandbox + && startup_created_policy_version == Some(snapshot.version), }, agent_proposals_enabled_from_settings(&snapshot.settings), snapshot.extension_authentication_enabled, @@ -3686,6 +3715,9 @@ enum LoadedPolicyOrigin { Gateway { revision: Option, has_last_valid_policy: bool, + /// This start wrote `revision` to the gateway, so it is the + /// supervisor's own revision to fail if the backend refuses it. + written_at_startup: bool, }, } @@ -4180,6 +4212,88 @@ fn enqueue_policy_status(sender: &UnboundedSender, update: P } } +/// The report that the backend refused, at Attach, the configuration this start +/// accepted. Only a refusal qualifies: `Unavailable` can succeed on the next start. +fn attach_refusal_report( + sandbox_id: &str, + origin: &LoadedPolicyOrigin, + error: &miette::Report, +) -> Option { + let LoadedPolicyOrigin::Gateway { + revision: Some(revision), + .. + } = origin + else { + return None; + }; + if error + .code() + .is_none_or(|code| code.to_string() != backend_setup::ATTACH_REFUSED) + { + return None; + } + Some(openshell_core::proto::ReportSandboxConfigurationRequest { + sandbox_id: sandbox_id.to_string(), + admission: Some(openshell_core::proto::SandboxConfigurationAdmission { + instance_id: revision.admission_instance_id.clone()?, + state: openshell_core::proto::ConfigurationAdmissionState::Rejected.into(), + policy_version: revision.version, + policy_hash: revision.policy_hash.clone(), + config_revision: revision.config_revision, + provider_env_revision: revision.provider_env_revision, + error: error.to_string(), + }), + expected_instance_id: String::new(), + refused_before_start: true, + }) +} + +/// Report an Attach refusal before the supervisor exits. The gateway records +/// the accepted configuration as refused, and a revision this start wrote ends +/// Failed with the backend's message. Both reports are best effort, and the +/// refusal report gets a single bounded attempt so exit is not held up. +async fn report_attach_refusal( + gateway: &openshell_core::grpc_client::GatewayEndpoint, + sandbox_id: &str, + origin: &LoadedPolicyOrigin, + error: &miette::Report, +) { + if let Some(request) = attach_refusal_report(sandbox_id, origin, error) + && let Err(report_error) = grpc_attempt( + "Attach refusal report", + openshell_core::grpc_client::report_sandbox_configuration_request(gateway, request), + ) + .await + { + warn!(error = %report_error, "Failed to report the isolation backend refusal"); + } + if let Some(revision) = refused_startup_revision(origin, error) { + report_initial_policy_failure(gateway, sandbox_id, Some(revision), error).await; + } +} + +/// The revision this start wrote that the backend refused at Attach, if any. +/// +/// Only a refusal marks it: `Unavailable` can succeed on the next start, and a +/// revision the user wrote stays as it was. +fn refused_startup_revision<'a>( + origin: &'a LoadedPolicyOrigin, + error: &miette::Report, +) -> Option<&'a LoadedPolicyRevision> { + let LoadedPolicyOrigin::Gateway { + revision: Some(revision), + written_at_startup: true, + .. + } = origin + else { + return None; + }; + error + .code() + .is_some_and(|code| code.to_string() == backend_setup::ATTACH_REFUSED) + .then_some(revision) +} + /// Best-effort `FAILED` acknowledgement when initial policy construction or /// conversion fails. /// @@ -5197,8 +5311,13 @@ network_policies: _sandbox: &str, _policy: &openshell_core::proto::SandboxPolicy, _workspace: &str, - ) -> Result { - self.snapshot("").await + ) -> Result { + self.snapshot("") + .await + .map(|snapshot| openshell_core::grpc_client::PolicySyncResult { + snapshot, + created_policy_version: None, + }) } async fn report( &self, @@ -5733,7 +5852,7 @@ network_policies: _sandbox: &str, policy: &openshell_core::proto::SandboxPolicy, _workspace: &str, - ) -> Result { + ) -> Result { self.calls.send(StartupCall::Sync).unwrap(); let mut desired = self.desired.lock().unwrap(); // The gateway checks for a global policy before it validates the @@ -5764,7 +5883,10 @@ network_policies: version, openshell_core::proto::PolicySource::Sandbox, ); - Ok(desired.clone()) + Ok(openshell_core::grpc_client::PolicySyncResult { + snapshot: desired.clone(), + created_policy_version: Some(version), + }) } async fn report( &self, @@ -6418,6 +6540,244 @@ network_policies: assert_eq!(revision.policy_source, PolicySource::Global); } + struct ConcurrentSyncStartupGateway { + inner: WriteRefusingStartupGateway, + after_sync: openshell_core::grpc_client::SettingsPollResult, + created_version: Option, + reject_first_acceptance: AtomicBool, + } + + impl StartupGateway for ConcurrentSyncStartupGateway { + async fn snapshot( + &self, + sandbox: &str, + ) -> Result { + self.inner.snapshot(sandbox).await + } + + async fn provider( + &self, + id: &str, + ) -> Result { + self.inner.provider(id).await + } + + async fn sync( + &self, + _sandbox: &str, + _policy: &openshell_core::proto::SandboxPolicy, + _workspace: &str, + ) -> Result { + self.inner.calls.send(StartupCall::Sync).unwrap(); + *self.inner.desired.lock().unwrap() = self.after_sync.clone(); + Ok(openshell_core::grpc_client::PolicySyncResult { + snapshot: self.after_sync.clone(), + created_policy_version: self.created_version, + }) + } + + async fn report( + &self, + id: &str, + instance_id: &str, + snapshot: Option<&openshell_core::grpc_client::SettingsPollResult>, + state: openshell_core::proto::ConfigurationAdmissionState, + error: &str, + ) -> Result<()> { + if state == openshell_core::proto::ConfigurationAdmissionState::Accepted + && self.reject_first_acceptance.swap(false, Ordering::SeqCst) + { + return Err(openshell_core::grpc_client::grpc_status_error( + tonic::Status::aborted("retry the same accepted configuration"), + )); + } + self.inner + .report(id, instance_id, snapshot, state, error) + .await + } + } + + #[tokio::test(start_paused = true)] + async fn startup_write_ownership_requires_a_matching_created_sandbox_revision() { + use openshell_core::proto::PolicySource; + + for (created_version, accepted_version, source, owned) in [ + (Some(2), 2, PolicySource::Sandbox, true), + (Some(2), 3, PolicySource::Sandbox, false), + (None, 2, PolicySource::Sandbox, false), + (Some(2), 2, PolicySource::Global, false), + ] { + let initial = proto_tcp_policy_fixture(); + let mut enriched = initial.clone(); + assert!(enrich_proto_baseline_paths(&mut enriched)); + let mut after_sync = settings_poll_result(Some(enriched), accepted_version, source); + if source == PolicySource::Global { + after_sync.global_policy_version = 7; + } + let (calls, _observed) = tokio::sync::mpsc::unbounded_channel(); + let gateway = ConcurrentSyncStartupGateway { + inner: WriteRefusingStartupGateway { + desired: Arc::new(std::sync::Mutex::new(settings_poll_result( + Some(initial), + 1, + PolicySource::Sandbox, + ))), + refusals: Arc::default(), + repair_after_refusal: None, + rejection_report_errors: Arc::default(), + calls, + }, + after_sync, + created_version, + reject_first_acceptance: AtomicBool::new(true), + }; + let bundle = load_policy_with_gateway( + Some("sandbox-id".into()), + Some("sandbox".into()), + Some(test_gateway()), + None, + None, + &openshell_extension_core::ExtensionCredentialStore::new(), + LocalPolicyIdentity::Required, + None, + Some(ImagePolicyDiscovery::Missing), + &gateway, + ) + .await + .expect("startup must retry and accept the canonical snapshot"); + let denial = miette::miette!(code = backend_setup::ATTACH_REFUSED, "policy denied"); + assert_eq!( + refused_startup_revision(&bundle.4, &denial).is_some(), + owned, + "ownership must survive a same-version retry but never pass to a later or deduplicated user revision" + ); + } + } + + #[tokio::test(start_paused = true)] + async fn startup_marks_only_the_revision_it_wrote() { + use openshell_core::proto::PolicySource; + + for (needs_baseline_paths, written) in [(true, true), (false, false)] { + let mut policy = proto_tcp_policy_fixture(); + if !needs_baseline_paths { + assert!(enrich_proto_baseline_paths(&mut policy)); + } + let (calls, _observed) = tokio::sync::mpsc::unbounded_channel(); + let gateway = WriteRefusingStartupGateway { + desired: Arc::new(std::sync::Mutex::new(settings_poll_result( + Some(policy), + 1, + PolicySource::Sandbox, + ))), + refusals: Arc::default(), + repair_after_refusal: None, + rejection_report_errors: Arc::default(), + calls, + }; + let bundle = load_policy_with_gateway( + Some("sandbox-id".to_string()), + Some("sandbox".to_string()), + Some(test_gateway()), + None, + None, + &openshell_extension_core::ExtensionCredentialStore::new(), + LocalPolicyIdentity::Required, + None, + Some(ImagePolicyDiscovery::Missing), + &gateway, + ) + .await + .expect("startup must accept the policy"); + let LoadedPolicyOrigin::Gateway { + revision: Some(revision), + written_at_startup, + .. + } = bundle.4 + else { + panic!("startup must bind the policy to its gateway revision"); + }; + assert_eq!(written_at_startup, written); + assert_eq!(revision.version, if written { 2 } else { 1 }); + } + } + + #[test] + fn attach_refusal_reports_the_accepted_generation_as_refused() { + use openshell_core::proto::ConfigurationAdmissionState; + + let mut revision = LoadedPolicyRevision::from_snapshot(&settings_poll_result( + Some(proto_policy_fixture()), + 3, + openshell_core::proto::PolicySource::Sandbox, + )); + revision.admission_instance_id = Some("instance-1".to_string()); + let origin = |revision: &LoadedPolicyRevision| LoadedPolicyOrigin::Gateway { + revision: Some(revision.clone()), + has_last_valid_policy: true, + written_at_startup: false, + }; + let refusal = miette::miette!(code = backend_setup::ATTACH_REFUSED, "refused"); + + let request = attach_refusal_report("sandbox-id", &origin(&revision), &refusal) + .expect("a refusal of the accepted generation is reported"); + assert!(request.refused_before_start); + let admission = request.admission.unwrap(); + assert_eq!( + admission.state, + i32::from(ConfigurationAdmissionState::Rejected) + ); + assert_eq!( + ( + admission.instance_id.as_str(), + admission.policy_version, + admission.policy_hash.as_str(), + admission.config_revision, + admission.provider_env_revision, + ), + ( + "instance-1", + revision.version, + revision.policy_hash.as_str(), + revision.config_revision, + revision.provider_env_revision, + ) + ); + + let transient = miette::miette!("unavailable"); + assert!(attach_refusal_report("sandbox-id", &origin(&revision), &transient).is_none()); + assert!( + attach_refusal_report("sandbox-id", &LoadedPolicyOrigin::LocalOverride, &refusal) + .is_none() + ); + revision.admission_instance_id = None; + assert!(attach_refusal_report("sandbox-id", &origin(&revision), &refusal).is_none()); + } + + #[test] + fn attach_refusal_fails_only_the_revision_this_start_wrote() { + let revision = LoadedPolicyRevision::from_snapshot(&settings_poll_result( + Some(proto_policy_fixture()), + 3, + openshell_core::proto::PolicySource::Sandbox, + )); + let origin = |written_at_startup| LoadedPolicyOrigin::Gateway { + revision: Some(revision.clone()), + has_last_valid_policy: true, + written_at_startup, + }; + let refusal = miette::miette!(code = backend_setup::ATTACH_REFUSED, "refused"); + let transient = miette::miette!("unavailable"); + + assert_eq!( + refused_startup_revision(&origin(true), &refusal).map(|revision| revision.version), + Some(3) + ); + assert!(refused_startup_revision(&origin(true), &transient).is_none()); + assert!(refused_startup_revision(&origin(false), &refusal).is_none()); + assert!(refused_startup_revision(&LoadedPolicyOrigin::LocalOverride, &refusal).is_none()); + } + #[tokio::test(start_paused = true)] async fn startup_policy_write_refusal_permanent_codes_still_exit() { use openshell_core::proto::{ConfigurationAdmissionState, PolicySource}; @@ -7227,6 +7587,7 @@ network_policies: &LoadedPolicyOrigin::Gateway { revision: Some(loaded), has_last_valid_policy: true, + written_at_startup: false, }, &canonical, ), @@ -7259,6 +7620,7 @@ network_policies: let origin = LoadedPolicyOrigin::Gateway { revision: None, has_last_valid_policy: true, + written_at_startup: false, }; assert_eq!( @@ -7366,6 +7728,7 @@ network_policies: &LoadedPolicyOrigin::Gateway { revision: None, has_last_valid_policy: true, + written_at_startup: false, }, true, true diff --git a/docs/extensibility/isolation-backends.mdx b/docs/extensibility/isolation-backends.mdx index 8dd5903176..dc60c244af 100644 --- a/docs/extensibility/isolation-backends.mdx +++ b/docs/extensibility/isolation-backends.mdx @@ -34,6 +34,15 @@ The supervisor's concrete RPC client, `OpenShellRuntimeBackend`, lives in The admission sequence is Attach, Confirm, then StartAgent. No workload code runs before StartAgent. +A backend refuses a policy it cannot enforce by returning `Denied` from Attach. +The sandbox then reports `ConfigurationReady: False (ConfigurationInvalid)`. +Transport and backend availability failures remain infrastructure failures. The +backend's reason is in the supervisor log. `openshell policy list` also shows it +when the gateway acknowledged that this supervisor created the refused startup +revision. A deduplicated write or a newer operator revision does not establish +that ownership. Delete the sandbox and create it again with a policy the backend +accepts. + External compute drivers advertise `openshell.compute.isolation-backend` to accept an operator-selected `IsolationBackendRegistration`, including its Unix, TCP, or vsock endpoint. The driver provisions the resource and delivers the @@ -50,3 +59,9 @@ backend's trusted launch metadata to the supervisor. For how `openshell-sandbox` enforces the boundary, refer to [Inside the Sandbox Boundary](/about/architecture#inside-the-sandbox-boundary). For driver-specific workload behavior, refer to [Runtimes](/how-it-works/sandboxes/runtimes). + +## Release compatibility + +These API changes target the next minor release and must not be backported into the `0.1.x` patch series. Upgrade gateway replicas and supervisors together before relying on startup refusal reporting. Older writers can drop the new refusal and policy-creation metadata. + +For raw Rust and TypeScript message changes, see the [SDK migration notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-sdk/README.md#next-minor-api-migration). diff --git a/proto/openshell.proto b/proto/openshell.proto index e498a84af9..8b31dca8b5 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -3018,6 +3018,11 @@ message UpdateConfigResponse { bool deleted = 4; // Sandbox metadata annotations after the update. Empty for global updates. map annotations = 5; + // True only when this request created a new sandbox policy revision. + // False for deduplicated policies, settings/global updates and older gateways. + // A supervisor may claim startup ownership only when this receipt's version + // matches the accepted sandbox-sourced configuration. + bool created_policy_revision = 6; } // Get sandbox policy status request. @@ -3100,6 +3105,12 @@ message ReportSandboxConfigurationRequest { SandboxConfigurationAdmission admission = 2; // Pending registration replaces only this previously observed instance. string expected_instance_id = 3; + // With a Rejected admission for the generation this instance already + // reported Accepted: the runtime refused it before the workload started, for + // example an isolation backend refusing Attach. The gateway then records the + // configuration as rejected instead of keeping the accepted state. Omitted or + // false keeps the accepted state; set with any other state is INVALID_ARGUMENT. + bool refused_before_start = 4; } message ReportSandboxConfigurationResponse {} diff --git a/sdk/go/openshell/v1/internal/converter/coverage_test.go b/sdk/go/openshell/v1/internal/converter/coverage_test.go index 8a457d0014..ad674fa428 100644 --- a/sdk/go/openshell/v1/internal/converter/coverage_test.go +++ b/sdk/go/openshell/v1/internal/converter/coverage_test.go @@ -462,3 +462,8 @@ func assertAllFieldsCovered( } } } + +func TestConverterCoversAllProtoFields_UpdateConfigResponse(t *testing.T) { + handled := fieldSet{"version": true, "policy_hash": true, "settings_revision": true, "deleted": true, "annotations": true, "created_policy_revision": true} + assertAllFieldsCovered(t, (&pb.UpdateConfigResponse{}).ProtoReflect().Descriptor(), handled, nil) +} diff --git a/sdk/go/openshell/v1/internal/converter/setting.go b/sdk/go/openshell/v1/internal/converter/setting.go index 57a0ae1242..eaa6897fc2 100644 --- a/sdk/go/openshell/v1/internal/converter/setting.go +++ b/sdk/go/openshell/v1/internal/converter/setting.go @@ -325,10 +325,11 @@ func ConfigUpdateResultFromProto(resp *pb.UpdateConfigResponse) *v1.ConfigUpdate return nil } return &v1.ConfigUpdateResult{ - Version: resp.GetVersion(), - PolicyHash: resp.GetPolicyHash(), - SettingsRevision: resp.GetSettingsRevision(), - Deleted: resp.GetDeleted(), - Annotations: CopyStringMap(resp.GetAnnotations()), + CreatedPolicyRevision: resp.GetCreatedPolicyRevision(), + Version: resp.GetVersion(), + PolicyHash: resp.GetPolicyHash(), + SettingsRevision: resp.GetSettingsRevision(), + Deleted: resp.GetDeleted(), + Annotations: CopyStringMap(resp.GetAnnotations()), } } diff --git a/sdk/go/openshell/v1/internal/converter/setting_test.go b/sdk/go/openshell/v1/internal/converter/setting_test.go index 5481235137..af332a45cd 100644 --- a/sdk/go/openshell/v1/internal/converter/setting_test.go +++ b/sdk/go/openshell/v1/internal/converter/setting_test.go @@ -533,17 +533,19 @@ func TestConfigUpdateToProto_NilPolicy(t *testing.T) { func TestConfigUpdateResultFromProto(t *testing.T) { resp := &pb.UpdateConfigResponse{ - Version: 10, - PolicyHash: "sha256:updated", - SettingsRevision: 55, - Deleted: true, - Annotations: map[string]string{"sandbox_id": "sb-123"}, + CreatedPolicyRevision: true, + Version: 10, + PolicyHash: "sha256:updated", + SettingsRevision: 55, + Deleted: true, + Annotations: map[string]string{"sandbox_id": "sb-123"}, } result := ConfigUpdateResultFromProto(resp) require.NotNil(t, result) assert.Equal(t, uint32(10), result.Version) + assert.True(t, result.CreatedPolicyRevision) assert.Equal(t, "sha256:updated", result.PolicyHash) assert.Equal(t, uint64(55), result.SettingsRevision) assert.True(t, result.Deleted) @@ -560,6 +562,7 @@ func TestConfigUpdateResultFromProto_DefaultValues(t *testing.T) { require.NotNil(t, result) assert.Zero(t, result.Version) + assert.False(t, result.CreatedPolicyRevision) assert.Empty(t, result.PolicyHash) assert.Zero(t, result.SettingsRevision) assert.False(t, result.Deleted) diff --git a/sdk/go/openshell/v1/types/setting.go b/sdk/go/openshell/v1/types/setting.go index 7dd8eff2f2..5cc75617dc 100644 --- a/sdk/go/openshell/v1/types/setting.go +++ b/sdk/go/openshell/v1/types/setting.go @@ -111,6 +111,9 @@ type ConfigUpdate struct { type ConfigUpdateResult struct { // Version is the assigned policy version. Version uint32 + // CreatedPolicyRevision is true only when this write created a sandbox policy revision. + // Deduplicated writes and older gateways leave it false. + CreatedPolicyRevision bool // PolicyHash is the SHA-256 of the serialized policy. PolicyHash string // SettingsRevision is the settings revision for the modified scope. diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 1ae1800784..1910797c0f 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -13052,9 +13052,14 @@ type UpdateConfigResponse struct { // True when a setting delete operation removed an existing key. Deleted bool `protobuf:"varint,4,opt,name=deleted,proto3" json:"deleted,omitempty"` // Sandbox metadata annotations after the update. Empty for global updates. - Annotations map[string]string `protobuf:"bytes,5,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Annotations map[string]string `protobuf:"bytes,5,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // True only when this request created a new sandbox policy revision. + // False for deduplicated policies, settings/global updates and older gateways. + // A supervisor may claim startup ownership only when this receipt's version + // matches the accepted sandbox-sourced configuration. + CreatedPolicyRevision bool `protobuf:"varint,6,opt,name=created_policy_revision,json=createdPolicyRevision,proto3" json:"created_policy_revision,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *UpdateConfigResponse) Reset() { @@ -13122,6 +13127,13 @@ func (x *UpdateConfigResponse) GetAnnotations() map[string]string { return nil } +func (x *UpdateConfigResponse) GetCreatedPolicyRevision() bool { + if x != nil { + return x.CreatedPolicyRevision + } + return false +} + // Get sandbox policy status request. type GetSandboxPolicyStatusRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -13596,6 +13608,12 @@ type ReportSandboxConfigurationRequest struct { Admission *SandboxConfigurationAdmission `protobuf:"bytes,2,opt,name=admission,proto3" json:"admission,omitempty"` // Pending registration replaces only this previously observed instance. ExpectedInstanceId string `protobuf:"bytes,3,opt,name=expected_instance_id,json=expectedInstanceId,proto3" json:"expected_instance_id,omitempty"` + // With a Rejected admission for the generation this instance already + // reported Accepted: the runtime refused it before the workload started, for + // example an isolation backend refusing Attach. The gateway then records the + // configuration as rejected instead of keeping the accepted state. Omitted or + // false keeps the accepted state; set with any other state is INVALID_ARGUMENT. + RefusedBeforeStart bool `protobuf:"varint,4,opt,name=refused_before_start,json=refusedBeforeStart,proto3" json:"refused_before_start,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -13651,6 +13669,13 @@ func (x *ReportSandboxConfigurationRequest) GetExpectedInstanceId() string { return "" } +func (x *ReportSandboxConfigurationRequest) GetRefusedBeforeStart() bool { + if x != nil { + return x.RefusedBeforeStart + } + return false +} + type ReportSandboxConfigurationResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -20707,14 +20732,15 @@ const file_openshell_proto_rawDesc = "" + "\x13RemoveNetworkBinary\x12\x1b\n" + "\trule_name\x18\x01 \x01(\tR\bruleName\x12\x1f\n" + "\vbinary_path\x18\x02 \x01(\tR\n" + - "binaryPath\"\xaf\x02\n" + + "binaryPath\"\xe7\x02\n" + "\x14UpdateConfigResponse\x12\x18\n" + "\aversion\x18\x01 \x01(\rR\aversion\x12\x1f\n" + "\vpolicy_hash\x18\x02 \x01(\tR\n" + "policyHash\x12+\n" + "\x11settings_revision\x18\x03 \x01(\x04R\x10settingsRevision\x12\x18\n" + "\adeleted\x18\x04 \x01(\bR\adeleted\x12U\n" + - "\vannotations\x18\x05 \x03(\v23.openshell.v1.UpdateConfigResponse.AnnotationsEntryR\vannotations\x1a>\n" + + "\vannotations\x18\x05 \x03(\v23.openshell.v1.UpdateConfigResponse.AnnotationsEntryR\vannotations\x126\n" + + "\x17created_policy_revision\x18\x06 \x01(\bR\x15createdPolicyRevision\x1a>\n" + "\x10AnnotationsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xbf\x01\n" + @@ -20753,12 +20779,13 @@ const file_openshell_proto_rawDesc = "" + "policyHash\x12'\n" + "\x0fconfig_revision\x18\x05 \x01(\x04R\x0econfigRevision\x122\n" + "\x15provider_env_revision\x18\x06 \x01(\x04R\x13providerEnvRevision\x12\x14\n" + - "\x05error\x18\a \x01(\tR\x05error\"\xbf\x01\n" + + "\x05error\x18\a \x01(\tR\x05error\"\xf1\x01\n" + "!ReportSandboxConfigurationRequest\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12I\n" + "\tadmission\x18\x02 \x01(\v2+.openshell.v1.SandboxConfigurationAdmissionR\tadmission\x120\n" + - "\x14expected_instance_id\x18\x03 \x01(\tR\x12expectedInstanceId\"$\n" + + "\x14expected_instance_id\x18\x03 \x01(\tR\x12expectedInstanceId\x120\n" + + "\x14refused_before_start\x18\x04 \x01(\bR\x12refusedBeforeStart\"$\n" + "\"ReportSandboxConfigurationResponse\"\x9b\x04\n" + "\x15SandboxPolicyRevision\x12\x18\n" + "\aversion\x18\x01 \x01(\rR\aversion\x12\x1f\n" + diff --git a/sdk/typescript/README.md b/sdk/typescript/README.md index 8b64c9768d..75d4338010 100644 --- a/sdk/typescript/README.md +++ b/sdk/typescript/README.md @@ -348,3 +348,7 @@ mise run e2e:sdk:ts:exec # public interactive helper against an isolated Dock ``` Formatting and linting are handled by [Biome](https://biomejs.dev) (`biome.json`): 2-space indent, single quotes, semicolons, 120-column width. Generated `src/gen/` is excluded. `sdk:ts:lint` runs in CI as part of `sdk:ts:ci`. + +## Next minor raw message migration + +Raw `ReportSandboxConfigurationRequest` messages add `refusedBeforeStart`, and `UpdateConfigResponse` adds `createdPolicyRevision`. Complete object literals need boolean defaults; use generated message builders when creating partial messages. This change belongs to the next minor release; it is not a patch-source-compatibility claim. Upgrade gateway writers together so older protobuf writers cannot discard the new metadata. From 8ba8ed732ee40e4e76247d3f0130ca685fcf24d0 Mon Sep 17 00:00:00 2001 From: Shiju Date: Wed, 4 Nov 2026 12:42:57 +0530 Subject: [PATCH 2/3] docs: remove startup policy bookkeeping from backend guidance Signed-off-by: Shiju --- docs/extensibility/isolation-backends.mdx | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/extensibility/isolation-backends.mdx b/docs/extensibility/isolation-backends.mdx index dc60c244af..a35e545561 100644 --- a/docs/extensibility/isolation-backends.mdx +++ b/docs/extensibility/isolation-backends.mdx @@ -37,11 +37,8 @@ runs before StartAgent. A backend refuses a policy it cannot enforce by returning `Denied` from Attach. The sandbox then reports `ConfigurationReady: False (ConfigurationInvalid)`. Transport and backend availability failures remain infrastructure failures. The -backend's reason is in the supervisor log. `openshell policy list` also shows it -when the gateway acknowledged that this supervisor created the refused startup -revision. A deduplicated write or a newer operator revision does not establish -that ownership. Delete the sandbox and create it again with a policy the backend -accepts. +backend's reason is in the supervisor log. Delete the sandbox and create it +again with a policy the backend accepts. External compute drivers advertise `openshell.compute.isolation-backend` to accept an operator-selected `IsolationBackendRegistration`, including its Unix, @@ -62,6 +59,6 @@ workload behavior, refer to [Runtimes](/how-it-works/sandboxes/runtimes). ## Release compatibility -These API changes target the next minor release and must not be backported into the `0.1.x` patch series. Upgrade gateway replicas and supervisors together before relying on startup refusal reporting. Older writers can drop the new refusal and policy-creation metadata. +These API changes target the next minor release and must not be backported into the `0.1.x` patch series. Upgrade gateway replicas and supervisors together before relying on startup refusal reporting. For raw Rust and TypeScript message changes, see the [SDK migration notes](https://github.com/NVIDIA/OpenShell/blob/main/crates/openshell-sdk/README.md#next-minor-api-migration). From 84587e502259d754932e99f3558e5c0974d7bbdb Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 3 Nov 2026 23:14:00 +0000 Subject: [PATCH 3/3] fix(tests): pin the k3s installer to a release (#5223) get.k3s.io serves the latest install.sh from k3s master, so the pinned checksum broke when upstream changed the script. Download install.sh from the release tag matching k3s_version instead and pin its checksum. Signed-off-by: Matthew Grossman Signed-off-by: Shiju --- tests/tmachine/ansible/playbooks/k3s.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/tmachine/ansible/playbooks/k3s.yaml b/tests/tmachine/ansible/playbooks/k3s.yaml index 8666b2ac6f..d27c6c487b 100644 --- a/tests/tmachine/ansible/playbooks/k3s.yaml +++ b/tests/tmachine/ansible/playbooks/k3s.yaml @@ -21,10 +21,12 @@ state: present update_cache: true + # get.k3s.io serves the latest script, so pin the one from the k3s_version + # release tag. Update the checksum whenever k3s_version changes. - name: Download K3s installer ansible.builtin.get_url: - url: https://get.k3s.io - checksum: sha256:e5cc3b3d9dfc1662c2d9be6da5abc9a4cd317d6abc3a5ffc02e3dd3248207fee + url: "https://raw.githubusercontent.com/k3s-io/k3s/v{{ k3s_version | urlencode }}/install.sh" + checksum: sha256:3ce239d57d43b2d836d2b561043433e6decae8b9dc41f5d13908c0fafb0340cd dest: /tmp/install-k3s.sh mode: "0755"