From cf39c71eabd227e3637133f8b2d07218db408797 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 3 Nov 2026 17:10:59 +0530 Subject: [PATCH 1/2] fix(supervisor): report bounded backend startup failures A backend can fail during construction, Attach or Confirm before the workload starts. This gives operators a fixed failure stage and recovery hint in sandbox status, keeps detailed errors in supervisor logs and bounds the time spent reporting the failure. Signed-off-by: Shiju --- crates/openshell-core/src/backend_startup.rs | 68 +++++++ crates/openshell-core/src/lib.rs | 1 + crates/openshell-server/src/grpc/policy.rs | 41 ++++- .../openshell-supervisor/src/backend_setup.rs | 72 ++++++-- .../src/backend_setup/tests.rs | 16 +- crates/openshell-supervisor/src/lib.rs | 172 +++++++++++++----- docs/extensibility/isolation-backends.mdx | 16 +- skills/debug-openshell-cluster/SKILL.md | 4 + 8 files changed, 310 insertions(+), 80 deletions(-) create mode 100644 crates/openshell-core/src/backend_startup.rs diff --git a/crates/openshell-core/src/backend_startup.rs b/crates/openshell-core/src/backend_startup.rs new file mode 100644 index 0000000000..5c1769b39f --- /dev/null +++ b/crates/openshell-core/src/backend_startup.rs @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Closed startup failure codes carried by pre-start configuration refusals. +//! +//! The supervisor reports a code in the existing admission error field. The +//! gateway projects only these fixed diagnostics, never backend error text. + +/// The trusted operation that failed before workload execution. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackendStartupStage { + /// Constructing the selected backend client. + Build, + /// Attaching the admitted runtime generation. + Attach, + /// Confirming the isolation boundary before starting workload code. + Confirm, +} + +impl BackendStartupStage { + /// Stable token used with the existing backend error-kind classification. + #[must_use] + pub const fn code(self) -> &'static str { + match self { + Self::Build => "build", + Self::Attach => "attach", + Self::Confirm => "confirm", + } + } +} + +/// Project an exact startup failure code into a bounded, credential-free +/// diagnostic. Unknown and legacy error strings have no projection. +/// +/// Only use this projection for an authenticated, generation-fenced +/// `refused_before_start` report; ordinary runtime errors are not startup claims. +#[must_use] +pub fn backend_startup_diagnostic(code: &str) -> Option { + let (stage, kind) = code.strip_prefix("openshell.backend.")?.split_once('.')?; + let stage = match stage { + "build" => "client construction", + "attach" => "attachment", + "confirm" => "isolation confirmation", + _ => return None, + }; + let recovery = match kind { + "invalid" => { + "the launch descriptor or admitted identity did not match; check the installation and recreate the sandbox" + } + "denied" => { + "the backend rejected the admitted policy or identity; review the policy and recreate the sandbox" + } + "unavailable" => { + "the backend was unavailable; check its service and host qualification, then retry after cleanup" + } + "unsupported" => { + "the backend does not support the requested operation; choose a compatible backend configuration" + } + "failed" => { + "the backend could not establish isolation; check the installation and host qualification before retrying after cleanup" + } + "terminated" => { + "the isolation boundary was lost before workload execution; check the backend service and recreate the sandbox" + } + _ => return None, + }; + Some(format!("Isolation backend {stage} failed: {recovery}")) +} diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 83faf18ef0..68928798c3 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -12,6 +12,7 @@ pub mod activity; pub mod app_listener; pub mod auth; +pub mod backend_startup; pub mod compute; pub mod config; pub mod container_paths; diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 5d3d1e918a..384745ea33 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -5091,6 +5091,9 @@ pub(super) async fn handle_report_sandbox_configuration( "refused_before_start requires a rejected admission", )); } + let startup_diagnostic = refused_before_start + .then(|| openshell_core::backend_startup::backend_startup_diagnostic(&admission.error)) + .flatten(); let Some(_mutation_guard) = state .compute .sandbox_mutation_guard_by_id(&sandbox_id) @@ -5172,14 +5175,17 @@ pub(super) async fn handle_report_sandbox_configuration( Ok(config) if !config.get_ref().configuration_error.is_empty() => config.into_inner().configuration_error, _ => "Effective configuration could not be activated; replace the policy or repair attached providers".to_string(), }; - if let Some(current) = current - && current.state == i32::from(ConfigurationAdmissionState::Accepted) - { - if refused_before_start && same_generation(current, &admission) { - // The runtime refused what it accepted, so the workload never - // started on it. Runtime text stays out of public status. - admission.error = "The isolation backend refused this configuration before the workload started; create the sandbox again with a policy the backend accepts".to_string(); - } else { + if let Some(current) = current { + let accepted = current.state == i32::from(ConfigurationAdmissionState::Accepted); + let rejected = current.state == i32::from(ConfigurationAdmissionState::Rejected); + if refused_before_start + && (accepted || rejected) + && same_generation(current, &admission) + { + // A retry of the same failed generation retains its diagnostic. + // Only exact known codes are projected; legacy/raw text stays out. + admission.error = startup_diagnostic.unwrap_or_else(|| "The isolation backend refused this configuration before the workload started; create the sandbox again with a policy the backend accepts".to_string()); + } else if accepted { // A rejected desired update does not invalidate an accepted runtime. let error = admission.error; admission = current.clone(); @@ -9121,7 +9127,8 @@ mod tests { assert!(same_generation(&refused, &refused)); assert!(!same_generation(&refused, &other_generation)); - handle_report_sandbox_configuration(&state, report(refused, true)) + refused.error = "openshell.backend.confirm.denied".to_string(); + handle_report_sandbox_configuration(&state, report(refused.clone(), true)) .await .unwrap(); let (admission_state, ready) = readiness().await; @@ -9131,10 +9138,24 @@ mod tests { ("False", "ConfigurationInvalid") ); assert!( - ready.message.contains("isolation backend refused"), + ready.message.contains("isolation confirmation"), "{ready:?}" ); assert!(!ready.message.contains("backend text"), "{ready:?}"); + let first_message = ready.message; + handle_report_sandbox_configuration(&state, report(refused.clone(), true)) + .await + .unwrap(); + let (_, repeated) = readiness().await; + assert_eq!(repeated.message, first_message); + + refused.error = "openshell.backend.confirm.denied\nBearer secret-value".to_string(); + handle_report_sandbox_configuration(&state, report(refused, true)) + .await + .unwrap(); + let (_, unknown) = readiness().await; + assert!(!unknown.message.contains("secret-value")); + assert!(unknown.message.contains("isolation backend refused")); } #[test] diff --git a/crates/openshell-supervisor/src/backend_setup.rs b/crates/openshell-supervisor/src/backend_setup.rs index ca8be97e38..684d056776 100644 --- a/crates/openshell-supervisor/src/backend_setup.rs +++ b/crates/openshell-supervisor/src/backend_setup.rs @@ -11,11 +11,13 @@ use std::sync::{Arc, Mutex}; use miette::Result; use openshell_core::SandboxSessionId; +use openshell_core::backend_startup::BackendStartupStage; use openshell_core::jwt::{SessionBearerTokenSlot, SupervisorAuthBundle}; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_isolation_interface::AgentSpec; use openshell_isolation_interface::contract::{ - BackendDescriptor, BackendError, ResolvedWorkloadIdentity, SandboxContext, SandboxPolicy, + BackendDescriptor, BackendError, BackendErrorKind, ResolvedWorkloadIdentity, SandboxContext, + SandboxPolicy, }; use openshell_sandbox_backend::BoundBoundary; @@ -85,9 +87,53 @@ pub trait PreparedBackend: Send + Sync { ) -> std::result::Result; } -/// Diagnostic code of an Attach error that is a backend refusal, as opposed to -/// a transient failure the next start can retry. -pub(crate) const ATTACH_REFUSED: &str = "openshell::backend::attach_refused"; +/// Keep the original typed error for refusal classification while exposing +/// only an allowlisted stage/kind code to the configuration report. +#[derive(Debug)] +struct StartupError { + stage: BackendStartupStage, + error: BackendError, +} + +impl std::fmt::Display for StartupError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.error, formatter) + } +} + +impl std::error::Error for StartupError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.error) + } +} + +impl miette::Diagnostic for StartupError { + fn code<'a>(&'a self) -> Option> { + let kind = match self.error.kind() { + BackendErrorKind::Invalid => "invalid", + BackendErrorKind::Denied => "denied", + BackendErrorKind::Unavailable => "unavailable", + BackendErrorKind::Unsupported => "unsupported", + BackendErrorKind::Failed => "failed", + BackendErrorKind::Terminated => "terminated", + }; + Some(Box::new(format!( + "openshell.backend.{}.{kind}", + self.stage.code() + ))) + } +} + +pub(crate) fn startup_error(stage: BackendStartupStage, error: BackendError) -> miette::Report { + miette::Report::new(StartupError { stage, error }) +} + +pub(crate) fn is_backend_refusal(error: &miette::Report) -> bool { + error.downcast_ref::().is_some_and(|failure| { + failure.stage == BackendStartupStage::Attach + && matches!(failure.error, BackendError::Denied(_)) + }) +} /// Created only after name and launch identity checks. Consuming attachment /// prevents reusing one prepared startup to build or attach a second client. @@ -125,7 +171,7 @@ impl SelectedBackend { let (identity, prepared) = setup .decode(&descriptor.payload) - .map_err(|error| miette::miette!(error.to_string()))?; + .map_err(|error| startup_error(BackendStartupStage::Build, error))?; if sandbox_id.is_none_or(|id| id.is_empty() || identity.sandbox_id != id) { return Err(miette::miette!( "runtime descriptor does not match admitted sandbox" @@ -184,11 +230,14 @@ impl SelectedBackend { let built = self .prepared .build(services) - .map_err(|error| miette::miette!(error.to_string()))?; + .map_err(|error| startup_error(BackendStartupStage::Build, error))?; self.descriptor.payload = built.payload; if built.backend.backend_name() != self.descriptor.backend_name { - return Err(miette::miette!( - "RPC client backend does not match admitted backend" + return Err(startup_error( + BackendStartupStage::Build, + BackendError::Descriptor( + "RPC client backend does not match admitted backend".into(), + ), )); } let backend = built.backend; @@ -204,12 +253,7 @@ impl SelectedBackend { }, ) .await - .map_err(|error| match error { - BackendError::Denied(_) => { - miette::miette!(code = ATTACH_REFUSED, "{error}") - } - error => miette::miette!("{error}"), - })?; + .map_err(|error| startup_error(BackendStartupStage::Attach, error))?; Ok((bound, built.proxy_listener)) } } diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 9602c7d68d..41c5ace138 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -600,7 +600,7 @@ async fn attach_refusal_is_marked_as_a_refusal() { .expect("an already attached boundary refuses attachment"); assert_eq!( error.code().map(|code| code.to_string()).as_deref(), - Some(ATTACH_REFUSED) + Some("openshell.backend.attach.denied") ); assert!(error.to_string().contains("already attached")); } @@ -619,11 +619,7 @@ async fn attach_transport_failures_are_not_policy_refusals() { .await .err() .expect("backend transport failure must fail startup"); - assert_ne!( - error.code().map(|code| code.to_string()).as_deref(), - Some(ATTACH_REFUSED), - "infrastructure failures must not be reported as policy denials" - ); + assert!(!is_backend_refusal(&error)); assert_eq!(setup.observed.starts.load(Ordering::SeqCst), 0); } } @@ -952,7 +948,13 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .observed .deny_confirmation .store(true, Ordering::SeqCst); - assert!(startup(false).await.is_err()); + let error = startup(false) + .await + .expect_err("confirmation must refuse startup"); + assert_eq!( + error.code().map(|code| code.to_string()).as_deref(), + Some("openshell.backend.confirm.unsupported") + ); assert_eq!( setup.observed.events(), [ diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index ae412b2000..e09fd07d7b 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -43,6 +43,7 @@ use std::time::Duration; use tracing::{Instrument as _, debug, info, warn}; use openshell_core::PolicyValidationFailureMode; +use openshell_core::backend_startup::{BackendStartupStage, backend_startup_diagnostic}; use openshell_supervisor_control::RegistrationTraceContext; use openshell_ocsf::{ @@ -1342,7 +1343,7 @@ pub async fn run_sandbox_instance( Ok(attached) => attached, Err(error) => { if let (Some(sandbox_id), Some(gateway)) = (sandbox_id.as_deref(), gateway.as_ref()) { - Box::pin(report_attach_refusal( + Box::pin(report_backend_startup_failure( gateway, sandbox_id, &loaded_policy_origin, @@ -1430,11 +1431,27 @@ pub async fn run_sandbox_instance( let adopts_running_agent = remote_boundary.0.adopts_running_agent(); let (bound, backend_name, ca_file_paths) = remote_boundary; let confirm = Box::pin(async { - let ready = bound + let ready = match bound .confirm() .instrument(tracing::info_span!(parent: &startup, "supervisor.boundary.confirm")) .await - .map_err(|error| miette::miette!(error.to_string()))?; + { + Ok(ready) => ready, + Err(error) => { + let error = backend_setup::startup_error(BackendStartupStage::Confirm, error); + if let (Some(sandbox_id), Some(gateway)) = (sandbox_id.as_deref(), gateway.as_ref()) + { + report_backend_startup_failure( + gateway, + sandbox_id, + &loaded_policy_origin, + &error, + ) + .await; + } + return Err(error); + } + }; info!(backend = %backend_name, "Isolation boundary enforcement confirmed"); Ok::<_, miette::Report>(ready) }); @@ -4212,9 +4229,8 @@ fn enqueue_policy_status(sender: &UnboundedSender, update: P } } -/// The report that the backend refused, at Attach, the configuration this start -/// accepted. Only a refusal qualifies: `Unavailable` can succeed on the next start. -fn attach_refusal_report( +/// Report a classified backend failure before the workload has started. +fn backend_startup_failure_report( sandbox_id: &str, origin: &LoadedPolicyOrigin, error: &miette::Report, @@ -4226,12 +4242,8 @@ fn attach_refusal_report( else { return None; }; - if error - .code() - .is_none_or(|code| code.to_string() != backend_setup::ATTACH_REFUSED) - { - return None; - } + let code = error.code()?.to_string(); + backend_startup_diagnostic(&code)?; Some(openshell_core::proto::ReportSandboxConfigurationRequest { sandbox_id: sandbox_id.to_string(), admission: Some(openshell_core::proto::SandboxConfigurationAdmission { @@ -4241,34 +4253,48 @@ fn attach_refusal_report( policy_hash: revision.policy_hash.clone(), config_revision: revision.config_revision, provider_env_revision: revision.provider_env_revision, - error: error.to_string(), + // Carry only a bounded machine code, never backend diagnostics. + error: code, }), expected_instance_id: String::new(), refused_before_start: true, }) } -/// Report an Attach refusal before the supervisor exits. The gateway records -/// the accepted configuration as refused, and a revision this start wrote ends -/// Failed with the backend's message. Both reports are best effort, and the -/// refusal report gets a single bounded attempt so exit is not held up. -async fn report_attach_refusal( +/// Report a pre-start failure without replacing the original error. The entire +/// best-effort operation, including any policy-history report, has one deadline. +async fn report_backend_startup_failure( gateway: &openshell_core::grpc_client::GatewayEndpoint, sandbox_id: &str, origin: &LoadedPolicyOrigin, error: &miette::Report, ) { - if let Some(request) = attach_refusal_report(sandbox_id, origin, error) - && let Err(report_error) = grpc_attempt( - "Attach refusal report", - openshell_core::grpc_client::report_sandbox_configuration_request(gateway, request), - ) - .await - { - warn!(error = %report_error, "Failed to report the isolation backend refusal"); - } - if let Some(revision) = refused_startup_revision(origin, error) { - report_initial_policy_failure(gateway, sandbox_id, Some(revision), error).await; + let Some(request) = backend_startup_failure_report(sandbox_id, origin, error) else { + return; + }; + let code = request + .admission + .as_ref() + .map_or("", |admission| admission.error.as_str()); + let safe_message = backend_startup_diagnostic(code).unwrap_or_else(|| { + "The isolation backend refused the admitted configuration before workload execution" + .to_string() + }); + let operation = async { + if let Err(report_error) = + openshell_core::grpc_client::report_sandbox_configuration_request(gateway, request) + .await + { + warn!(error = %report_error, "Failed to report the isolation backend failure"); + } + if let Some(revision) = refused_startup_revision(origin, error) { + let safe_error = miette::miette!(safe_message); + report_initial_policy_failure(gateway, sandbox_id, Some(revision), &safe_error).await; + } + Ok(()) + }; + if let Err(report_error) = grpc_attempt("Backend startup failure reports", operation).await { + warn!(error = %report_error, "Backend startup failure reporting did not complete"); } } @@ -4288,10 +4314,7 @@ fn refused_startup_revision<'a>( else { return None; }; - error - .code() - .is_some_and(|code| code.to_string() == backend_setup::ATTACH_REFUSED) - .then_some(revision) + backend_setup::is_backend_refusal(error).then_some(revision) } /// Best-effort `FAILED` acknowledgement when initial policy construction or @@ -6645,7 +6668,12 @@ network_policies: ) .await .expect("startup must retry and accept the canonical snapshot"); - let denial = miette::miette!(code = backend_setup::ATTACH_REFUSED, "policy denied"); + let denial = backend_setup::startup_error( + BackendStartupStage::Attach, + openshell_isolation_interface::contract::BackendError::Denied( + "policy denied".into(), + ), + ); assert_eq!( refused_startup_revision(&bundle.4, &denial).is_some(), owned, @@ -6705,6 +6733,7 @@ network_policies: #[test] fn attach_refusal_reports_the_accepted_generation_as_refused() { use openshell_core::proto::ConfigurationAdmissionState; + use openshell_isolation_interface::contract::BackendError; let mut revision = LoadedPolicyRevision::from_snapshot(&settings_poll_result( Some(proto_policy_fixture()), @@ -6717,9 +6746,12 @@ network_policies: has_last_valid_policy: true, written_at_startup: false, }; - let refusal = miette::miette!(code = backend_setup::ATTACH_REFUSED, "refused"); + let refusal = backend_setup::startup_error( + BackendStartupStage::Attach, + BackendError::Denied("refused".into()), + ); - let request = attach_refusal_report("sandbox-id", &origin(&revision), &refusal) + let request = backend_startup_failure_report("sandbox-id", &origin(&revision), &refusal) .expect("a refusal of the accepted generation is reported"); assert!(request.refused_before_start); let admission = request.admission.unwrap(); @@ -6744,18 +6776,51 @@ network_policies: ) ); + for (stage, failure) in [ + ( + BackendStartupStage::Build, + BackendError::Descriptor("secret launch payload".into()), + ), + ( + BackendStartupStage::Attach, + BackendError::Unavailable("secret endpoint".into()), + ), + ( + BackendStartupStage::Confirm, + BackendError::Denied("secret policy".into()), + ), + ] { + let error = backend_setup::startup_error(stage, failure); + let request = backend_startup_failure_report("sandbox-id", &origin(&revision), &error) + .expect("classified startup failures must be reported"); + let code = request.admission.unwrap().error; + assert!(code.starts_with(&format!("openshell.backend.{}.", stage.code()))); + assert!(!code.contains("secret")); + assert!(backend_startup_diagnostic(&code).is_some()); + } + let transient = miette::miette!("unavailable"); - assert!(attach_refusal_report("sandbox-id", &origin(&revision), &transient).is_none()); assert!( - attach_refusal_report("sandbox-id", &LoadedPolicyOrigin::LocalOverride, &refusal) - .is_none() + backend_startup_failure_report("sandbox-id", &origin(&revision), &transient).is_none() + ); + assert!( + backend_startup_failure_report( + "sandbox-id", + &LoadedPolicyOrigin::LocalOverride, + &refusal + ) + .is_none() ); revision.admission_instance_id = None; - assert!(attach_refusal_report("sandbox-id", &origin(&revision), &refusal).is_none()); + assert!( + backend_startup_failure_report("sandbox-id", &origin(&revision), &refusal).is_none() + ); } #[test] fn attach_refusal_fails_only_the_revision_this_start_wrote() { + use openshell_isolation_interface::contract::BackendError; + let revision = LoadedPolicyRevision::from_snapshot(&settings_poll_result( Some(proto_policy_fixture()), 3, @@ -6766,7 +6831,10 @@ network_policies: has_last_valid_policy: true, written_at_startup, }; - let refusal = miette::miette!(code = backend_setup::ATTACH_REFUSED, "refused"); + let refusal = backend_setup::startup_error( + BackendStartupStage::Attach, + BackendError::Denied("refused".into()), + ); let transient = miette::miette!("unavailable"); assert_eq!( @@ -6776,6 +6844,28 @@ network_policies: assert!(refused_startup_revision(&origin(true), &transient).is_none()); assert!(refused_startup_revision(&origin(false), &refusal).is_none()); assert!(refused_startup_revision(&LoadedPolicyOrigin::LocalOverride, &refusal).is_none()); + for stage in [ + BackendStartupStage::Build, + BackendStartupStage::Attach, + BackendStartupStage::Confirm, + ] { + for failure in [ + BackendError::Denied("policy rejected".into()), + BackendError::Attach("mediation transport failed".into()), + BackendError::Unavailable("retry after cleanup".into()), + BackendError::Confirm("confirmation failed".into()), + BackendError::Process("process failed".into()), + ] { + let refused = stage == BackendStartupStage::Attach + && matches!(failure, BackendError::Denied(_)); + let error = backend_setup::startup_error(stage, failure); + assert_eq!( + refused_startup_revision(&origin(true), &error).is_some(), + refused + ); + assert_eq!(backend_setup::is_backend_refusal(&error), refused); + } + } } #[tokio::test(start_paused = true)] diff --git a/docs/extensibility/isolation-backends.mdx b/docs/extensibility/isolation-backends.mdx index dc60c244af..4a1a7c0389 100644 --- a/docs/extensibility/isolation-backends.mdx +++ b/docs/extensibility/isolation-backends.mdx @@ -34,14 +34,14 @@ The supervisor's concrete RPC client, `OpenShellRuntimeBackend`, lives in The admission sequence is Attach, Confirm, then StartAgent. No workload code runs before StartAgent. -A backend refuses a policy it cannot enforce by returning `Denied` from Attach. -The sandbox then reports `ConfigurationReady: False (ConfigurationInvalid)`. -Transport and backend availability failures remain infrastructure failures. The -backend's reason is in the supervisor log. `openshell policy list` also shows it -when the gateway acknowledged that this supervisor created the refused startup -revision. A deduplicated write or a newer operator revision does not establish -that ownership. Delete the sandbox and create it again with a policy the backend -accepts. +A backend failure during client construction, Attach, or Confirm reports +`ConfigurationReady: False (ConfigurationInvalid)` before workload execution. +Sandbox status identifies the failed stage and gives a bounded recovery hint. +The supervisor log retains detailed backend diagnostics. A policy revision the +gateway acknowledged this startup created, and which the backend refused, shows +the safe reason in `openshell policy list`. Deduplicated writes and newer +operator revisions do not establish startup ownership. Follow the recovery hint +and wait for cleanup before retrying or recreating the sandbox. External compute drivers advertise `openshell.compute.isolation-backend` to accept an operator-selected `IsolationBackendRegistration`, including its Unix, diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 69a340fcf0..eab0821d3a 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -242,6 +242,10 @@ the image/effective policy or provider configuration. The supervisor remains alive while the workload stays unstarted. Inspect `openshell sandbox get` and repair the desired configuration with a complete policy replacement or provider change; do not treat a healthy container as proof that the workload is ready. +If the reported failure names backend client construction, attachment, or +isolation confirmation, the supervisor exits before workload execution. Follow +the bounded recovery hint and inspect cleanup status before restarting or +recreating the sandbox; this failure does not imply in-place repair is available. If the 300-second provisioning repair window expires, the gateway records `ProvisioningTimedOut` and stops workload and supervisor compute. Inspect `provisioning` in sandbox JSON and TUI NOTES to distinguish cleanup pending from From 8a0ec8443eef1656bceed0c9e8ce63113fd1de6f Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 3 Nov 2026 18:47:27 +0530 Subject: [PATCH 2/2] test(server): keep static pool members alive Signed-off-by: Shiju --- crates/openshell-server/src/supervisor_pool.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/openshell-server/src/supervisor_pool.rs b/crates/openshell-server/src/supervisor_pool.rs index d042736c7f..0ad45039b7 100644 --- a/crates/openshell-server/src/supervisor_pool.rs +++ b/crates/openshell-server/src/supervisor_pool.rs @@ -2968,6 +2968,8 @@ mod tests { replica_id.to_string(), ); let unique = Arc::get_mut(&mut pool).expect("unique"); + // Seeded members do not heartbeat; expiry tests use timed_replica. + unique.member_ttl = Duration::MAX; unique.unowned_grace = Duration::ZERO; unique.candidate_grace = Duration::ZERO; unique.takeovers_from = tokio::time::Instant::now();