Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions crates/openshell-core/src/backend_startup.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Closed startup failure codes carried by pre-start configuration refusals.
//!
//! The supervisor reports a code in the existing admission error field. The
//! gateway projects only these fixed diagnostics, never backend error text.

/// The trusted operation that failed before workload execution.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BackendStartupStage {
/// Constructing the selected backend client.
Build,
/// Attaching the admitted runtime generation.
Attach,
/// Confirming the isolation boundary before starting workload code.
Confirm,
}

impl BackendStartupStage {
/// Stable token used with the existing backend error-kind classification.
#[must_use]
pub const fn code(self) -> &'static str {
match self {
Self::Build => "build",
Self::Attach => "attach",
Self::Confirm => "confirm",
}
}
}

/// Project an exact startup failure code into a bounded, credential-free
/// diagnostic. Unknown and legacy error strings have no projection.
///
/// Only use this projection for an authenticated, generation-fenced
/// `refused_before_start` report; ordinary runtime errors are not startup claims.
#[must_use]
pub fn backend_startup_diagnostic(code: &str) -> Option<String> {
let (stage, kind) = code.strip_prefix("openshell.backend.")?.split_once('.')?;
let stage = match stage {
"build" => "client construction",
"attach" => "attachment",
"confirm" => "isolation confirmation",
_ => return None,
};
let recovery = match kind {
"invalid" => {
"the launch descriptor or admitted identity did not match; check the installation and recreate the sandbox"
}
"denied" => {
"the backend rejected the admitted policy or identity; review the policy and recreate the sandbox"
}
"unavailable" => {
"the backend was unavailable; check its service and host qualification, then retry after cleanup"
}
"unsupported" => {
"the backend does not support the requested operation; choose a compatible backend configuration"
}
"failed" => {
"the backend could not establish isolation; check the installation and host qualification before retrying after cleanup"
}
"terminated" => {
"the isolation boundary was lost before workload execution; check the backend service and recreate the sandbox"
}
_ => return None,
};
Some(format!("Isolation backend {stage} failed: {recovery}"))
}
1 change: 1 addition & 0 deletions crates/openshell-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
pub mod activity;
pub mod app_listener;
pub mod auth;
pub mod backend_startup;
pub mod compute;
pub mod config;
pub mod container_paths;
Expand Down
41 changes: 31 additions & 10 deletions crates/openshell-server/src/grpc/policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5091,6 +5091,9 @@ pub(super) async fn handle_report_sandbox_configuration(
"refused_before_start requires a rejected admission",
));
}
let startup_diagnostic = refused_before_start
.then(|| openshell_core::backend_startup::backend_startup_diagnostic(&admission.error))
.flatten();
let Some(_mutation_guard) = state
.compute
.sandbox_mutation_guard_by_id(&sandbox_id)
Expand Down Expand Up @@ -5172,14 +5175,17 @@ pub(super) async fn handle_report_sandbox_configuration(
Ok(config) if !config.get_ref().configuration_error.is_empty() => config.into_inner().configuration_error,
_ => "Effective configuration could not be activated; replace the policy or repair attached providers".to_string(),
};
if let Some(current) = current
&& current.state == i32::from(ConfigurationAdmissionState::Accepted)
{
if refused_before_start && same_generation(current, &admission) {
// The runtime refused what it accepted, so the workload never
// started on it. Runtime text stays out of public status.
admission.error = "The isolation backend refused this configuration before the workload started; create the sandbox again with a policy the backend accepts".to_string();
} else {
if let Some(current) = current {
let accepted = current.state == i32::from(ConfigurationAdmissionState::Accepted);
let rejected = current.state == i32::from(ConfigurationAdmissionState::Rejected);
if refused_before_start
&& (accepted || rejected)
&& same_generation(current, &admission)
{
// A retry of the same failed generation retains its diagnostic.
// Only exact known codes are projected; legacy/raw text stays out.
admission.error = startup_diagnostic.unwrap_or_else(|| "The isolation backend refused this configuration before the workload started; create the sandbox again with a policy the backend accepts".to_string());
} else if accepted {
// A rejected desired update does not invalidate an accepted runtime.
let error = admission.error;
admission = current.clone();
Expand Down Expand Up @@ -9121,7 +9127,8 @@ mod tests {
assert!(same_generation(&refused, &refused));
assert!(!same_generation(&refused, &other_generation));

handle_report_sandbox_configuration(&state, report(refused, true))
refused.error = "openshell.backend.confirm.denied".to_string();
handle_report_sandbox_configuration(&state, report(refused.clone(), true))
.await
.unwrap();
let (admission_state, ready) = readiness().await;
Expand All @@ -9131,10 +9138,24 @@ mod tests {
("False", "ConfigurationInvalid")
);
assert!(
ready.message.contains("isolation backend refused"),
ready.message.contains("isolation confirmation"),
"{ready:?}"
);
assert!(!ready.message.contains("backend text"), "{ready:?}");
let first_message = ready.message;
handle_report_sandbox_configuration(&state, report(refused.clone(), true))
.await
.unwrap();
let (_, repeated) = readiness().await;
assert_eq!(repeated.message, first_message);

refused.error = "openshell.backend.confirm.denied\nBearer secret-value".to_string();
handle_report_sandbox_configuration(&state, report(refused, true))
.await
.unwrap();
let (_, unknown) = readiness().await;
assert!(!unknown.message.contains("secret-value"));
assert!(unknown.message.contains("isolation backend refused"));
}

#[test]
Expand Down
2 changes: 2 additions & 0 deletions crates/openshell-server/src/supervisor_pool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2968,6 +2968,8 @@ mod tests {
replica_id.to_string(),
);
let unique = Arc::get_mut(&mut pool).expect("unique");
// Seeded members do not heartbeat; expiry tests use timed_replica.
unique.member_ttl = Duration::MAX;
unique.unowned_grace = Duration::ZERO;
unique.candidate_grace = Duration::ZERO;
unique.takeovers_from = tokio::time::Instant::now();
Expand Down
72 changes: 58 additions & 14 deletions crates/openshell-supervisor/src/backend_setup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,13 @@ use std::sync::{Arc, Mutex};

use miette::Result;
use openshell_core::SandboxSessionId;
use openshell_core::backend_startup::BackendStartupStage;
use openshell_core::jwt::{SessionBearerTokenSlot, SupervisorAuthBundle};
use openshell_core::provider_credentials::ProviderCredentialState;
use openshell_isolation_interface::AgentSpec;
use openshell_isolation_interface::contract::{
BackendDescriptor, BackendError, ResolvedWorkloadIdentity, SandboxContext, SandboxPolicy,
BackendDescriptor, BackendError, BackendErrorKind, ResolvedWorkloadIdentity, SandboxContext,
SandboxPolicy,
};
use openshell_sandbox_backend::BoundBoundary;

Expand Down Expand Up @@ -85,9 +87,53 @@ pub trait PreparedBackend: Send + Sync {
) -> std::result::Result<BuiltBackend, BackendError>;
}

/// Diagnostic code of an Attach error that is a backend refusal, as opposed to
/// a transient failure the next start can retry.
pub(crate) const ATTACH_REFUSED: &str = "openshell::backend::attach_refused";
/// Keep the original typed error for refusal classification while exposing
/// only an allowlisted stage/kind code to the configuration report.
#[derive(Debug)]
struct StartupError {
stage: BackendStartupStage,
error: BackendError,
}

impl std::fmt::Display for StartupError {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.error, formatter)
}
}

impl std::error::Error for StartupError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
}
}

impl miette::Diagnostic for StartupError {
fn code<'a>(&'a self) -> Option<Box<dyn std::fmt::Display + 'a>> {
let kind = match self.error.kind() {
BackendErrorKind::Invalid => "invalid",
BackendErrorKind::Denied => "denied",
BackendErrorKind::Unavailable => "unavailable",
BackendErrorKind::Unsupported => "unsupported",
BackendErrorKind::Failed => "failed",
BackendErrorKind::Terminated => "terminated",
};
Some(Box::new(format!(
"openshell.backend.{}.{kind}",
self.stage.code()
)))
}
}

pub(crate) fn startup_error(stage: BackendStartupStage, error: BackendError) -> miette::Report {
miette::Report::new(StartupError { stage, error })
}

pub(crate) fn is_backend_refusal(error: &miette::Report) -> bool {
error.downcast_ref::<StartupError>().is_some_and(|failure| {
failure.stage == BackendStartupStage::Attach
&& matches!(failure.error, BackendError::Denied(_))
})
}

/// Created only after name and launch identity checks. Consuming attachment
/// prevents reusing one prepared startup to build or attach a second client.
Expand Down Expand Up @@ -125,7 +171,7 @@ impl SelectedBackend {

let (identity, prepared) = setup
.decode(&descriptor.payload)
.map_err(|error| miette::miette!(error.to_string()))?;
.map_err(|error| startup_error(BackendStartupStage::Build, error))?;
if sandbox_id.is_none_or(|id| id.is_empty() || identity.sandbox_id != id) {
return Err(miette::miette!(
"runtime descriptor does not match admitted sandbox"
Expand Down Expand Up @@ -184,11 +230,14 @@ impl SelectedBackend {
let built = self
.prepared
.build(services)
.map_err(|error| miette::miette!(error.to_string()))?;
.map_err(|error| startup_error(BackendStartupStage::Build, error))?;
self.descriptor.payload = built.payload;
if built.backend.backend_name() != self.descriptor.backend_name {
return Err(miette::miette!(
"RPC client backend does not match admitted backend"
return Err(startup_error(
BackendStartupStage::Build,
BackendError::Descriptor(
"RPC client backend does not match admitted backend".into(),
),
));
}
let backend = built.backend;
Expand All @@ -204,12 +253,7 @@ impl SelectedBackend {
},
)
.await
.map_err(|error| match error {
BackendError::Denied(_) => {
miette::miette!(code = ATTACH_REFUSED, "{error}")
}
error => miette::miette!("{error}"),
})?;
.map_err(|error| startup_error(BackendStartupStage::Attach, error))?;
Ok((bound, built.proxy_listener))
}
}
Expand Down
16 changes: 9 additions & 7 deletions crates/openshell-supervisor/src/backend_setup/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -600,7 +600,7 @@ async fn attach_refusal_is_marked_as_a_refusal() {
.expect("an already attached boundary refuses attachment");
assert_eq!(
error.code().map(|code| code.to_string()).as_deref(),
Some(ATTACH_REFUSED)
Some("openshell.backend.attach.denied")
);
assert!(error.to_string().contains("already attached"));
}
Expand All @@ -619,11 +619,7 @@ async fn attach_transport_failures_are_not_policy_refusals() {
.await
.err()
.expect("backend transport failure must fail startup");
assert_ne!(
error.code().map(|code| code.to_string()).as_deref(),
Some(ATTACH_REFUSED),
"infrastructure failures must not be reported as policy denials"
);
assert!(!is_backend_refusal(&error));
assert_eq!(setup.observed.starts.load(Ordering::SeqCst), 0);
}
}
Expand Down Expand Up @@ -952,7 +948,13 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() {
.observed
.deny_confirmation
.store(true, Ordering::SeqCst);
assert!(startup(false).await.is_err());
let error = startup(false)
.await
.expect_err("confirmation must refuse startup");
assert_eq!(
error.code().map(|code| code.to_string()).as_deref(),
Some("openshell.backend.confirm.unsupported")
);
assert_eq!(
setup.observed.events(),
[
Expand Down
Loading
Loading