From 8de41ead1c5bba49f1f283ddc66e86f29fdd8dd9 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:35:33 +1000 Subject: [PATCH 01/98] chore: add reqsign workspace dependencies for azure --- Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 262f32b86a..54b0775cf3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,6 +117,11 @@ nom = "8.0.0" nom-language = "0.1.0" num_cpus = "1.16.0" opendal = { version = "0.56.0", default-features = false } +reqsign-core = "3.0.1" +reqsign-azure-storage = "3.0.1" +reqsign-command-execute-tokio = "3.0.1" +reqsign-file-read-tokio = "3.0.1" +reqsign-http-send-reqwest = "4.0.1" once_cell = "1.21.3" open = "5" openidconnect = { version = "4", default-features = false } From c2a8875e109009ffcc854649255f2363d256473a Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:37:33 +1000 Subject: [PATCH 02/98] feat(index): add azure feature enabling opendal azblob --- Cargo.lock | 84 +++++++++++++++++++++++++++++++++ crates/rattler_index/Cargo.toml | 13 +++++ 2 files changed, 97 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 993446eb78..210f5bb604 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4045,6 +4045,7 @@ checksum = "97b31d3d8e99a85d83b73ec26647f5607b80578ed9375810b6e44ffa3590a236" dependencies = [ "opendal-core", "opendal-layer-retry", + "opendal-service-azblob", "opendal-service-fs", "opendal-service-s3", ] @@ -4088,6 +4089,37 @@ dependencies = [ "opendal-core", ] +[[package]] +name = "opendal-service-azblob" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7452bf3ec61cfd81ac9ad9ada17825931e9e371d44a045c6bfab9596c0a2ac3b" +dependencies = [ + "base64 0.22.1", + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.38.4", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.10.9", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb0e45d6c8dcf66ce2da20e241bcb80e6e540e109a4ff20f318f6c9b4c54e0c" +dependencies = [ + "http 1.4.2", + "opendal-core", +] + [[package]] name = "opendal-service-fs" version = "0.56.0" @@ -5169,6 +5201,11 @@ dependencies = [ "rattler_networking", "rattler_package_streaming", "rattler_s3", + "reqsign-azure-storage", + "reqsign-command-execute-tokio", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-http-send-reqwest", "reqwest 0.13.4", "retry-policies", "rmp-serde", @@ -5730,6 +5767,37 @@ dependencies = [ "sha1 0.11.0", ] +[[package]] +name = "reqsign-azure-storage" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b96928e73ad984de1d99e382749d09e5dab7dd707b767974f7e40aa926b82f" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "http 1.4.2", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-command-execute-tokio" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13460892332fd1ae7bc344ca063379beae3f0db50e7066af72467c938c3eea9f" +dependencies = [ + "reqsign-core", + "tokio", +] + [[package]] name = "reqsign-core" version = "3.0.1" @@ -5766,6 +5834,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "reqsign-http-send-reqwest" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6089b23d9ba77aa6a8e5cf38d11550da23458d1c455645ae59e52294ffe7d26f" +dependencies = [ + "anyhow", + "bytes", + "futures-channel", + "http 1.4.2", + "http-body-util", + "reqsign-core", + "reqwest 0.13.4", + "wasm-bindgen-futures", +] + [[package]] name = "reqwest" version = "0.12.28" diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index 584983f14d..42211b2022 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -28,6 +28,14 @@ rustls = [ "opendal/reqwest-rustls-tls", ] s3 = ["opendal/services-s3", "dep:rattler_s3"] +azure = [ + "opendal/services-azblob", + "dep:reqsign-core", + "dep:reqsign-azure-storage", + "dep:reqsign-command-execute-tokio", + "dep:reqsign-file-read-tokio", + "dep:reqsign-http-send-reqwest", +] [[bin]] name = "rattler-index" @@ -59,6 +67,11 @@ rattler_digest = { workspace = true, default-features = false } rattler_package_streaming = { workspace = true, default-features = false } hex = { workspace = true } rattler_s3 = { workspace = true, optional = true, features = ["clap"] } +reqsign-core = { workspace = true, optional = true } +reqsign-azure-storage = { workspace = true, optional = true } +reqsign-command-execute-tokio = { workspace = true, optional = true } +reqsign-file-read-tokio = { workspace = true, optional = true } +reqsign-http-send-reqwest = { workspace = true, optional = true } reqwest = { workspace = true, default-features = false, features = [ "http2", "system-proxy", From d138be1d8e7aae903cffeb6a930c490f9d4305cf Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:39:52 +1000 Subject: [PATCH 03/98] feat(config): add AzureOptions and AzureOptionsMap --- crates/rattler_config/src/config.rs | 1 + crates/rattler_config/src/config/azure.rs | 167 ++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 crates/rattler_config/src/config/azure.rs diff --git a/crates/rattler_config/src/config.rs b/crates/rattler_config/src/config.rs index 30ffec1dfa..b96477b762 100644 --- a/crates/rattler_config/src/config.rs +++ b/crates/rattler_config/src/config.rs @@ -12,6 +12,7 @@ use crate::config::{ repodata_config::RepodataConfig, run_post_link_scripts::RunPostLinkScripts, }; +pub mod azure; pub mod build; pub mod channel_config; pub mod concurrency; diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs new file mode 100644 index 0000000000..2647362858 --- /dev/null +++ b/crates/rattler_config/src/config/azure.rs @@ -0,0 +1,167 @@ +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::config::Config; +#[cfg(feature = "edit")] +use crate::edit::ConfigEditError; + +#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct AzureOptionsMap(pub IndexMap); + +impl AzureOptionsMap { + /// Returns `true` if no Azure containers are configured. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct AzureOptions { + /// Storage account name → host `{account}.blob.core.windows.net`. + pub account: String, + + /// Optional full endpoint override for sovereign clouds / Azurite. + /// Defaults to `https://{account}.blob.core.windows.net`. + pub endpoint_url: Option, +} + +impl Config for AzureOptionsMap { + fn is_default(&self) -> bool { + self.0.is_empty() + } + + fn merge_config(self, other: &Self) -> Result { + let mut merged = self.0.clone(); + for (key, value) in &other.0 { + merged.insert(key.clone(), value.clone()); + } + Ok(AzureOptionsMap(merged)) + } + + #[cfg(feature = "edit")] + fn set(&mut self, key: &str, value: Option) -> Result<(), ConfigEditError> { + if key == "azure-options" { + let value = value.ok_or_else(|| ConfigEditError::MissingValue { + key: key.to_string(), + })?; + self.0 = + serde_json::de::from_str(&value).map_err(|e| ConfigEditError::JsonParseError { + key: key.to_string(), + source: e, + })?; + return Ok(()); + } + let Some(subkey) = key.strip_prefix("azure-options.") else { + return Err(ConfigEditError::UnknownKey { + key: key.to_string(), + supported_keys: "".to_string(), + }); + }; + if let Some((container, rest)) = subkey.split_once('.') { + if !self.0.contains_key(container) { + return Err(ConfigEditError::BucketNotFound { + bucket: container.to_string(), + }); + } + let container_config = self.0.get_mut(container).unwrap(); + match rest { + "account" => { + container_config.account = + value.ok_or_else(|| ConfigEditError::MissingValue { + key: key.to_string(), + })?; + } + "endpoint-url" => { + let value = value.ok_or_else(|| ConfigEditError::MissingValue { + key: key.to_string(), + })?; + container_config.endpoint_url = Some(Url::parse(&value).map_err(|e| { + ConfigEditError::UrlParseError { + key: key.to_string(), + source: e, + } + })?); + } + _ => { + return Err(ConfigEditError::UnknownKey { + key: key.to_string(), + supported_keys: "".to_string(), + }); + } + } + } else { + let value = value.ok_or_else(|| ConfigEditError::MissingValue { + key: key.to_string(), + })?; + let azure_options: AzureOptions = + serde_json::de::from_str(&value).map_err(|e| ConfigEditError::JsonParseError { + key: key.to_string(), + source: e, + })?; + self.0.insert(subkey.to_string(), azure_options); + } + Ok(()) + } + + fn get_extension_name(&self) -> String { + "azure-options".to_string() + } + + fn validate(&self) -> Result<(), super::ValidationError> { + Ok(()) + } + + fn keys(&self) -> Vec { + self.0.keys().map(ToString::to_string).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deserializes_account_and_optional_endpoint() { + let toml = r#" + [mychannel] + account = "myacct" + + [other] + account = "acct2" + endpoint-url = "https://acct2.blob.core.windows.net" + "#; + let map: AzureOptionsMap = toml::from_str(toml).unwrap(); + let mychannel = map.0.get("mychannel").unwrap(); + assert_eq!(mychannel.account, "myacct"); + assert_eq!(mychannel.endpoint_url, None); + let other = map.0.get("other").unwrap(); + assert_eq!( + other.endpoint_url.as_ref().unwrap().as_str(), + "https://acct2.blob.core.windows.net/" + ); + } + + #[test] + fn is_default_when_empty() { + let map = AzureOptionsMap::default(); + assert!(map.is_default()); + } + + #[test] + fn merge_overwrites_existing_keys() { + let mut base = AzureOptionsMap::default(); + base.0.insert( + "c".to_string(), + AzureOptions { account: "old".into(), endpoint_url: None }, + ); + let mut other = AzureOptionsMap::default(); + other.0.insert( + "c".to_string(), + AzureOptions { account: "new".into(), endpoint_url: None }, + ); + let merged = base.merge_config(&other).unwrap(); + assert_eq!(merged.0.get("c").unwrap().account, "new"); + } +} From 7c1ee723dae1e71f92201e8340bf5e5b6cdff757 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:41:16 +1000 Subject: [PATCH 04/98] feat(config): wire azure-options into ConfigBase --- crates/rattler_config/src/config.rs | 27 +++++++++++++++++++++++++++ crates/rattler_config/src/edit.rs | 4 ++++ 2 files changed, 31 insertions(+) diff --git a/crates/rattler_config/src/config.rs b/crates/rattler_config/src/config.rs index b96477b762..02840f964b 100644 --- a/crates/rattler_config/src/config.rs +++ b/crates/rattler_config/src/config.rs @@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use thiserror::Error; use url::Url; +use crate::config::azure::AzureOptionsMap; use crate::config::s3::S3OptionsMap; use crate::config::{ build::BuildConfig, concurrency::ConcurrencyConfig, index::IndexConfig, proxy::ProxyConfig, @@ -127,6 +128,10 @@ pub struct ConfigBase { #[serde(skip_serializing_if = "S3OptionsMap::is_default")] pub s3_options: S3OptionsMap, + #[serde(default)] + #[serde(skip_serializing_if = "AzureOptionsMap::is_default")] + pub azure_options: AzureOptionsMap, + /// Per-channel configuration for `rattler-index`. #[serde(default, skip_serializing_if = "IndexConfig::is_empty")] pub index_config: IndexConfig, @@ -171,6 +176,7 @@ where concurrency: ConcurrencyConfig::default(), proxy_config: ProxyConfig::default(), s3_options: S3OptionsMap::default(), + azure_options: AzureOptionsMap::default(), index_config: IndexConfig::default(), run_post_link_scripts: None, extensions: T::default(), @@ -265,6 +271,7 @@ where fn merge_config(self, other: &Self) -> Result { Ok(Self { s3_options: self.s3_options.merge_config(&other.s3_options)?, + azure_options: self.azure_options.merge_config(&other.azure_options)?, // Use the other configuration's default channels if available default_channels: other .default_channels @@ -327,6 +334,7 @@ where keys.extend(get_keys(&self.proxy_config)); keys.extend(get_keys(&self.extensions)); keys.extend(get_keys(&self.s3_options)); + keys.extend(get_keys(&self.azure_options)); keys.extend(get_keys(&self.index_config)); keys.push("default_channels".to_string()); @@ -349,3 +357,22 @@ pub fn load_config Deserialize<'de>>( let config: ConfigBase = toml::from_str(&config_content)?; Ok(config) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn azure_options_round_trips_through_config_base() { + let toml = r#" + [azure-options.mychannel] + account = "myacct" + "#; + let cfg: ConfigBase<()> = toml::from_str(toml).unwrap(); + assert_eq!( + cfg.azure_options.0.get("mychannel").unwrap().account, + "myacct" + ); + assert!(!cfg.azure_options.is_default()); + } +} diff --git a/crates/rattler_config/src/edit.rs b/crates/rattler_config/src/edit.rs index 933d597ae9..d3ccd34ec7 100644 --- a/crates/rattler_config/src/edit.rs +++ b/crates/rattler_config/src/edit.rs @@ -140,6 +140,10 @@ where self.s3_options.set(key, value)?; Ok(()) } + key if key.starts_with("azure-options") => { + self.azure_options.set(key, value)?; + Ok(()) + } key if key.starts_with("concurrency.") => { self.concurrency.set(key, value)?; Ok(()) From 90f434f000d1a000d0f13bc0096c573ab7b19373 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:43:17 +1000 Subject: [PATCH 05/98] feat(networking): add azure feature deps --- Cargo.lock | 5 +++++ crates/rattler_networking/Cargo.toml | 15 ++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 210f5bb604..f407ce6017 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5333,6 +5333,11 @@ dependencies = [ "keyring-core", "netrc-rs", "rattler_config", + "reqsign-azure-storage", + "reqsign-command-execute-tokio", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-http-send-reqwest", "reqwest 0.13.4", "retry-policies", "rstest", diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 5c59bec6f8..2dc4bbb304 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -17,6 +17,14 @@ native-tls = ["reqwest/native-tls", "ambient-id/native-tls"] rustls = ["reqwest/rustls", "ambient-id/rustls"] gcs = ["google-cloud-auth", "tokio/sync"] s3 = ["aws-config", "aws-sdk-s3", "aws-smithy-http-client"] +azure = [ + "dep:reqsign-core", + "dep:reqsign-azure-storage", + "dep:reqsign-command-execute-tokio", + "dep:reqsign-file-read-tokio", + "dep:reqsign-http-send-reqwest", + "rattler_config", +] system-integration = ["keyring", "netrc-rs", "dirs"] keyring = [ "keyring-core", @@ -26,7 +34,7 @@ keyring = [ ] [package.metadata.docs.rs] -features = ["gcs", "s3"] +features = ["gcs", "s3", "azure"] [dependencies] ambient-id = { workspace = true, features = ["reqwest-middleware"] } @@ -65,6 +73,11 @@ tokio = { workspace = true, optional = true } tracing = { workspace = true } url = { workspace = true } rattler_config = { workspace = true, optional = true } +reqsign-core = { workspace = true, optional = true } +reqsign-azure-storage = { workspace = true, optional = true } +reqsign-command-execute-tokio = { workspace = true, optional = true } +reqsign-file-read-tokio = { workspace = true, optional = true } +reqsign-http-send-reqwest = { workspace = true, optional = true } [target.'cfg( target_arch = "wasm32" )'.dependencies] getrandom = { workspace = true, features = ["wasm_js"] } From 9988ce0de3e19dcf1a0ab8c8624559bdd3da5261 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:44:06 +1000 Subject: [PATCH 06/98] feat(networking): add AzureMiddleware url rewriting --- .../src/azure_middleware.rs | 134 ++++++++++++++++++ crates/rattler_networking/src/lib.rs | 5 + 2 files changed, 139 insertions(+) create mode 100644 crates/rattler_networking/src/azure_middleware.rs diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs new file mode 100644 index 0000000000..9b45539fb5 --- /dev/null +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -0,0 +1,134 @@ +//! Middleware to handle `az://` URLs to pull artifacts from Azure Blob Storage. +use std::collections::HashMap; + +use async_trait::async_trait; +use rattler_config::config::azure::AzureOptions; +use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; +use reqsign_command_execute_tokio::TokioCommandExecute; +use reqsign_core::{Context, OsEnv, Signer}; +use reqsign_file_read_tokio::TokioFileRead; +use reqsign_http_send_reqwest::ReqwestHttpSend; +use reqwest::{Client, Request, Response}; +use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; +use url::Url; + +/// The Azure Storage REST API version sent on every request. +const X_MS_VERSION: &str = "2021-12-02"; + +/// Middleware that rewrites `az://{container}/{path}` URLs to HTTPS Azure Blob +/// Storage URLs and signs them via reqsign's Azure `DefaultCredentialProvider`. +#[derive(Clone)] +pub struct AzureMiddleware { + /// Container name -> addressing options (account, optional endpoint). + config: HashMap, + /// reqsign signer; caches the resolved credential internally. + signer: Signer, +} + +impl AzureMiddleware { + /// Create a new Azure middleware from a container -> options map. + pub fn new(config: HashMap) -> Self { + let client = Client::new(); + let ctx = Context::new() + .with_file_read(TokioFileRead) + .with_http_send(ReqwestHttpSend::new(client)) + .with_command_execute(TokioCommandExecute) + .with_env(OsEnv); + let signer = Signer::new(ctx, DefaultCredentialProvider::new(), RequestSigner::new()); + Self { config, signer } + } + + /// Resolve the HTTPS base host for a container from config. + /// Returns the endpoint origin, e.g. `https://acct.blob.core.windows.net`. + fn endpoint_for(&self, container: &str) -> MiddlewareResult { + let options = self.config.get(container).ok_or_else(|| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "no azure-options configured for container '{container}'" + )) + })?; + let endpoint = match &options.endpoint_url { + Some(url) => url.clone(), + None => Url::parse(&format!( + "https://{}.blob.core.windows.net", + options.account + )) + .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e)))?, + }; + Ok(endpoint) + } + + /// Rewrite an `az://{container}/{path}` URL to its HTTPS equivalent. + fn rewrite_url(&self, az_url: &Url) -> MiddlewareResult { + let container = az_url.host_str().ok_or_else(|| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "container should be present in az URL, got: {az_url}" + )) + })?; + let endpoint = self.endpoint_for(container)?; + let new_url = format!( + "{}/{}{}", + endpoint.as_str().trim_end_matches('/'), + container, + az_url.path() + ); + Url::parse(&new_url).map_err(|e| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "failed to parse constructed azure URL '{new_url}': {e}" + )) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn opts(account: &str, endpoint: Option<&str>) -> AzureOptions { + AzureOptions { + account: account.to_string(), + endpoint_url: endpoint.map(|e| Url::parse(e).unwrap()), + } + } + + #[test] + fn rewrites_default_endpoint() { + let mw = AzureMiddleware::new(HashMap::from([( + "mychannel".to_string(), + opts("myacct", None), + )])); + let rewritten = mw + .rewrite_url(&Url::parse("az://mychannel/noarch/repodata.json").unwrap()) + .unwrap(); + assert_eq!( + rewritten.as_str(), + "https://myacct.blob.core.windows.net/mychannel/noarch/repodata.json" + ); + } + + #[test] + fn rewrites_override_endpoint_for_azurite() { + let mw = AzureMiddleware::new(HashMap::from([( + "devstoreaccount1".to_string(), + opts( + "devstoreaccount1", + Some("http://127.0.0.1:10000/devstoreaccount1"), + ), + )])); + let rewritten = mw + .rewrite_url(&Url::parse("az://devstoreaccount1/noarch/repodata.json").unwrap()) + .unwrap(); + assert_eq!( + rewritten.as_str(), + "http://127.0.0.1:10000/devstoreaccount1/devstoreaccount1/noarch/repodata.json" + ); + } + + #[test] + fn errors_when_container_not_configured() { + let mw = AzureMiddleware::new(HashMap::new()); + let err = mw + .rewrite_url(&Url::parse("az://missing/noarch/repodata.json").unwrap()) + .unwrap_err(); + assert!(err.to_string().contains("missing")); + } +} diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs index d889be90c0..faca577f47 100644 --- a/crates/rattler_networking/src/lib.rs +++ b/crates/rattler_networking/src/lib.rs @@ -17,6 +17,11 @@ pub mod s3_middleware; #[cfg(feature = "s3")] pub use s3_middleware::S3Middleware; +#[cfg(feature = "azure")] +pub mod azure_middleware; +#[cfg(feature = "azure")] +pub use azure_middleware::AzureMiddleware; + pub mod authentication_middleware; pub mod authentication_storage; pub mod oauth_refresh; From a4d264d510abc29d3169b816fb9e11df60e028d8 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:44:53 +1000 Subject: [PATCH 07/98] feat(networking): sign az requests via reqsign --- .../src/azure_middleware.rs | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 9b45539fb5..668050ab01 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -79,6 +79,61 @@ impl AzureMiddleware { } } +impl AzureMiddleware { + /// Sign a reqwest `Request` in place using reqsign. + async fn sign(&self, req: &mut Request) -> MiddlewareResult<()> { + if !req.headers().contains_key("x-ms-version") { + req.headers_mut() + .insert("x-ms-version", X_MS_VERSION.parse().unwrap()); + } + + let mut builder = http::Request::builder() + .method(req.method().clone()) + .uri(req.url().as_str()); + for (name, value) in req.headers() { + builder = builder.header(name, value); + } + let http_req = builder.body(()).map_err(|e| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "failed to build http request for signing: {e}" + )) + })?; + let (mut parts, ()) = http_req.into_parts(); + + self.signer + .sign(&mut parts, None) + .await + .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e)))?; + + *req.headers_mut() = parts.headers; + let signed_url = Url::parse(&parts.uri.to_string()).map_err(|e| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "failed to parse signed azure URL '{}': {e}", + parts.uri + )) + })?; + *req.url_mut() = signed_url; + Ok(()) + } +} + +#[async_trait] +impl Middleware for AzureMiddleware { + async fn handle( + &self, + mut req: Request, + extensions: &mut http::Extensions, + next: Next<'_>, + ) -> MiddlewareResult { + if req.url().scheme() == "az" { + let https_url = self.rewrite_url(&req.url().clone())?; + *req.url_mut() = https_url; + self.sign(&mut req).await?; + } + next.run(req, extensions).await + } +} + #[cfg(test)] mod tests { use super::*; @@ -131,4 +186,21 @@ mod tests { .unwrap_err(); assert!(err.to_string().contains("missing")); } + + #[tokio::test] + async fn passes_through_non_az_schemes_unchanged() { + use reqwest_middleware::ClientBuilder; + let mw = AzureMiddleware::new(HashMap::new()); + let client = ClientBuilder::new(Client::new()).with(mw).build(); + let result = client + .get("https://this-host-does-not-exist.invalid/x") + .send() + .await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + !err.to_string().contains("azure-options"), + "non-az request must not hit azure config lookup: {err}" + ); + } } From a4fbf11d5ad3d658d0182df8cbf4d18097bcaf94 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:47:21 +1000 Subject: [PATCH 08/98] feat(bin): register AzureMiddleware behind azure feature --- crates/rattler-bin/Cargo.toml | 1 + crates/rattler-bin/src/commands/client.rs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index 4d50c4c40f..263944cb55 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -33,6 +33,7 @@ rustls = [ ] s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] +azure = ["rattler_networking/azure"] oauth = ["rattler/oauth"] [dependencies] diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index 7ffa9603a4..dd2e061f5b 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -30,6 +30,10 @@ pub fn create_client_with_middleware() -> miette::Result Date: Fri, 12 Jun 2026 09:51:08 +1000 Subject: [PATCH 09/98] feat(index): add azure indexing functions --- crates/rattler_index/src/lib.rs | 159 +++++++++++++++++++++++++++++++- 1 file changed, 157 insertions(+), 2 deletions(-) diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index fd731099aa..30d2f26030 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -23,8 +23,10 @@ use fs_err::{self as fs}; use futures::{StreamExt, stream::FuturesUnordered}; use indexmap::IndexMap; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -#[cfg(feature = "s3")] +#[cfg(any(feature = "s3", feature = "azure"))] use opendal::layers::RetryLayer; +#[cfg(feature = "azure")] +use opendal::services::AzblobConfig; #[cfg(feature = "s3")] use opendal::services::S3Config; use opendal::{Configurator, Operator, services::FsConfig}; @@ -54,7 +56,7 @@ use serde::Serialize; use sha2::{Digest, Sha256}; use tokio::sync::Semaphore; use tracing::Instrument; -#[cfg(feature = "s3")] +#[cfg(any(feature = "s3", feature = "azure"))] use url::Url; /// Channel metadata written into generated repodata. @@ -1793,6 +1795,131 @@ pub async fn ensure_channel_initialized_s3_with_channel_metadata( ensure_channel_initialized_with_channel_metadata(&op, channel_metadata).await } +/// Configuration for `index_azure`. +#[cfg(feature = "azure")] +pub struct IndexAzureConfig { + /// The channel to index, e.g. `az://my-container/my-channel`. + pub channel: Url, + /// The Azure Storage account name. + pub account: String, + /// Optional endpoint override (sovereign clouds / Azurite). Defaults to + /// `https://{account}.blob.core.windows.net`. + pub endpoint_url: Option, + /// The target platform to index. + pub target_platform: Option, + /// The path to a repodata patch to apply to the index. + pub repodata_patch: Option, + /// Whether to write the repodata as a zstd-compressed file. + pub write_zst: bool, + /// Whether to write the repodata shards. + pub write_shards: bool, + /// Repodata revisions to advertise in generated repodata. + pub repodata_revisions: Vec, + /// How packages are assigned to repodata revisions. + pub package_revision_assignment: PackageRevisionAssignment, + /// Whether to force the index to be written. + pub force: bool, + /// The maximum number of parallel tasks to run. + pub max_parallel: usize, + /// The multi-progress bar to use for the index. + pub multi_progress: Option, + /// Configuration for precondition checks during file operations. + pub precondition_checks: PreconditionChecks, +} + +/// Build an OpenDAL `AzblobConfig` for the given account/endpoint/channel. +/// +/// Deliberately sets **no** `account_key` or `sas_token`: with only +/// `account_name` + `endpoint`, OpenDAL's azblob backend authenticates through +/// reqsign's `DefaultCredentialProvider` chain (env → Azure CLI → managed +/// identity → ...), which is exactly what we want for Entra ID. +#[cfg(feature = "azure")] +fn azblob_config( + account: &str, + endpoint_url: Option<&Url>, + channel: &Url, +) -> Result { + let mut cfg = AzblobConfig::default(); + cfg.root = Some(channel.path().to_string()); + cfg.container = channel + .host_str() + .ok_or_else(|| anyhow::anyhow!("No container in az URL"))? + .to_string(); + cfg.account_name = Some(account.to_string()); + cfg.endpoint = Some(match endpoint_url { + Some(url) => url.as_str().trim_end_matches('/').to_string(), + None => format!("https://{account}.blob.core.windows.net"), + }); + Ok(cfg) +} + +/// Create a new `repodata.json` for all packages in the channel at the given +/// Azure Blob URL. +#[cfg(feature = "azure")] +pub async fn index_azure(config: IndexAzureConfig) -> anyhow::Result<()> { + index_azure_with_channel_metadata(config, ChannelMetadata::default()).await +} + +/// Create a new `repodata.json` for all packages in the channel at the given +/// Azure Blob URL and write channel metadata into the generated repodata. +#[cfg(feature = "azure")] +pub async fn index_azure_with_channel_metadata( + IndexAzureConfig { + channel, + account, + endpoint_url, + target_platform, + repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force, + max_parallel, + multi_progress, + precondition_checks, + }: IndexAzureConfig, + channel_metadata: ChannelMetadata, +) -> anyhow::Result<()> { + let cfg = azblob_config(&account, endpoint_url.as_ref(), &channel)?; + let op = Operator::new(cfg.into_builder())? + .layer(RetryLayer::new()) + .finish(); + + index_with_channel_metadata( + target_platform, + op, + repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force, + max_parallel, + multi_progress, + precondition_checks, + channel_metadata, + ) + .await + .map(|_| ()) +} + +/// Ensures that an Azure Blob channel has a valid `noarch/repodata.json` file. +/// +/// See [`ensure_channel_initialized`] for details. +#[cfg(feature = "azure")] +pub async fn ensure_channel_initialized_azure( + channel: &Url, + account: &str, + endpoint_url: Option<&Url>, +) -> anyhow::Result<()> { + let cfg = azblob_config(account, endpoint_url, channel)?; + let op = Operator::new(cfg.into_builder())? + .layer(RetryLayer::new()) + .finish(); + ensure_channel_initialized_with_channel_metadata(&op, ChannelMetadata::default()).await +} + #[cfg(test)] mod tests { use std::str::FromStr; @@ -1860,4 +1987,32 @@ mod tests { assert!(conda_packages.is_empty()); assert!(v3.whl.contains_key(&identifier)); } + + #[cfg(feature = "azure")] + #[test] + fn azblob_config_uses_container_and_root_no_credentials() { + let channel = Url::parse("az://mychannel/my/sub/path").unwrap(); + let cfg = azblob_config("myacct", None, &channel).unwrap(); + assert_eq!(cfg.container, "mychannel"); + assert_eq!(cfg.root.as_deref(), Some("/my/sub/path")); + assert_eq!(cfg.account_name.as_deref(), Some("myacct")); + assert_eq!( + cfg.endpoint.as_deref(), + Some("https://myacct.blob.core.windows.net") + ); + assert!(cfg.account_key.is_none()); + assert!(cfg.sas_token.is_none()); + } + + #[cfg(feature = "azure")] + #[test] + fn azblob_config_respects_endpoint_override() { + let channel = Url::parse("az://devstoreaccount1/ch").unwrap(); + let endpoint = Url::parse("http://127.0.0.1:10000/devstoreaccount1").unwrap(); + let cfg = azblob_config("devstoreaccount1", Some(&endpoint), &channel).unwrap(); + assert_eq!( + cfg.endpoint.as_deref(), + Some("http://127.0.0.1:10000/devstoreaccount1") + ); + } } From 175aa103249b89f08cc8a1ee49461c552fa2cf26 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:52:36 +1000 Subject: [PATCH 10/98] feat(index): add rattler-index azure subcommand --- crates/rattler_index/src/main.rs | 72 ++++++++++++++++++++++++++++++-- 1 file changed, 68 insertions(+), 4 deletions(-) diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 98108df575..0d5e044754 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -10,13 +10,17 @@ use rattler_config::config::{ use rattler_index::{ ChannelMetadata, IndexFsConfig, PackageRevisionAssignment, index_fs_with_channel_metadata, }; +#[cfg(feature = "azure")] +use rattler_index::{IndexAzureConfig, index_azure_with_channel_metadata}; +#[cfg(any(feature = "s3", feature = "azure"))] +use rattler_index::PreconditionChecks; #[cfg(feature = "s3")] -use rattler_index::{IndexS3Config, PreconditionChecks, index_s3_with_channel_metadata}; +use rattler_index::{IndexS3Config, index_s3_with_channel_metadata}; #[cfg(feature = "s3")] use rattler_networking::AuthenticationStorage; #[cfg(feature = "s3")] use rattler_s3::S3Credentials; -#[cfg(feature = "s3")] +#[cfg(any(feature = "s3", feature = "azure"))] use url::Url; #[cfg(feature = "s3")] @@ -31,6 +35,18 @@ fn parse_s3_url(value: &str) -> Result { } } +#[cfg(feature = "azure")] +fn parse_az_url(value: &str) -> Result { + let url: Url = Url::parse(value).map_err(|e| format!("`{value}` isn't a valid URL: {e}"))?; + if url.scheme() == "az" && url.host_str().is_some() { + Ok(url) + } else { + Err(format!( + "Only Azure URLs of format az://container/... can be used, not `{value}`" + )) + } +} + /// The `rattler-index` CLI. #[derive(Parser)] #[command(name = "rattler-index", version, about, long_about = None)] @@ -66,7 +82,7 @@ struct Cli { /// Use this flag if your S3 backend doesn't fully support conditional requests, /// or if you're certain no concurrent indexing processes are running. /// Warning: Disabling this removes protection against concurrent modifications. - #[cfg(feature = "s3")] + #[cfg(any(feature = "s3", feature = "azure"))] #[arg(long, default_value = "false", global = true)] disable_precondition_checks: bool, @@ -99,6 +115,22 @@ enum Commands { #[clap(flatten)] credentials: rattler_s3::clap::S3CredentialsOpts, }, + + /// Index a channel stored in an Azure Blob Storage container. + #[cfg(feature = "azure")] + Azure { + /// The Azure channel URL, e.g. `az://my-container/my-channel`. + #[arg(value_parser = parse_az_url)] + channel: Url, + + /// The Azure Storage account name. + #[arg(long, env = "AZURE_STORAGE_ACCOUNT")] + account: String, + + /// Optional endpoint override (sovereign clouds / Azurite). + #[arg(long, env = "AZURE_ENDPOINT_URL")] + endpoint_url: Option, + }, } /// The configuration type for rattler-index - just extends rattler config and @@ -127,7 +159,7 @@ async fn main() -> anyhow::Result<()> { .or(config.as_ref().map(|c| c.concurrency.downloads)) .unwrap_or_else(default_max_concurrent_solves); - #[cfg(feature = "s3")] + #[cfg(any(feature = "s3", feature = "azure"))] let precondition_checks = if cli.disable_precondition_checks { PreconditionChecks::Disabled } else { @@ -213,6 +245,38 @@ async fn main() -> anyhow::Result<()> { ) .await } + #[cfg(feature = "azure")] + Commands::Azure { + channel, + account, + endpoint_url, + } => { + let target = channel.to_string(); + let resolved = resolve_index_channel_config(&config, &target); + let (write_zst, write_shards, repodata_revisions, package_revision_assignment) = + effective_index_options(&resolved); + let channel_metadata = ChannelMetadata::from_index_config(&resolved); + + index_azure_with_channel_metadata( + IndexAzureConfig { + channel, + account, + endpoint_url, + target_platform: cli.target_platform, + repodata_patch: cli.repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force: cli.force, + max_parallel, + multi_progress: Some(multi_progress), + precondition_checks, + }, + channel_metadata, + ) + .await + } }?; println!("Finished indexing channel."); Ok(()) From 464031aefb813ab84b56e91d93a35957e2465f35 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:54:08 +1000 Subject: [PATCH 11/98] fix(index): satisfy clippy for azblob_config --- crates/rattler_index/src/lib.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 30d2f26030..97f54efcb6 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -1827,13 +1827,14 @@ pub struct IndexAzureConfig { pub precondition_checks: PreconditionChecks, } -/// Build an OpenDAL `AzblobConfig` for the given account/endpoint/channel. +/// Build an `OpenDAL` `AzblobConfig` for the given account/endpoint/channel. /// /// Deliberately sets **no** `account_key` or `sas_token`: with only -/// `account_name` + `endpoint`, OpenDAL's azblob backend authenticates through -/// reqsign's `DefaultCredentialProvider` chain (env → Azure CLI → managed -/// identity → ...), which is exactly what we want for Entra ID. +/// `account_name` + `endpoint`, `OpenDAL`'s azblob backend authenticates +/// through reqsign's `DefaultCredentialProvider` chain (env → Azure CLI → +/// managed identity → ...), which is exactly what we want for Entra ID. #[cfg(feature = "azure")] +#[allow(clippy::field_reassign_with_default)] fn azblob_config( account: &str, endpoint_url: Option<&Url>, From b4b24d9c364bd5acd5140c73f45841e6a948303a Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:56:18 +1000 Subject: [PATCH 12/98] feat(upload): add azure feature --- crates/rattler_upload/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/rattler_upload/Cargo.toml b/crates/rattler_upload/Cargo.toml index 3c560d2755..d3d6d0a419 100644 --- a/crates/rattler_upload/Cargo.toml +++ b/crates/rattler_upload/Cargo.toml @@ -29,6 +29,7 @@ native-tls = [ "sigstore-sign?/native-tls", ] s3 = ["rattler_networking/s3", "rattler_s3", "dep:opendal"] +azure = ["rattler_networking/azure", "dep:opendal", "opendal/services-azblob"] sigstore-sign = ["dep:sigstore-sign", "dep:sigstore-trust-root"] [package.metadata.docs.rs] From 3cc485d5a2f15809fd39f8c6e01b2bf8e0547fcf Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:56:56 +1000 Subject: [PATCH 13/98] feat(upload): add upload_package_to_azure --- crates/rattler_upload/src/upload/azure.rs | 141 ++++++++++++++++++++++ crates/rattler_upload/src/upload/mod.rs | 4 + 2 files changed, 145 insertions(+) create mode 100644 crates/rattler_upload/src/upload/azure.rs diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs new file mode 100644 index 0000000000..bad705f8f0 --- /dev/null +++ b/crates/rattler_upload/src/upload/azure.rs @@ -0,0 +1,141 @@ +use std::path::PathBuf; + +use miette::IntoDiagnostic; +use opendal::{Configurator, ErrorKind, Operator, services::AzblobConfig}; +use rattler_digest::{HashingReader, Md5, Sha256}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio_util::bytes::BytesMut; +use url::Url; + +use crate::upload::package::ExtractedPackage; + +const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; + +/// Uploads packages to a channel in an Azure Blob Storage container. +/// +/// Credentials are resolved ambiently by OpenDAL's azblob backend via reqsign's +/// `DefaultCredentialProvider` (env → Azure CLI → managed identity → ...): this +/// function sets only the account name and endpoint, never an account key or +/// SAS token. +pub async fn upload_package_to_azure( + channel: Url, + account: String, + endpoint_url: Option, + package_files: &Vec, + force: bool, +) -> miette::Result<()> { + let container = channel + .host_str() + .ok_or(miette::miette!("No container in Azure URL"))?; + + // Create the Azblob configuration for opendal. + let mut cfg = AzblobConfig::default(); + cfg.root = Some(channel.path().to_string()); + cfg.container = container.to_string(); + cfg.account_name = Some(account.clone()); + cfg.endpoint = Some(match &endpoint_url { + Some(url) => url.as_str().trim_end_matches('/').to_string(), + None => format!("https://{account}.blob.core.windows.net"), + }); + + let builder = cfg.into_builder(); + let op = Operator::new(builder).into_diagnostic()?.finish(); + + for package_file in package_files { + let package = ExtractedPackage::from_package_file(package_file)?; + let subdir = package + .subdir() + .ok_or_else(|| miette::miette!("Failed to get subdir"))?; + let filename = package + .filename() + .ok_or_else(|| miette::miette!("Failed to get filename"))?; + let key = format!("{subdir}/{filename}"); + + // Compute the hash of the package by streaming its content. + let file = tokio::io::BufReader::new( + fs_err::tokio::File::open(package_file) + .await + .into_diagnostic()?, + ); + let sha256_reader = HashingReader::<_, Sha256>::new(file); + let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); + let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) + .await + .into_diagnostic()?; + let (sha256_reader, md5hash) = md5_reader.finalize(); + let (mut file, sha256hash) = sha256_reader.finalize(); + + // Rewind the file to the beginning. + file.rewind().await.into_diagnostic()?; + + // Construct a writer for the package. + let mut writer = match op + .writer_with(&key) + .content_disposition(&format!("attachment; filename={filename}")) + .if_not_exists(!force) + .user_metadata([ + (String::from("package-sha256"), hex::encode(sha256hash)), + (String::from("package-md5"), hex::encode(md5hash)), + ]) + .await + { + Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { + miette::bail!( + "Package az://{container}{}/{key} already exists. Use --force to overwrite.", + channel.path().to_string() + ); + } + Ok(writer) => writer, + Err(e) => { + return Err(e).into_diagnostic(); + } + }; + + // Write the contents to the writer. We do this in a more complex way than just + // using `io::copy` because some underlying storage providers expect to receive + // the data in specifically sized chunks. The code below guarantees chunks of + // equal size except for maybe the last chunk. + let mut remaining_size = size as usize; + loop { + // Allocate memory for this chunk + let chunk_size = remaining_size.min(DESIRED_CHUNK_SIZE); + let mut chunk = BytesMut::with_capacity(chunk_size); + // SAFE: because we do not care about the bytes that are currently in the buffer + unsafe { chunk.set_len(chunk_size) }; + + // Fill the chunk with data. This reads exactly the number of bytes we want. No + // more, no less. + let bytes_read = file.read_exact(&mut chunk[..]).await.into_diagnostic()?; + debug_assert_eq!(bytes_read, chunk.len()); + + // Write the writes directly to storage + writer.write(chunk.freeze()).await.into_diagnostic()?; + + // Update the number of remaining bytes + remaining_size = remaining_size.saturating_sub(bytes_read); + if remaining_size == 0 { + break; + } + } + + match writer.close().await { + Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { + miette::bail!( + "Package az://{container}{}/{key} already exists. Use --force to overwrite.", + channel.path().to_string() + ); + } + Ok(_) => { + tracing::info!( + "Uploaded package to az://{container}{}/{key}", + channel.path().to_string() + ); + } + Err(e) => { + return Err(e).into_diagnostic(); + } + } + } + + Ok(()) +} diff --git a/crates/rattler_upload/src/upload/mod.rs b/crates/rattler_upload/src/upload/mod.rs index 92ceaf7ff3..663042d7b3 100644 --- a/crates/rattler_upload/src/upload/mod.rs +++ b/crates/rattler_upload/src/upload/mod.rs @@ -38,6 +38,10 @@ mod prefix; mod s3; #[cfg(feature = "s3")] pub use s3::upload_package_to_s3; +#[cfg(feature = "azure")] +mod azure; +#[cfg(feature = "azure")] +pub use azure::upload_package_to_azure; pub use anaconda::AnacondaError; pub use cloudsmith::CloudsmithError; From 4edb5ce3a1316f92397b6d751d43cb9bd23d7d57 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:58:10 +1000 Subject: [PATCH 14/98] feat(upload): add azure server type and dispatch --- crates/rattler_upload/src/lib.rs | 12 ++++ crates/rattler_upload/src/upload/opt.rs | 91 +++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index b7ad07b200..0143a23383 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -81,6 +81,18 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { ) .await } + #[cfg(feature = "azure")] + ServerType::Azure(azure_opts) => { + let data = upload::opt::AzureData::from(azure_opts); + upload::upload_package_to_azure( + url::Url::from(data.channel), + data.account, + data.endpoint_url, + &args.package_files, + data.force, + ) + .await + } ServerType::CondaForge(conda_forge_opts) => { let conda_forge_data = CondaForgeData::from(conda_forge_opts); upload::conda_forge::upload_packages_to_conda_forge( diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 6a94c7a4eb..f23bc2cab3 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -124,6 +124,8 @@ pub enum ServerType { Cloudsmith(CloudsmithOpts), #[cfg(feature = "s3")] S3(S3Opts), + #[cfg(feature = "azure")] + Azure(AzureOpts), #[clap(hide = true)] CondaForge(CondaForgeOpts), } @@ -401,6 +403,63 @@ pub struct S3Opts { pub force: bool, } +#[cfg(feature = "azure")] +fn parse_az_url(value: &str) -> Result { + let url: Url = + Url::parse(value).map_err(|err| format!("`{value}` isn't a valid URL: {err}"))?; + if url.scheme() == "az" && url.host_str().is_some() { + Ok(url) + } else { + Err(format!( + "Only Azure URLs of format az://container/... can be used, not `{value}`" + )) + } +} + +/// Options for uploading to Azure Blob Storage. +#[cfg(feature = "azure")] +#[derive(Clone, Debug, PartialEq, Parser)] +pub struct AzureOpts { + /// The channel URL in the Azure container, e.g. + /// `az://my-container/my-channel`. + #[arg(short, long, env = "AZURE_CHANNEL", value_parser = parse_az_url)] + pub channel: Url, + + /// The Azure Storage account name. + #[arg(long, env = "AZURE_STORAGE_ACCOUNT")] + pub account: String, + + /// Optional endpoint override (sovereign clouds / Azurite). + #[arg(long, env = "AZURE_ENDPOINT_URL")] + pub endpoint_url: Option, + + /// Replace files if they already exist. + #[arg(long)] + pub force: bool, +} + +#[cfg(feature = "azure")] +#[derive(Debug)] +#[allow(missing_docs)] +pub struct AzureData { + pub channel: UrlWithTrailingSlash, + pub account: String, + pub endpoint_url: Option, + pub force: bool, +} + +#[cfg(feature = "azure")] +impl From for AzureData { + fn from(value: AzureOpts) -> Self { + Self { + channel: value.channel.into(), + account: value.account, + endpoint_url: value.endpoint_url, + force: value.force, + } + } +} + #[derive(Debug)] #[allow(missing_docs)] pub struct AnacondaData { @@ -588,3 +647,35 @@ impl CondaForgeData { } } } + +#[cfg(all(test, feature = "azure"))] +mod azure_opt_tests { + use super::*; + + #[test] + fn parse_az_url_accepts_az_scheme() { + let url = parse_az_url("az://my-container/my-channel").unwrap(); + assert_eq!(url.scheme(), "az"); + assert_eq!(url.host_str(), Some("my-container")); + } + + #[test] + fn parse_az_url_rejects_other_schemes() { + assert!(parse_az_url("s3://b/c").is_err()); + assert!(parse_az_url("https://x/y").is_err()); + } + + #[test] + fn azure_data_from_opts_preserves_fields() { + let opts = AzureOpts { + channel: Url::parse("az://c/ch").unwrap(), + account: "acct".to_string(), + endpoint_url: None, + force: true, + }; + let data = AzureData::from(opts); + assert_eq!(data.account, "acct"); + assert!(data.force); + assert_eq!(data.channel.as_str(), "az://c/ch/"); + } +} From 444c0276d7ae1d39f589cd28f56a6f8ec1d59b5d Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:58:49 +1000 Subject: [PATCH 15/98] feat(bin): enable rattler_upload azure passthrough --- crates/rattler-bin/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index 263944cb55..a3c8a0042e 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -33,7 +33,7 @@ rustls = [ ] s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] -azure = ["rattler_networking/azure"] +azure = ["rattler_networking/azure", "rattler_upload/azure"] oauth = ["rattler/oauth"] [dependencies] From feb3f6ffdfeb3049a49ff149c058342051994ffa Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 09:59:55 +1000 Subject: [PATCH 16/98] fix(upload): satisfy clippy in azure upload --- crates/rattler_upload/src/upload/azure.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index bad705f8f0..1932cfdb0a 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -13,7 +13,7 @@ const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; /// Uploads packages to a channel in an Azure Blob Storage container. /// -/// Credentials are resolved ambiently by OpenDAL's azblob backend via reqsign's +/// Credentials are resolved ambiently by `OpenDAL`'s azblob backend via reqsign's /// `DefaultCredentialProvider` (env → Azure CLI → managed identity → ...): this /// function sets only the account name and endpoint, never an account key or /// SAS token. @@ -29,14 +29,16 @@ pub async fn upload_package_to_azure( .ok_or(miette::miette!("No container in Azure URL"))?; // Create the Azblob configuration for opendal. - let mut cfg = AzblobConfig::default(); - cfg.root = Some(channel.path().to_string()); - cfg.container = container.to_string(); - cfg.account_name = Some(account.clone()); - cfg.endpoint = Some(match &endpoint_url { - Some(url) => url.as_str().trim_end_matches('/').to_string(), - None => format!("https://{account}.blob.core.windows.net"), - }); + let cfg = AzblobConfig { + root: Some(channel.path().to_string()), + container: container.to_string(), + account_name: Some(account.clone()), + endpoint: Some(match &endpoint_url { + Some(url) => url.as_str().trim_end_matches('/').to_string(), + None => format!("https://{account}.blob.core.windows.net"), + }), + ..Default::default() + }; let builder = cfg.into_builder(); let op = Operator::new(builder).into_diagnostic()?.finish(); From 9bfca47ff419956d1878a6cc344b64f36b3b96a3 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 10:03:30 +1000 Subject: [PATCH 17/98] test(index): add azurite azure integration test --- crates/rattler_index/tests/azure_azurite.rs | 78 +++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 crates/rattler_index/tests/azure_azurite.rs diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs new file mode 100644 index 0000000000..9019744851 --- /dev/null +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -0,0 +1,78 @@ +//! Integration test against a local Azurite emulator. +//! +//! Run with: +//! docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ +//! azurite-blob --blobHost 0.0.0.0 +//! AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== \ +//! cargo test -p rattler_index --features azure --test azure_azurite -- --ignored --nocapture +#![cfg(feature = "azure")] + +use opendal::{services::AzblobConfig, Configurator, Operator}; +use url::Url; + +const ACCOUNT: &str = "devstoreaccount1"; +const CONTAINER: &str = "test-channel"; +const ENDPOINT: &str = "http://127.0.0.1:10000/devstoreaccount1"; + +fn azurite_operator(root: &str) -> Operator { + let key = std::env::var("AZURE_STORAGE_ACCOUNT_KEY").expect("AZURE_STORAGE_ACCOUNT_KEY"); + let mut cfg = AzblobConfig::default(); + cfg.root = Some(root.to_string()); + cfg.container = CONTAINER.to_string(); + cfg.account_name = Some(ACCOUNT.to_string()); + cfg.account_key = Some(key); + cfg.endpoint = Some(ENDPOINT.to_string()); + Operator::new(cfg.into_builder()).unwrap().finish() +} + +fn test_package_path() -> std::path::PathBuf { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../test-data/packages") + .join(test_package_name()) +} + +fn test_package_name() -> &'static str { + "empty-0.1.0-h4616a5c_0.conda" +} + +#[tokio::test] +#[ignore = "requires a running Azurite emulator"] +async fn index_azure_creates_repodata() { + let channel_root = format!("/{CONTAINER}-chan"); + let op = azurite_operator(&channel_root); + let pkg_bytes = std::fs::read(test_package_path()).unwrap(); + op.write(&format!("noarch/{}", test_package_name()), pkg_bytes) + .await + .unwrap(); + + let channel = Url::parse(&format!("az://{CONTAINER}{channel_root}")).unwrap(); + rattler_index::index_azure(rattler_index::IndexAzureConfig { + channel: channel.clone(), + account: ACCOUNT.to_string(), + endpoint_url: Some(Url::parse(ENDPOINT).unwrap()), + target_platform: None, + repodata_patch: None, + write_zst: false, + write_shards: false, + repodata_revisions: Vec::new(), + package_revision_assignment: Default::default(), + force: true, + max_parallel: 4, + multi_progress: None, + precondition_checks: rattler_index::PreconditionChecks::Disabled, + }) + .await + .unwrap(); + + let repodata = op.read("noarch/repodata.json").await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&repodata.to_vec()).unwrap(); + let has_packages = json["packages"] + .as_object() + .map(|m| !m.is_empty()) + .unwrap_or(false) + || json["packages.conda"] + .as_object() + .map(|m| !m.is_empty()) + .unwrap_or(false); + assert!(has_packages, "repodata should list the seeded package: {json}"); +} From 6c26b8bb2c70bb64d8727f43c6f065df611e2f45 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 10:05:28 +1000 Subject: [PATCH 18/98] docs(azure): document azure blob channel support --- crates/rattler_index/README.md | 45 +++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index 6dbe16d017..bbf6364f40 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -2,7 +2,10 @@ `rattler_index` creates or updates conda channel indexes by writing `repodata.json`, optional compressed repodata, and optional sharded repodata for -packages stored on a local filesystem or in S3. +packages stored on a local filesystem, in S3, or in Azure Blob Storage. + +S3 support requires the `s3` feature; Azure Blob Storage support requires the +`azure` feature. ## CLI Usage @@ -18,14 +21,50 @@ Index an S3 channel: rattler-index --config ./rattler-config.toml s3 s3://my-bucket/my-channel ``` +Index an Azure Blob Storage channel: + +```shell +rattler-index --config ./rattler-config.toml azure \ + --channel az://my-container/my-channel \ + --account my-storage-account \ + [--endpoint-url https://my-storage-account.blob.core.windows.net] +``` + +Azure channels need no secrets on the command line. For local development run +`az login`; in CI, authentication is resolved through reqsign's +`DefaultCredentialProvider` chain, which covers managed identity, workload +identity, and service-principal environment variables. The `--endpoint-url` +flag is only needed for sovereign clouds or a local Azurite emulator. + The `--config` flag points at the same TOML configuration file used by pixi. It -configures S3 credentials, concurrency, and per-channel index options under the -`[index-config]` section. +configures S3 and Azure credentials, concurrency, and per-channel index options +under the `[index-config]` section. When `--config` is omitted, `rattler-index` falls back to its built-in defaults (`write-zst = true`, `write-shards = true`, no advertised repodata revisions, `from-index-json` revision assignment, no channel metadata). +## Remote storage credentials + +S3 endpoint and region settings live under `[s3-options.]`, keyed by +bucket name. Azure Blob Storage uses the analogous `[azure-options.]` +block, keyed by container name. Each Azure entry records the storage account and +an optional endpoint override (no secrets are stored — credentials are resolved +at runtime through `az login` locally or the `DefaultCredentialProvider` chain +in CI): + +```toml +[s3-options.my-bucket] +endpoint-url = "https://my-bucket.s3.amazonaws.com" +region = "eu-central-1" +force-path-style = false + +[azure-options.my-container] +account = "my-storage-account" +# Optional; defaults to https://{account}.blob.core.windows.net +endpoint-url = "https://my-storage-account.blob.core.windows.net" +``` + ## Per-channel index configuration Index options live in `[index-config]` and follow the same shape as From 7a9eab26c1609f9c207f20ff58bf4426c82343c7 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 10:12:56 +1000 Subject: [PATCH 19/98] chore(azure): polish header value and help text --- crates/rattler_index/src/main.rs | 2 +- crates/rattler_networking/src/azure_middleware.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 0d5e044754..b225c1c9b8 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -79,7 +79,7 @@ struct Cli { repodata_patch: Option, /// Disable precondition checks (`ETags`, timestamps) during file operations. - /// Use this flag if your S3 backend doesn't fully support conditional requests, + /// Use this flag if your S3 or Azure Blob Storage backend doesn't fully support conditional requests, /// or if you're certain no concurrent indexing processes are running. /// Warning: Disabling this removes protection against concurrent modifications. #[cfg(any(feature = "s3", feature = "azure"))] diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 668050ab01..5e36b9b03a 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -83,8 +83,10 @@ impl AzureMiddleware { /// Sign a reqwest `Request` in place using reqsign. async fn sign(&self, req: &mut Request) -> MiddlewareResult<()> { if !req.headers().contains_key("x-ms-version") { - req.headers_mut() - .insert("x-ms-version", X_MS_VERSION.parse().unwrap()); + req.headers_mut().insert( + "x-ms-version", + http::HeaderValue::from_static(X_MS_VERSION), + ); } let mut builder = http::Request::builder() From 12df59a700abfdef611eb22bd592e7677b9899e6 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:55:03 +1000 Subject: [PATCH 20/98] test(networking): add azurite azure middleware fetch test --- .../tests/azure_azurite_fetch.rs | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 crates/rattler_networking/tests/azure_azurite_fetch.rs diff --git a/crates/rattler_networking/tests/azure_azurite_fetch.rs b/crates/rattler_networking/tests/azure_azurite_fetch.rs new file mode 100644 index 0000000000..dd1ac43801 --- /dev/null +++ b/crates/rattler_networking/tests/azure_azurite_fetch.rs @@ -0,0 +1,66 @@ +//! Live read-path integration test against a local Azurite emulator. +//! +//! Exercises `AzureMiddleware`: it rewrites an `az://` URL to the Azurite +//! HTTPS-equivalent and signs it via reqsign's shared-key credential, then we +//! actually fetch a blob that was seeded into the emulator. +//! +//! Prereqs (mirrors the index-side test at `rattler_index/tests/azure_azurite.rs`): +//! +//! ```text +//! docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ +//! azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck +//! # seed a `cli-channel` container with `noarch/repodata.json` +//! AZURE_STORAGE_ACCOUNT_NAME=devstoreaccount1 \ +//! AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== \ +//! cargo test -p rattler_networking --features azure --test azure_azurite_fetch -- --ignored --nocapture +//! ``` +#![cfg(feature = "azure")] + +use std::collections::HashMap; + +use rattler_config::config::azure::AzureOptions; +use rattler_networking::AzureMiddleware; +use reqwest_middleware::ClientBuilder; +use url::Url; + +const CONTAINER: &str = "cli-channel"; +const ENDPOINT: &str = "http://127.0.0.1:10000/devstoreaccount1"; + +#[tokio::test] +#[ignore = "requires a running Azurite emulator seeded with cli-channel/noarch/repodata.json"] +async fn azure_middleware_fetches_repodata() { + let mut config = HashMap::new(); + config.insert( + CONTAINER.to_string(), + AzureOptions { + account: "devstoreaccount1".to_string(), + endpoint_url: Some(Url::parse(ENDPOINT).unwrap()), + }, + ); + + let client = ClientBuilder::new(reqwest::Client::new()) + .with(AzureMiddleware::new(config)) + .build(); + + let resp = client + .get(format!("az://{CONTAINER}/noarch/repodata.json")) + .send() + .await + .expect("request through azure middleware failed"); + + assert!( + resp.status().is_success(), + "unexpected status {} fetching repodata", + resp.status() + ); + + let json: serde_json::Value = resp.json().await.expect("repodata was not valid json"); + let has_pkg = json["packages.conda"] + .as_object() + .is_some_and(|m| !m.is_empty()) + || json["packages"].as_object().is_some_and(|m| !m.is_empty()); + assert!( + has_pkg, + "repodata fetched via az:// should list a package: {json}" + ); +} From 72fe5ff186538494b5db5c4605ad7e826bfb4f17 Mon Sep 17 00:00:00 2001 From: Tenzin Platter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 12 Jun 2026 13:46:46 +1000 Subject: [PATCH 21/98] test(networking): add real-azure AAD middleware fetch test --- .../tests/azure_real_fetch.rs | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 crates/rattler_networking/tests/azure_real_fetch.rs diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs new file mode 100644 index 0000000000..0d024c46ac --- /dev/null +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -0,0 +1,71 @@ +//! Live read-path test against a *real* Azure Blob Storage account. +//! +//! Unlike `azure_azurite_fetch.rs` (shared-key vs. the emulator), this exercises +//! the AAD / token path of reqsign's `DefaultCredentialProvider` — i.e. the +//! credential `az login` provides — against a real `*.blob.core.windows.net` +//! endpoint. It is purely a read; it never writes. +//! +//! Configure via env and run (requires `az login` with at least +//! `Storage Blob Data Reader` on the container): +//! +//! ```text +//! AZURE_TEST_ACCOUNT=stgrcondachannel \ +//! AZURE_TEST_CONTAINER=general \ +//! AZURE_TEST_PATH=noarch/repodata.json \ +//! cargo test -p rattler_networking --features azure --test azure_real_fetch -- --ignored --nocapture +//! ``` +#![cfg(feature = "azure")] + +use std::collections::HashMap; + +use rattler_config::config::azure::AzureOptions; +use rattler_networking::AzureMiddleware; +use reqwest_middleware::ClientBuilder; +use url::Url; + +#[tokio::test] +#[ignore = "requires az login and AZURE_TEST_* env vars pointing at a real account"] +async fn azure_middleware_fetches_real_repodata() { + let account = std::env::var("AZURE_TEST_ACCOUNT").expect("AZURE_TEST_ACCOUNT"); + let container = std::env::var("AZURE_TEST_CONTAINER").expect("AZURE_TEST_CONTAINER"); + let path = + std::env::var("AZURE_TEST_PATH").unwrap_or_else(|_| "noarch/repodata.json".to_string()); + + let endpoint_url = std::env::var("AZURE_TEST_ENDPOINT") + .ok() + .map(|e| Url::parse(&e).expect("invalid AZURE_TEST_ENDPOINT")); + + let mut config = HashMap::new(); + config.insert( + container.clone(), + AzureOptions { + account, + endpoint_url, + }, + ); + + let client = ClientBuilder::new(reqwest::Client::new()) + .with(AzureMiddleware::new(config)) + .build(); + + let url = format!("az://{container}/{path}"); + println!("fetching {url}"); + let resp = client + .get(&url) + .send() + .await + .expect("request through azure middleware failed"); + + let status = resp.status(); + let body = resp.bytes().await.expect("failed to read body"); + println!("status={status} bytes={}", body.len()); + assert!(status.is_success(), "unexpected status {status}: {url}"); + + let json: serde_json::Value = + serde_json::from_slice(&body).expect("real repodata was not valid json"); + assert!( + json.get("info").is_some() || json.get("packages").is_some(), + "fetched body does not look like repodata: {json}" + ); + println!("info = {}", json["info"]); +} From afaf2beff23712801426eb5bca0fe48d99bf3466 Mon Sep 17 00:00:00 2001 From: Tenzin <143778894+tenzinplatter@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:59:41 +1000 Subject: [PATCH 22/98] feat(rattler_index): add Azure Blob backend (#2) * feat(rattler_index): add Azure Blob backend * fix(rattler_index): disable preconditions on azure (opendal azblob lacks conditional writes) * fix(rattler_index): re-index packages whose blob size changed --- Cargo.lock | 115 +++++++++ Cargo.toml | 1 + crates/rattler_azure/Cargo.toml | 20 ++ crates/rattler_azure/src/clap.rs | 83 ++++++ crates/rattler_azure/src/lib.rs | 24 ++ crates/rattler_index/Cargo.toml | 4 +- crates/rattler_index/src/lib.rs | 238 +++++++++++++++++- crates/rattler_index/src/main.rs | 72 +++++- .../tests/integration/basic_indexing.rs | 89 +++++++ 9 files changed, 636 insertions(+), 10 deletions(-) create mode 100644 crates/rattler_azure/Cargo.toml create mode 100644 crates/rattler_azure/src/clap.rs create mode 100644 crates/rattler_azure/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 38de7cf088..c097f5afb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4071,6 +4071,7 @@ checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" dependencies = [ "opendal-core", "opendal-layer-retry", + "opendal-service-azblob", "opendal-service-fs", "opendal-service-s3", ] @@ -4114,6 +4115,37 @@ dependencies = [ "opendal-core", ] +[[package]] +name = "opendal-service-azblob" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +dependencies = [ + "base64 0.22.1", + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.39.4", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.11.0", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +dependencies = [ + "http 1.4.2", + "opendal-core", +] + [[package]] name = "opendal-service-fs" version = "0.57.0" @@ -4353,6 +4385,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pem" version = "3.0.6" @@ -4510,6 +4552,21 @@ dependencies = [ "spki", ] +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -4517,6 +4574,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", + "pkcs5", + "rand_core 0.6.4", "spki", ] @@ -4986,6 +5045,16 @@ dependencies = [ "url", ] +[[package]] +name = "rattler_azure" +version = "0.1.0" +dependencies = [ + "clap", + "serde", + "thiserror 2.0.18", + "url", +] + [[package]] name = "rattler_cache" version = "0.10.3" @@ -5158,6 +5227,7 @@ dependencies = [ "indicatif", "jiff", "opendal", + "rattler_azure", "rattler_conda_types", "rattler_config", "rattler_digest", @@ -5730,6 +5800,27 @@ dependencies = [ "sha1 0.11.0", ] +[[package]] +name = "reqsign-azure-storage" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee8b9e5d0fc927551a6ac25ba5dc6518860c54ddb592fecf685523e528e77bd" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "http 1.4.2", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + [[package]] name = "reqsign-core" version = "3.1.0" @@ -5747,6 +5838,9 @@ dependencies = [ "jiff", "log", "percent-encoding", + "rsa", + "serde", + "serde_json", "sha1 0.11.0", "sha2 0.11.0", "windows-sys 0.61.2", @@ -5898,6 +5992,7 @@ dependencies = [ "pkcs1", "pkcs8", "rand_core 0.6.4", + "sha2 0.10.9", "signature", "spki", "subtle", @@ -6101,6 +6196,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6149,6 +6253,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sec1" version = "0.7.3" diff --git a/Cargo.toml b/Cargo.toml index f425fa3848..5b7c7b14f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -210,6 +210,7 @@ coalesced_map = { path = "crates/coalesced_map", version = "=0.1.4", default-fea file_url = { path = "crates/file_url", version = "=0.3.2", default-features = false } path_resolver = { path = "crates/path_resolver", version = "=0.2.12", default-features = false } rattler = { path = "crates/rattler", version = "=0.47.1", default-features = false } +rattler_azure = { path = "crates/rattler_azure", version = "=0.1.0", default-features = false } rattler_cache = { path = "crates/rattler_cache", version = "=0.10.3", default-features = false } rattler_conda_types = { path = "crates/rattler_conda_types", version = "=0.48.1", default-features = false } rattler_config = { path = "crates/rattler_config", version = "=0.6.1", default-features = false } diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml new file mode 100644 index 0000000000..86492ab7d3 --- /dev/null +++ b/crates/rattler_azure/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rattler_azure" +version = "0.1.0" +description = "A crate to streamline interaction with Azure Blob storage for rattler" +categories.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +edition.workspace = true +readme.workspace = true + +[features] +default = [] + +[dependencies] +clap = { workspace = true, optional = true } +thiserror = { workspace = true } +url = { workspace = true } + +serde = { workspace = true, optional = true } diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs new file mode 100644 index 0000000000..2be874a591 --- /dev/null +++ b/crates/rattler_azure/src/clap.rs @@ -0,0 +1,83 @@ +use clap::Parser; + +use crate::{AzureCredentials, MissingAzureCredentials}; + +/// Manually specified Azure Blob credentials. +/// +/// See [`super::AzureCredentials`] for details on how these credentials are +/// used. `--account-key` and `--sas-token` are mutually exclusive; exactly one +/// must be supplied. +#[derive(Clone, Debug, PartialEq, Parser)] +pub struct AzureCredentialsOpts { + /// The Azure Storage account key. + #[arg( + long, + env = "AZURE_STORAGE_KEY", + conflicts_with = "sas_token", + help_heading = "Azure Credentials" + )] + pub account_key: Option, + + /// A shared access signature (SAS) token, with or without a leading `?`. + #[arg( + long, + env = "AZURE_STORAGE_SAS_TOKEN", + conflicts_with = "account_key", + help_heading = "Azure Credentials" + )] + pub sas_token: Option, +} + +impl TryFrom for AzureCredentials { + type Error = MissingAzureCredentials; + + fn try_from(value: AzureCredentialsOpts) -> Result { + // `conflicts_with` guarantees at most one of the two is set, so the + // order of these checks doesn't matter. + if let Some(account_key) = value.account_key { + Ok(AzureCredentials::AccountKey(account_key)) + } else if let Some(sas_token) = value.sas_token { + Ok(AzureCredentials::SasToken(sas_token)) + } else { + Err(MissingAzureCredentials) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn account_key_resolves() { + let opts = AzureCredentialsOpts { + account_key: Some("key".into()), + sas_token: None, + }; + assert!(matches!( + AzureCredentials::try_from(opts), + Ok(AzureCredentials::AccountKey(k)) if k == "key" + )); + } + + #[test] + fn sas_token_resolves() { + let opts = AzureCredentialsOpts { + account_key: None, + sas_token: Some("sv=...".into()), + }; + assert!(matches!( + AzureCredentials::try_from(opts), + Ok(AzureCredentials::SasToken(t)) if t == "sv=..." + )); + } + + #[test] + fn neither_is_rejected() { + let opts = AzureCredentialsOpts { + account_key: None, + sas_token: None, + }; + assert!(AzureCredentials::try_from(opts).is_err()); + } +} diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs new file mode 100644 index 0000000000..4a1b962ea3 --- /dev/null +++ b/crates/rattler_azure/src/lib.rs @@ -0,0 +1,24 @@ +#[cfg(feature = "clap")] +pub mod clap; + +/// Credentials for authenticating to Azure Blob storage. +/// +/// Exactly one authentication method is carried, so the ambiguous "both a key +/// and a SAS token" and "neither" states are unrepresentable. The storage +/// account name, endpoint, and container are not stored here: they are fully +/// determined by the channel URL (`https://.blob.core.windows.net//...`) +/// and derived by the consumer. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum AzureCredentials { + /// A shared storage account key. + AccountKey(String), + + /// A shared access signature (SAS) token. + SasToken(String), +} + +/// Error returned when no Azure credentials were supplied. +#[derive(Debug, thiserror::Error)] +#[error("no Azure credentials supplied: pass either an account key or a SAS token")] +pub struct MissingAzureCredentials; diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index a10a784650..adbe673ae0 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -13,7 +13,7 @@ readme.workspace = true default-run = "rattler-index" [features] -default = ["rustls", "s3"] +default = ["rustls", "s3", "azure"] native-tls = [ "reqwest/native-tls", "rattler_package_streaming/native-tls", @@ -28,6 +28,7 @@ rustls = [ "opendal/reqwest-rustls-tls", ] s3 = ["opendal/services-s3", "dep:rattler_s3"] +azure = ["opendal/services-azblob", "dep:rattler_azure"] [[bin]] name = "rattler-index" @@ -58,6 +59,7 @@ rattler_conda_types = { workspace = true, default-features = false } rattler_digest = { workspace = true, default-features = false } rattler_package_streaming = { workspace = true, default-features = false } hex = { workspace = true } +rattler_azure = { workspace = true, optional = true, features = ["clap"] } rattler_s3 = { workspace = true, optional = true, features = ["clap"] } reqwest = { workspace = true, default-features = false, features = [ "http2", diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index be5c547bb2..79148f1173 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -23,11 +23,15 @@ use fs_err::{self as fs}; use futures::{StreamExt, stream::FuturesUnordered}; use indexmap::IndexMap; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -#[cfg(feature = "s3")] +#[cfg(any(feature = "s3", feature = "azure"))] use opendal::layers::RetryLayer; +#[cfg(feature = "azure")] +use opendal::services::AzblobConfig; #[cfg(feature = "s3")] use opendal::services::S3Config; use opendal::{Configurator, Operator, services::FsConfig}; +#[cfg(feature = "azure")] +use rattler_azure::AzureCredentials; use rattler_conda_types::{ ChannelInfo, ChannelRelations, PackageRecord, PatchInstructions, Platform, RepoData, Shard, ShardedRepodata, ShardedSubdirInfo, UrlOrPath, V3Packages, WhlPackageRecord, @@ -54,7 +58,7 @@ use serde::Serialize; use sha2::{Digest, Sha256}; use tokio::sync::Semaphore; use tracing::Instrument; -#[cfg(feature = "s3")] +#[cfg(any(feature = "s3", feature = "azure"))] use url::Url; /// Channel metadata written into generated repodata. @@ -767,21 +771,26 @@ async fn index_subdir_inner( } }; - // List all the packages in the subdirectory. - let uploaded_packages: HashSet = op + // List all the packages in the subdirectory, keeping each blob's size. Both + // the azblob and fs listers populate content_length for free (see their + // `with_content_length` calls), so this needs no extra round-trips. + let uploaded_sizes: HashMap = op .list_with(&format!("{}/", subdir.as_str())) .await? .iter() .filter_map(|entry| { - if entry.metadata().mode().is_file() { - let filename = entry.name().to_string(); + let meta = entry.metadata(); + if meta.mode().is_file() { // Check if the file is an archive package file. - DistArchiveIdentifier::try_from_filename(&filename) + DistArchiveIdentifier::try_from_filename(entry.name()) + .map(|id| (id, meta.content_length())) } else { None } }) .collect(); + let uploaded_packages: HashSet = + uploaded_sizes.keys().cloned().collect(); tracing::debug!( "Found {} already uploaded packages in subdir {}.", @@ -809,6 +818,36 @@ async fn index_subdir_inner( registered_packages.remove(filename); } + // Re-index packages whose blob no longer matches the size recorded in the + // previous repodata. `.conda`/`.tar.bz2` archives aren't reproducible, so a + // package rebuilt and republished under the same filename has different + // bytes; without this, the stale record's sha256/size are kept and clients + // hit a hash mismatch on download. Dropping the mismatched entry here moves + // it into `packages_to_add` below, which re-reads and re-hashes it. + // ponytail: size only — a rebuild that lands on the exact same byte count + // slips through. Upgrade path: reindex with `--force`, or bump the build + // number per rebuild so filenames are genuinely immutable. + let stale = registered_packages + .iter() + .filter(|(id, pkg)| match (uploaded_sizes.get(id), pkg.record.size) { + (Some(current), Some(recorded)) => *current != recorded, + (Some(_), None) => true, // no recorded size to trust + (None, _) => false, // absent from the channel: handled above + }) + .map(|(id, _)| id.clone()) + .collect::>(); + + if !stale.is_empty() { + tracing::info!( + "Re-indexing {} packages in subdir {} whose blob size changed since the last index.", + stale.len(), + subdir + ); + } + for filename in &stale { + registered_packages.remove(filename); + } + let packages_to_add = uploaded_packages .difference(®istered_packages.keys().cloned().collect::>()) .cloned() @@ -1513,6 +1552,139 @@ pub async fn index_s3_with_channel_metadata( .map(|_| ()) } +/// Configuration for `index_azure` +#[cfg(feature = "azure")] +pub struct IndexAzureConfig { + /// The channel to index, as an Azure Blob URL + /// (`https://.blob.core.windows.net//`). + pub channel: Url, + /// The credentials to use for Azure Blob access. + pub credentials: AzureCredentials, + /// The target platform to index. + pub target_platform: Option, + /// The path to a repodata patch to apply to the index. + pub repodata_patch: Option, + /// Whether to write the repodata as a zstd-compressed file. + pub write_zst: bool, + /// Whether to write the repodata shards. + pub write_shards: bool, + /// Repodata revisions to advertise in generated repodata. + pub repodata_revisions: Vec, + /// How packages are assigned to repodata revisions. + pub package_revision_assignment: PackageRevisionAssignment, + /// Whether to force the index to be written. + pub force: bool, + /// The maximum number of parallel tasks to run. + pub max_parallel: usize, + /// The multi-progress bar to use for the index. + pub multi_progress: Option, + // NOTE: no `precondition_checks` field. opendal's azblob service does not + // support conditional (`if_match`) writes, so precondition checks can only + // ever be disabled here; the Azure path hardcodes `Disabled` rather than + // exposing a knob whose enabled state always fails. +} + +/// Build an opendal `AzblobConfig` from a channel URL and credentials. +/// +/// The account name, endpoint, container, and root prefix are all derived from +/// the URL (`https://.blob.core.windows.net//`); the +/// credentials supply only the account key or SAS token. +#[cfg(feature = "azure")] +fn azblob_config( + credentials: &AzureCredentials, + channel: &Url, +) -> Result { + let host = channel + .host_str() + .ok_or_else(|| anyhow::anyhow!("No host in Azure blob URL"))?; + let account_name = host + .split('.') + .next() + .filter(|name| !name.is_empty()) + .ok_or_else(|| anyhow::anyhow!("Could not derive account name from Azure blob URL"))?; + + let mut segments = channel + .path_segments() + .ok_or_else(|| anyhow::anyhow!("No path in Azure blob URL"))?; + let container = segments + .next() + .filter(|segment| !segment.is_empty()) + .ok_or_else(|| anyhow::anyhow!("No container in Azure blob URL"))?; + let root = format!("/{}", segments.collect::>().join("/")); + + // Preserve a non-default port so custom endpoints (e.g. the Azurite + // emulator on :10000) work; real Azure uses the scheme default (443). + let authority = match channel.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + }; + + let (account_key, sas_token) = match credentials { + AzureCredentials::AccountKey(key) => (Some(key.clone()), None), + AzureCredentials::SasToken(token) => (None, Some(token.clone())), + }; + + Ok(AzblobConfig { + endpoint: Some(format!("{}://{}", channel.scheme(), authority)), + account_name: Some(account_name.to_string()), + container: container.to_string(), + root: Some(root), + account_key, + sas_token, + ..Default::default() + }) +} + +/// Create a new `repodata.json` for all packages in the channel at the given +/// Azure Blob URL. +#[cfg(feature = "azure")] +pub async fn index_azure(config: IndexAzureConfig) -> anyhow::Result<()> { + index_azure_with_channel_metadata(config, ChannelMetadata::default()).await +} + +/// Create a new `repodata.json` for all packages in the channel at the given +/// Azure Blob URL and write channel metadata into the generated repodata. +#[cfg(feature = "azure")] +pub async fn index_azure_with_channel_metadata( + IndexAzureConfig { + channel, + credentials, + target_platform, + repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force, + max_parallel, + multi_progress, + }: IndexAzureConfig, + channel_metadata: ChannelMetadata, +) -> anyhow::Result<()> { + let azblob_config = azblob_config(&credentials, &channel)?; + let builder = azblob_config.into_builder(); + let op = Operator::new(builder)?.layer(RetryLayer::new()).finish(); + + index_with_channel_metadata( + target_platform, + op, + repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force, + max_parallel, + multi_progress, + // opendal's azblob service can't do conditional writes, so preconditions + // must be disabled (matching the filesystem backend). + PreconditionChecks::Disabled, + channel_metadata, + ) + .await + .map(|_| ()) +} + /// Create a new `repodata.json` for all packages in the given operator's root. /// /// If `target_platform` is `Some`, only that specific subdir is indexed. @@ -1814,6 +1986,58 @@ mod tests { use super::*; + #[cfg(feature = "azure")] + #[test] + fn azblob_config_derives_fields_from_url() { + let channel = + Url::parse("https://stgrcondachannel.blob.core.windows.net/general/sub/dir").unwrap(); + let credentials = AzureCredentials::SasToken("sv=token".to_string()); + + let config = azblob_config(&credentials, &channel).unwrap(); + + assert_eq!( + config.endpoint.as_deref(), + Some("https://stgrcondachannel.blob.core.windows.net") + ); + assert_eq!(config.account_name.as_deref(), Some("stgrcondachannel")); + assert_eq!(config.container, "general"); + assert_eq!(config.root.as_deref(), Some("/sub/dir")); + assert_eq!(config.sas_token.as_deref(), Some("sv=token")); + assert_eq!(config.account_key, None); + } + + #[cfg(feature = "azure")] + #[test] + fn azblob_config_container_only_url() { + let channel = Url::parse("https://stgrcondachannel.blob.core.windows.net/general").unwrap(); + let credentials = AzureCredentials::AccountKey("key".to_string()); + + let config = azblob_config(&credentials, &channel).unwrap(); + + assert_eq!(config.container, "general"); + assert_eq!(config.root.as_deref(), Some("/")); + assert_eq!(config.account_key.as_deref(), Some("key")); + assert_eq!(config.sas_token, None); + } + + #[cfg(feature = "azure")] + #[test] + fn azblob_config_preserves_non_default_port() { + let channel = + Url::parse("http://devstoreaccount1.blob.localhost:10000/testcontainer/ch").unwrap(); + let credentials = AzureCredentials::AccountKey("key".to_string()); + + let config = azblob_config(&credentials, &channel).unwrap(); + + assert_eq!( + config.endpoint.as_deref(), + Some("http://devstoreaccount1.blob.localhost:10000") + ); + assert_eq!(config.account_name.as_deref(), Some("devstoreaccount1")); + assert_eq!(config.container, "testcontainer"); + assert_eq!(config.root.as_deref(), Some("/ch")); + } + #[test] fn package_records_from_repodata_preserves_v3_wheels() { let identifier = ArchiveIdentifier::from_str("demo-1.0-py_0").unwrap(); diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index ba09ef2a5e..671fd9d2f5 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -1,22 +1,29 @@ use std::path::PathBuf; +#[cfg(feature = "s3")] use anyhow::Context; use clap::{Parser, Subcommand}; use clap_verbosity_flag::Verbosity; +#[cfg(feature = "azure")] +use rattler_azure::AzureCredentials; use rattler_conda_types::Platform; use rattler_config::config::{ concurrency::default_max_concurrent_solves, index::IndexChannelConfig, }; +#[cfg(feature = "s3")] +use rattler_index::PreconditionChecks; use rattler_index::{ ChannelMetadata, IndexFsConfig, PackageRevisionAssignment, index_fs_with_channel_metadata, }; +#[cfg(feature = "azure")] +use rattler_index::{IndexAzureConfig, index_azure_with_channel_metadata}; #[cfg(feature = "s3")] -use rattler_index::{IndexS3Config, PreconditionChecks, index_s3_with_channel_metadata}; +use rattler_index::{IndexS3Config, index_s3_with_channel_metadata}; #[cfg(feature = "s3")] use rattler_networking::AuthenticationStorage; #[cfg(feature = "s3")] use rattler_s3::S3Credentials; -#[cfg(feature = "s3")] +#[cfg(any(feature = "s3", feature = "azure"))] use url::Url; #[cfg(feature = "s3")] @@ -31,6 +38,24 @@ fn parse_s3_url(value: &str) -> Result { } } +#[cfg(feature = "azure")] +fn parse_azure_url(value: &str) -> Result { + let url: Url = Url::parse(value).map_err(|e| format!("`{value}` isn't a valid URL: {e}"))?; + // Require host + a container segment, e.g. + // https://.blob.core.windows.net//. + let has_container = url + .path_segments() + .and_then(|mut segments| segments.next()) + .is_some_and(|segment| !segment.is_empty()); + if matches!(url.scheme(), "http" | "https") && url.host_str().is_some() && has_container { + Ok(url) + } else { + Err(format!( + "Only Azure Blob URLs of format https://.blob.core.windows.net//... can be used, not `{value}`" + )) + } +} + /// The `rattler-index` CLI. #[derive(Parser)] #[command(name = "rattler-index", version, about, long_about = None)] @@ -99,6 +124,18 @@ enum Commands { #[clap(flatten)] credentials: rattler_s3::clap::S3CredentialsOpts, }, + + /// Index a channel stored in an Azure Blob container. + #[cfg(feature = "azure")] + Azblob { + /// The Azure Blob channel URL, e.g. + /// `https://.blob.core.windows.net//`. + #[arg(value_parser = parse_azure_url)] + channel: Url, + + #[clap(flatten)] + credentials: rattler_azure::clap::AzureCredentialsOpts, + }, } /// The configuration type for rattler-index - just extends rattler config and @@ -213,6 +250,37 @@ async fn main() -> anyhow::Result<()> { ) .await } + #[cfg(feature = "azure")] + Commands::Azblob { + channel, + credentials, + } => { + let target = channel.to_string(); + let resolved = resolve_index_channel_config(&config, &target); + let (write_zst, write_shards, repodata_revisions, package_revision_assignment) = + effective_index_options(&resolved); + let channel_metadata = ChannelMetadata::from_index_config(&resolved); + + let credentials = AzureCredentials::try_from(credentials)?; + + index_azure_with_channel_metadata( + IndexAzureConfig { + channel, + credentials, + target_platform: cli.target_platform, + repodata_patch: cli.repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force: cli.force, + max_parallel, + multi_progress: Some(multi_progress), + }, + channel_metadata, + ) + .await + } }?; println!("Finished indexing channel."); Ok(()) diff --git a/crates/rattler_index/tests/integration/basic_indexing.rs b/crates/rattler_index/tests/integration/basic_indexing.rs index c2062b6b0a..b87b83074d 100644 --- a/crates/rattler_index/tests/integration/basic_indexing.rs +++ b/crates/rattler_index/tests/integration/basic_indexing.rs @@ -378,6 +378,95 @@ async fn test_index_repodata_revision_from_index_json() { assert_eq!(revision["newest"], 1710000000000i64); } +/// Regression: a package rebuilt and republished under the same filename (new +/// bytes) must be re-hashed on an incremental index. Keying only on filename +/// left the previous build's sha256/size in repodata, so clients downloading the +/// current blob hit a hash mismatch. +#[tokio::test] +async fn test_incremental_reindexes_replaced_package() { + let temp_dir = tempfile::tempdir().unwrap(); + let subdir_path = temp_dir.path().join("noarch"); + fs::create_dir(&subdir_path).unwrap(); + let package_name = "stale-demo-1.0.0-h123_0.tar.bz2"; + + // Build a tar.bz2 package with a payload of the given size and write it into + // the channel under a fixed filename, overwriting any previous build. + let build = |payload_len: usize| { + let build_dir = temp_dir.path().join(format!("build-{payload_len}")); + let info_dir = build_dir.join("info"); + fs::create_dir_all(&info_dir).unwrap(); + fs::write( + info_dir.join("index.json"), + r#"{"build":"h123_0","build_number":0,"name":"stale-demo","noarch":"generic","subdir":"noarch","timestamp":1710000000000,"version":"1.0.0"}"#, + ) + .unwrap(); + fs::write(build_dir.join("payload.txt"), "x".repeat(payload_len)).unwrap(); + let writer = File::create(subdir_path.join(package_name)).unwrap(); + write_tar_bz2_package( + writer, + &build_dir, + &[info_dir.join("index.json"), build_dir.join("payload.txt")], + CompressionLevel::Default, + None, + None, + ) + .unwrap(); + }; + + // Incremental index (force: false) — the path that carried the bug. + async fn index(channel: &Path) { + index_fs(IndexFsConfig { + channel: channel.into(), + target_platform: Some(Platform::NoArch), + repodata_patch: None, + write_zst: false, + write_shards: false, + repodata_revisions: Vec::new(), + package_revision_assignment: PackageRevisionAssignment::default(), + force: false, + max_parallel: 1, + multi_progress: None, + }) + .await + .unwrap(); + } + + let recorded = |field: &str| -> Value { + let repodata: Value = + serde_json::from_reader(File::open(subdir_path.join("repodata.json")).unwrap()) + .unwrap(); + repodata + .pointer(&format!("/packages/{package_name}/{field}")) + .expect("package present in repodata") + .clone() + }; + + // Baseline: first build, indexed. + build(64); + index(temp_dir.path()).await; + let disk1 = fs::metadata(subdir_path.join(package_name)).unwrap().len(); + assert_eq!(recorded("size").as_u64(), Some(disk1)); + let sha1 = recorded("sha256"); + + // Rebuild under the same filename with very different bytes, then reindex + // incrementally. + build(65536); + index(temp_dir.path()).await; + let disk2 = fs::metadata(subdir_path.join(package_name)).unwrap().len(); + + assert_ne!(disk1, disk2, "test setup: rebuild must change the file size"); + assert_eq!( + recorded("size").as_u64(), + Some(disk2), + "repodata size must match the rebuilt file" + ); + assert_ne!( + recorded("sha256"), + sha1, + "repodata sha256 must reflect the rebuilt bytes" + ); +} + #[tokio::test] async fn test_index_writes_channel_metadata() { let temp_dir = tempfile::tempdir().unwrap(); From f57e47f2143b83d7570ad3a8685923fc4001947e Mon Sep 17 00:00:00 2001 From: Tenzin <143778894+tenzinplatter@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:21:26 +1000 Subject: [PATCH 23/98] feat: azure index md5 (#3) * fix(rattler_index): re-index packages whose blob size changed * fix: move to md5 checks instead of size * feat(rattler_index): add Azure Blob backend * fix(rattler_index): disable preconditions on azure (opendal azblob lacks conditional writes) * fix(rattler_index): base64-decode backend md5 before comparing * test(rattler_index): probe azure content_md5 on list * chore: lint --- Cargo.lock | 1 + crates/rattler_index/Cargo.toml | 1 + crates/rattler_index/src/lib.rs | 48 ++++++----- crates/rattler_index/tests/azure_md5_probe.rs | 80 +++++++++++++++++++ .../tests/integration/basic_indexing.rs | 5 +- 5 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 crates/rattler_index/tests/azure_md5_probe.rs diff --git a/Cargo.lock b/Cargo.lock index c097f5afb2..3a13e70c4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5216,6 +5216,7 @@ version = "0.30.9" dependencies = [ "ahash", "anyhow", + "base64 0.22.1", "bytes", "clap", "clap-verbosity-flag", diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index adbe673ae0..b46e33dcd1 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -38,6 +38,7 @@ path = "src/main.rs" thiserror = { workspace = true } ahash = { workspace = true } anyhow = { workspace = true } +base64 = { workspace = true } bytes = { workspace = true } indexmap = { workspace = true } jiff = { workspace = true } diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 79148f1173..18baab653f 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -18,6 +18,7 @@ use std::{ }; use anyhow::{Context, Result}; +use base64::Engine; use bytes::buf::Buf; use fs_err::{self as fs}; use futures::{StreamExt, stream::FuturesUnordered}; @@ -771,26 +772,28 @@ async fn index_subdir_inner( } }; - // List all the packages in the subdirectory, keeping each blob's size. Both - // the azblob and fs listers populate content_length for free (see their - // `with_content_length` calls), so this needs no extra round-trips. - let uploaded_sizes: HashMap = op - .list_with(&format!("{}/", subdir.as_str())) - .await? + let existing = op.list_with(&format!("{}/", subdir.as_str())).await?; + // get the md5 hashes of each uploaded .conda, storing an optional md5 hash and a archive size + // for each, so that we can later check if the contents has changed + let uploaded_hashes: HashMap> = existing .iter() .filter_map(|entry| { let meta = entry.metadata(); if meta.mode().is_file() { // Check if the file is an archive package file. DistArchiveIdentifier::try_from_filename(entry.name()) - .map(|id| (id, meta.content_length())) + // opendal populates content_md5 from the backend's Content-MD5 + // header on list (verified for azure/azblob via the azure_md5_probe + // test). md5 is documented best-effort, so backends that omit it + // yield None here and those packages get re-indexed. + .map(|id| (id, meta.content_md5())) } else { None } }) .collect(); let uploaded_packages: HashSet = - uploaded_sizes.keys().cloned().collect(); + uploaded_hashes.keys().cloned().collect(); tracing::debug!( "Found {} already uploaded packages in subdir {}.", @@ -818,28 +821,35 @@ async fn index_subdir_inner( registered_packages.remove(filename); } - // Re-index packages whose blob no longer matches the size recorded in the + // Re-index packages whose file no longer matches the md5 recorded in the // previous repodata. `.conda`/`.tar.bz2` archives aren't reproducible, so a // package rebuilt and republished under the same filename has different - // bytes; without this, the stale record's sha256/size are kept and clients + // bytes; without this the stale record's sha256/size are kept and clients // hit a hash mismatch on download. Dropping the mismatched entry here moves // it into `packages_to_add` below, which re-reads and re-hashes it. - // ponytail: size only — a rebuild that lands on the exact same byte count - // slips through. Upgrade path: reindex with `--force`, or bump the build - // number per rebuild so filenames are genuinely immutable. let stale = registered_packages .iter() - .filter(|(id, pkg)| match (uploaded_sizes.get(id), pkg.record.size) { - (Some(current), Some(recorded)) => *current != recorded, - (Some(_), None) => true, // no recorded size to trust - (None, _) => false, // absent from the channel: handled above + .filter(|(id, pkg)| { + // Not stale only if the backend's md5 matches the one in the previous + // repodata. opendal exposes content_md5 as the base64 Content-MD5 header + // (e.g. azure), so decode it to raw bytes before comparing to the record's + // 16-byte digest. Missing/undecodable md5 => treat as stale and re-index. + if let Some(Some(new_md5)) = uploaded_hashes.get(id) + && let Some(old_md5) = pkg.record.md5 + && let Ok(new_md5) = base64::engine::general_purpose::STANDARD.decode(new_md5) + && old_md5.as_slice() == new_md5.as_slice() + { + false + } else { + true + } }) .map(|(id, _)| id.clone()) .collect::>(); if !stale.is_empty() { - tracing::info!( - "Re-indexing {} packages in subdir {} whose blob size changed since the last index.", + tracing::warn!( + "Re-indexing {} packages in subdir {} whose md5 changed since the last index.", stale.len(), subdir ); diff --git a/crates/rattler_index/tests/azure_md5_probe.rs b/crates/rattler_index/tests/azure_md5_probe.rs new file mode 100644 index 0000000000..e32f50db62 --- /dev/null +++ b/crates/rattler_index/tests/azure_md5_probe.rs @@ -0,0 +1,80 @@ +//! Probe: does opendal's azblob backend populate `content_md5` on list? +//! +//! This is exactly the signal the md5-based re-index check in `index_subdir_inner` +//! relies on (`entry.metadata().content_md5()`). If azure doesn't set it on a plain +//! `list_with`, that check silently forces a full re-index of the channel. +//! +//! Run with: +//! ```text +//! docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ +//! azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck +//! AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== \ +//! cargo test -p rattler_index --features azure --test azure_md5_probe -- --ignored --nocapture +//! ``` +#![cfg(feature = "azure")] + +use base64::Engine; +use opendal::{Configurator, Operator, services::AzblobConfig}; + +const ACCOUNT: &str = "devstoreaccount1"; +const CONTAINER: &str = "md5-probe"; +const ENDPOINT: &str = "http://127.0.0.1:10000/devstoreaccount1"; + +fn azurite_operator() -> Operator { + let key = std::env::var("AZURE_STORAGE_ACCOUNT_KEY").expect("AZURE_STORAGE_ACCOUNT_KEY"); + let cfg = AzblobConfig { + container: CONTAINER.to_string(), + account_name: Some(ACCOUNT.to_string()), + account_key: Some(key), + endpoint: Some(ENDPOINT.to_string()), + ..Default::default() + }; + Operator::new(cfg.into_builder()).unwrap().finish() +} + +#[tokio::test] +#[ignore = "requires a running Azurite emulator"] +async fn azure_populates_content_md5_on_list() { + let op = azurite_operator(); + let _ = op.create_dir("noarch/").await; + + let name = "noarch/empty-0.1.0-h4616a5c_0.conda"; + let bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../test-data/packages/empty-0.1.0-h4616a5c_0.conda"), + ) + .unwrap(); + op.write(name, bytes).await.unwrap(); + + // Exactly what index_subdir_inner does: plain list, then read content_md5 off + // each entry's listing metadata. + let entries = op.list_with("noarch/").await.unwrap(); + let mut saw_file = false; + for entry in &entries { + let meta = entry.metadata(); + if !meta.mode().is_file() { + continue; + } + saw_file = true; + println!( + "LIST {} -> content_md5 = {:?}", + entry.name(), + meta.content_md5() + ); + } + assert!(saw_file, "no file entry found in listing"); + + // For contrast: does an explicit stat populate it? + let stat = op.stat(name).await.unwrap(); + println!("STAT {name} -> content_md5 = {:?}", stat.content_md5()); + + // The re-index check decodes this base64 header to compare against the record's + // raw 16-byte md5, so lock in that it is standard-base64 of exactly 16 bytes. + let md5 = stat + .content_md5() + .expect("azure did not populate content_md5"); + let raw = base64::engine::general_purpose::STANDARD + .decode(md5) + .expect("content_md5 was not valid base64"); + assert_eq!(raw.len(), 16, "decoded md5 was not 16 bytes: {raw:?}"); +} diff --git a/crates/rattler_index/tests/integration/basic_indexing.rs b/crates/rattler_index/tests/integration/basic_indexing.rs index b87b83074d..fb143734d3 100644 --- a/crates/rattler_index/tests/integration/basic_indexing.rs +++ b/crates/rattler_index/tests/integration/basic_indexing.rs @@ -454,7 +454,10 @@ async fn test_incremental_reindexes_replaced_package() { index(temp_dir.path()).await; let disk2 = fs::metadata(subdir_path.join(package_name)).unwrap().len(); - assert_ne!(disk1, disk2, "test setup: rebuild must change the file size"); + assert_ne!( + disk1, disk2, + "test setup: rebuild must change the file size" + ); assert_eq!( recorded("size").as_u64(), Some(disk2), From 3f45d5500e72d28b4761ce7d3bdc9b3bd99a1f1b Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:31:51 +1000 Subject: [PATCH 24/98] feat(upload): add azure upload path on AzureCredentials --- Cargo.lock | 1 + crates/rattler_upload/Cargo.toml | 2 + crates/rattler_upload/src/lib.rs | 16 ++ crates/rattler_upload/src/upload/azure.rs | 207 ++++++++++++++++++++++ crates/rattler_upload/src/upload/mod.rs | 4 + crates/rattler_upload/src/upload/opt.rs | 41 +++++ 6 files changed, 271 insertions(+) create mode 100644 crates/rattler_upload/src/upload/azure.rs diff --git a/Cargo.lock b/Cargo.lock index 3a13e70c4d..1dc372e5b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5625,6 +5625,7 @@ dependencies = [ "indicatif", "miette", "opendal", + "rattler_azure", "rattler_conda_types", "rattler_config", "rattler_digest", diff --git a/crates/rattler_upload/Cargo.toml b/crates/rattler_upload/Cargo.toml index 95e570e807..74f5ce6aa9 100644 --- a/crates/rattler_upload/Cargo.toml +++ b/crates/rattler_upload/Cargo.toml @@ -29,6 +29,7 @@ native-tls = [ "sigstore-sign?/native-tls", ] s3 = ["rattler_networking/s3", "rattler_s3", "dep:opendal"] +azure = ["dep:rattler_azure", "dep:opendal", "opendal/services-azblob"] sigstore-sign = ["dep:sigstore-sign", "dep:sigstore-trust-root"] [package.metadata.docs.rs] @@ -44,6 +45,7 @@ rattler_package_streaming = { workspace = true, default-features = false } rattler_config = { workspace = true, default-features = false } rattler_solve = { workspace = true } rattler_s3 = { workspace = true, optional = true, features = ["clap"] } +rattler_azure = { workspace = true, optional = true, features = ["clap"] } miette = { workspace = true, features = ["fancy"] } clap = { workspace = true, features = ["env"] } fs-err = { workspace = true, features = ["tokio"] } diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index b7ad07b200..7b965ad2e8 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -81,6 +81,22 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { ) .await } + #[cfg(feature = "azure")] + ServerType::Azure(azure_opts) => { + let credentials = + rattler_azure::AzureCredentials::try_from(azure_opts.credentials).map_err(|err| { + miette::miette!( + "{err}. Provide either --account-key or --sas-token (or the corresponding environment variables)." + ) + })?; + upload::upload_package_to_azure( + azure_opts.channel, + credentials, + &args.package_files, + azure_opts.force, + ) + .await + } ServerType::CondaForge(conda_forge_opts) => { let conda_forge_data = CondaForgeData::from(conda_forge_opts); upload::conda_forge::upload_packages_to_conda_forge( diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs new file mode 100644 index 0000000000..320a1ced42 --- /dev/null +++ b/crates/rattler_upload/src/upload/azure.rs @@ -0,0 +1,207 @@ +use std::path::{Path, PathBuf}; + +use futures::StreamExt; +use miette::IntoDiagnostic; +use opendal::{Configurator, ErrorKind, Operator, services::AzblobConfig}; +use rattler_azure::AzureCredentials; +use rattler_digest::{HashingReader, Md5, Sha256}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio_util::bytes::BytesMut; +use url::Url; + +use crate::upload::package::ExtractedPackage; + +/// Size of a single chunk handed to the writer. Azure Blob storage expects data +/// to be uploaded in reasonably sized blocks; we buffer into 10 MiB chunks. +const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; + +/// Number of packages that are uploaded concurrently. +const PACKAGE_CONCURRENCY: usize = 4; + +/// Uploads packages to a channel in an Azure Blob Storage container. +/// +/// The channel URL is expected to be of the form +/// `https://.blob.core.windows.net//`; the account +/// name, endpoint, container, and root prefix are all derived from it (see +/// [`azblob_config`]). The [`AzureCredentials`] supply only the account key or +/// SAS token. +pub async fn upload_package_to_azure( + channel: Url, + credentials: AzureCredentials, + package_files: &[PathBuf], + force: bool, +) -> miette::Result<()> { + let config = azblob_config(&credentials, &channel)?; + let container = config.container.clone(); + + let builder = config.into_builder(); + let op = Operator::new(builder).into_diagnostic()?.finish(); + + // Upload multiple packages concurrently. Each package is written to its own + // key, so the individual uploads are independent. + futures::stream::iter(package_files.iter()) + .map(|package_file| { + let op = op.clone(); + let channel = &channel; + let container = container.as_str(); + async move { upload_single_package(&op, channel, container, package_file, force).await } + }) + .buffer_unordered(PACKAGE_CONCURRENCY) + .collect::>() + .await + .into_iter() + .collect::>>()?; + + Ok(()) +} + +/// Uploads a single package file to the Azure Blob container via the given operator. +async fn upload_single_package( + op: &Operator, + channel: &Url, + container: &str, + package_file: &Path, + force: bool, +) -> miette::Result<()> { + let package = ExtractedPackage::from_package_file(package_file)?; + let subdir = package + .subdir() + .ok_or_else(|| miette::miette!("Failed to get subdir"))?; + let filename = package + .filename() + .ok_or_else(|| miette::miette!("Failed to get filename"))?; + let key = format!("{subdir}/{filename}"); + + // Compute the hash of the package by streaming its content. + let file = tokio::io::BufReader::new( + fs_err::tokio::File::open(package_file) + .await + .into_diagnostic()?, + ); + let sha256_reader = HashingReader::<_, Sha256>::new(file); + let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); + let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) + .await + .into_diagnostic()?; + let (sha256_reader, md5hash) = md5_reader.finalize(); + let (mut file, sha256hash) = sha256_reader.finalize(); + + // Rewind the file to the beginning. + file.rewind().await.into_diagnostic()?; + + // Construct a writer for the package. `if_not_exists(!force)` maps to an + // `If-None-Match: *` precondition so an existing blob is not silently + // overwritten unless `--force` was passed. + let mut writer = match op + .writer_with(&key) + .content_disposition(&format!("attachment; filename={filename}")) + .if_not_exists(!force) + .user_metadata([ + (String::from("package-sha256"), hex::encode(sha256hash)), + (String::from("package-md5"), hex::encode(md5hash)), + ]) + .await + { + Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { + miette::bail!( + "Package az://{container}{}/{key} already exists. Use --force to overwrite.", + channel.path().to_string() + ); + } + Ok(writer) => writer, + Err(e) => { + return Err(e).into_diagnostic(); + } + }; + + // Stream the file to the writer in `DESIRED_CHUNK_SIZE` chunks. We do this in + // a more complex way than a plain `io::copy` because the underlying storage + // provider expects to receive the data in specifically sized chunks. The code + // below guarantees chunks of equal size except for maybe the last chunk. + let mut remaining_size = size as usize; + while remaining_size > 0 { + // Allocate memory for this chunk. + let chunk_size = remaining_size.min(DESIRED_CHUNK_SIZE); + let mut chunk = BytesMut::with_capacity(chunk_size); + // SAFE: because we do not care about the bytes that are currently in the buffer + unsafe { chunk.set_len(chunk_size) }; + + // Fill the chunk with data. This reads exactly the number of bytes we want. No + // more, no less. + let bytes_read = file.read_exact(&mut chunk[..]).await.into_diagnostic()?; + debug_assert_eq!(bytes_read, chunk.len()); + + // Hand the chunk to the writer. + writer.write(chunk.freeze()).await.into_diagnostic()?; + + // Update the number of remaining bytes. + remaining_size = remaining_size.saturating_sub(bytes_read); + } + + match writer.close().await { + Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { + miette::bail!( + "Package az://{container}{}/{key} already exists. Use --force to overwrite.", + channel.path().to_string() + ); + } + Ok(_) => { + tracing::info!( + "Uploaded package to az://{container}{}/{key}", + channel.path().to_string() + ); + } + Err(e) => { + return Err(e).into_diagnostic(); + } + } + + Ok(()) +} + +/// Build an opendal [`AzblobConfig`] from a channel URL and credentials. +/// +/// The account name, endpoint, container, and root prefix are all derived from +/// the URL (`https://.blob.core.windows.net//`); the +/// credentials supply only the account key or SAS token. +fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Result { + let host = channel + .host_str() + .ok_or_else(|| miette::miette!("No host in Azure blob URL"))?; + let account_name = host + .split('.') + .next() + .filter(|name| !name.is_empty()) + .ok_or_else(|| miette::miette!("Could not derive account name from Azure blob URL"))?; + + let mut segments = channel + .path_segments() + .ok_or_else(|| miette::miette!("No path in Azure blob URL"))?; + let container = segments + .next() + .filter(|segment| !segment.is_empty()) + .ok_or_else(|| miette::miette!("No container in Azure blob URL"))?; + let root = format!("/{}", segments.collect::>().join("/")); + + // Preserve a non-default port so custom endpoints (e.g. the Azurite + // emulator on :10000) work; real Azure uses the scheme default (443). + let authority = match channel.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + }; + + let (account_key, sas_token) = match credentials { + AzureCredentials::AccountKey(key) => (Some(key.clone()), None), + AzureCredentials::SasToken(token) => (None, Some(token.clone())), + }; + + Ok(AzblobConfig { + endpoint: Some(format!("{}://{}", channel.scheme(), authority)), + account_name: Some(account_name.to_string()), + container: container.to_string(), + root: Some(root), + account_key, + sas_token, + ..Default::default() + }) +} diff --git a/crates/rattler_upload/src/upload/mod.rs b/crates/rattler_upload/src/upload/mod.rs index e3de54bea4..875cb05720 100644 --- a/crates/rattler_upload/src/upload/mod.rs +++ b/crates/rattler_upload/src/upload/mod.rs @@ -28,6 +28,10 @@ use crate::upload::package::{ExtractedPackage, sha256_sum}; pub(crate) mod test_utils; mod anaconda; +#[cfg(feature = "azure")] +mod azure; +#[cfg(feature = "azure")] +pub use azure::upload_package_to_azure; #[cfg(feature = "sigstore-sign")] pub mod attestation; mod cloudsmith; diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 1cc5dae935..e26fe17f11 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -123,6 +123,8 @@ pub enum ServerType { Cloudsmith(CloudsmithOpts), #[cfg(feature = "s3")] S3(S3Opts), + #[cfg(feature = "azure")] + Azure(AzureOpts), #[clap(hide = true)] CondaForge(CondaForgeOpts), } @@ -416,6 +418,45 @@ pub struct S3Opts { pub force: bool, } +#[cfg(feature = "azure")] +fn parse_azure_url(value: &str) -> Result { + let url: Url = + Url::parse(value).map_err(|err| format!("`{value}` isn't a valid URL: {err}"))?; + // Require host + a container segment, e.g. + // https://.blob.core.windows.net//. + let has_container = url + .path_segments() + .and_then(|mut segments| segments.next()) + .is_some_and(|segment| !segment.is_empty()); + if matches!(url.scheme(), "http" | "https") && url.host_str().is_some() && has_container { + Ok(url) + } else { + Err(format!( + "Only Azure Blob URLs of format https://.blob.core.windows.net//... can be used, not `{value}`" + )) + } +} + +/// Options for uploading to Azure Blob Storage. +/// +/// Authentication is supplied with either an account key or a shared access +/// signature (SAS) token; the two are mutually exclusive. +#[cfg(feature = "azure")] +#[derive(Clone, Debug, PartialEq, Parser)] +pub struct AzureOpts { + /// The channel URL in the Azure Blob container to upload the package to, + /// e.g., `https://myaccount.blob.core.windows.net/my-container/my-channel` + #[arg(short, long, env = "AZURE_CHANNEL", value_parser = parse_azure_url)] + pub channel: Url, + + #[clap(flatten)] + pub credentials: rattler_azure::clap::AzureCredentialsOpts, + + /// Replace files if it already exists. + #[arg(long)] + pub force: bool, +} + #[derive(Debug)] #[allow(missing_docs)] pub struct AnacondaData { From 425b46ae6a3e3c6da6af2b4f4d78e78b844994e1 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:32:35 +1000 Subject: [PATCH 25/98] feat(networking): add azure fetch middleware --- Cargo.lock | 31 ++ Cargo.toml | 5 + crates/rattler_config/src/config.rs | 12 + crates/rattler_config/src/config/azure.rs | 103 +++++++ crates/rattler_networking/Cargo.toml | 16 +- .../src/azure_middleware.rs | 269 ++++++++++++++++++ crates/rattler_networking/src/lib.rs | 5 + 7 files changed, 440 insertions(+), 1 deletion(-) create mode 100644 crates/rattler_config/src/config/azure.rs create mode 100644 crates/rattler_networking/src/azure_middleware.rs diff --git a/Cargo.lock b/Cargo.lock index 3a13e70c4d..c6070bfb28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5366,6 +5366,11 @@ dependencies = [ "netrc-rs", "rattler_config", "regex", + "reqsign-azure-storage", + "reqsign-command-execute-tokio", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-http-send-reqwest", "reqwest", "retry-policies", "rstest", @@ -5822,6 +5827,16 @@ dependencies = [ "sha1 0.11.0", ] +[[package]] +name = "reqsign-command-execute-tokio" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf28fcf0aef694448cb235d393fbec7b5543d25050eacc0ccc2118596618c52" +dependencies = [ + "reqsign-core", + "tokio", +] + [[package]] name = "reqsign-core" version = "3.1.0" @@ -5858,6 +5873,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "reqsign-http-send-reqwest" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b4509b12b73289cdeed349ba494553404e33782abc09bc0ae4e09f741807cb" +dependencies = [ + "anyhow", + "bytes", + "futures-channel", + "http 1.4.2", + "http-body-util", + "reqsign-core", + "reqwest", + "wasm-bindgen-futures", +] + [[package]] name = "reqwest" version = "0.13.4" diff --git a/Cargo.toml b/Cargo.toml index 5b7c7b14f5..6dd149a369 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,11 @@ google-cloud-auth = { version = "1", default-features = false, features = [ aws-config = { version = "1", default-features = false } aws-sdk-s3 = { version = "1", default-features = false } aws-smithy-http-client = { version = "1", default-features = false } +reqsign-core = "3.0.1" +reqsign-azure-storage = "3.0.1" +reqsign-command-execute-tokio = "3.0.1" +reqsign-file-read-tokio = "3.0.1" +reqsign-http-send-reqwest = "4.0.1" aws-credential-types = { version = "1", default-features = false } hashbrown = "0.17" hex = "0.4" diff --git a/crates/rattler_config/src/config.rs b/crates/rattler_config/src/config.rs index d514a3ae6a..c90b76c29a 100644 --- a/crates/rattler_config/src/config.rs +++ b/crates/rattler_config/src/config.rs @@ -20,12 +20,14 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use thiserror::Error; use url::Url; +use crate::config::azure::AzureOptionsMap; use crate::config::s3::S3OptionsMap; use crate::config::{ build::BuildConfig, concurrency::ConcurrencyConfig, index::IndexConfig, proxy::ProxyConfig, repodata_config::RepodataConfig, run_post_link_scripts::RunPostLinkScripts, }; +pub mod azure; pub mod build; pub mod channel_config; pub mod concurrency; @@ -194,6 +196,11 @@ pub struct CommonConfig { #[serde(skip_serializing_if = "S3OptionsMap::is_default")] pub s3_options: S3OptionsMap, + /// Configuration for Azure Blob Storage. + #[serde(default)] + #[serde(skip_serializing_if = "AzureOptionsMap::is_default")] + pub azure_options: AzureOptionsMap, + /// Per-channel configuration for `rattler-index`. #[serde(default, skip_serializing_if = "IndexConfig::is_empty")] pub index_config: IndexConfig, @@ -251,6 +258,7 @@ impl Default for CommonConfig { concurrency: ConcurrencyConfig::default(), proxy_config: ProxyConfig::default(), s3_options: S3OptionsMap::default(), + azure_options: AzureOptionsMap::default(), index_config: IndexConfig::default(), run_post_link_scripts: None, allow_symbolic_links: None, @@ -305,6 +313,7 @@ impl Config for CommonConfig { concurrency: self.concurrency.merge_config(&other.concurrency)?, proxy_config: self.proxy_config.merge_config(&other.proxy_config)?, s3_options: self.s3_options.merge_config(&other.s3_options)?, + azure_options: self.azure_options.merge_config(&other.azure_options)?, index_config: self.index_config.merge_config(&other.index_config)?, run_post_link_scripts: other .run_post_link_scripts @@ -322,6 +331,7 @@ impl Config for CommonConfig { self.concurrency.validate()?; self.proxy_config.validate()?; self.s3_options.validate()?; + self.azure_options.validate()?; self.index_config.validate()?; Ok(()) } @@ -338,6 +348,7 @@ impl Config for CommonConfig { "allow-hard-links".to_string(), "allow-ref-links".to_string(), "s3-options".to_string(), + "azure-options".to_string(), "index-config".to_string(), ]; keys.extend(prefixed_keys("build", self.build.keys())); @@ -348,6 +359,7 @@ impl Config for CommonConfig { keys.extend(prefixed_keys("concurrency", self.concurrency.keys())); keys.extend(prefixed_keys("proxy-config", self.proxy_config.keys())); keys.extend(prefixed_keys("s3-options", self.s3_options.keys())); + keys.extend(prefixed_keys("azure-options", self.azure_options.keys())); keys } } diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs new file mode 100644 index 0000000000..df927f39f3 --- /dev/null +++ b/crates/rattler_config/src/config/azure.rs @@ -0,0 +1,103 @@ +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::config::Config; + +#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct AzureOptionsMap(pub IndexMap); + +impl AzureOptionsMap { + /// Returns `true` if no Azure containers are configured. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub struct AzureOptions { + /// Storage account name → host `{account}.blob.core.windows.net`. + pub account: String, + + /// Optional full endpoint override for sovereign clouds / Azurite. + /// Defaults to `https://{account}.blob.core.windows.net`. + pub endpoint_url: Option, +} + +impl Config for AzureOptionsMap { + fn is_default(&self) -> bool { + self.0.is_empty() + } + + fn merge_config(self, other: &Self) -> Result { + // Merge the two AzureOptionsMaps, overwriting existing keys + let mut merged = self.0.clone(); + for (key, value) in &other.0 { + merged.insert(key.clone(), value.clone()); + } + Ok(AzureOptionsMap(merged)) + } + + fn validate(&self) -> Result<(), super::ValidationError> { + Ok(()) + } + + fn keys(&self) -> Vec { + self.0.keys().map(ToString::to_string).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn deserializes_account_and_optional_endpoint() { + let toml = r#" + [mychannel] + account = "myacct" + + [other] + account = "acct2" + endpoint-url = "https://acct2.blob.core.windows.net" + "#; + let map: AzureOptionsMap = toml::from_str(toml).unwrap(); + let mychannel = map.0.get("mychannel").unwrap(); + assert_eq!(mychannel.account, "myacct"); + assert_eq!(mychannel.endpoint_url, None); + let other = map.0.get("other").unwrap(); + assert_eq!( + other.endpoint_url.as_ref().unwrap().as_str(), + "https://acct2.blob.core.windows.net/" + ); + } + + #[test] + fn is_default_when_empty() { + let map = AzureOptionsMap::default(); + assert!(map.is_default()); + } + + #[test] + fn merge_overwrites_existing_keys() { + let mut base = AzureOptionsMap::default(); + base.0.insert( + "c".to_string(), + AzureOptions { + account: "old".into(), + endpoint_url: None, + }, + ); + let mut other = AzureOptionsMap::default(); + other.0.insert( + "c".to_string(), + AzureOptions { + account: "new".into(), + endpoint_url: None, + }, + ); + let merged = base.merge_config(&other).unwrap(); + assert_eq!(merged.0.get("c").unwrap().account, "new"); + } +} diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 0a12ccb4f5..8337fbb79e 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -17,6 +17,15 @@ native-tls = ["reqwest/native-tls", "ambient-id/native-tls"] rustls = ["reqwest/rustls", "ambient-id/rustls"] gcs = ["google-cloud-auth", "tokio/sync"] s3 = ["aws-config", "aws-sdk-s3", "aws-smithy-http-client"] +azure = [ + "dep:reqsign-core", + "dep:reqsign-azure-storage", + "dep:reqsign-command-execute-tokio", + "dep:reqsign-file-read-tokio", + "dep:reqsign-http-send-reqwest", + "rattler_config", + "tokio", +] system-integration = ["keyring", "netrc-rs", "dirs"] keyring = [ "keyring-core", @@ -27,7 +36,7 @@ keyring = [ ] [package.metadata.docs.rs] -features = ["gcs", "s3"] +features = ["gcs", "s3", "azure"] [dependencies] ambient-id = { workspace = true, features = ["reqwest-middleware"] } @@ -68,6 +77,11 @@ tokio = { workspace = true, optional = true } tracing = { workspace = true } url = { workspace = true } rattler_config = { workspace = true, optional = true } +reqsign-core = { workspace = true, optional = true } +reqsign-azure-storage = { workspace = true, optional = true } +reqsign-command-execute-tokio = { workspace = true, optional = true } +reqsign-file-read-tokio = { workspace = true, optional = true } +reqsign-http-send-reqwest = { workspace = true, optional = true } [target.'cfg( target_arch = "wasm32" )'.dependencies] getrandom = { workspace = true, features = ["wasm_js"] } diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs new file mode 100644 index 0000000000..e34602640b --- /dev/null +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -0,0 +1,269 @@ +//! Middleware to handle `az://` URLs to pull artifacts from Azure Blob Storage. +use std::collections::HashMap; + +use async_trait::async_trait; +use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; +use reqsign_command_execute_tokio::TokioCommandExecute; +use reqsign_core::{Context, OsEnv, Signer}; +use reqsign_file_read_tokio::TokioFileRead; +use reqsign_http_send_reqwest::ReqwestHttpSend; +use reqwest::{Client, Request, Response}; +use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; +use url::Url; + +/// The Azure Storage REST API version sent on every request. +const X_MS_VERSION: &str = "2021-12-02"; + +/// Per-container addressing configuration for the Azure middleware. +/// +/// This mirrors [`crate::s3_middleware::S3Config`] in spirit: it holds only the +/// information needed to *address* a container (account name and an optional +/// endpoint override). Credentials are resolved separately at request time via +/// reqsign's [`DefaultCredentialProvider`], so they are deliberately kept out +/// of this struct. +#[derive(Clone, Debug)] +pub struct AzureConfig { + /// Storage account name → host `{account}.blob.core.windows.net`. + pub account: String, + /// Optional full endpoint override for sovereign clouds / Azurite. + /// Defaults to `https://{account}.blob.core.windows.net`. + pub endpoint_url: Option, +} + +#[cfg(feature = "rattler_config")] +/// Compute the Azure configuration from the given Azure options. +pub fn compute_azure_config(azure_options: &M) -> HashMap +where + M: IntoIterator + Clone, +{ + azure_options + .clone() + .into_iter() + .map(|(k, v)| { + ( + k, + AzureConfig { + account: v.account, + endpoint_url: v.endpoint_url, + }, + ) + }) + .collect() +} + +#[cfg(feature = "rattler_config")] +/// Compute the Azure configuration from the `azure-options` of the shared +/// rattler configuration (see [`rattler_config`]). +/// +/// Accepts a [`rattler_config::config::CommonConfig`]; a `&ConfigBase` of +/// any extension coerces into it. +pub fn compute_azure_config_from_config( + config: &rattler_config::config::CommonConfig, +) -> HashMap { + config + .azure_options + .0 + .iter() + .map(|(container, options)| { + ( + container.clone(), + AzureConfig { + account: options.account.clone(), + endpoint_url: options.endpoint_url.clone(), + }, + ) + }) + .collect() +} + +/// Middleware that rewrites `az://{container}/{path}` URLs to HTTPS Azure Blob +/// Storage URLs and signs them via reqsign's Azure `DefaultCredentialProvider`. +#[derive(Clone)] +pub struct AzureMiddleware { + /// Container name -> addressing options (account, optional endpoint). + config: HashMap, + /// reqsign signer; caches the resolved credential internally. + signer: Signer, +} + +impl AzureMiddleware { + /// Create a new Azure middleware from a container -> config map. + pub fn new(config: HashMap) -> Self { + tracing::trace!("Creating Azure middleware using {:?}", config); + let client = Client::new(); + let ctx = Context::new() + .with_file_read(TokioFileRead) + .with_http_send(ReqwestHttpSend::new(client)) + .with_command_execute(TokioCommandExecute) + .with_env(OsEnv); + let signer = Signer::new(ctx, DefaultCredentialProvider::new(), RequestSigner::new()); + Self { config, signer } + } + + /// Resolve the HTTPS base host for a container from config. + /// Returns the endpoint origin, e.g. `https://acct.blob.core.windows.net`. + fn endpoint_for(&self, container: &str) -> MiddlewareResult { + let options = self.config.get(container).ok_or_else(|| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "no azure-options configured for container '{container}'" + )) + })?; + let endpoint = match &options.endpoint_url { + Some(url) => url.clone(), + None => Url::parse(&format!( + "https://{}.blob.core.windows.net", + options.account + )) + .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e)))?, + }; + Ok(endpoint) + } + + /// Rewrite an `az://{container}/{path}` URL to its HTTPS equivalent. + fn rewrite_url(&self, az_url: &Url) -> MiddlewareResult { + let container = az_url.host_str().ok_or_else(|| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "container should be present in az URL, got: {az_url}" + )) + })?; + let endpoint = self.endpoint_for(container)?; + let new_url = format!( + "{}/{}{}", + endpoint.as_str().trim_end_matches('/'), + container, + az_url.path() + ); + Url::parse(&new_url).map_err(|e| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "failed to parse constructed azure URL '{new_url}': {e}" + )) + }) + } + + /// Sign a reqwest `Request` in place using reqsign. + async fn sign(&self, req: &mut Request) -> MiddlewareResult<()> { + if !req.headers().contains_key("x-ms-version") { + req.headers_mut() + .insert("x-ms-version", http::HeaderValue::from_static(X_MS_VERSION)); + } + + let mut builder = http::Request::builder() + .method(req.method().clone()) + .uri(req.url().as_str()); + for (name, value) in req.headers() { + builder = builder.header(name, value); + } + let http_req = builder.body(()).map_err(|e| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "failed to build http request for signing: {e}" + )) + })?; + let (mut parts, ()) = http_req.into_parts(); + + self.signer + .sign(&mut parts, None) + .await + .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e)))?; + + *req.headers_mut() = parts.headers; + let signed_url = Url::parse(&parts.uri.to_string()).map_err(|e| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "failed to parse signed azure URL '{}': {e}", + parts.uri + )) + })?; + *req.url_mut() = signed_url; + Ok(()) + } +} + +#[async_trait] +impl Middleware for AzureMiddleware { + async fn handle( + &self, + mut req: Request, + extensions: &mut http::Extensions, + next: Next<'_>, + ) -> MiddlewareResult { + // Only intercept `az://` requests. + if req.url().scheme() != "az" { + return next.run(req, extensions).await; + } + + let https_url = self.rewrite_url(&req.url().clone())?; + *req.url_mut() = https_url; + self.sign(&mut req).await?; + next.run(req, extensions).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn opts(account: &str, endpoint: Option<&str>) -> AzureConfig { + AzureConfig { + account: account.to_string(), + endpoint_url: endpoint.map(|e| Url::parse(e).unwrap()), + } + } + + #[test] + fn rewrites_default_endpoint() { + let mw = AzureMiddleware::new(HashMap::from([( + "mychannel".to_string(), + opts("myacct", None), + )])); + let rewritten = mw + .rewrite_url(&Url::parse("az://mychannel/noarch/repodata.json").unwrap()) + .unwrap(); + assert_eq!( + rewritten.as_str(), + "https://myacct.blob.core.windows.net/mychannel/noarch/repodata.json" + ); + } + + #[test] + fn rewrites_override_endpoint_for_azurite() { + let mw = AzureMiddleware::new(HashMap::from([( + "devstoreaccount1".to_string(), + opts( + "devstoreaccount1", + Some("http://127.0.0.1:10000/devstoreaccount1"), + ), + )])); + let rewritten = mw + .rewrite_url(&Url::parse("az://devstoreaccount1/noarch/repodata.json").unwrap()) + .unwrap(); + assert_eq!( + rewritten.as_str(), + "http://127.0.0.1:10000/devstoreaccount1/devstoreaccount1/noarch/repodata.json" + ); + } + + #[test] + fn errors_when_container_not_configured() { + let mw = AzureMiddleware::new(HashMap::new()); + let err = mw + .rewrite_url(&Url::parse("az://missing/noarch/repodata.json").unwrap()) + .unwrap_err(); + assert!(err.to_string().contains("missing")); + } + + #[tokio::test] + async fn passes_through_non_az_schemes_unchanged() { + use reqwest_middleware::ClientBuilder; + let mw = AzureMiddleware::new(HashMap::new()); + let client = ClientBuilder::new(Client::new()).with(mw).build(); + let result = client + .get("https://this-host-does-not-exist.invalid/x") + .send() + .await; + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!( + !err.to_string().contains("azure-options"), + "non-az request must not hit azure config lookup: {err}" + ); + } +} diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs index c11d21fdd6..b24a557056 100644 --- a/crates/rattler_networking/src/lib.rs +++ b/crates/rattler_networking/src/lib.rs @@ -21,6 +21,11 @@ pub mod s3_middleware; #[cfg(feature = "s3")] pub use s3_middleware::S3Middleware; +#[cfg(feature = "azure")] +pub mod azure_middleware; +#[cfg(feature = "azure")] +pub use azure_middleware::AzureMiddleware; + pub mod authentication_middleware; pub mod authentication_storage; pub mod challenge_middleware; From b55a4a208eb11159b684963dc37829d219c76f17 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:37:50 +1000 Subject: [PATCH 26/98] feat(bin): enable azure feature and register AzureMiddleware --- crates/rattler-bin/Cargo.toml | 3 ++- crates/rattler-bin/src/commands/client.rs | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index b637e5c91d..a9b7d872e2 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -16,7 +16,7 @@ name = "rattler" path = "src/main.rs" [features] -default = ["rustls", "s3", "gcs", "oauth"] +default = ["rustls", "s3", "gcs", "azure", "oauth"] native-tls = [ "reqwest/native-tls", "rattler/native-tls", @@ -33,6 +33,7 @@ rustls = [ ] s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] +azure = ["rattler_networking/azure", "rattler_upload/azure"] oauth = ["rattler/oauth"] [dependencies] diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index 6897a16d15..22062293f1 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -57,6 +57,11 @@ pub fn create_client_with_middleware( )); #[cfg(feature = "gcs")] let client = client.with(rattler_networking::GCSMiddleware::default()); + // Empty map like S3 above — rattler-bin loads no rattler_config, so there are + // no per-container azure-options to plumb. Real consumers (e.g. pixi) build the + // map via rattler_networking::compute_azure_config_from_config. + #[cfg(feature = "azure")] + let client = client.with(rattler_networking::AzureMiddleware::new(HashMap::new())); Ok(client.build()) } From 895bf1599f3cc9f1bc5f95b22f67f110ceb71c03 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Tue, 21 Jul 2026 18:12:31 +1000 Subject: [PATCH 27/98] fix(azure): address review findings (host validation, enum config, streaming) --- crates/rattler_config/src/config/azure.rs | 9 +- crates/rattler_index/src/lib.rs | 4 +- crates/rattler_index/src/main.rs | 12 ++- .../src/azure_middleware.rs | 93 +++++++++++-------- crates/rattler_upload/src/upload/azure.rs | 44 +++++---- crates/rattler_upload/src/upload/opt.rs | 12 ++- 6 files changed, 109 insertions(+), 65 deletions(-) diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs index df927f39f3..54691b9b76 100644 --- a/crates/rattler_config/src/config/azure.rs +++ b/crates/rattler_config/src/config/azure.rs @@ -17,11 +17,14 @@ impl AzureOptionsMap { #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub struct AzureOptions { - /// Storage account name → host `{account}.blob.core.windows.net`. + /// Storage account name. When `endpoint-url` is not set, the endpoint is + /// derived from it as `https://{account}.blob.core.windows.net`. pub account: String, - /// Optional full endpoint override for sovereign clouds / Azurite. - /// Defaults to `https://{account}.blob.core.windows.net`. + /// Optional full endpoint override for sovereign clouds / custom endpoints + /// (e.g. Azurite). When set, this is used verbatim as the endpoint and + /// `account` is then only used to construct the default endpoint when this + /// is absent — i.e. it has no effect on addressing while an override is set. pub endpoint_url: Option, } diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 18baab653f..85517bd05a 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -1622,8 +1622,8 @@ fn azblob_config( .ok_or_else(|| anyhow::anyhow!("No container in Azure blob URL"))?; let root = format!("/{}", segments.collect::>().join("/")); - // Preserve a non-default port so custom endpoints (e.g. the Azurite - // emulator on :10000) work; real Azure uses the scheme default (443). + // Preserve a non-default port if one is present; real Azure uses the scheme + // default (443). let authority = match channel.port() { Some(port) => format!("{host}:{port}"), None => host.to_string(), diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 671fd9d2f5..436a87af77 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -41,13 +41,19 @@ fn parse_s3_url(value: &str) -> Result { #[cfg(feature = "azure")] fn parse_azure_url(value: &str) -> Result { let url: Url = Url::parse(value).map_err(|e| format!("`{value}` isn't a valid URL: {e}"))?; - // Require host + a container segment, e.g. - // https://.blob.core.windows.net//. + // Require an `.blob.` host and a container segment, e.g. + // https://.blob.core.windows.net//. The index + // path derives the storage account from the host, so it must be a dotted + // domain (real Azure and sovereign clouds always are). IP literals and + // single-label hosts (localhost, the Azurite emulator) are rejected here + // because no account can be derived from them. + let has_dotted_domain = + matches!(url.host(), Some(url::Host::Domain(host)) if host.contains('.')); let has_container = url .path_segments() .and_then(|mut segments| segments.next()) .is_some_and(|segment| !segment.is_empty()); - if matches!(url.scheme(), "http" | "https") && url.host_str().is_some() && has_container { + if matches!(url.scheme(), "http" | "https") && has_dotted_domain && has_container { Ok(url) } else { Err(format!( diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index e34602640b..30aadadd7e 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -16,18 +16,22 @@ const X_MS_VERSION: &str = "2021-12-02"; /// Per-container addressing configuration for the Azure middleware. /// -/// This mirrors [`crate::s3_middleware::S3Config`] in spirit: it holds only the -/// information needed to *address* a container (account name and an optional -/// endpoint override). Credentials are resolved separately at request time via -/// reqsign's [`DefaultCredentialProvider`], so they are deliberately kept out -/// of this struct. +/// Like [`crate::s3_middleware::S3Config`] this is an enum, so exactly one of +/// the two addressing modes is representable: either a storage account (from +/// which the default endpoint is derived) or a full endpoint override. It holds +/// only the information needed to *address* a container; credentials are +/// resolved separately at request time via reqsign's +/// [`DefaultCredentialProvider`], so they are kept out of this type. +/// +/// Unlike `S3Config`, there is no default-provider fallback: a container that is +/// not present in the middleware's config map is an error at request time. #[derive(Clone, Debug)] -pub struct AzureConfig { - /// Storage account name → host `{account}.blob.core.windows.net`. - pub account: String, - /// Optional full endpoint override for sovereign clouds / Azurite. - /// Defaults to `https://{account}.blob.core.windows.net`. - pub endpoint_url: Option, +pub enum AzureConfig { + /// Address the container via `https://{account}.blob.core.windows.net`. + Account(String), + /// Address the container via a full endpoint override (sovereign clouds / + /// custom endpoints such as Azurite). + Endpoint(Url), } #[cfg(feature = "rattler_config")] @@ -42,10 +46,8 @@ where .map(|(k, v)| { ( k, - AzureConfig { - account: v.account, - endpoint_url: v.endpoint_url, - }, + v.endpoint_url + .map_or_else(|| AzureConfig::Account(v.account), AzureConfig::Endpoint), ) }) .collect() @@ -67,20 +69,26 @@ pub fn compute_azure_config_from_config( .map(|(container, options)| { ( container.clone(), - AzureConfig { - account: options.account.clone(), - endpoint_url: options.endpoint_url.clone(), - }, + options.endpoint_url.clone().map_or_else( + || AzureConfig::Account(options.account.clone()), + AzureConfig::Endpoint, + ), ) }) .collect() } /// Middleware that rewrites `az://{container}/{path}` URLs to HTTPS Azure Blob -/// Storage URLs and signs them via reqsign's Azure `DefaultCredentialProvider`. +/// Storage URLs and signs them. +/// +/// Credentials are resolved by reqsign's [`DefaultCredentialProvider`] chain, in +/// its usual order: environment variables, then workload/managed identity, then +/// the Azure CLI (`az login`). rattler's [`crate::AuthenticationStorage`] is not +/// consulted for Azure — there is no `Authentication` Azure variant — so +/// per-host credentials configured there do not apply to `az://` requests. #[derive(Clone)] pub struct AzureMiddleware { - /// Container name -> addressing options (account, optional endpoint). + /// Container name -> addressing config (account or full endpoint override). config: HashMap, /// reqsign signer; caches the resolved credential internally. signer: Signer, @@ -100,23 +108,25 @@ impl AzureMiddleware { Self { config, signer } } - /// Resolve the HTTPS base host for a container from config. - /// Returns the endpoint origin, e.g. `https://acct.blob.core.windows.net`. + /// Resolve the endpoint base URL for a container from config. + /// + /// For an [`AzureConfig::Account`] this is the endpoint origin + /// `https://{account}.blob.core.windows.net`. For an + /// [`AzureConfig::Endpoint`] the configured override is returned verbatim, + /// which may carry a path component (e.g. Azurite endpoints). fn endpoint_for(&self, container: &str) -> MiddlewareResult { - let options = self.config.get(container).ok_or_else(|| { + let config = self.config.get(container).ok_or_else(|| { reqwest_middleware::Error::Middleware(anyhow::anyhow!( "no azure-options configured for container '{container}'" )) })?; - let endpoint = match &options.endpoint_url { - Some(url) => url.clone(), - None => Url::parse(&format!( - "https://{}.blob.core.windows.net", - options.account - )) - .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e)))?, - }; - Ok(endpoint) + match config { + AzureConfig::Endpoint(url) => Ok(url.clone()), + AzureConfig::Account(account) => { + Url::parse(&format!("https://{account}.blob.core.windows.net")) + .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e))) + } + } } /// Rewrite an `az://{container}/{path}` URL to its HTTPS equivalent. @@ -127,11 +137,18 @@ impl AzureMiddleware { )) })?; let endpoint = self.endpoint_for(container)?; + let query = az_url.query().map(|q| format!("?{q}")).unwrap_or_default(); + let fragment = az_url + .fragment() + .map(|f| format!("#{f}")) + .unwrap_or_default(); let new_url = format!( - "{}/{}{}", + "{}/{}{}{}{}", endpoint.as_str().trim_end_matches('/'), container, - az_url.path() + az_url.path(), + query, + fragment ); Url::parse(&new_url).map_err(|e| { reqwest_middleware::Error::Middleware(anyhow::anyhow!( @@ -202,9 +219,9 @@ mod tests { use super::*; fn opts(account: &str, endpoint: Option<&str>) -> AzureConfig { - AzureConfig { - account: account.to_string(), - endpoint_url: endpoint.map(|e| Url::parse(e).unwrap()), + match endpoint { + Some(e) => AzureConfig::Endpoint(Url::parse(e).unwrap()), + None => AzureConfig::Account(account.to_string()), } } diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 320a1ced42..dc66a14967 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -11,20 +11,27 @@ use url::Url; use crate::upload::package::ExtractedPackage; -/// Size of a single chunk handed to the writer. Azure Blob storage expects data -/// to be uploaded in reasonably sized blocks; we buffer into 10 MiB chunks. +/// Size of a single block handed to the writer. Azure Blob block uploads keep +/// the number of blocks low with larger blocks; we use 10 MiB. const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; +/// Number of blocks of a single package that are uploaded concurrently. +const PART_CONCURRENCY: usize = 4; + /// Number of packages that are uploaded concurrently. const PACKAGE_CONCURRENCY: usize = 4; /// Uploads packages to a channel in an Azure Blob Storage container. /// -/// The channel URL is expected to be of the form +/// The channel URL must be of the form /// `https://.blob.core.windows.net//`; the account /// name, endpoint, container, and root prefix are all derived from it (see -/// [`azblob_config`]). The [`AzureCredentials`] supply only the account key or -/// SAS token. +/// [`azblob_config`]). Because the account is derived from the host, upload +/// requires this dotted `.blob...` form and does not support +/// path-style or emulator (Azurite) endpoints. (The fetch middleware, by +/// contrast, does support custom endpoints — including Azurite — via +/// `azure-options..endpoint-url`.) The [`AzureCredentials`] supply +/// only the account key or SAS token. pub async fn upload_package_to_azure( channel: Url, credentials: AzureCredentials, @@ -89,12 +96,17 @@ async fn upload_single_package( // Rewind the file to the beginning. file.rewind().await.into_diagnostic()?; - // Construct a writer for the package. `if_not_exists(!force)` maps to an - // `If-None-Match: *` precondition so an existing blob is not silently - // overwritten unless `--force` was passed. + // Construct a writer for the package. Setting `chunk` and `concurrent` + // enables opendal's concurrent block upload: data is buffered into + // `DESIRED_CHUNK_SIZE` blocks and up to `PART_CONCURRENCY` blocks are + // uploaded in parallel. `if_not_exists(!force)` maps to an `If-None-Match: *` + // precondition so an existing blob is not silently overwritten unless + // `--force` was passed. (opendal's azblob backend does not support setting + // `content_disposition` on write, so it is not set here.) let mut writer = match op .writer_with(&key) - .content_disposition(&format!("attachment; filename={filename}")) + .chunk(DESIRED_CHUNK_SIZE) + .concurrent(PART_CONCURRENCY) .if_not_exists(!force) .user_metadata([ (String::from("package-sha256"), hex::encode(sha256hash)), @@ -114,10 +126,9 @@ async fn upload_single_package( } }; - // Stream the file to the writer in `DESIRED_CHUNK_SIZE` chunks. We do this in - // a more complex way than a plain `io::copy` because the underlying storage - // provider expects to receive the data in specifically sized chunks. The code - // below guarantees chunks of equal size except for maybe the last chunk. + // Stream the file to the writer in `DESIRED_CHUNK_SIZE` chunks. opendal takes + // care of buffering these into correctly sized blocks and uploading them + // concurrently. let mut remaining_size = size as usize; while remaining_size > 0 { // Allocate memory for this chunk. @@ -131,7 +142,8 @@ async fn upload_single_package( let bytes_read = file.read_exact(&mut chunk[..]).await.into_diagnostic()?; debug_assert_eq!(bytes_read, chunk.len()); - // Hand the chunk to the writer. + // Hand the chunk to the writer. With concurrent writes enabled this returns + // as soon as the chunk is queued rather than fully uploaded. writer.write(chunk.freeze()).await.into_diagnostic()?; // Update the number of remaining bytes. @@ -183,8 +195,8 @@ fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Resul .ok_or_else(|| miette::miette!("No container in Azure blob URL"))?; let root = format!("/{}", segments.collect::>().join("/")); - // Preserve a non-default port so custom endpoints (e.g. the Azurite - // emulator on :10000) work; real Azure uses the scheme default (443). + // Preserve a non-default port if one is present; real Azure uses the scheme + // default (443). let authority = match channel.port() { Some(port) => format!("{host}:{port}"), None => host.to_string(), diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index e26fe17f11..324066f032 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -422,13 +422,19 @@ pub struct S3Opts { fn parse_azure_url(value: &str) -> Result { let url: Url = Url::parse(value).map_err(|err| format!("`{value}` isn't a valid URL: {err}"))?; - // Require host + a container segment, e.g. - // https://.blob.core.windows.net//. + // Require an `.blob.` host and a container segment, e.g. + // https://.blob.core.windows.net//. The upload + // path derives the storage account from the host, so it must be a dotted + // domain (real Azure and sovereign clouds always are). IP literals and + // single-label hosts (localhost, the Azurite emulator) are rejected here + // because no account can be derived from them. + let has_dotted_domain = + matches!(url.host(), Some(url::Host::Domain(host)) if host.contains('.')); let has_container = url .path_segments() .and_then(|mut segments| segments.next()) .is_some_and(|segment| !segment.is_empty()); - if matches!(url.scheme(), "http" | "https") && url.host_str().is_some() && has_container { + if matches!(url.scheme(), "http" | "https") && has_dotted_domain && has_container { Ok(url) } else { Err(format!( From 67f27df74a1e9dc99356483aadb7cec59f80959a Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:40:01 +1000 Subject: [PATCH 28/98] feat(azure): mint user-delegation SAS from az login for write paths --- Cargo.lock | 1 + crates/rattler_azure/Cargo.toml | 7 ++ crates/rattler_azure/src/clap.rs | 118 ++++++++++++++++++---- crates/rattler_azure/src/lib.rs | 101 +++++++++++++++++- crates/rattler_index/README.md | 36 +++++-- crates/rattler_index/src/main.rs | 32 +++++- crates/rattler_upload/src/lib.rs | 11 +- crates/rattler_upload/src/upload/azure.rs | 45 ++++++--- crates/rattler_upload/src/upload/mod.rs | 2 + 9 files changed, 301 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d8fd75c39..fe7608b99f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5050,6 +5050,7 @@ name = "rattler_azure" version = "0.1.0" dependencies = [ "clap", + "jiff", "serde", "thiserror 2.0.18", "url", diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index 86492ab7d3..45a17c1498 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -11,9 +11,16 @@ readme.workspace = true [features] default = [] +# Minting a short-lived user-delegation SAS by shelling out to the Azure CLI. +azure-cli-sas = ["dep:jiff"] +# CLI credential options; resolving them may mint a SAS via the Azure CLI. +clap = ["dep:clap", "azure-cli-sas"] +# Serialize/deserialize support for the credential types. +serde = ["dep:serde", "serde/derive"] [dependencies] clap = { workspace = true, optional = true } +jiff = { workspace = true, optional = true } thiserror = { workspace = true } url = { workspace = true } diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index 2be874a591..77d65d14a2 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -1,19 +1,41 @@ +use std::time::Duration; + use clap::Parser; -use crate::{AzureCredentials, MissingAzureCredentials}; +use crate::{AzureCliSasError, AzureCredentials, mint_user_delegation_sas}; + +/// How long a SAS minted from an `az login` session stays valid. +/// +/// SAS tokens are deliberately short-lived: a SAS cannot be individually +/// revoked, so a short lifetime keeps the blast radius small if one leaks. Thirty +/// minutes comfortably covers an index or upload run. +const AZURE_CLI_SAS_TTL: Duration = Duration::from_secs(30 * 60); + +/// Errors that can occur while resolving [`AzureCredentialsOpts`] into +/// [`AzureCredentials`]. +#[derive(Debug, thiserror::Error)] +pub enum AzureCredentialsError { + /// No credential source was supplied. + #[error("no Azure credentials supplied: pass --account-key, --sas-token, or --azure-cli")] + Missing, + + /// Minting a SAS via the Azure CLI failed. + #[error("failed to mint a user-delegation SAS from the Azure CLI")] + Cli(#[from] AzureCliSasError), +} /// Manually specified Azure Blob credentials. /// /// See [`super::AzureCredentials`] for details on how these credentials are -/// used. `--account-key` and `--sas-token` are mutually exclusive; exactly one -/// must be supplied. +/// used. `--account-key`, `--sas-token`, and `--azure-cli` are mutually +/// exclusive; exactly one must be supplied. #[derive(Clone, Debug, PartialEq, Parser)] pub struct AzureCredentialsOpts { /// The Azure Storage account key. #[arg( long, env = "AZURE_STORAGE_KEY", - conflicts_with = "sas_token", + conflicts_with_all = ["sas_token", "azure_cli"], help_heading = "Azure Credentials" )] pub account_key: Option, @@ -22,24 +44,46 @@ pub struct AzureCredentialsOpts { #[arg( long, env = "AZURE_STORAGE_SAS_TOKEN", - conflicts_with = "account_key", + conflicts_with_all = ["account_key", "azure_cli"], help_heading = "Azure Credentials" )] pub sas_token: Option, -} -impl TryFrom for AzureCredentials { - type Error = MissingAzureCredentials; + /// Mint a short-lived user-delegation SAS from the current `az login` + /// session (requires the Azure CLI). + #[arg( + long, + conflicts_with_all = ["account_key", "sas_token"], + help_heading = "Azure Credentials" + )] + pub azure_cli: bool, +} - fn try_from(value: AzureCredentialsOpts) -> Result { - // `conflicts_with` guarantees at most one of the two is set, so the +impl AzureCredentialsOpts { + /// Resolve the supplied options into a concrete [`AzureCredentials`]. + /// + /// `account`, `container`, and `permissions` are only used by the + /// `--azure-cli` path, which mints a short-lived user-delegation SAS scoped + /// to that container with those permissions. The account and container are + /// derived by the caller from the channel URL. + pub fn resolve( + self, + account: &str, + container: &str, + permissions: &str, + ) -> Result { + // `conflicts_with_all` guarantees at most one source is set, so the // order of these checks doesn't matter. - if let Some(account_key) = value.account_key { + if let Some(account_key) = self.account_key { Ok(AzureCredentials::AccountKey(account_key)) - } else if let Some(sas_token) = value.sas_token { + } else if let Some(sas_token) = self.sas_token { Ok(AzureCredentials::SasToken(sas_token)) + } else if self.azure_cli { + let token = + mint_user_delegation_sas(account, container, permissions, AZURE_CLI_SAS_TTL)?; + Ok(AzureCredentials::SasToken(token)) } else { - Err(MissingAzureCredentials) + Err(AzureCredentialsError::Missing) } } } @@ -53,9 +97,10 @@ mod tests { let opts = AzureCredentialsOpts { account_key: Some("key".into()), sas_token: None, + azure_cli: false, }; assert!(matches!( - AzureCredentials::try_from(opts), + opts.resolve("acct", "container", "cw"), Ok(AzureCredentials::AccountKey(k)) if k == "key" )); } @@ -65,19 +110,58 @@ mod tests { let opts = AzureCredentialsOpts { account_key: None, sas_token: Some("sv=...".into()), + azure_cli: false, }; assert!(matches!( - AzureCredentials::try_from(opts), + opts.resolve("acct", "container", "cw"), Ok(AzureCredentials::SasToken(t)) if t == "sv=..." )); } #[test] - fn neither_is_rejected() { + fn none_is_rejected() { let opts = AzureCredentialsOpts { account_key: None, sas_token: None, + azure_cli: false, }; - assert!(AzureCredentials::try_from(opts).is_err()); + assert!(matches!( + opts.resolve("acct", "container", "cw"), + Err(AzureCredentialsError::Missing) + )); + } + + // The `--azure-cli` variant shells out to `az`, which isn't available in the + // test environment, so we only assert the flag is wired through the parser + // (its conflicts are enforced by clap, exercised below) rather than invoking + // the mint path. + #[test] + fn azure_cli_flag_parses() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + creds: AzureCredentialsOpts, + } + + let cli = Cli::try_parse_from(["test", "--azure-cli"]).expect("should parse"); + assert!(cli.creds.azure_cli); + assert!(cli.creds.account_key.is_none()); + assert!(cli.creds.sas_token.is_none()); + } + + #[test] + fn azure_cli_conflicts_with_other_sources() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + creds: AzureCredentialsOpts, + } + + assert!(Cli::try_parse_from(["test", "--azure-cli", "--account-key", "k"]).is_err()); + assert!(Cli::try_parse_from(["test", "--azure-cli", "--sas-token", "s"]).is_err()); } } diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 4a1b962ea3..4faff916c4 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -18,7 +18,102 @@ pub enum AzureCredentials { SasToken(String), } -/// Error returned when no Azure credentials were supplied. +/// Errors that can occur while minting a user-delegation SAS via the Azure CLI. +#[cfg(feature = "azure-cli-sas")] #[derive(Debug, thiserror::Error)] -#[error("no Azure credentials supplied: pass either an account key or a SAS token")] -pub struct MissingAzureCredentials; +pub enum AzureCliSasError { + /// The SAS expiry timestamp could not be computed. + #[error("failed to compute the SAS expiry timestamp: {0}")] + Expiry(String), + + /// The `az` executable could not be found on `PATH`. + #[error("could not find the Azure CLI (`az`) on PATH; install it and run `az login`")] + AzNotFound(#[source] std::io::Error), + + /// The `az` process could not be spawned. + #[error("failed to run the Azure CLI (`az`)")] + Spawn(#[source] std::io::Error), + + /// `az` exited with a non-zero status. + #[error("the Azure CLI failed to generate a user-delegation SAS (is `az login` current?): {0}")] + CommandFailed(String), + + /// `az` succeeded but produced no SAS token. + #[error("the Azure CLI returned an empty SAS token")] + EmptyOutput, +} + +/// Mint a short-lived user-delegation SAS for a container by shelling out to the +/// Azure CLI. +/// +/// opendal's azblob backend (used by the index and upload write paths) only +/// accepts a shared account key or a SAS token, not an AAD bearer token. To let +/// users authenticate writes with their `az login` session, this converts that +/// session into a SAS via: +/// +/// ```text +/// az storage container generate-sas --account-name --name +/// --permissions --expiry --auth-mode login --as-user +/// --https-only -o tsv +/// ``` +/// +/// `permissions` is the Azure SAS permission string (e.g. `"cw"`). The returned +/// token has no leading `?`. Requires `az` on `PATH` and a prior `az login`. +/// +/// This blocks the calling thread while the `az` process runs; it is meant to be +/// called once at setup time. +#[cfg(feature = "azure-cli-sas")] +pub fn mint_user_delegation_sas( + account: &str, + container: &str, + permissions: &str, + valid_for: std::time::Duration, +) -> Result { + let signed = jiff::SignedDuration::try_from(valid_for) + .map_err(|err| AzureCliSasError::Expiry(err.to_string()))?; + let expiry = jiff::Timestamp::now() + .checked_add(signed) + .map_err(|err| AzureCliSasError::Expiry(err.to_string()))?; + // `az` expects an ISO-8601 UTC timestamp; minute precision is sufficient. + let expiry = expiry.strftime("%Y-%m-%dT%H:%MZ").to_string(); + + let output = std::process::Command::new("az") + .args([ + "storage", + "container", + "generate-sas", + "--account-name", + account, + "--name", + container, + "--permissions", + permissions, + "--expiry", + &expiry, + "--auth-mode", + "login", + "--as-user", + "--https-only", + "-o", + "tsv", + ]) + .output() + .map_err(|err| { + if err.kind() == std::io::ErrorKind::NotFound { + AzureCliSasError::AzNotFound(err) + } else { + AzureCliSasError::Spawn(err) + } + })?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return Err(AzureCliSasError::CommandFailed(stderr)); + } + + let token = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if token.is_empty() { + return Err(AzureCliSasError::EmptyOutput); + } + Ok(token) +} diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index bbf6364f40..39dd139dd7 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -24,17 +24,35 @@ rattler-index --config ./rattler-config.toml s3 s3://my-bucket/my-channel Index an Azure Blob Storage channel: ```shell -rattler-index --config ./rattler-config.toml azure \ - --channel az://my-container/my-channel \ - --account my-storage-account \ - [--endpoint-url https://my-storage-account.blob.core.windows.net] +rattler-index --config ./rattler-config.toml azblob \ + https://my-storage-account.blob.core.windows.net/my-container/my-channel \ + --azure-cli ``` -Azure channels need no secrets on the command line. For local development run -`az login`; in CI, authentication is resolved through reqsign's -`DefaultCredentialProvider` chain, which covers managed identity, workload -identity, and service-principal environment variables. The `--endpoint-url` -flag is only needed for sovereign clouds or a local Azurite emulator. +Indexing writes to the container through opendal, which only accepts a storage +account key or a shared access signature (SAS) token — it cannot use an `az +login` AAD bearer token directly. Supply one of: + +- `--azure-cli`: mint a short-lived user-delegation SAS from the current `az + login` session automatically. Requires the Azure CLI (`az`) on `PATH` and a + prior `az login`. The minted SAS is scoped to the target container, granted + only the permissions indexing needs, and expires after 30 minutes (a SAS + cannot be individually revoked, so it is kept short-lived on purpose). +- `--account-key` / `AZURE_STORAGE_KEY`: a storage account key. +- `--sas-token` / `AZURE_STORAGE_SAS_TOKEN`: a SAS token you supply yourself. + +To mint a SAS manually instead of using `--azure-cli` (for example, to reuse it +across several commands), generate one from your `az login` session and pass it +via `--sas-token`: + +```shell +export AZURE_STORAGE_SAS_TOKEN=$(az storage container generate-sas \ + --account-name my-storage-account --name my-container \ + --permissions rwlc --expiry 2026-01-01T00:00Z \ + --auth-mode login --as-user --https-only -o tsv) +rattler-index --config ./rattler-config.toml azblob \ + https://my-storage-account.blob.core.windows.net/my-container/my-channel +``` The `--config` flag points at the same TOML configuration file used by pixi. It configures S3 and Azure credentials, concurrency, and per-channel index options diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 436a87af77..98720aec70 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -4,8 +4,6 @@ use std::path::PathBuf; use anyhow::Context; use clap::{Parser, Subcommand}; use clap_verbosity_flag::Verbosity; -#[cfg(feature = "azure")] -use rattler_azure::AzureCredentials; use rattler_conda_types::Platform; use rattler_config::config::{ concurrency::default_max_concurrent_solves, index::IndexChannelConfig, @@ -62,6 +60,32 @@ fn parse_azure_url(value: &str) -> Result { } } +/// SAS permissions requested when minting a user-delegation SAS for indexing. +/// Indexing does a read-modify-write of repodata and lists/reads packages, so it +/// needs read, write, list, and create (`r` + `w` + `l` + `c`). +#[cfg(feature = "azure")] +const AZURE_INDEX_SAS_PERMISSIONS: &str = "rwlc"; + +/// Derive the storage account name and container from an Azure Blob channel URL +/// of the form `https://.blob.core.windows.net//`. +#[cfg(feature = "azure")] +fn azure_account_and_container(channel: &Url) -> anyhow::Result<(String, String)> { + let host = channel + .host_str() + .ok_or_else(|| anyhow::anyhow!("No host in Azure blob URL"))?; + let account_name = host + .split('.') + .next() + .filter(|name| !name.is_empty()) + .ok_or_else(|| anyhow::anyhow!("Could not derive account name from Azure blob URL"))?; + let container = channel + .path_segments() + .and_then(|mut segments| segments.next()) + .filter(|segment| !segment.is_empty()) + .ok_or_else(|| anyhow::anyhow!("No container in Azure blob URL"))?; + Ok((account_name.to_string(), container.to_string())) +} + /// The `rattler-index` CLI. #[derive(Parser)] #[command(name = "rattler-index", version, about, long_about = None)] @@ -267,7 +291,9 @@ async fn main() -> anyhow::Result<()> { effective_index_options(&resolved); let channel_metadata = ChannelMetadata::from_index_config(&resolved); - let credentials = AzureCredentials::try_from(credentials)?; + let (account, container) = azure_account_and_container(&channel)?; + let credentials = + credentials.resolve(&account, &container, AZURE_INDEX_SAS_PERMISSIONS)?; index_azure_with_channel_metadata( IndexAzureConfig { diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index 7b965ad2e8..3275c35797 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -83,12 +83,11 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { } #[cfg(feature = "azure")] ServerType::Azure(azure_opts) => { - let credentials = - rattler_azure::AzureCredentials::try_from(azure_opts.credentials).map_err(|err| { - miette::miette!( - "{err}. Provide either --account-key or --sas-token (or the corresponding environment variables)." - ) - })?; + let (account, container) = upload::azure_account_and_container(&azure_opts.channel)?; + let credentials = azure_opts + .credentials + .resolve(&account, &container, upload::AZURE_UPLOAD_SAS_PERMISSIONS) + .into_diagnostic()?; upload::upload_package_to_azure( azure_opts.channel, credentials, diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index dc66a14967..32de022c03 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -21,6 +21,29 @@ const PART_CONCURRENCY: usize = 4; /// Number of packages that are uploaded concurrently. const PACKAGE_CONCURRENCY: usize = 4; +/// SAS permissions requested when minting a user-delegation SAS for uploads. +/// Uploading only needs to create and write blobs (`c` + `w`). +pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "cw"; + +/// Derive the storage account name and container from an Azure Blob channel URL +/// of the form `https://.blob.core.windows.net//`. +pub(crate) fn azure_account_and_container(channel: &Url) -> miette::Result<(String, String)> { + let host = channel + .host_str() + .ok_or_else(|| miette::miette!("No host in Azure blob URL"))?; + let account_name = host + .split('.') + .next() + .filter(|name| !name.is_empty()) + .ok_or_else(|| miette::miette!("Could not derive account name from Azure blob URL"))?; + let container = channel + .path_segments() + .and_then(|mut segments| segments.next()) + .filter(|segment| !segment.is_empty()) + .ok_or_else(|| miette::miette!("No container in Azure blob URL"))?; + Ok((account_name.to_string(), container.to_string())) +} + /// Uploads packages to a channel in an Azure Blob Storage container. /// /// The channel URL must be of the form @@ -177,26 +200,20 @@ async fn upload_single_package( /// the URL (`https://.blob.core.windows.net//`); the /// credentials supply only the account key or SAS token. fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Result { - let host = channel - .host_str() - .ok_or_else(|| miette::miette!("No host in Azure blob URL"))?; - let account_name = host - .split('.') - .next() - .filter(|name| !name.is_empty()) - .ok_or_else(|| miette::miette!("Could not derive account name from Azure blob URL"))?; + let (account_name, container) = azure_account_and_container(channel)?; let mut segments = channel .path_segments() .ok_or_else(|| miette::miette!("No path in Azure blob URL"))?; - let container = segments - .next() - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| miette::miette!("No container in Azure blob URL"))?; + // Skip the container segment; the remainder is the root prefix. + segments.next(); let root = format!("/{}", segments.collect::>().join("/")); // Preserve a non-default port if one is present; real Azure uses the scheme // default (443). + let host = channel + .host_str() + .ok_or_else(|| miette::miette!("No host in Azure blob URL"))?; let authority = match channel.port() { Some(port) => format!("{host}:{port}"), None => host.to_string(), @@ -209,8 +226,8 @@ fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Resul Ok(AzblobConfig { endpoint: Some(format!("{}://{}", channel.scheme(), authority)), - account_name: Some(account_name.to_string()), - container: container.to_string(), + account_name: Some(account_name), + container, root: Some(root), account_key, sas_token, diff --git a/crates/rattler_upload/src/upload/mod.rs b/crates/rattler_upload/src/upload/mod.rs index 875cb05720..a35d86face 100644 --- a/crates/rattler_upload/src/upload/mod.rs +++ b/crates/rattler_upload/src/upload/mod.rs @@ -32,6 +32,8 @@ mod anaconda; mod azure; #[cfg(feature = "azure")] pub use azure::upload_package_to_azure; +#[cfg(feature = "azure")] +pub(crate) use azure::{AZURE_UPLOAD_SAS_PERMISSIONS, azure_account_and_container}; #[cfg(feature = "sigstore-sign")] pub mod attestation; mod cloudsmith; From 18b740de581eb70cc3017f089304d98f7418a7ee Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:11:52 +1000 Subject: [PATCH 29/98] fix(azure): typed auth source, configurable SAS ttl, windows az, dedup url helper --- crates/rattler_azure/Cargo.toml | 7 +- crates/rattler_azure/src/clap.rs | 250 +++++++++++++++------- crates/rattler_azure/src/lib.rs | 94 +++++++- crates/rattler_index/src/lib.rs | 22 +- crates/rattler_index/src/main.rs | 26 +-- crates/rattler_upload/src/lib.rs | 8 +- crates/rattler_upload/src/upload/azure.rs | 27 +-- crates/rattler_upload/src/upload/mod.rs | 4 +- 8 files changed, 292 insertions(+), 146 deletions(-) diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index 45a17c1498..44ced641c2 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -11,10 +11,9 @@ readme.workspace = true [features] default = [] -# Minting a short-lived user-delegation SAS by shelling out to the Azure CLI. -azure-cli-sas = ["dep:jiff"] -# CLI credential options; resolving them may mint a SAS via the Azure CLI. -clap = ["dep:clap", "azure-cli-sas"] +# CLI credential options; resolving them may mint a short-lived user-delegation +# SAS by shelling out to the Azure CLI (which needs `jiff` for the expiry). +clap = ["dep:clap", "dep:jiff"] # Serialize/deserialize support for the credential types. serde = ["dep:serde", "serde/derive"] diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index 77d65d14a2..36337157a5 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -2,14 +2,14 @@ use std::time::Duration; use clap::Parser; -use crate::{AzureCliSasError, AzureCredentials, mint_user_delegation_sas}; +use crate::{AzureCliSasError, AzureCredentials, AzureUrlError, mint_user_delegation_sas}; -/// How long a SAS minted from an `az login` session stays valid. +/// Default lifetime, in minutes, of a SAS minted from an `az login` session. /// -/// SAS tokens are deliberately short-lived: a SAS cannot be individually -/// revoked, so a short lifetime keeps the blast radius small if one leaks. Thirty -/// minutes comfortably covers an index or upload run. -const AZURE_CLI_SAS_TTL: Duration = Duration::from_secs(30 * 60); +/// SAS tokens are deliberately short-lived: a SAS cannot be individually revoked, +/// so a short lifetime keeps the blast radius small if one leaks. Thirty minutes +/// comfortably covers a typical index or upload run. +const DEFAULT_AZURE_CLI_SAS_TTL_MINUTES: u64 = 30; /// Errors that can occur while resolving [`AzureCredentialsOpts`] into /// [`AzureCredentials`]. @@ -19,72 +19,146 @@ pub enum AzureCredentialsError { #[error("no Azure credentials supplied: pass --account-key, --sas-token, or --azure-cli")] Missing, + /// The channel URL required to mint a SAS could not be parsed. + #[error(transparent)] + Url(#[from] AzureUrlError), + /// Minting a SAS via the Azure CLI failed. #[error("failed to mint a user-delegation SAS from the Azure CLI")] Cli(#[from] AzureCliSasError), } +/// A resolved, unambiguous choice of authentication source. +/// +/// [`AzureCredentialsOpts`] can express several inputs at once (an exported +/// `AZURE_STORAGE_KEY` and an explicit `--azure-cli`, say); this enum is the +/// single winner after precedence is applied, so downstream code never has to +/// reason about combinations. Only [`AzureAuthSource::AzureCli`] carries state +/// (the minting TTL), which is why account/container derivation is needed for +/// that arm alone. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AzureAuthSource { + /// Use a shared storage account key verbatim. + AccountKey(String), + + /// Use a supplied SAS token verbatim. + SasToken(String), + + /// Mint a short-lived user-delegation SAS from the current `az login` + /// session, valid for `ttl`. + AzureCli { + /// How long the minted SAS should remain valid. + ttl: Duration, + }, +} + +impl AzureAuthSource { + /// Resolve this source into concrete [`AzureCredentials`]. + /// + /// `permissions` and `cli_context` are consulted **only** for the + /// [`AzureAuthSource::AzureCli`] arm, which mints a SAS scoped to the + /// container returned by `cli_context` with those permissions. The account + /// key and SAS token arms never invoke `cli_context`, so callers pay for + /// account/container derivation only on the minting path. + pub fn resolve( + self, + permissions: &str, + cli_context: impl FnOnce() -> Result<(String, String), AzureCredentialsError>, + ) -> Result { + match self { + AzureAuthSource::AccountKey(key) => Ok(AzureCredentials::AccountKey(key)), + AzureAuthSource::SasToken(token) => Ok(AzureCredentials::SasToken(token)), + AzureAuthSource::AzureCli { ttl } => { + let (account, container) = cli_context()?; + let token = mint_user_delegation_sas(&account, &container, permissions, ttl)?; + Ok(AzureCredentials::SasToken(token)) + } + } + } +} + /// Manually specified Azure Blob credentials. /// -/// See [`super::AzureCredentials`] for details on how these credentials are -/// used. `--account-key`, `--sas-token`, and `--azure-cli` are mutually -/// exclusive; exactly one must be supplied. +/// See [`super::AzureCredentials`] for details on how these credentials are used. +/// Several inputs may be present at once (for example when `AZURE_STORAGE_KEY` is +/// exported *and* `--azure-cli` is passed), so [`AzureCredentialsOpts::source`] +/// applies an explicit precedence rather than treating the combination as an +/// error — see that method for the exact ordering. #[derive(Clone, Debug, PartialEq, Parser)] pub struct AzureCredentialsOpts { /// The Azure Storage account key. - #[arg( - long, - env = "AZURE_STORAGE_KEY", - conflicts_with_all = ["sas_token", "azure_cli"], - help_heading = "Azure Credentials" - )] + #[arg(long, env = "AZURE_STORAGE_KEY", help_heading = "Azure Credentials")] pub account_key: Option, /// A shared access signature (SAS) token, with or without a leading `?`. #[arg( long, env = "AZURE_STORAGE_SAS_TOKEN", - conflicts_with_all = ["account_key", "azure_cli"], help_heading = "Azure Credentials" )] pub sas_token: Option, /// Mint a short-lived user-delegation SAS from the current `az login` /// session (requires the Azure CLI). + /// + /// Takes precedence over AZURE_STORAGE_KEY / AZURE_STORAGE_SAS_TOKEN, so it + /// can be used to override ambient credentials picked up from the + /// environment. + #[allow(clippy::doc_markdown)] + #[arg(long, help_heading = "Azure Credentials")] + pub azure_cli: bool, + + /// Lifetime, in minutes, of the SAS minted for `--azure-cli`. + /// + /// The default keeps the token short-lived. Raise it for very large index or + /// upload runs: if the SAS expires mid-run, subsequent requests fail with a + /// 403 and the run aborts, potentially leaving a partial index behind. #[arg( long, - conflicts_with_all = ["account_key", "sas_token"], + default_value_t = DEFAULT_AZURE_CLI_SAS_TTL_MINUTES, help_heading = "Azure Credentials" )] - pub azure_cli: bool, + pub azure_cli_sas_ttl_minutes: u64, } impl AzureCredentialsOpts { - /// Resolve the supplied options into a concrete [`AzureCredentials`]. + /// Collapse the supplied options into a single, unambiguous + /// [`AzureAuthSource`]. + /// + /// When more than one input is present the following precedence applies, + /// highest first: + /// + /// 1. `--azure-cli` — an explicit opt-in, so it wins over anything picked up + /// from the environment. + /// 2. `--sas-token` / `AZURE_STORAGE_SAS_TOKEN`. + /// 3. `--account-key` / `AZURE_STORAGE_KEY`. /// - /// `account`, `container`, and `permissions` are only used by the - /// `--azure-cli` path, which mints a short-lived user-delegation SAS scoped - /// to that container with those permissions. The account and container are - /// derived by the caller from the channel URL. + /// If none are set, returns [`AzureCredentialsError::Missing`]. + pub fn source(&self) -> Result { + if self.azure_cli { + Ok(AzureAuthSource::AzureCli { + ttl: Duration::from_secs(self.azure_cli_sas_ttl_minutes * 60), + }) + } else if let Some(sas_token) = &self.sas_token { + Ok(AzureAuthSource::SasToken(sas_token.clone())) + } else if let Some(account_key) = &self.account_key { + Ok(AzureAuthSource::AccountKey(account_key.clone())) + } else { + Err(AzureCredentialsError::Missing) + } + } + + /// Resolve the supplied options into concrete [`AzureCredentials`]. + /// + /// Precedence is applied by [`AzureCredentialsOpts::source`]. `permissions` + /// and `cli_context` are consulted only when the winning source is + /// `--azure-cli`; see [`AzureAuthSource::resolve`]. pub fn resolve( self, - account: &str, - container: &str, permissions: &str, + cli_context: impl FnOnce() -> Result<(String, String), AzureCredentialsError>, ) -> Result { - // `conflicts_with_all` guarantees at most one source is set, so the - // order of these checks doesn't matter. - if let Some(account_key) = self.account_key { - Ok(AzureCredentials::AccountKey(account_key)) - } else if let Some(sas_token) = self.sas_token { - Ok(AzureCredentials::SasToken(sas_token)) - } else if self.azure_cli { - let token = - mint_user_delegation_sas(account, container, permissions, AZURE_CLI_SAS_TTL)?; - Ok(AzureCredentials::SasToken(token)) - } else { - Err(AzureCredentialsError::Missing) - } + self.source()?.resolve(permissions, cli_context) } } @@ -92,67 +166,85 @@ impl AzureCredentialsOpts { mod tests { use super::*; + fn opts( + account_key: Option<&str>, + sas_token: Option<&str>, + azure_cli: bool, + ) -> AzureCredentialsOpts { + AzureCredentialsOpts { + account_key: account_key.map(str::to_string), + sas_token: sas_token.map(str::to_string), + azure_cli, + azure_cli_sas_ttl_minutes: DEFAULT_AZURE_CLI_SAS_TTL_MINUTES, + } + } + + /// `cli_context` must not be invoked for the account-key/SAS-token paths. + fn unreachable_context() -> Result<(String, String), AzureCredentialsError> { + panic!("cli_context should not be called for non-azure-cli sources"); + } + #[test] fn account_key_resolves() { - let opts = AzureCredentialsOpts { - account_key: Some("key".into()), - sas_token: None, - azure_cli: false, - }; assert!(matches!( - opts.resolve("acct", "container", "cw"), + opts(Some("key"), None, false).resolve("cw", unreachable_context), Ok(AzureCredentials::AccountKey(k)) if k == "key" )); } #[test] fn sas_token_resolves() { - let opts = AzureCredentialsOpts { - account_key: None, - sas_token: Some("sv=...".into()), - azure_cli: false, - }; assert!(matches!( - opts.resolve("acct", "container", "cw"), + opts(None, Some("sv=..."), false).resolve("cw", unreachable_context), Ok(AzureCredentials::SasToken(t)) if t == "sv=..." )); } #[test] fn none_is_rejected() { - let opts = AzureCredentialsOpts { - account_key: None, - sas_token: None, - azure_cli: false, - }; assert!(matches!( - opts.resolve("acct", "container", "cw"), + opts(None, None, false).resolve("cw", unreachable_context), Err(AzureCredentialsError::Missing) )); } - // The `--azure-cli` variant shells out to `az`, which isn't available in the - // test environment, so we only assert the flag is wired through the parser - // (its conflicts are enforced by clap, exercised below) rather than invoking - // the mint path. #[test] - fn azure_cli_flag_parses() { - use clap::Parser; - - #[derive(Parser)] - struct Cli { - #[command(flatten)] - creds: AzureCredentialsOpts, - } + fn azure_cli_beats_sas_beats_account_key() { + // All three present: `--azure-cli` wins. + assert!(matches!( + opts(Some("key"), Some("sv=..."), true).source(), + Ok(AzureAuthSource::AzureCli { .. }) + )); + // SAS token beats an account key when `--azure-cli` is absent. + assert!(matches!( + opts(Some("key"), Some("sv=..."), false).source(), + Ok(AzureAuthSource::SasToken(t)) if t == "sv=..." + )); + // Account key is the last resort. + assert!(matches!( + opts(Some("key"), None, false).source(), + Ok(AzureAuthSource::AccountKey(k)) if k == "key" + )); + } - let cli = Cli::try_parse_from(["test", "--azure-cli"]).expect("should parse"); - assert!(cli.creds.azure_cli); - assert!(cli.creds.account_key.is_none()); - assert!(cli.creds.sas_token.is_none()); + #[test] + fn azure_cli_ttl_is_carried_through() { + let mut opts = opts(None, None, true); + opts.azure_cli_sas_ttl_minutes = 45; + assert_eq!( + opts.source().unwrap(), + AzureAuthSource::AzureCli { + ttl: Duration::from_secs(45 * 60), + } + ); } + // The `--azure-cli` resolve path shells out to `az`, which isn't available in + // the test environment, so we only assert the flag and its TTL parse through + // clap and that precedence selects the CLI source; the mint itself is not + // exercised here. #[test] - fn azure_cli_conflicts_with_other_sources() { + fn azure_cli_flag_and_ttl_parse() { use clap::Parser; #[derive(Parser)] @@ -161,7 +253,15 @@ mod tests { creds: AzureCredentialsOpts, } - assert!(Cli::try_parse_from(["test", "--azure-cli", "--account-key", "k"]).is_err()); - assert!(Cli::try_parse_from(["test", "--azure-cli", "--sas-token", "s"]).is_err()); + let cli = Cli::try_parse_from(["test", "--azure-cli", "--azure-cli-sas-ttl-minutes", "90"]) + .expect("should parse"); + assert!(cli.creds.azure_cli); + assert_eq!(cli.creds.azure_cli_sas_ttl_minutes, 90); + + let default = Cli::try_parse_from(["test", "--azure-cli"]).expect("should parse"); + assert_eq!( + default.creds.azure_cli_sas_ttl_minutes, + DEFAULT_AZURE_CLI_SAS_TTL_MINUTES + ); } } diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 4faff916c4..7276e52ac2 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -1,6 +1,8 @@ #[cfg(feature = "clap")] pub mod clap; +use url::Url; + /// Credentials for authenticating to Azure Blob storage. /// /// Exactly one authentication method is carried, so the ambiguous "both a key @@ -18,8 +20,65 @@ pub enum AzureCredentials { SasToken(String), } +/// Strip a single leading `?` from a SAS token. +/// +/// `--sas-token` may be supplied with or without a leading `?`, but a SAS minted +/// by [`mint_user_delegation_sas`] never has one. Normalizing at the single point +/// where a token is handed to opendal means both sources behave identically. +pub fn normalize_sas_token(token: &str) -> &str { + token.strip_prefix('?').unwrap_or(token) +} + +/// Errors that can occur while deriving Azure Blob coordinates from a channel +/// URL. +#[derive(Debug, thiserror::Error)] +pub enum AzureUrlError { + /// The URL has no host component. + #[error("no host in Azure blob URL")] + NoHost, + + /// The host is not a dotted domain, so no storage account can be derived. + #[error( + "Azure blob URL host `{0}` is not a dotted domain of the form `.blob.`; \ + IP literals and single-label hosts have no derivable storage account" + )] + InvalidHost(String), + + /// The account name (first host label) is empty. + #[error("could not derive account name from Azure blob URL")] + NoAccount, + + /// The URL has no container path segment. + #[error("no container in Azure blob URL")] + NoContainer, +} + +/// Derive the storage account name and container from an Azure Blob channel URL +/// of the form `https://.blob.core.windows.net//`. +/// +/// The account name is the first label of the host, so the host must be a dotted +/// domain; IP-literal and single-label hosts (e.g. `localhost` or the Azurite +/// emulator) are rejected because no account can be derived from them. +pub fn account_and_container(url: &Url) -> Result<(String, String), AzureUrlError> { + let host = url.host_str().ok_or(AzureUrlError::NoHost)?; + if !matches!(url.host(), Some(url::Host::Domain(domain)) if domain.contains('.')) { + return Err(AzureUrlError::InvalidHost(host.to_string())); + } + let account = host + .split('.') + .next() + .filter(|name| !name.is_empty()) + .ok_or(AzureUrlError::NoAccount)?; + let container = url + .path_segments() + .and_then(|mut segments| segments.next()) + .filter(|segment| !segment.is_empty()) + .ok_or(AzureUrlError::NoContainer)?; + Ok((account.to_string(), container.to_string())) +} + /// Errors that can occur while minting a user-delegation SAS via the Azure CLI. -#[cfg(feature = "azure-cli-sas")] +#[cfg(feature = "clap")] #[derive(Debug, thiserror::Error)] pub enum AzureCliSasError { /// The SAS expiry timestamp could not be computed. @@ -62,22 +121,28 @@ pub enum AzureCliSasError { /// /// This blocks the calling thread while the `az` process runs; it is meant to be /// called once at setup time. -#[cfg(feature = "azure-cli-sas")] +#[cfg(feature = "clap")] pub fn mint_user_delegation_sas( account: &str, container: &str, permissions: &str, valid_for: std::time::Duration, ) -> Result { - let signed = jiff::SignedDuration::try_from(valid_for) + /// Extra slack added to the requested lifetime so a slightly fast client + /// clock (the expiry is computed from *this* machine's time) does not shrink + /// the usable window toward zero at the Azure end. + const CLOCK_SKEW_HEADROOM: std::time::Duration = std::time::Duration::from_secs(120); + + let signed = jiff::SignedDuration::try_from(valid_for.saturating_add(CLOCK_SKEW_HEADROOM)) .map_err(|err| AzureCliSasError::Expiry(err.to_string()))?; let expiry = jiff::Timestamp::now() .checked_add(signed) .map_err(|err| AzureCliSasError::Expiry(err.to_string()))?; - // `az` expects an ISO-8601 UTC timestamp; minute precision is sufficient. - let expiry = expiry.strftime("%Y-%m-%dT%H:%MZ").to_string(); + // `az` expects an ISO-8601 UTC timestamp; keep second precision so the window + // is not floored down to the enclosing whole minute. + let expiry = expiry.strftime("%Y-%m-%dT%H:%M:%SZ").to_string(); - let output = std::process::Command::new("az") + let output = az_command() .args([ "storage", "container", @@ -117,3 +182,20 @@ pub fn mint_user_delegation_sas( } Ok(token) } + +/// Build the [`std::process::Command`] used to invoke the Azure CLI. +/// +/// On Windows the Azure CLI is a `az.cmd` batch shim; `std::process` does not +/// honor `PATHEXT`, so a bare `az` fails to resolve it. Going through the command +/// interpreter (`cmd /C az ...`) lets Windows apply `PATHEXT` and find the shim. +#[cfg(all(feature = "clap", windows))] +fn az_command() -> std::process::Command { + let mut command = std::process::Command::new("cmd"); + command.args(["/C", "az"]); + command +} + +#[cfg(all(feature = "clap", not(windows)))] +fn az_command() -> std::process::Command { + std::process::Command::new("az") +} diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 85517bd05a..8836bade81 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -1604,22 +1604,17 @@ fn azblob_config( credentials: &AzureCredentials, channel: &Url, ) -> Result { + let (account_name, container) = rattler_azure::account_and_container(channel)?; + let host = channel .host_str() .ok_or_else(|| anyhow::anyhow!("No host in Azure blob URL"))?; - let account_name = host - .split('.') - .next() - .filter(|name| !name.is_empty()) - .ok_or_else(|| anyhow::anyhow!("Could not derive account name from Azure blob URL"))?; let mut segments = channel .path_segments() .ok_or_else(|| anyhow::anyhow!("No path in Azure blob URL"))?; - let container = segments - .next() - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| anyhow::anyhow!("No container in Azure blob URL"))?; + // Skip the container segment; the remainder is the root prefix. + segments.next(); let root = format!("/{}", segments.collect::>().join("/")); // Preserve a non-default port if one is present; real Azure uses the scheme @@ -1631,13 +1626,16 @@ fn azblob_config( let (account_key, sas_token) = match credentials { AzureCredentials::AccountKey(key) => (Some(key.clone()), None), - AzureCredentials::SasToken(token) => (None, Some(token.clone())), + AzureCredentials::SasToken(token) => ( + None, + Some(rattler_azure::normalize_sas_token(token).to_string()), + ), }; Ok(AzblobConfig { endpoint: Some(format!("{}://{}", channel.scheme(), authority)), - account_name: Some(account_name.to_string()), - container: container.to_string(), + account_name: Some(account_name), + container, root: Some(root), account_key, sas_token, diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 98720aec70..f0e19b2dc7 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -66,26 +66,6 @@ fn parse_azure_url(value: &str) -> Result { #[cfg(feature = "azure")] const AZURE_INDEX_SAS_PERMISSIONS: &str = "rwlc"; -/// Derive the storage account name and container from an Azure Blob channel URL -/// of the form `https://.blob.core.windows.net//`. -#[cfg(feature = "azure")] -fn azure_account_and_container(channel: &Url) -> anyhow::Result<(String, String)> { - let host = channel - .host_str() - .ok_or_else(|| anyhow::anyhow!("No host in Azure blob URL"))?; - let account_name = host - .split('.') - .next() - .filter(|name| !name.is_empty()) - .ok_or_else(|| anyhow::anyhow!("Could not derive account name from Azure blob URL"))?; - let container = channel - .path_segments() - .and_then(|mut segments| segments.next()) - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| anyhow::anyhow!("No container in Azure blob URL"))?; - Ok((account_name.to_string(), container.to_string())) -} - /// The `rattler-index` CLI. #[derive(Parser)] #[command(name = "rattler-index", version, about, long_about = None)] @@ -291,9 +271,9 @@ async fn main() -> anyhow::Result<()> { effective_index_options(&resolved); let channel_metadata = ChannelMetadata::from_index_config(&resolved); - let (account, container) = azure_account_and_container(&channel)?; - let credentials = - credentials.resolve(&account, &container, AZURE_INDEX_SAS_PERMISSIONS)?; + let credentials = credentials.resolve(AZURE_INDEX_SAS_PERMISSIONS, || { + Ok(rattler_azure::account_and_container(&channel)?) + })?; index_azure_with_channel_metadata( IndexAzureConfig { diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index 3275c35797..4d806235e4 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -83,13 +83,15 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { } #[cfg(feature = "azure")] ServerType::Azure(azure_opts) => { - let (account, container) = upload::azure_account_and_container(&azure_opts.channel)?; + let channel = azure_opts.channel; let credentials = azure_opts .credentials - .resolve(&account, &container, upload::AZURE_UPLOAD_SAS_PERMISSIONS) + .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, || { + Ok(rattler_azure::account_and_container(&channel)?) + }) .into_diagnostic()?; upload::upload_package_to_azure( - azure_opts.channel, + channel, credentials, &args.package_files, azure_opts.force, diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 32de022c03..9e64389151 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -25,25 +25,6 @@ const PACKAGE_CONCURRENCY: usize = 4; /// Uploading only needs to create and write blobs (`c` + `w`). pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "cw"; -/// Derive the storage account name and container from an Azure Blob channel URL -/// of the form `https://.blob.core.windows.net//`. -pub(crate) fn azure_account_and_container(channel: &Url) -> miette::Result<(String, String)> { - let host = channel - .host_str() - .ok_or_else(|| miette::miette!("No host in Azure blob URL"))?; - let account_name = host - .split('.') - .next() - .filter(|name| !name.is_empty()) - .ok_or_else(|| miette::miette!("Could not derive account name from Azure blob URL"))?; - let container = channel - .path_segments() - .and_then(|mut segments| segments.next()) - .filter(|segment| !segment.is_empty()) - .ok_or_else(|| miette::miette!("No container in Azure blob URL"))?; - Ok((account_name.to_string(), container.to_string())) -} - /// Uploads packages to a channel in an Azure Blob Storage container. /// /// The channel URL must be of the form @@ -200,7 +181,8 @@ async fn upload_single_package( /// the URL (`https://.blob.core.windows.net//`); the /// credentials supply only the account key or SAS token. fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Result { - let (account_name, container) = azure_account_and_container(channel)?; + let (account_name, container) = + rattler_azure::account_and_container(channel).into_diagnostic()?; let mut segments = channel .path_segments() @@ -221,7 +203,10 @@ fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Resul let (account_key, sas_token) = match credentials { AzureCredentials::AccountKey(key) => (Some(key.clone()), None), - AzureCredentials::SasToken(token) => (None, Some(token.clone())), + AzureCredentials::SasToken(token) => ( + None, + Some(rattler_azure::normalize_sas_token(token).to_string()), + ), }; Ok(AzblobConfig { diff --git a/crates/rattler_upload/src/upload/mod.rs b/crates/rattler_upload/src/upload/mod.rs index a35d86face..309417c174 100644 --- a/crates/rattler_upload/src/upload/mod.rs +++ b/crates/rattler_upload/src/upload/mod.rs @@ -31,9 +31,9 @@ mod anaconda; #[cfg(feature = "azure")] mod azure; #[cfg(feature = "azure")] -pub use azure::upload_package_to_azure; +pub(crate) use azure::AZURE_UPLOAD_SAS_PERMISSIONS; #[cfg(feature = "azure")] -pub(crate) use azure::{AZURE_UPLOAD_SAS_PERMISSIONS, azure_account_and_container}; +pub use azure::upload_package_to_azure; #[cfg(feature = "sigstore-sign")] pub mod attestation; mod cloudsmith; From 79e10ca8d021afa09bacdb4b6cab0d9ace4e1eef Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:39:31 +1000 Subject: [PATCH 30/98] test(networking): live az-login azure fetch against real account --- .../tests/azure_real_fetch.rs | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 crates/rattler_networking/tests/azure_real_fetch.rs diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs new file mode 100644 index 0000000000..eb2183bf89 --- /dev/null +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -0,0 +1,68 @@ +//! Live read-path test against a *real* Azure Blob Storage account. +//! +//! Exercises the AAD / token path of reqsign's `DefaultCredentialProvider` — +//! i.e. the credential `az login` provides — against a real +//! `*.blob.core.windows.net` endpoint. It is purely a read; it never writes. +//! +//! Configure via env and run (requires `az login` with at least +//! `Storage Blob Data Reader` on the container): +//! +//! ```text +//! AZURE_TEST_ACCOUNT=stgrcondachannel \ +//! AZURE_TEST_CONTAINER=general \ +//! AZURE_TEST_PATH=noarch/repodata.json \ +//! cargo test -p rattler_networking --features azure --test azure_real_fetch -- --ignored --nocapture +//! ``` +#![cfg(feature = "azure")] + +use std::collections::HashMap; + +use rattler_networking::{AzureMiddleware, azure_middleware::AzureConfig}; +use reqwest_middleware::ClientBuilder; +use url::Url; + +#[tokio::test] +#[ignore = "requires az login and AZURE_TEST_* env vars pointing at a real account"] +async fn azure_middleware_fetches_real_repodata() { + let account = std::env::var("AZURE_TEST_ACCOUNT").expect("AZURE_TEST_ACCOUNT"); + let container = std::env::var("AZURE_TEST_CONTAINER").expect("AZURE_TEST_CONTAINER"); + let path = + std::env::var("AZURE_TEST_PATH").unwrap_or_else(|_| "noarch/repodata.json".to_string()); + + // An explicit endpoint override addresses sovereign clouds / emulators; the + // default (account form) targets `https://{account}.blob.core.windows.net`. + let azure_config = match std::env::var("AZURE_TEST_ENDPOINT").ok() { + Some(endpoint) => { + AzureConfig::Endpoint(Url::parse(&endpoint).expect("invalid AZURE_TEST_ENDPOINT")) + } + None => AzureConfig::Account(account), + }; + + let mut config = HashMap::new(); + config.insert(container.clone(), azure_config); + + let client = ClientBuilder::new(reqwest::Client::new()) + .with(AzureMiddleware::new(config)) + .build(); + + let url = format!("az://{container}/{path}"); + println!("fetching {url}"); + let resp = client + .get(&url) + .send() + .await + .expect("request through azure middleware failed"); + + let status = resp.status(); + let body = resp.bytes().await.expect("failed to read body"); + println!("status={status} bytes={}", body.len()); + assert!(status.is_success(), "unexpected status {status}: {url}"); + + let json: serde_json::Value = + serde_json::from_slice(&body).expect("real repodata was not valid json"); + assert!( + json.get("info").is_some() || json.get("packages").is_some(), + "fetched body does not look like repodata: {json}" + ); + println!("info = {}", json["info"]); +} From 7da64cef209107cd6e375854327e86ab94339744 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:03:25 +1000 Subject: [PATCH 31/98] feat(gateway): accept az:// channel scheme --- crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs b/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs index e522d7b804..38c8612a53 100644 --- a/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs +++ b/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs @@ -55,6 +55,7 @@ impl<'g> SubdirBuilder<'g> { || url.scheme() == "gcs" || url.scheme() == "oci" || url.scheme() == "s3" + || url.scheme() == "az" { let source_config = self.gateway.channel_config.get(&self.channel.base_url); From 25fe5daa65dbd5a889010ab5a7e47a8a2c010c66 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:12:03 +1000 Subject: [PATCH 32/98] fix(gateway): honor 304 on shard index revalidation --- .../src/gateway/sharded_subdir/tokio/index.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs index 669d34af5e..9d6a6343da 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs @@ -264,6 +264,29 @@ pub async fn fetch_index( return Err(create_subdir_not_found_error(channel_base_url)); } + // A `304 Not Modified` means the cached shard index is + // still valid, so serve it directly. Servers that omit + // `Cache-Control` on the shard index (e.g. Azure Blob + // Storage) leave the cached policy with no freshness + // lifetime, so every request revalidates and answers with + // 304; without this guard the 304 falls through to + // `from_response`, which rejects it as an unexpected status. + if response.status() == reqwest::StatusCode::NOT_MODIFIED { + match read_shard_index_from_reader(&mut cache_reader).await { + Ok(shard_index) => { + tracing::debug!("shard index revalidated (304 Not Modified)"); + if let Some((reporter, index)) = download_reporter { + reporter.on_download_complete(response.url(), index); + } + return Ok(shard_index); + } + Err(e) => { + // Cache unreadable; fall through to a fresh fetch. + tracing::warn!("the cached shard index has been corrupted: {e}"); + } + } + } + match cache_header.policy.after_response( &state_request, &response, From d9e4334a795f3775ef5cdb2cb5ab3b0e6f3f67ec Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:19:36 +1000 Subject: [PATCH 33/98] refactor(azure): address az:// by full blob host, drop config layer --- crates/rattler-bin/src/commands/client.rs | 7 +- crates/rattler_config/src/config.rs | 12 - crates/rattler_config/src/config/azure.rs | 106 --------- crates/rattler_networking/Cargo.toml | 1 - .../src/azure_middleware.rs | 208 +++++------------- .../tests/azure_real_fetch.rs | 28 +-- 6 files changed, 63 insertions(+), 299 deletions(-) delete mode 100644 crates/rattler_config/src/config/azure.rs diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index 22062293f1..3f78557bce 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -57,11 +57,10 @@ pub fn create_client_with_middleware( )); #[cfg(feature = "gcs")] let client = client.with(rattler_networking::GCSMiddleware::default()); - // Empty map like S3 above — rattler-bin loads no rattler_config, so there are - // no per-container azure-options to plumb. Real consumers (e.g. pixi) build the - // map via rattler_networking::compute_azure_config_from_config. + // `az://` URLs carry the full blob endpoint, so the middleware needs no + // configuration — it just swaps the scheme and signs via reqsign. #[cfg(feature = "azure")] - let client = client.with(rattler_networking::AzureMiddleware::new(HashMap::new())); + let client = client.with(rattler_networking::AzureMiddleware::new()); Ok(client.build()) } diff --git a/crates/rattler_config/src/config.rs b/crates/rattler_config/src/config.rs index c90b76c29a..d514a3ae6a 100644 --- a/crates/rattler_config/src/config.rs +++ b/crates/rattler_config/src/config.rs @@ -20,14 +20,12 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use thiserror::Error; use url::Url; -use crate::config::azure::AzureOptionsMap; use crate::config::s3::S3OptionsMap; use crate::config::{ build::BuildConfig, concurrency::ConcurrencyConfig, index::IndexConfig, proxy::ProxyConfig, repodata_config::RepodataConfig, run_post_link_scripts::RunPostLinkScripts, }; -pub mod azure; pub mod build; pub mod channel_config; pub mod concurrency; @@ -196,11 +194,6 @@ pub struct CommonConfig { #[serde(skip_serializing_if = "S3OptionsMap::is_default")] pub s3_options: S3OptionsMap, - /// Configuration for Azure Blob Storage. - #[serde(default)] - #[serde(skip_serializing_if = "AzureOptionsMap::is_default")] - pub azure_options: AzureOptionsMap, - /// Per-channel configuration for `rattler-index`. #[serde(default, skip_serializing_if = "IndexConfig::is_empty")] pub index_config: IndexConfig, @@ -258,7 +251,6 @@ impl Default for CommonConfig { concurrency: ConcurrencyConfig::default(), proxy_config: ProxyConfig::default(), s3_options: S3OptionsMap::default(), - azure_options: AzureOptionsMap::default(), index_config: IndexConfig::default(), run_post_link_scripts: None, allow_symbolic_links: None, @@ -313,7 +305,6 @@ impl Config for CommonConfig { concurrency: self.concurrency.merge_config(&other.concurrency)?, proxy_config: self.proxy_config.merge_config(&other.proxy_config)?, s3_options: self.s3_options.merge_config(&other.s3_options)?, - azure_options: self.azure_options.merge_config(&other.azure_options)?, index_config: self.index_config.merge_config(&other.index_config)?, run_post_link_scripts: other .run_post_link_scripts @@ -331,7 +322,6 @@ impl Config for CommonConfig { self.concurrency.validate()?; self.proxy_config.validate()?; self.s3_options.validate()?; - self.azure_options.validate()?; self.index_config.validate()?; Ok(()) } @@ -348,7 +338,6 @@ impl Config for CommonConfig { "allow-hard-links".to_string(), "allow-ref-links".to_string(), "s3-options".to_string(), - "azure-options".to_string(), "index-config".to_string(), ]; keys.extend(prefixed_keys("build", self.build.keys())); @@ -359,7 +348,6 @@ impl Config for CommonConfig { keys.extend(prefixed_keys("concurrency", self.concurrency.keys())); keys.extend(prefixed_keys("proxy-config", self.proxy_config.keys())); keys.extend(prefixed_keys("s3-options", self.s3_options.keys())); - keys.extend(prefixed_keys("azure-options", self.azure_options.keys())); keys } } diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs deleted file mode 100644 index 54691b9b76..0000000000 --- a/crates/rattler_config/src/config/azure.rs +++ /dev/null @@ -1,106 +0,0 @@ -use indexmap::IndexMap; -use serde::{Deserialize, Serialize}; -use url::Url; - -use crate::config::Config; - -#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -pub struct AzureOptionsMap(pub IndexMap); - -impl AzureOptionsMap { - /// Returns `true` if no Azure containers are configured. - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -#[serde(rename_all = "kebab-case")] -pub struct AzureOptions { - /// Storage account name. When `endpoint-url` is not set, the endpoint is - /// derived from it as `https://{account}.blob.core.windows.net`. - pub account: String, - - /// Optional full endpoint override for sovereign clouds / custom endpoints - /// (e.g. Azurite). When set, this is used verbatim as the endpoint and - /// `account` is then only used to construct the default endpoint when this - /// is absent — i.e. it has no effect on addressing while an override is set. - pub endpoint_url: Option, -} - -impl Config for AzureOptionsMap { - fn is_default(&self) -> bool { - self.0.is_empty() - } - - fn merge_config(self, other: &Self) -> Result { - // Merge the two AzureOptionsMaps, overwriting existing keys - let mut merged = self.0.clone(); - for (key, value) in &other.0 { - merged.insert(key.clone(), value.clone()); - } - Ok(AzureOptionsMap(merged)) - } - - fn validate(&self) -> Result<(), super::ValidationError> { - Ok(()) - } - - fn keys(&self) -> Vec { - self.0.keys().map(ToString::to_string).collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn deserializes_account_and_optional_endpoint() { - let toml = r#" - [mychannel] - account = "myacct" - - [other] - account = "acct2" - endpoint-url = "https://acct2.blob.core.windows.net" - "#; - let map: AzureOptionsMap = toml::from_str(toml).unwrap(); - let mychannel = map.0.get("mychannel").unwrap(); - assert_eq!(mychannel.account, "myacct"); - assert_eq!(mychannel.endpoint_url, None); - let other = map.0.get("other").unwrap(); - assert_eq!( - other.endpoint_url.as_ref().unwrap().as_str(), - "https://acct2.blob.core.windows.net/" - ); - } - - #[test] - fn is_default_when_empty() { - let map = AzureOptionsMap::default(); - assert!(map.is_default()); - } - - #[test] - fn merge_overwrites_existing_keys() { - let mut base = AzureOptionsMap::default(); - base.0.insert( - "c".to_string(), - AzureOptions { - account: "old".into(), - endpoint_url: None, - }, - ); - let mut other = AzureOptionsMap::default(); - other.0.insert( - "c".to_string(), - AzureOptions { - account: "new".into(), - endpoint_url: None, - }, - ); - let merged = base.merge_config(&other).unwrap(); - assert_eq!(merged.0.get("c").unwrap().account, "new"); - } -} diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 8337fbb79e..c87ec271ca 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -23,7 +23,6 @@ azure = [ "dep:reqsign-command-execute-tokio", "dep:reqsign-file-read-tokio", "dep:reqsign-http-send-reqwest", - "rattler_config", "tokio", ] system-integration = ["keyring", "netrc-rs", "dirs"] diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 30aadadd7e..64d7d3d15c 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -1,6 +1,4 @@ //! Middleware to handle `az://` URLs to pull artifacts from Azure Blob Storage. -use std::collections::HashMap; - use async_trait::async_trait; use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; use reqsign_command_execute_tokio::TokioCommandExecute; @@ -14,72 +12,15 @@ use url::Url; /// The Azure Storage REST API version sent on every request. const X_MS_VERSION: &str = "2021-12-02"; -/// Per-container addressing configuration for the Azure middleware. -/// -/// Like [`crate::s3_middleware::S3Config`] this is an enum, so exactly one of -/// the two addressing modes is representable: either a storage account (from -/// which the default endpoint is derived) or a full endpoint override. It holds -/// only the information needed to *address* a container; credentials are -/// resolved separately at request time via reqsign's -/// [`DefaultCredentialProvider`], so they are kept out of this type. -/// -/// Unlike `S3Config`, there is no default-provider fallback: a container that is -/// not present in the middleware's config map is an error at request time. -#[derive(Clone, Debug)] -pub enum AzureConfig { - /// Address the container via `https://{account}.blob.core.windows.net`. - Account(String), - /// Address the container via a full endpoint override (sovereign clouds / - /// custom endpoints such as Azurite). - Endpoint(Url), -} - -#[cfg(feature = "rattler_config")] -/// Compute the Azure configuration from the given Azure options. -pub fn compute_azure_config(azure_options: &M) -> HashMap -where - M: IntoIterator + Clone, -{ - azure_options - .clone() - .into_iter() - .map(|(k, v)| { - ( - k, - v.endpoint_url - .map_or_else(|| AzureConfig::Account(v.account), AzureConfig::Endpoint), - ) - }) - .collect() -} - -#[cfg(feature = "rattler_config")] -/// Compute the Azure configuration from the `azure-options` of the shared -/// rattler configuration (see [`rattler_config`]). +/// Middleware that rewrites `az://` URLs to HTTPS Azure Blob Storage URLs and +/// signs them. /// -/// Accepts a [`rattler_config::config::CommonConfig`]; a `&ConfigBase` of -/// any extension coerces into it. -pub fn compute_azure_config_from_config( - config: &rattler_config::config::CommonConfig, -) -> HashMap { - config - .azure_options - .0 - .iter() - .map(|(container, options)| { - ( - container.clone(), - options.endpoint_url.clone().map_or_else( - || AzureConfig::Account(options.account.clone()), - AzureConfig::Endpoint, - ), - ) - }) - .collect() -} - -/// Middleware that rewrites `az://{container}/{path}` URLs to HTTPS Azure Blob -/// Storage URLs and signs them. +/// The `az://` URL carries the full blob endpoint in its host, so rewriting is a +/// plain scheme swap: `az://{host}/{path}` → `https://{host}/{path}`. A conda +/// channel is therefore addressed the same way it is on the wire, e.g. +/// `az://myaccount.blob.core.windows.net/mycontainer` — no separate account or +/// endpoint configuration is needed. Sovereign clouds and emulators (Azurite) +/// work automatically because the endpoint is spelled out in the host. /// /// Credentials are resolved by reqsign's [`DefaultCredentialProvider`] chain, in /// its usual order: environment variables, then workload/managed identity, then @@ -88,16 +29,19 @@ pub fn compute_azure_config_from_config( /// per-host credentials configured there do not apply to `az://` requests. #[derive(Clone)] pub struct AzureMiddleware { - /// Container name -> addressing config (account or full endpoint override). - config: HashMap, /// reqsign signer; caches the resolved credential internally. signer: Signer, } +impl Default for AzureMiddleware { + fn default() -> Self { + Self::new() + } +} + impl AzureMiddleware { - /// Create a new Azure middleware from a container -> config map. - pub fn new(config: HashMap) -> Self { - tracing::trace!("Creating Azure middleware using {:?}", config); + /// Create a new Azure middleware. + pub fn new() -> Self { let client = Client::new(); let ctx = Context::new() .with_file_read(TokioFileRead) @@ -105,54 +49,16 @@ impl AzureMiddleware { .with_command_execute(TokioCommandExecute) .with_env(OsEnv); let signer = Signer::new(ctx, DefaultCredentialProvider::new(), RequestSigner::new()); - Self { config, signer } + Self { signer } } - /// Resolve the endpoint base URL for a container from config. - /// - /// For an [`AzureConfig::Account`] this is the endpoint origin - /// `https://{account}.blob.core.windows.net`. For an - /// [`AzureConfig::Endpoint`] the configured override is returned verbatim, - /// which may carry a path component (e.g. Azurite endpoints). - fn endpoint_for(&self, container: &str) -> MiddlewareResult { - let config = self.config.get(container).ok_or_else(|| { + /// Rewrite an `az://{host}/{path}` URL to its HTTPS equivalent by swapping + /// the scheme. Host, path, query and fragment are preserved verbatim. + fn rewrite_url(az_url: &Url) -> MiddlewareResult { + let https = az_url.as_str().replacen("az://", "https://", 1); + Url::parse(&https).map_err(|e| { reqwest_middleware::Error::Middleware(anyhow::anyhow!( - "no azure-options configured for container '{container}'" - )) - })?; - match config { - AzureConfig::Endpoint(url) => Ok(url.clone()), - AzureConfig::Account(account) => { - Url::parse(&format!("https://{account}.blob.core.windows.net")) - .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e))) - } - } - } - - /// Rewrite an `az://{container}/{path}` URL to its HTTPS equivalent. - fn rewrite_url(&self, az_url: &Url) -> MiddlewareResult { - let container = az_url.host_str().ok_or_else(|| { - reqwest_middleware::Error::Middleware(anyhow::anyhow!( - "container should be present in az URL, got: {az_url}" - )) - })?; - let endpoint = self.endpoint_for(container)?; - let query = az_url.query().map(|q| format!("?{q}")).unwrap_or_default(); - let fragment = az_url - .fragment() - .map(|f| format!("#{f}")) - .unwrap_or_default(); - let new_url = format!( - "{}/{}{}{}{}", - endpoint.as_str().trim_end_matches('/'), - container, - az_url.path(), - query, - fragment - ); - Url::parse(&new_url).map_err(|e| { - reqwest_middleware::Error::Middleware(anyhow::anyhow!( - "failed to parse constructed azure URL '{new_url}': {e}" + "failed to parse constructed azure URL '{https}': {e}" )) }) } @@ -207,7 +113,7 @@ impl Middleware for AzureMiddleware { return next.run(req, extensions).await; } - let https_url = self.rewrite_url(&req.url().clone())?; + let https_url = Self::rewrite_url(&req.url().clone())?; *req.url_mut() = https_url; self.sign(&mut req).await?; next.run(req, extensions).await @@ -218,22 +124,13 @@ impl Middleware for AzureMiddleware { mod tests { use super::*; - fn opts(account: &str, endpoint: Option<&str>) -> AzureConfig { - match endpoint { - Some(e) => AzureConfig::Endpoint(Url::parse(e).unwrap()), - None => AzureConfig::Account(account.to_string()), - } - } - #[test] - fn rewrites_default_endpoint() { - let mw = AzureMiddleware::new(HashMap::from([( - "mychannel".to_string(), - opts("myacct", None), - )])); - let rewritten = mw - .rewrite_url(&Url::parse("az://mychannel/noarch/repodata.json").unwrap()) - .unwrap(); + fn swaps_scheme_to_https() { + let rewritten = AzureMiddleware::rewrite_url( + &Url::parse("az://myacct.blob.core.windows.net/mychannel/noarch/repodata.json") + .unwrap(), + ) + .unwrap(); assert_eq!( rewritten.as_str(), "https://myacct.blob.core.windows.net/mychannel/noarch/repodata.json" @@ -241,46 +138,41 @@ mod tests { } #[test] - fn rewrites_override_endpoint_for_azurite() { - let mw = AzureMiddleware::new(HashMap::from([( - "devstoreaccount1".to_string(), - opts( - "devstoreaccount1", - Some("http://127.0.0.1:10000/devstoreaccount1"), - ), - )])); - let rewritten = mw - .rewrite_url(&Url::parse("az://devstoreaccount1/noarch/repodata.json").unwrap()) - .unwrap(); + fn preserves_query_and_fragment() { + let rewritten = AzureMiddleware::rewrite_url( + &Url::parse("az://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc#frag").unwrap(), + ) + .unwrap(); assert_eq!( rewritten.as_str(), - "http://127.0.0.1:10000/devstoreaccount1/devstoreaccount1/noarch/repodata.json" + "https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc#frag" ); } #[test] - fn errors_when_container_not_configured() { - let mw = AzureMiddleware::new(HashMap::new()); - let err = mw - .rewrite_url(&Url::parse("az://missing/noarch/repodata.json").unwrap()) - .unwrap_err(); - assert!(err.to_string().contains("missing")); + fn rewrites_azurite_style_host_and_port() { + let rewritten = AzureMiddleware::rewrite_url( + &Url::parse("az://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json").unwrap(), + ) + .unwrap(); + assert_eq!( + rewritten.as_str(), + "https://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" + ); } #[tokio::test] async fn passes_through_non_az_schemes_unchanged() { use reqwest_middleware::ClientBuilder; - let mw = AzureMiddleware::new(HashMap::new()); - let client = ClientBuilder::new(Client::new()).with(mw).build(); + let client = ClientBuilder::new(Client::new()) + .with(AzureMiddleware::new()) + .build(); + // A non-`az` request must not be rewritten; it should be attempted as-is + // (and fail on DNS), proving the middleware left it untouched. let result = client .get("https://this-host-does-not-exist.invalid/x") .send() .await; assert!(result.is_err()); - let err = result.unwrap_err(); - assert!( - !err.to_string().contains("azure-options"), - "non-az request must not hit azure config lookup: {err}" - ); } } diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs index eb2183bf89..c6902d913f 100644 --- a/crates/rattler_networking/tests/azure_real_fetch.rs +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -13,13 +13,13 @@ //! AZURE_TEST_PATH=noarch/repodata.json \ //! cargo test -p rattler_networking --features azure --test azure_real_fetch -- --ignored --nocapture //! ``` +//! +//! Set `AZURE_TEST_HOST` to target a sovereign cloud or emulator directly +//! (overrides the default `{account}.blob.core.windows.net` host). #![cfg(feature = "azure")] -use std::collections::HashMap; - -use rattler_networking::{AzureMiddleware, azure_middleware::AzureConfig}; +use rattler_networking::AzureMiddleware; use reqwest_middleware::ClientBuilder; -use url::Url; #[tokio::test] #[ignore = "requires az login and AZURE_TEST_* env vars pointing at a real account"] @@ -28,24 +28,16 @@ async fn azure_middleware_fetches_real_repodata() { let container = std::env::var("AZURE_TEST_CONTAINER").expect("AZURE_TEST_CONTAINER"); let path = std::env::var("AZURE_TEST_PATH").unwrap_or_else(|_| "noarch/repodata.json".to_string()); - - // An explicit endpoint override addresses sovereign clouds / emulators; the - // default (account form) targets `https://{account}.blob.core.windows.net`. - let azure_config = match std::env::var("AZURE_TEST_ENDPOINT").ok() { - Some(endpoint) => { - AzureConfig::Endpoint(Url::parse(&endpoint).expect("invalid AZURE_TEST_ENDPOINT")) - } - None => AzureConfig::Account(account), - }; - - let mut config = HashMap::new(); - config.insert(container.clone(), azure_config); + let host = std::env::var("AZURE_TEST_HOST") + .unwrap_or_else(|_| format!("{account}.blob.core.windows.net")); let client = ClientBuilder::new(reqwest::Client::new()) - .with(AzureMiddleware::new(config)) + .with(AzureMiddleware::new()) .build(); - let url = format!("az://{container}/{path}"); + // The `az://` host carries the full blob endpoint — same form used in a + // channel URL, e.g. `az://stgrcondachannel.blob.core.windows.net/general`. + let url = format!("az://{host}/{container}/{path}"); println!("fetching {url}"); let resp = client .get(&url) From 02641fd7c7f17d6f9e57a7cb9e26851bad1c8a8f Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:34:00 +1000 Subject: [PATCH 34/98] fix(networking): send az:// unsigned on no credential, skip pre-signed SAS, use configured client --- crates/rattler-bin/src/commands/client.rs | 10 +- .../src/azure_middleware.rs | 137 ++++++++++++++++-- .../tests/azure_real_fetch.rs | 2 +- 3 files changed, 130 insertions(+), 19 deletions(-) diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index 3f78557bce..dc7a7d69ca 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -49,7 +49,9 @@ pub fn create_client_with_middleware( ))) .with_arc(Arc::new(AuthChallengeMiddleware::default())); - let client = client.with(rattler_networking::OciMiddleware::new(download_client)); + let client = client.with(rattler_networking::OciMiddleware::new( + download_client.clone(), + )); #[cfg(feature = "s3")] let client = client.with(rattler_networking::S3Middleware::new( HashMap::new(), @@ -58,9 +60,11 @@ pub fn create_client_with_middleware( #[cfg(feature = "gcs")] let client = client.with(rattler_networking::GCSMiddleware::default()); // `az://` URLs carry the full blob endpoint, so the middleware needs no - // configuration — it just swaps the scheme and signs via reqsign. + // configuration — it just swaps the scheme and signs via reqsign. It reuses + // the configured download client for reqsign's credential resolution so + // proxy/CA/TLS settings carry through. #[cfg(feature = "azure")] - let client = client.with(rattler_networking::AzureMiddleware::new()); + let client = client.with(rattler_networking::AzureMiddleware::new(download_client)); Ok(client.build()) } diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 64d7d3d15c..39280cf24e 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; use reqsign_command_execute_tokio::TokioCommandExecute; -use reqsign_core::{Context, OsEnv, Signer}; +use reqsign_core::{Context, ErrorKind, OsEnv, ProvideCredential, Signer}; use reqsign_file_read_tokio::TokioFileRead; use reqsign_http_send_reqwest::ReqwestHttpSend; use reqwest::{Client, Request, Response}; @@ -27,28 +27,42 @@ const X_MS_VERSION: &str = "2021-12-02"; /// the Azure CLI (`az login`). rattler's [`crate::AuthenticationStorage`] is not /// consulted for Azure — there is no `Authentication` Azure variant — so /// per-host credentials configured there do not apply to `az://` requests. +/// +/// When no credential resolves at all, the request is sent **unsigned** rather +/// than failing, so public/anonymous containers remain reachable with zero +/// ambient credentials. #[derive(Clone)] pub struct AzureMiddleware { /// reqsign signer; caches the resolved credential internally. signer: Signer, } -impl Default for AzureMiddleware { - fn default() -> Self { - Self::new() - } -} - impl AzureMiddleware { /// Create a new Azure middleware. - pub fn new() -> Self { - let client = Client::new(); + /// + /// `client` is used for reqsign's credential resolution (IMDS / managed + /// identity / AAD token fetches), so it must be the caller's configured + /// client — proxy, CA bundle, and TLS settings carry through to those + /// requests. + pub fn new(client: Client) -> Self { + Self::with_credential_provider(client, DefaultCredentialProvider::new()) + } + + /// Build the middleware around an explicit credential provider. + /// + /// [`AzureMiddleware::new`] wires up the [`DefaultCredentialProvider`] chain; + /// tests use this seam to inject a deterministic provider (e.g. an empty + /// chain, or a static key) without touching the ambient environment. + fn with_credential_provider( + client: Client, + provider: impl ProvideCredential + 'static, + ) -> Self { let ctx = Context::new() .with_file_read(TokioFileRead) .with_http_send(ReqwestHttpSend::new(client)) .with_command_execute(TokioCommandExecute) .with_env(OsEnv); - let signer = Signer::new(ctx, DefaultCredentialProvider::new(), RequestSigner::new()); + let signer = Signer::new(ctx, provider, RequestSigner::new()); Self { signer } } @@ -63,8 +77,26 @@ impl AzureMiddleware { }) } + /// Whether the URL already carries an explicit SAS token (a `sig` query + /// parameter). Such a URL is self-authenticating and must not be re-signed. + fn has_sas_token(url: &Url) -> bool { + url.query_pairs().any(|(key, _)| key == "sig") + } + /// Sign a reqwest `Request` in place using reqsign. + /// + /// Two cases short-circuit without touching the request: + /// - The URL already carries an explicit SAS (`?...&sig=...`). Signing would + /// add an `Authorization` header that Azure prefers over the SAS, silently + /// overriding the caller's explicit token. + /// - No credential resolves. reqsign surfaces this as + /// [`ErrorKind::CredentialInvalid`]; the request is then sent unsigned so + /// public/anonymous containers stay reachable. async fn sign(&self, req: &mut Request) -> MiddlewareResult<()> { + if Self::has_sas_token(req.url()) { + return Ok(()); + } + if !req.headers().contains_key("x-ms-version") { req.headers_mut() .insert("x-ms-version", http::HeaderValue::from_static(X_MS_VERSION)); @@ -83,10 +115,14 @@ impl AzureMiddleware { })?; let (mut parts, ()) = http_req.into_parts(); - self.signer - .sign(&mut parts, None) - .await - .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e)))?; + match self.signer.sign(&mut parts, None).await { + Ok(()) => {} + Err(e) if e.kind() == ErrorKind::CredentialInvalid => { + tracing::debug!("no Azure credential resolved; sending `az://` request unsigned"); + return Ok(()); + } + Err(e) => return Err(reqwest_middleware::Error::Middleware(anyhow::anyhow!(e))), + } *req.headers_mut() = parts.headers; let signed_url = Url::parse(&parts.uri.to_string()).map_err(|e| { @@ -165,7 +201,7 @@ mod tests { async fn passes_through_non_az_schemes_unchanged() { use reqwest_middleware::ClientBuilder; let client = ClientBuilder::new(Client::new()) - .with(AzureMiddleware::new()) + .with(AzureMiddleware::new(Client::new())) .build(); // A non-`az` request must not be rewritten; it should be attempted as-is // (and fail on DNS), proving the middleware left it untouched. @@ -175,4 +211,75 @@ mod tests { .await; assert!(result.is_err()); } + + #[test] + fn detects_sas_token_in_query() { + assert!(AzureMiddleware::has_sas_token( + &Url::parse("https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc").unwrap() + )); + assert!(!AzureMiddleware::has_sas_token( + &Url::parse("https://acct.blob.core.windows.net/c/x.json?sv=2021").unwrap() + )); + } + + /// With no credential resolvable, a signable `az://` request must be passed + /// through UNSIGNED (not errored), so public/anonymous containers work with + /// zero ambient credentials. An empty provider chain resolves nothing, which + /// reqsign reports as `CredentialInvalid`. + #[tokio::test] + async fn passes_request_through_unsigned_when_no_credential() { + use reqsign_core::ProvideCredentialChain; + + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + ProvideCredentialChain::::new(), + ); + let mut req = Client::new() + .get("https://acct.blob.core.windows.net/pub/noarch/repodata.json") + .build() + .unwrap(); + + middleware + .sign(&mut req) + .await + .expect("a request with no resolvable credential must pass through unsigned"); + + assert!( + req.headers().get(http::header::AUTHORIZATION).is_none(), + "unsigned request must not carry an Authorization header" + ); + assert!( + !req.url().query_pairs().any(|(k, _)| k == "sig"), + "unsigned request must not gain a SAS query parameter" + ); + } + + /// A URL that already carries a SAS token must not be re-signed even when a + /// credential is available: no `Authorization` header is added. + #[tokio::test] + async fn does_not_sign_url_that_already_has_sas() { + use reqsign_azure_storage::StaticCredentialProvider; + + // A valid base64 account key so the static provider yields a usable + // SharedKey credential that would otherwise sign the request. + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + StaticCredentialProvider::new_shared_key("acct", "dGVzdF9rZXk="), + ); + let mut req = Client::new() + .get("https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc") + .build() + .unwrap(); + + middleware.sign(&mut req).await.unwrap(); + + assert!( + req.headers().get(http::header::AUTHORIZATION).is_none(), + "a URL carrying an explicit SAS must not be re-signed" + ); + assert!( + !req.headers().contains_key("x-ms-version"), + "a self-authenticating SAS URL is left untouched" + ); + } } diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs index c6902d913f..3f0a4114c9 100644 --- a/crates/rattler_networking/tests/azure_real_fetch.rs +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -32,7 +32,7 @@ async fn azure_middleware_fetches_real_repodata() { .unwrap_or_else(|_| format!("{account}.blob.core.windows.net")); let client = ClientBuilder::new(reqwest::Client::new()) - .with(AzureMiddleware::new()) + .with(AzureMiddleware::new(reqwest::Client::new())) .build(); // The `az://` host carries the full blob endpoint — same form used in a From 50b8903be7f5f6e7191618d0779b855923c5957c Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:34:07 +1000 Subject: [PATCH 35/98] fix(gateway): synthesize shard-index freshness and persist refreshed policy on 304 --- .../src/gateway/sharded_subdir/tokio/index.rs | 90 ++++++++++++++++--- 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs index 9d6a6343da..e99d67feb2 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs @@ -29,6 +29,39 @@ use tokio::{ }; use url::Url; +/// `Cache-Control` synthesized for a shard index served without one. +/// +/// Azure Blob Storage sends no `Cache-Control` on the shard index, so the +/// derived [`CachePolicy`] has a zero freshness lifetime and every fetch issues +/// a conditional revalidation round-trip forever. Granting a small `max-age` +/// (60 seconds) lets repeated fetches inside the window be served straight from +/// the local cache; once it lapses the retained validators (`ETag` / +/// `Last-Modified`) still drive a cheap revalidation. Responses that carry their +/// own `Cache-Control` are used verbatim, so an origin's explicit freshness +/// policy is never weakened. +const SHARD_INDEX_SYNTHETIC_CACHE_CONTROL: &str = "max-age=60"; + +/// Build a [`CachePolicy`] for a shard-index response. +/// +/// Responses that already carry a `Cache-Control` header are used as-is. +/// Responses without one (notably Azure Blob Storage) are given a small +/// synthetic `max-age` so freshness accumulates instead of forcing a +/// revalidation on every fetch. Validators are preserved either way. +fn shard_index_cache_policy(request: &SimpleRequest, response: &Response) -> CachePolicy { + if response.headers().contains_key(http::header::CACHE_CONTROL) { + return CachePolicy::new(request, response); + } + + let mut synthetic = http::Response::new(()); + *synthetic.status_mut() = response.status(); + *synthetic.headers_mut() = response.headers().clone(); + synthetic.headers_mut().insert( + http::header::CACHE_CONTROL, + http::HeaderValue::from_static(SHARD_INDEX_SYNTHETIC_CACHE_CONTROL), + ); + CachePolicy::new(request, &synthetic) +} + /// Creates a `SubdirNotFoundError` for when sharded repodata is not available. fn create_subdir_not_found_error(channel_base_url: &Url) -> GatewayError { GatewayError::SubdirNotFoundError(Box::new(SubdirNotFoundError { @@ -268,17 +301,36 @@ pub async fn fetch_index( // still valid, so serve it directly. Servers that omit // `Cache-Control` on the shard index (e.g. Azure Blob // Storage) leave the cached policy with no freshness - // lifetime, so every request revalidates and answers with - // 304; without this guard the 304 falls through to + // lifetime, so without this guard the 304 falls through to // `from_response`, which rejects it as an unexpected status. + // + // Persist a refreshed policy from this revalidation so the + // next fetch is a local cache hit rather than yet another + // conditional round-trip: `shard_index_cache_policy` + // synthesizes a small `max-age` for the Cache-Control-less + // Azure case, so freshness accumulates. if response.status() == reqwest::StatusCode::NOT_MODIFIED { - match read_shard_index_from_reader(&mut cache_reader).await { - Ok(shard_index) => { + match read_cached_body(&mut cache_reader).await { + Ok(body) => { tracing::debug!("shard index revalidated (304 Not Modified)"); + let refreshed = + shard_index_cache_policy(&canonical_request, &response); + let mut guard = cache_reader.into_inner(); + if let Err(e) = write_shard_index_cache( + guard.inner_mut(), + refreshed, + Bytes::from(body.clone()), + ) + .await + { + tracing::warn!( + "failed to persist refreshed shard index cache policy: {e}" + ); + } if let Some((reporter, index)) = download_reporter { reporter.on_download_complete(response.url(), index); } - return Ok(shard_index); + return parse_shard_index(body).await; } Err(e) => { // Cache unreadable; fall through to a fresh fetch. @@ -314,9 +366,13 @@ pub async fn fetch_index( } } } - AfterResponse::Modified(policy, _) => { + AfterResponse::Modified(_, _) => { // Close the old file so we can create a new one. tracing::debug!("shard index cache has become stale"); + // Synthesize freshness for a Cache-Control-less + // response so the re-cached index does not revalidate + // on every subsequent fetch. + let policy = shard_index_cache_policy(&canonical_request, &response); return from_response( cache_reader.into_inner(), &cache_path, @@ -403,7 +459,7 @@ pub async fn fetch_index( return Err(create_subdir_not_found_error(channel_base_url)); } - let policy = CachePolicy::new(&canonical_request, &response); + let policy = shard_index_cache_policy(&canonical_request, &response); from_response( cache_reader.into_inner(), &cache_path, @@ -483,18 +539,20 @@ async fn write_not_found_cache(cache_file: &mut File, policy: CachePolicy) -> st .await } -/// Read the shard index from a reader and deserialize it. -pub async fn read_shard_index_from_reader( +/// Read the remaining bytes (the cached shard-index body) from a reader. +async fn read_cached_body( reader: &mut BufReader, -) -> Result { - // Read the file to memory +) -> Result, GatewayError> { let mut bytes = Vec::new(); reader .read_to_end(&mut bytes) .await .map_err(|e| GatewayError::IoError("failed to read shard index buffer".to_string(), e))?; + Ok(bytes) +} - // Deserialize the bytes +/// Deserialize a shard index from raw `msgpack` bytes. +async fn parse_shard_index(bytes: Vec) -> Result { run_blocking_task(move || { rmp_serde::from_slice(&bytes) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) @@ -503,6 +561,14 @@ pub async fn read_shard_index_from_reader( .await } +/// Read the shard index from a reader and deserialize it. +pub async fn read_shard_index_from_reader( + reader: &mut BufReader, +) -> Result { + let bytes = read_cached_body(reader).await?; + parse_shard_index(bytes).await +} + /// Cache information stored at the start of the cache file. #[derive(Clone, Debug, Serialize, Deserialize)] struct CacheHeader { From 67a6ee40e0601ba7a44e3b1aaa8b000c84cfff0f Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:34:07 +1000 Subject: [PATCH 36/98] refactor(index,upload): validate az URLs via rattler_azure::account_and_container --- crates/rattler_index/src/main.rs | 27 ++++++++++--------------- crates/rattler_upload/src/upload/opt.rs | 27 ++++++++++--------------- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index f0e19b2dc7..f3557b90ce 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -40,24 +40,19 @@ fn parse_s3_url(value: &str) -> Result { fn parse_azure_url(value: &str) -> Result { let url: Url = Url::parse(value).map_err(|e| format!("`{value}` isn't a valid URL: {e}"))?; // Require an `.blob.` host and a container segment, e.g. - // https://.blob.core.windows.net//. The index - // path derives the storage account from the host, so it must be a dotted - // domain (real Azure and sovereign clouds always are). IP literals and - // single-label hosts (localhost, the Azurite emulator) are rejected here - // because no account can be derived from them. - let has_dotted_domain = - matches!(url.host(), Some(url::Host::Domain(host)) if host.contains('.')); - let has_container = url - .path_segments() - .and_then(|mut segments| segments.next()) - .is_some_and(|segment| !segment.is_empty()); - if matches!(url.scheme(), "http" | "https") && has_dotted_domain && has_container { - Ok(url) - } else { - Err(format!( + // https://.blob.core.windows.net//. The account + // and container are derived exactly as the index path derives them, via + // `rattler_azure::account_and_container`, which rejects IP literals and + // single-label hosts (localhost, the Azurite emulator) and container-less + // URLs. + if !matches!(url.scheme(), "http" | "https") + || rattler_azure::account_and_container(&url).is_err() + { + return Err(format!( "Only Azure Blob URLs of format https://.blob.core.windows.net//... can be used, not `{value}`" - )) + )); } + Ok(url) } /// SAS permissions requested when minting a user-delegation SAS for indexing. diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 324066f032..4686fa21e4 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -423,24 +423,19 @@ fn parse_azure_url(value: &str) -> Result { let url: Url = Url::parse(value).map_err(|err| format!("`{value}` isn't a valid URL: {err}"))?; // Require an `.blob.` host and a container segment, e.g. - // https://.blob.core.windows.net//. The upload - // path derives the storage account from the host, so it must be a dotted - // domain (real Azure and sovereign clouds always are). IP literals and - // single-label hosts (localhost, the Azurite emulator) are rejected here - // because no account can be derived from them. - let has_dotted_domain = - matches!(url.host(), Some(url::Host::Domain(host)) if host.contains('.')); - let has_container = url - .path_segments() - .and_then(|mut segments| segments.next()) - .is_some_and(|segment| !segment.is_empty()); - if matches!(url.scheme(), "http" | "https") && has_dotted_domain && has_container { - Ok(url) - } else { - Err(format!( + // https://.blob.core.windows.net//. The account + // and container are derived exactly as the upload path derives them, via + // `rattler_azure::account_and_container`, which rejects IP literals and + // single-label hosts (localhost, the Azurite emulator) and container-less + // URLs. + if !matches!(url.scheme(), "http" | "https") + || rattler_azure::account_and_container(&url).is_err() + { + return Err(format!( "Only Azure Blob URLs of format https://.blob.core.windows.net//... can be used, not `{value}`" - )) + )); } + Ok(url) } /// Options for uploading to Azure Blob Storage. From 0c406510e93274c4018489a69e7014c37ac5baa8 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:34:07 +1000 Subject: [PATCH 37/98] docs(index): document URL-derived azure model, drop removed azure-options block --- crates/rattler_index/README.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index 39dd139dd7..3de5fecc41 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -65,24 +65,23 @@ When `--config` is omitted, `rattler-index` falls back to its built-in defaults ## Remote storage credentials S3 endpoint and region settings live under `[s3-options.]`, keyed by -bucket name. Azure Blob Storage uses the analogous `[azure-options.]` -block, keyed by container name. Each Azure entry records the storage account and -an optional endpoint override (no secrets are stored — credentials are resolved -at runtime through `az login` locally or the `DefaultCredentialProvider` chain -in CI): +bucket name: ```toml [s3-options.my-bucket] endpoint-url = "https://my-bucket.s3.amazonaws.com" region = "eu-central-1" force-path-style = false - -[azure-options.my-container] -account = "my-storage-account" -# Optional; defaults to https://{account}.blob.core.windows.net -endpoint-url = "https://my-storage-account.blob.core.windows.net" ``` +Azure Blob Storage needs no such block: the storage account and blob endpoint +are read directly from the channel URL +(`https://.blob.core.windows.net//`), so the account, +container, and endpoint (including sovereign clouds) are fully determined by the +URL you pass. Credentials are never stored in the config — they are resolved at +runtime from `--account-key` / `--sas-token`, an `az login` session +(`--azure-cli`), or the `DefaultCredentialProvider` chain. + ## Per-channel index configuration Index options live in `[index-config]` and follow the same shape as From f6bcf0d968e0e57f917cfa3f4d08a108c4856278 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:34:13 +1000 Subject: [PATCH 38/98] refactor(azure): typed coordinates, key/SAS exclusion, TTL cap, drop serde, fix blob URLs & prefix encoding --- Cargo.lock | 3 +- crates/rattler_azure/Cargo.toml | 4 -- crates/rattler_azure/src/clap.rs | 78 +++++++++++++++++++++-- crates/rattler_azure/src/lib.rs | 24 ++++++- crates/rattler_index/Cargo.toml | 3 +- crates/rattler_index/src/lib.rs | 18 +++++- crates/rattler_upload/Cargo.toml | 3 +- crates/rattler_upload/src/upload/azure.rs | 59 +++++++++-------- 8 files changed, 146 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe7608b99f..a74638c791 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5051,7 +5051,6 @@ version = "0.1.0" dependencies = [ "clap", "jiff", - "serde", "thiserror 2.0.18", "url", ] @@ -5229,6 +5228,7 @@ dependencies = [ "indicatif", "jiff", "opendal", + "percent-encoding", "rattler_azure", "rattler_conda_types", "rattler_config", @@ -5631,6 +5631,7 @@ dependencies = [ "indicatif", "miette", "opendal", + "percent-encoding", "rattler_azure", "rattler_conda_types", "rattler_config", diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index 44ced641c2..674fffc9d8 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -14,13 +14,9 @@ default = [] # CLI credential options; resolving them may mint a short-lived user-delegation # SAS by shelling out to the Azure CLI (which needs `jiff` for the expiry). clap = ["dep:clap", "dep:jiff"] -# Serialize/deserialize support for the credential types. -serde = ["dep:serde", "serde/derive"] [dependencies] clap = { workspace = true, optional = true } jiff = { workspace = true, optional = true } thiserror = { workspace = true } url = { workspace = true } - -serde = { workspace = true, optional = true } diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index 36337157a5..93957dd457 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -2,7 +2,9 @@ use std::time::Duration; use clap::Parser; -use crate::{AzureCliSasError, AzureCredentials, AzureUrlError, mint_user_delegation_sas}; +use crate::{ + AzureCliSasError, AzureCoordinates, AzureCredentials, AzureUrlError, mint_user_delegation_sas, +}; /// Default lifetime, in minutes, of a SAS minted from an `az login` session. /// @@ -11,6 +13,13 @@ use crate::{AzureCliSasError, AzureCredentials, AzureUrlError, mint_user_delegat /// comfortably covers a typical index or upload run. const DEFAULT_AZURE_CLI_SAS_TTL_MINUTES: u64 = 30; +/// Upper bound, in minutes, accepted for `--azure-cli-sas-ttl-minutes`. +/// +/// A SAS is meant to be short-lived; one week is already generous. Capping the +/// value at the clap layer also keeps `minutes * 60` well clear of overflowing +/// the [`Duration`] arithmetic in [`AzureCredentialsOpts::source`]. +const MAX_AZURE_CLI_SAS_TTL_MINUTES: u64 = 7 * 24 * 60; + /// Errors that can occur while resolving [`AzureCredentialsOpts`] into /// [`AzureCredentials`]. #[derive(Debug, thiserror::Error)] @@ -63,13 +72,13 @@ impl AzureAuthSource { pub fn resolve( self, permissions: &str, - cli_context: impl FnOnce() -> Result<(String, String), AzureCredentialsError>, + cli_context: impl FnOnce() -> Result, ) -> Result { match self { AzureAuthSource::AccountKey(key) => Ok(AzureCredentials::AccountKey(key)), AzureAuthSource::SasToken(token) => Ok(AzureCredentials::SasToken(token)), AzureAuthSource::AzureCli { ttl } => { - let (account, container) = cli_context()?; + let AzureCoordinates { account, container } = cli_context()?; let token = mint_user_delegation_sas(&account, &container, permissions, ttl)?; Ok(AzureCredentials::SasToken(token)) } @@ -87,7 +96,16 @@ impl AzureAuthSource { #[derive(Clone, Debug, PartialEq, Parser)] pub struct AzureCredentialsOpts { /// The Azure Storage account key. - #[arg(long, env = "AZURE_STORAGE_KEY", help_heading = "Azure Credentials")] + /// + /// Mutually exclusive with `--sas-token`: supplying both is a usage error + /// rather than silently discarding one. `--azure-cli` layers on top of both + /// (see [`AzureCredentialsOpts::source`]). + #[arg( + long, + env = "AZURE_STORAGE_KEY", + conflicts_with = "sas_token", + help_heading = "Azure Credentials" + )] pub account_key: Option, /// A shared access signature (SAS) token, with or without a leading `?`. @@ -116,6 +134,7 @@ pub struct AzureCredentialsOpts { #[arg( long, default_value_t = DEFAULT_AZURE_CLI_SAS_TTL_MINUTES, + value_parser = clap::value_parser!(u64).range(1..=MAX_AZURE_CLI_SAS_TTL_MINUTES), help_heading = "Azure Credentials" )] pub azure_cli_sas_ttl_minutes: u64, @@ -137,7 +156,7 @@ impl AzureCredentialsOpts { pub fn source(&self) -> Result { if self.azure_cli { Ok(AzureAuthSource::AzureCli { - ttl: Duration::from_secs(self.azure_cli_sas_ttl_minutes * 60), + ttl: Duration::from_secs(self.azure_cli_sas_ttl_minutes.saturating_mul(60)), }) } else if let Some(sas_token) = &self.sas_token { Ok(AzureAuthSource::SasToken(sas_token.clone())) @@ -156,7 +175,7 @@ impl AzureCredentialsOpts { pub fn resolve( self, permissions: &str, - cli_context: impl FnOnce() -> Result<(String, String), AzureCredentialsError>, + cli_context: impl FnOnce() -> Result, ) -> Result { self.source()?.resolve(permissions, cli_context) } @@ -180,7 +199,7 @@ mod tests { } /// `cli_context` must not be invoked for the account-key/SAS-token paths. - fn unreachable_context() -> Result<(String, String), AzureCredentialsError> { + fn unreachable_context() -> Result { panic!("cli_context should not be called for non-azure-cli sources"); } @@ -264,4 +283,49 @@ mod tests { DEFAULT_AZURE_CLI_SAS_TTL_MINUTES ); } + + /// `--account-key` and `--sas-token` are mutually exclusive: passing both is + /// a clap error rather than silently discarding one. + #[test] + fn account_key_and_sas_token_conflict() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + creds: AzureCredentialsOpts, + } + + let err = Cli::try_parse_from(["test", "--account-key", "k", "--sas-token", "sv=..."]) + .map(|_| ()) + .expect_err("passing both --account-key and --sas-token must be rejected"); + assert_eq!(err.kind(), clap::error::ErrorKind::ArgumentConflict); + } + + /// A zero TTL is rejected, and the maximum is capped so `minutes * 60` + /// cannot overflow. + #[test] + fn ttl_range_is_enforced() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + creds: AzureCredentialsOpts, + } + + assert!( + Cli::try_parse_from(["test", "--azure-cli", "--azure-cli-sas-ttl-minutes", "0"]) + .is_err() + ); + assert!( + Cli::try_parse_from([ + "test", + "--azure-cli", + "--azure-cli-sas-ttl-minutes", + &(MAX_AZURE_CLI_SAS_TTL_MINUTES + 1).to_string(), + ]) + .is_err() + ); + } } diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 7276e52ac2..d0397846ff 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -10,8 +10,10 @@ use url::Url; /// account name, endpoint, and container are not stored here: they are fully /// determined by the channel URL (`https://.blob.core.windows.net//...`) /// and derived by the consumer. +/// +/// The type deliberately has no `Serialize`/`Deserialize`: it holds raw account +/// keys and SAS tokens, so serialization would risk leaking secrets to disk. #[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum AzureCredentials { /// A shared storage account key. AccountKey(String), @@ -53,13 +55,26 @@ pub enum AzureUrlError { NoContainer, } +/// The storage account and container an Azure Blob channel URL resolves to. +/// +/// A named pair rather than a bare `(String, String)` so the two cannot be +/// silently transposed at a call site. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AzureCoordinates { + /// The storage account name (first label of the host). + pub account: String, + + /// The blob container name (first path segment). + pub container: String, +} + /// Derive the storage account name and container from an Azure Blob channel URL /// of the form `https://.blob.core.windows.net//`. /// /// The account name is the first label of the host, so the host must be a dotted /// domain; IP-literal and single-label hosts (e.g. `localhost` or the Azurite /// emulator) are rejected because no account can be derived from them. -pub fn account_and_container(url: &Url) -> Result<(String, String), AzureUrlError> { +pub fn account_and_container(url: &Url) -> Result { let host = url.host_str().ok_or(AzureUrlError::NoHost)?; if !matches!(url.host(), Some(url::Host::Domain(domain)) if domain.contains('.')) { return Err(AzureUrlError::InvalidHost(host.to_string())); @@ -74,7 +89,10 @@ pub fn account_and_container(url: &Url) -> Result<(String, String), AzureUrlErro .and_then(|mut segments| segments.next()) .filter(|segment| !segment.is_empty()) .ok_or(AzureUrlError::NoContainer)?; - Ok((account.to_string(), container.to_string())) + Ok(AzureCoordinates { + account: account.to_string(), + container: container.to_string(), + }) } /// Errors that can occur while minting a user-delegation SAS via the Azure CLI. diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index b46e33dcd1..7dd92cd6f2 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -28,7 +28,7 @@ rustls = [ "opendal/reqwest-rustls-tls", ] s3 = ["opendal/services-s3", "dep:rattler_s3"] -azure = ["opendal/services-azblob", "dep:rattler_azure"] +azure = ["opendal/services-azblob", "dep:rattler_azure", "dep:percent-encoding"] [[bin]] name = "rattler-index" @@ -76,6 +76,7 @@ tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } url = { workspace = true } +percent-encoding = { workspace = true, optional = true } zstd = { workspace = true } retry-policies = { workspace = true } diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 8836bade81..86ae4cb4f1 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -1604,7 +1604,10 @@ fn azblob_config( credentials: &AzureCredentials, channel: &Url, ) -> Result { - let (account_name, container) = rattler_azure::account_and_container(channel)?; + let rattler_azure::AzureCoordinates { + account: account_name, + container, + } = rattler_azure::account_and_container(channel)?; let host = channel .host_str() @@ -1613,9 +1616,18 @@ fn azblob_config( let mut segments = channel .path_segments() .ok_or_else(|| anyhow::anyhow!("No path in Azure blob URL"))?; - // Skip the container segment; the remainder is the root prefix. + // Skip the container segment; the remainder is the root prefix. Percent-decode + // each segment before joining: `path_segments()` yields still-encoded segments, + // and opendal percent-encodes the root again, so passing them through verbatim + // would double-encode prefixes containing spaces or `+`. segments.next(); - let root = format!("/{}", segments.collect::>().join("/")); + let root = format!( + "/{}", + segments + .map(|segment| percent_encoding::percent_decode_str(segment).decode_utf8_lossy()) + .collect::>() + .join("/") + ); // Preserve a non-default port if one is present; real Azure uses the scheme // default (443). diff --git a/crates/rattler_upload/Cargo.toml b/crates/rattler_upload/Cargo.toml index 74f5ce6aa9..b0bcb034dc 100644 --- a/crates/rattler_upload/Cargo.toml +++ b/crates/rattler_upload/Cargo.toml @@ -29,7 +29,7 @@ native-tls = [ "sigstore-sign?/native-tls", ] s3 = ["rattler_networking/s3", "rattler_s3", "dep:opendal"] -azure = ["dep:rattler_azure", "dep:opendal", "opendal/services-azblob"] +azure = ["dep:rattler_azure", "dep:opendal", "opendal/services-azblob", "dep:percent-encoding"] sigstore-sign = ["dep:sigstore-sign", "dep:sigstore-trust-root"] [package.metadata.docs.rs] @@ -63,6 +63,7 @@ reqwest = { workspace = true, default-features = false, features = [ "stream", ] } url = { workspace = true } +percent-encoding = { workspace = true, optional = true } tracing = { workspace = true } reqwest-middleware = { workspace = true, features = ["json"] } serde_yaml = { workspace = true } diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 9e64389151..ca3140c9d8 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -32,10 +32,9 @@ pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "cw"; /// name, endpoint, container, and root prefix are all derived from it (see /// [`azblob_config`]). Because the account is derived from the host, upload /// requires this dotted `.blob...` form and does not support -/// path-style or emulator (Azurite) endpoints. (The fetch middleware, by -/// contrast, does support custom endpoints — including Azurite — via -/// `azure-options..endpoint-url`.) The [`AzureCredentials`] supply -/// only the account key or SAS token. +/// path-style or emulator (Azurite) endpoints. The full blob host lives in the +/// channel URL itself, so no separate account/endpoint configuration is needed. +/// The [`AzureCredentials`] supply only the account key or SAS token. pub async fn upload_package_to_azure( channel: Url, credentials: AzureCredentials, @@ -43,7 +42,6 @@ pub async fn upload_package_to_azure( force: bool, ) -> miette::Result<()> { let config = azblob_config(&credentials, &channel)?; - let container = config.container.clone(); let builder = config.into_builder(); let op = Operator::new(builder).into_diagnostic()?.finish(); @@ -54,8 +52,7 @@ pub async fn upload_package_to_azure( .map(|package_file| { let op = op.clone(); let channel = &channel; - let container = container.as_str(); - async move { upload_single_package(&op, channel, container, package_file, force).await } + async move { upload_single_package(&op, channel, package_file, force).await } }) .buffer_unordered(PACKAGE_CONCURRENCY) .collect::>() @@ -70,7 +67,6 @@ pub async fn upload_package_to_azure( async fn upload_single_package( op: &Operator, channel: &Url, - container: &str, package_file: &Path, force: bool, ) -> miette::Result<()> { @@ -83,6 +79,15 @@ async fn upload_single_package( .ok_or_else(|| miette::miette!("Failed to get filename"))?; let key = format!("{subdir}/{filename}"); + // The blob's on-the-wire address, used only for diagnostics. `channel.path()` + // already carries `//`, so the full `az://` URL is the host + // followed by that path and the key; do not prepend the container again. + let blob_url = format!( + "az://{}{}/{key}", + channel.host_str().unwrap_or_default(), + channel.path() + ); + // Compute the hash of the package by streaming its content. let file = tokio::io::BufReader::new( fs_err::tokio::File::open(package_file) @@ -119,10 +124,7 @@ async fn upload_single_package( .await { Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { - miette::bail!( - "Package az://{container}{}/{key} already exists. Use --force to overwrite.", - channel.path().to_string() - ); + miette::bail!("Package {blob_url} already exists. Use --force to overwrite."); } Ok(writer) => writer, Err(e) => { @@ -138,8 +140,8 @@ async fn upload_single_package( // Allocate memory for this chunk. let chunk_size = remaining_size.min(DESIRED_CHUNK_SIZE); let mut chunk = BytesMut::with_capacity(chunk_size); - // SAFE: because we do not care about the bytes that are currently in the buffer - unsafe { chunk.set_len(chunk_size) }; + // Zero-fill up to `chunk_size`; `read_exact` below overwrites every byte. + chunk.resize(chunk_size, 0); // Fill the chunk with data. This reads exactly the number of bytes we want. No // more, no less. @@ -156,16 +158,10 @@ async fn upload_single_package( match writer.close().await { Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { - miette::bail!( - "Package az://{container}{}/{key} already exists. Use --force to overwrite.", - channel.path().to_string() - ); + miette::bail!("Package {blob_url} already exists. Use --force to overwrite."); } Ok(_) => { - tracing::info!( - "Uploaded package to az://{container}{}/{key}", - channel.path().to_string() - ); + tracing::info!("Uploaded package to {blob_url}"); } Err(e) => { return Err(e).into_diagnostic(); @@ -181,15 +177,26 @@ async fn upload_single_package( /// the URL (`https://.blob.core.windows.net//`); the /// credentials supply only the account key or SAS token. fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Result { - let (account_name, container) = - rattler_azure::account_and_container(channel).into_diagnostic()?; + let rattler_azure::AzureCoordinates { + account: account_name, + container, + } = rattler_azure::account_and_container(channel).into_diagnostic()?; let mut segments = channel .path_segments() .ok_or_else(|| miette::miette!("No path in Azure blob URL"))?; - // Skip the container segment; the remainder is the root prefix. + // Skip the container segment; the remainder is the root prefix. Percent-decode + // each segment before joining: `path_segments()` yields still-encoded segments, + // and opendal percent-encodes the root again, so passing them through verbatim + // would double-encode prefixes containing spaces or `+`. segments.next(); - let root = format!("/{}", segments.collect::>().join("/")); + let root = format!( + "/{}", + segments + .map(|segment| percent_encoding::percent_decode_str(segment).decode_utf8_lossy()) + .collect::>() + .join("/") + ); // Preserve a non-default port if one is present; real Azure uses the scheme // default (443). From 29dfa2b139e3dd4e11c679f63c90ea0f486ef5cd Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 13:40:03 +1000 Subject: [PATCH 39/98] feat(py): add azure middleware bindings --- py-rattler/Cargo.lock | 217 +++++++++++++++++- py-rattler/Cargo.toml | 1 + py-rattler/rattler/networking/__init__.py | 2 + py-rattler/rattler/networking/client.py | 2 + py-rattler/rattler/networking/middleware.py | 25 ++ py-rattler/src/lib.rs | 5 +- py-rattler/src/networking/client.rs | 7 +- py-rattler/src/networking/middleware.rs | 14 ++ .../tests/unit/test_package_streaming.py | 6 +- 9 files changed, 273 insertions(+), 6 deletions(-) diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index 6305348ce2..586b761b2d 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -1281,7 +1281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ "digest 0.10.7", - "spin", + "spin 0.10.1", ] [[package]] @@ -2873,6 +2873,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.9", +] [[package]] name = "libbz2-rs-sys" @@ -3116,6 +3119,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.7", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -3168,6 +3187,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3200,6 +3220,7 @@ checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" dependencies = [ "opendal-core", "opendal-layer-retry", + "opendal-service-azblob", "opendal-service-fs", "opendal-service-s3", ] @@ -3243,6 +3264,37 @@ dependencies = [ "opendal-core", ] +[[package]] +name = "opendal-service-azblob" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +dependencies = [ + "base64", + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.39.4", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.11.0", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +dependencies = [ + "http 1.4.2", + "opendal-core", +] + [[package]] name = "opendal-service-fs" version = "0.57.0" @@ -3416,6 +3468,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3510,6 +3582,32 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -3517,6 +3615,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", + "pkcs5", + "rand_core 0.6.4", "spki", ] @@ -4012,6 +4112,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "rattler_azure" +version = "0.1.0" +dependencies = [ + "clap", + "jiff", + "thiserror 2.0.18", + "url", +] + [[package]] name = "rattler_cache" version = "0.10.3" @@ -4125,6 +4235,7 @@ version = "0.30.9" dependencies = [ "ahash", "anyhow", + "base64", "bytes", "clap", "clap-verbosity-flag", @@ -4136,6 +4247,8 @@ dependencies = [ "indicatif", "jiff", "opendal", + "percent-encoding", + "rattler_azure", "rattler_conda_types", "rattler_config", "rattler_digest", @@ -4247,6 +4360,11 @@ dependencies = [ "keyring-core", "netrc-rs", "regex", + "reqsign-azure-storage", + "reqsign-command-execute-tokio", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-http-send-reqwest", "reqwest", "retry-policies", "serde", @@ -4577,6 +4695,37 @@ dependencies = [ "sha1 0.11.0", ] +[[package]] +name = "reqsign-azure-storage" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee8b9e5d0fc927551a6ac25ba5dc6518860c54ddb592fecf685523e528e77bd" +dependencies = [ + "anyhow", + "base64", + "bytes", + "form_urlencoded", + "http 1.4.2", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-command-execute-tokio" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf28fcf0aef694448cb235d393fbec7b5543d25050eacc0ccc2118596618c52" +dependencies = [ + "reqsign-core", + "tokio", +] + [[package]] name = "reqsign-core" version = "3.1.0" @@ -4594,6 +4743,9 @@ dependencies = [ "jiff", "log", "percent-encoding", + "rsa", + "serde", + "serde_json", "sha1 0.11.0", "sha2 0.11.0", "windows-sys 0.61.2", @@ -4610,6 +4762,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "reqsign-http-send-reqwest" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b4509b12b73289cdeed349ba494553404e33782abc09bc0ae4e09f741807cb" +dependencies = [ + "anyhow", + "bytes", + "futures-channel", + "http 1.4.2", + "http-body-util", + "reqsign-core", + "reqwest", + "wasm-bindgen-futures", +] + [[package]] name = "reqwest" version = "0.13.4" @@ -4726,6 +4894,27 @@ dependencies = [ "serde", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -4877,6 +5066,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -4925,6 +5123,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sec1" version = "0.7.3" @@ -5303,6 +5512,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "spin" version = "0.10.1" diff --git a/py-rattler/Cargo.toml b/py-rattler/Cargo.toml index 16f1870c22..00d3067452 100644 --- a/py-rattler/Cargo.toml +++ b/py-rattler/Cargo.toml @@ -44,6 +44,7 @@ rattler_digest = { path = "../crates/rattler_digest" } rattler_networking = { path = "../crates/rattler_networking", default-features = false, features = [ "gcs", "s3", + "azure", "system-integration", ] } rattler_shell = { path = "../crates/rattler_shell", default-features = false } diff --git a/py-rattler/rattler/networking/__init__.py b/py-rattler/rattler/networking/__init__.py index 580072acda..81897aabf2 100644 --- a/py-rattler/rattler/networking/__init__.py +++ b/py-rattler/rattler/networking/__init__.py @@ -3,6 +3,7 @@ from rattler.networking.middleware import ( AddHeadersMiddleware, AuthenticationMiddleware, + AzureMiddleware, GCSMiddleware, MirrorMiddleware, OciMiddleware, @@ -22,4 +23,5 @@ "OciMiddleware", "S3Middleware", "GCSMiddleware", + "AzureMiddleware", ] diff --git a/py-rattler/rattler/networking/client.py b/py-rattler/rattler/networking/client.py index e18d08f982..5ef7d84b82 100644 --- a/py-rattler/rattler/networking/client.py +++ b/py-rattler/rattler/networking/client.py @@ -3,6 +3,7 @@ from rattler.networking.middleware import ( AddHeadersMiddleware, AuthenticationMiddleware, + AzureMiddleware, GCSMiddleware, MirrorMiddleware, OciMiddleware, @@ -27,6 +28,7 @@ def __init__( | MirrorMiddleware | OciMiddleware | GCSMiddleware + | AzureMiddleware | S3Middleware ] | None diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index 6a791de516..4389518483 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -5,6 +5,7 @@ from rattler.rattler import ( PyAddHeadersMiddleware, PyAuthenticationMiddleware, + PyAzureMiddleware, PyGCSMiddleware, PyMirrorMiddleware, PyOciMiddleware, @@ -156,6 +157,30 @@ def __repr__(self) -> str: return f"{type(self).__name__}()" +class AzureMiddleware: + """ + Middleware to work with az:// URLs + + Examples + -------- + ```python + >>> from rattler.networking import Client + >>> middleware = AzureMiddleware() + >>> middleware + AzureMiddleware() + >>> Client([middleware]) + Client() + >>> + ``` + """ + + def __init__(self) -> None: + self._middleware = PyAzureMiddleware() + + def __repr__(self) -> str: + return f"{type(self).__name__}()" + + class S3Config: """ Middleware to work with s3:// URLs diff --git a/py-rattler/src/lib.rs b/py-rattler/src/lib.rs index a8c386aa90..df26e81a44 100644 --- a/py-rattler/src/lib.rs +++ b/py-rattler/src/lib.rs @@ -59,8 +59,8 @@ use match_spec::PyMatchSpec; use meta::get_rattler_version; use nameless_match_spec::PyNamelessMatchSpec; use networking::middleware::{ - PyAddHeadersMiddleware, PyAuthenticationMiddleware, PyGCSMiddleware, PyMirrorMiddleware, - PyOciMiddleware, PyRetryMiddleware, PyS3Config, PyS3Middleware, + PyAddHeadersMiddleware, PyAuthenticationMiddleware, PyAzureMiddleware, PyGCSMiddleware, + PyMirrorMiddleware, PyOciMiddleware, PyRetryMiddleware, PyS3Config, PyS3Middleware, }; use networking::{client::PyClientWithMiddleware, py_fetch_repo_data}; use no_arch_type::PyNoArchType; @@ -119,6 +119,7 @@ fn rattler<'py>(py: Python<'py>, m: Bound<'py, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index 1c6580cb4e..bfc65c5b29 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -4,8 +4,8 @@ use crate::{ }; use pyo3::{PyResult, pyclass, pymethods}; use rattler_networking::{ - AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, LazyClient, MirrorMiddleware, - OciMiddleware, S3Middleware, + AuthenticationMiddleware, AuthenticationStorage, AzureMiddleware, GCSMiddleware, LazyClient, + MirrorMiddleware, OciMiddleware, S3Middleware, }; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use reqwest_middleware::ClientWithMiddleware; @@ -84,6 +84,9 @@ impl PyClientWithMiddleware { PyMiddleware::Gcs(middleware) => { client = client.with(GCSMiddleware::from(middleware)); } + PyMiddleware::Azure(_middleware) => { + client = client.with(AzureMiddleware::new(reqwest_client.clone())); + } PyMiddleware::S3(middleware) => { client = client.with(S3Middleware::new( middleware diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index 92b12d14ac..66428068ef 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -20,6 +20,7 @@ pub enum PyMiddleware { Retry(PyRetryMiddleware), Oci(PyOciMiddleware), Gcs(PyGCSMiddleware), + Azure(PyAzureMiddleware), S3(PyS3Middleware), AddHeaders(PyAddHeadersMiddleware), } @@ -124,6 +125,19 @@ impl From for GCSMiddleware { } } +#[pyclass(from_py_object)] +#[repr(transparent)] +#[derive(Clone)] +pub struct PyAzureMiddleware {} + +#[pymethods] +impl PyAzureMiddleware { + #[new] + pub fn __init__() -> Self { + Self {} + } +} + #[derive(Clone)] #[pyclass(from_py_object)] pub struct PyS3Config { diff --git a/py-rattler/tests/unit/test_package_streaming.py b/py-rattler/tests/unit/test_package_streaming.py index a9797637c7..12ef2a391d 100644 --- a/py-rattler/tests/unit/test_package_streaming.py +++ b/py-rattler/tests/unit/test_package_streaming.py @@ -2,7 +2,7 @@ import pytest from pathlib import Path -from rattler.networking.middleware import MirrorMiddleware, OciMiddleware, GCSMiddleware +from rattler.networking.middleware import AzureMiddleware, MirrorMiddleware, OciMiddleware, GCSMiddleware from rattler.package_streaming import ( download_and_extract, download_bytes, @@ -139,3 +139,7 @@ async def test_download_from_oci(tmpdir: Path) -> None: def test_instantiate_gcs_middleware() -> None: _client = Client([GCSMiddleware()]) + + +def test_instantiate_azure_middleware() -> None: + _client = Client([AzureMiddleware()]) From c6eaf26ef30c31d860de451aff262adf2d388207 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 13:44:16 +1000 Subject: [PATCH 40/98] fix(index): hoist nested async fn above statements in test --- .../tests/integration/basic_indexing.rs | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/crates/rattler_index/tests/integration/basic_indexing.rs b/crates/rattler_index/tests/integration/basic_indexing.rs index fb143734d3..bf8b9a7e97 100644 --- a/crates/rattler_index/tests/integration/basic_indexing.rs +++ b/crates/rattler_index/tests/integration/basic_indexing.rs @@ -384,6 +384,24 @@ async fn test_index_repodata_revision_from_index_json() { /// current blob hit a hash mismatch. #[tokio::test] async fn test_incremental_reindexes_replaced_package() { + // Incremental index (force: false) — the path that carried the bug. + async fn index(channel: &Path) { + index_fs(IndexFsConfig { + channel: channel.into(), + target_platform: Some(Platform::NoArch), + repodata_patch: None, + write_zst: false, + write_shards: false, + repodata_revisions: Vec::new(), + package_revision_assignment: PackageRevisionAssignment::default(), + force: false, + max_parallel: 1, + multi_progress: None, + }) + .await + .unwrap(); + } + let temp_dir = tempfile::tempdir().unwrap(); let subdir_path = temp_dir.path().join("noarch"); fs::create_dir(&subdir_path).unwrap(); @@ -413,24 +431,6 @@ async fn test_incremental_reindexes_replaced_package() { .unwrap(); }; - // Incremental index (force: false) — the path that carried the bug. - async fn index(channel: &Path) { - index_fs(IndexFsConfig { - channel: channel.into(), - target_platform: Some(Platform::NoArch), - repodata_patch: None, - write_zst: false, - write_shards: false, - repodata_revisions: Vec::new(), - package_revision_assignment: PackageRevisionAssignment::default(), - force: false, - max_parallel: 1, - multi_progress: None, - }) - .await - .unwrap(); - } - let recorded = |field: &str| -> Value { let repodata: Value = serde_json::from_reader(File::open(subdir_path.join("repodata.json")).unwrap()) From 4ba47b6cdf11b6a4057029429750f119d20c3aa8 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:59:19 +1000 Subject: [PATCH 41/98] fix(azure): reject userinfo, gate az coord charset, invoke az directly on windows --- Cargo.lock | 1 + crates/rattler_azure/Cargo.toml | 3 +- crates/rattler_azure/src/lib.rs | 115 ++++++++++++++++++++++++++++++-- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a74638c791..a2a2b7e735 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5053,6 +5053,7 @@ dependencies = [ "jiff", "thiserror 2.0.18", "url", + "which", ] [[package]] diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index 674fffc9d8..be57685f93 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -13,10 +13,11 @@ readme.workspace = true default = [] # CLI credential options; resolving them may mint a short-lived user-delegation # SAS by shelling out to the Azure CLI (which needs `jiff` for the expiry). -clap = ["dep:clap", "dep:jiff"] +clap = ["dep:clap", "dep:jiff", "dep:which"] [dependencies] clap = { workspace = true, optional = true } jiff = { workspace = true, optional = true } thiserror = { workspace = true } url = { workspace = true } +which = { workspace = true, optional = true } diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index d0397846ff..151b04a1e8 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -1,3 +1,15 @@ +//! Helpers for deriving Azure Blob coordinates from channel URLs and for minting +//! short-lived credentials for them. +//! +//! # Trusted-host model +//! +//! [`account_and_container`] trusts the URL host verbatim: whatever host is +//! named is taken to be the storage endpoint, and any ambient AAD credentials +//! (an `az login` session) are sent to that host. Userinfo (`user:pass@host`) is +//! rejected because it is a host-spoofing vector, but an honest, arbitrary host +//! is the caller's responsibility — this crate does not police which hosts are +//! legitimate Azure endpoints. + #[cfg(feature = "clap")] pub mod clap; @@ -39,6 +51,13 @@ pub enum AzureUrlError { #[error("no host in Azure blob URL")] NoHost, + /// The URL carries userinfo (`user:pass@host`). + #[error( + "Azure blob URL must not contain userinfo (`user:pass@host`): the `user@host` form is a \ + host-spoofing vector that can disguise the real target host" + )] + UserInfoNotAllowed, + /// The host is not a dotted domain, so no storage account can be derived. #[error( "Azure blob URL host `{0}` is not a dotted domain of the form `.blob.`; \ @@ -53,6 +72,13 @@ pub enum AzureUrlError { /// The URL has no container path segment. #[error("no container in Azure blob URL")] NoContainer, + + /// The derived account or container contains characters outside `[a-z0-9-]`. + #[error( + "Azure blob URL component `{0}` contains characters outside [a-z0-9-]; account and \ + container names are restricted to that set" + )] + InvalidCharacters(String), } /// The storage account and container an Azure Blob channel URL resolves to. @@ -74,7 +100,19 @@ pub struct AzureCoordinates { /// The account name is the first label of the host, so the host must be a dotted /// domain; IP-literal and single-label hosts (e.g. `localhost` or the Azurite /// emulator) are rejected because no account can be derived from them. +/// +/// The host is otherwise trusted verbatim (see the [crate-level docs] for the +/// trusted-host model): userinfo (`user:pass@host`) is rejected as a +/// host-spoofing vector, but an honest, arbitrary host is the caller's +/// responsibility. The derived account and container are additionally restricted +/// to `[a-z0-9-]` so that argument-injection-shaped values can never reach the +/// `az` subprocess. +/// +/// [crate-level docs]: crate pub fn account_and_container(url: &Url) -> Result { + if !url.username().is_empty() || url.password().is_some() { + return Err(AzureUrlError::UserInfoNotAllowed); + } let host = url.host_str().ok_or(AzureUrlError::NoHost)?; if !matches!(url.host(), Some(url::Host::Domain(domain)) if domain.contains('.')) { return Err(AzureUrlError::InvalidHost(host.to_string())); @@ -89,6 +127,14 @@ pub fn account_and_container(url: &Url) -> Result std::process::Command { - let mut command = std::process::Command::new("cmd"); - command.args(["/C", "az"]); - command +fn az_command() -> Result { + let path = which::which("az").map_err(AzureCliSasError::AzResolve)?; + Ok(std::process::Command::new(path)) } #[cfg(all(feature = "clap", not(windows)))] fn az_command() -> std::process::Command { std::process::Command::new("az") } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normal_url_resolves() { + let url = Url::parse("https://acct.blob.core.windows.net/general/noarch").unwrap(); + assert_eq!( + account_and_container(&url).unwrap(), + AzureCoordinates { + account: "acct".to_string(), + container: "general".to_string(), + } + ); + } + + #[test] + fn userinfo_is_rejected() { + let url = Url::parse("https://acct.blob.core.windows.net@evil.example/general").unwrap(); + assert!(matches!( + account_and_container(&url), + Err(AzureUrlError::UserInfoNotAllowed) + )); + } + + #[test] + fn invalid_charset_container_is_rejected() { + let url = Url::parse("https://acct.blob.core.windows.net/general;evil/noarch").unwrap(); + assert!(matches!( + account_and_container(&url), + Err(AzureUrlError::InvalidCharacters(_)) + )); + } +} From 36bd7397115c82338ecac6ff2df12bd93c3cc721 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:00:34 +1000 Subject: [PATCH 42/98] fix(azure): reject az:// userinfo and error on broken credential --- .../src/azure_middleware.rs | 173 ++++++++++++++++-- 1 file changed, 162 insertions(+), 11 deletions(-) diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 39280cf24e..3ff947d063 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -12,6 +12,45 @@ use url::Url; /// The Azure Storage REST API version sent on every request. const X_MS_VERSION: &str = "2021-12-02"; +/// Whether an Azure credential source appears configured in the process +/// environment. +/// +/// Used to distinguish "no credential at all" (fall back to an unsigned, +/// anonymous request) from "a credential is configured but signing failed" +/// (a hard error). It is a presence check only — it does not validate that the +/// values are usable; reqsign does that when it actually signs. +/// +/// NOTE: an interactive `az login` CLI session is intentionally NOT detected +/// here. Doing so would require shelling out to `az` (or parsing its token +/// cache), which is more than a cheap env probe. The residual gap: a machine +/// authenticated only via `az login` whose session is broken will still fall +/// back to an unsigned request rather than erroring. +fn azure_credential_source_present() -> bool { + // Explicit Shared Key or SAS token. + if std::env::var_os("AZURE_STORAGE_ACCOUNT_KEY").is_some() + || std::env::var_os("AZURE_STORAGE_SAS_TOKEN").is_some() + { + return true; + } + // Service-principal flows need both a client and a tenant to be meaningful. + if std::env::var_os("AZURE_CLIENT_ID").is_some() + && std::env::var_os("AZURE_TENANT_ID").is_some() + { + return true; + } + // Workload identity federation (e.g. AKS). + if std::env::var_os("AZURE_FEDERATED_TOKEN_FILE").is_some() { + return true; + } + // Managed identity endpoints (App Service / Functions / Cloud Shell / IMDS + // override). + if std::env::var_os("MSI_ENDPOINT").is_some() || std::env::var_os("IDENTITY_ENDPOINT").is_some() + { + return true; + } + false +} + /// Middleware that rewrites `az://` URLs to HTTPS Azure Blob Storage URLs and /// signs them. /// @@ -22,19 +61,39 @@ const X_MS_VERSION: &str = "2021-12-02"; /// endpoint configuration is needed. Sovereign clouds and emulators (Azurite) /// work automatically because the endpoint is spelled out in the host. /// +/// # Trust model +/// +/// The URL host is **trusted verbatim**: it becomes the HTTPS request target +/// unchanged, with no allow-list of accounts or endpoints. Whatever ambient +/// AAD / Shared-Key credential resolves below is applied to that host — so a +/// channel author who controls the `az://` URL controls where the request, and +/// any credential material, is sent. Because of this, **userinfo is rejected**: +/// an `az://user:pass@host/...` authority is a host-spoofing vector (the real +/// host can be hidden behind userinfo) and such requests are refused before any +/// rewrite or signing. +/// /// Credentials are resolved by reqsign's [`DefaultCredentialProvider`] chain, in /// its usual order: environment variables, then workload/managed identity, then /// the Azure CLI (`az login`). rattler's [`crate::AuthenticationStorage`] is not /// consulted for Azure — there is no `Authentication` Azure variant — so /// per-host credentials configured there do not apply to `az://` requests. /// -/// When no credential resolves at all, the request is sent **unsigned** rather -/// than failing, so public/anonymous containers remain reachable with zero -/// ambient credentials. +/// When **no credential source is detected**, the request is sent **unsigned** +/// rather than failing, so public/anonymous containers remain reachable with +/// zero ambient credentials. When a credential source *is* configured +/// (see [`azure_credential_source_present`]) but signing fails, that is a +/// **hard error** — reqsign collapses "no credential" and "broken credential" +/// into the same [`ErrorKind::CredentialInvalid`], so a broken credential must +/// not be silently downgraded to an anonymous request. #[derive(Clone)] pub struct AzureMiddleware { /// reqsign signer; caches the resolved credential internally. signer: Signer, + /// Whether an Azure credential source appears configured in the process + /// environment. Captured at construction. When `true`, a signing failure is + /// propagated as a hard error instead of falling back to an unsigned + /// request. + credential_source_present: bool, } impl AzureMiddleware { @@ -45,17 +104,24 @@ impl AzureMiddleware { /// client — proxy, CA bundle, and TLS settings carry through to those /// requests. pub fn new(client: Client) -> Self { - Self::with_credential_provider(client, DefaultCredentialProvider::new()) + Self::with_credential_provider( + client, + DefaultCredentialProvider::new(), + azure_credential_source_present(), + ) } /// Build the middleware around an explicit credential provider. /// - /// [`AzureMiddleware::new`] wires up the [`DefaultCredentialProvider`] chain; - /// tests use this seam to inject a deterministic provider (e.g. an empty - /// chain, or a static key) without touching the ambient environment. + /// [`AzureMiddleware::new`] wires up the [`DefaultCredentialProvider`] chain + /// and detects the credential source from the environment; tests use this + /// seam to inject a deterministic provider (e.g. an empty chain, or a static + /// key) and an explicit `credential_source_present` flag without touching the + /// ambient environment. fn with_credential_provider( client: Client, provider: impl ProvideCredential + 'static, + credential_source_present: bool, ) -> Self { let ctx = Context::new() .with_file_read(TokioFileRead) @@ -63,7 +129,10 @@ impl AzureMiddleware { .with_command_execute(TokioCommandExecute) .with_env(OsEnv); let signer = Signer::new(ctx, provider, RequestSigner::new()); - Self { signer } + Self { + signer, + credential_source_present, + } } /// Rewrite an `az://{host}/{path}` URL to its HTTPS equivalent by swapping @@ -83,15 +152,26 @@ impl AzureMiddleware { url.query_pairs().any(|(key, _)| key == "sig") } + /// Whether the URL carries userinfo (`user` and/or `:pass` before the host). + /// Because the host is trusted verbatim, a `user:pass@host` authority is a + /// host-spoofing vector and must be refused. + fn has_userinfo(url: &Url) -> bool { + !url.username().is_empty() || url.password().is_some() + } + /// Sign a reqwest `Request` in place using reqsign. /// /// Two cases short-circuit without touching the request: /// - The URL already carries an explicit SAS (`?...&sig=...`). Signing would /// add an `Authorization` header that Azure prefers over the SAS, silently /// overriding the caller's explicit token. - /// - No credential resolves. reqsign surfaces this as + /// - No credential source is configured. reqsign surfaces this as /// [`ErrorKind::CredentialInvalid`]; the request is then sent unsigned so /// public/anonymous containers stay reachable. + /// + /// If a credential source *is* configured but signing still fails with + /// [`ErrorKind::CredentialInvalid`] (a broken key/token, not an absent one), + /// the error is propagated rather than downgraded to an unsigned request. async fn sign(&self, req: &mut Request) -> MiddlewareResult<()> { if Self::has_sas_token(req.url()) { return Ok(()); @@ -117,8 +197,16 @@ impl AzureMiddleware { match self.signer.sign(&mut parts, None).await { Ok(()) => {} - Err(e) if e.kind() == ErrorKind::CredentialInvalid => { - tracing::debug!("no Azure credential resolved; sending `az://` request unsigned"); + // reqsign reports both "no credential configured" and "credential is + // broken" as `CredentialInvalid`. Only fall back to unsigned when no + // credential source was detected; otherwise a broken credential must + // surface as a hard error instead of silently going anonymous. + Err(e) + if e.kind() == ErrorKind::CredentialInvalid && !self.credential_source_present => + { + tracing::debug!( + "no Azure credential source detected; sending `az://` request unsigned" + ); return Ok(()); } Err(e) => return Err(reqwest_middleware::Error::Middleware(anyhow::anyhow!(e))), @@ -149,6 +237,18 @@ impl Middleware for AzureMiddleware { return next.run(req, extensions).await; } + // The host is trusted verbatim as the request target, so userinfo is a + // host-spoofing vector (`az://user:pass@real.host/...` can hide the real + // authority). Reject it before rewriting or signing. This mirrors the + // same rejection in `rattler_azure::account_and_container`; the check is + // inlined here because this middleware does not depend on rattler_azure. + if Self::has_userinfo(req.url()) { + return Err(reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "userinfo is not allowed in `az://` URLs (host-spoofing vector); \ + remove the `user:pass@` component" + ))); + } + let https_url = Self::rewrite_url(&req.url().clone())?; *req.url_mut() = https_url; self.sign(&mut req).await?; @@ -233,6 +333,7 @@ mod tests { let middleware = AzureMiddleware::with_credential_provider( Client::new(), ProvideCredentialChain::::new(), + false, ); let mut req = Client::new() .get("https://acct.blob.core.windows.net/pub/noarch/repodata.json") @@ -265,6 +366,7 @@ mod tests { let middleware = AzureMiddleware::with_credential_provider( Client::new(), StaticCredentialProvider::new_shared_key("acct", "dGVzdF9rZXk="), + true, ); let mut req = Client::new() .get("https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc") @@ -282,4 +384,53 @@ mod tests { "a self-authenticating SAS URL is left untouched" ); } + + /// A URL carrying userinfo must be recognised so the fetch path can reject + /// it: the host is trusted verbatim, so `user:pass@host` is a host-spoofing + /// vector. (A request built through reqwest's client strips userinfo into a + /// header before the middleware runs, so the predicate — not the whole + /// client path — is what guards direct `Request` construction.) + #[test] + fn detects_userinfo_in_url() { + assert!(AzureMiddleware::has_userinfo( + &Url::parse("az://user:pass@acct.blob.core.windows.net/c/x.json").unwrap() + )); + assert!(AzureMiddleware::has_userinfo( + &Url::parse("az://user@acct.blob.core.windows.net/c/x.json").unwrap() + )); + assert!(!AzureMiddleware::has_userinfo( + &Url::parse("az://acct.blob.core.windows.net/c/x.json").unwrap() + )); + } + + /// When a credential source is detected but signing fails + /// (`CredentialInvalid`), the failure must be a hard error rather than an + /// unsigned fallback: a broken credential must not silently go anonymous. An + /// empty provider chain yields `CredentialInvalid`, standing in for a broken + /// credential. + #[tokio::test] + async fn errors_when_credential_source_present_but_signing_fails() { + use reqsign_core::ProvideCredentialChain; + + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + ProvideCredentialChain::::new(), + true, + ); + let mut req = Client::new() + .get("https://acct.blob.core.windows.net/c/noarch/repodata.json") + .build() + .unwrap(); + + let result = middleware.sign(&mut req).await; + + assert!( + result.is_err(), + "a configured-but-failing credential must be a hard error, not unsigned" + ); + assert!( + req.headers().get(http::header::AUTHORIZATION).is_none(), + "a failed signing attempt must not leave a partial Authorization header" + ); + } } From fab4ec4cd713d8b835c86797a2ecd3a4abd8a0c6 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:03:34 +1000 Subject: [PATCH 43/98] fix(azure): stat-guard overwrites, warn on unmetadata'd block uploads, wire azure feature --- crates/rattler_upload/Cargo.toml | 8 +- crates/rattler_upload/src/upload/azure.rs | 174 +++++++++++++++++++--- 2 files changed, 162 insertions(+), 20 deletions(-) diff --git a/crates/rattler_upload/Cargo.toml b/crates/rattler_upload/Cargo.toml index b0bcb034dc..98f2e93af5 100644 --- a/crates/rattler_upload/Cargo.toml +++ b/crates/rattler_upload/Cargo.toml @@ -14,7 +14,7 @@ license.workspace = true readme.workspace = true [features] -default = ["rustls", "s3", "sigstore-sign"] +default = ["rustls", "s3", "azure", "sigstore-sign"] rustls = [ "reqwest/rustls", "rattler_networking/rustls", @@ -28,7 +28,7 @@ native-tls = [ "rattler_package_streaming/native-tls", "sigstore-sign?/native-tls", ] -s3 = ["rattler_networking/s3", "rattler_s3", "dep:opendal"] +s3 = ["rattler_networking/s3", "rattler_s3", "dep:opendal", "opendal/services-s3"] azure = ["dep:rattler_azure", "dep:opendal", "opendal/services-azblob", "dep:percent-encoding"] sigstore-sign = ["dep:sigstore-sign", "dep:sigstore-trust-root"] @@ -52,7 +52,6 @@ fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } indicatif = { workspace = true } opendal = { workspace = true, optional = true, default-features = false, features = [ - "services-s3", "executors-tokio", ] } reqwest-retry = { workspace = true } @@ -84,3 +83,6 @@ sha2 = { workspace = true } [dev-dependencies] axum = { workspace = true } +opendal = { workspace = true, default-features = false, features = [ + "services-memory", +] } diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index ca3140c9d8..f71ab4d12f 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -88,22 +88,40 @@ async fn upload_single_package( channel.path() ); - // Compute the hash of the package by streaming its content. - let file = tokio::io::BufReader::new( - fs_err::tokio::File::open(package_file) - .await - .into_diagnostic()?, - ); - let sha256_reader = HashingReader::<_, Sha256>::new(file); - let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); - let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) - .await - .into_diagnostic()?; - let (sha256_reader, md5hash) = md5_reader.finalize(); - let (mut file, sha256hash) = sha256_reader.finalize(); + // Guard against overwriting an existing blob when `--force` was not passed. + // opendal 0.57 only honours `if_not_exists` on the single-shot Put Blob path, + // never the multi-block Put Block List path used for packages larger than a + // single block, so the writer-level `if_not_exists(!force)` below silently + // does nothing for large uploads. An explicit `stat` closes that gap at all + // sizes. The residual stat->write TOCTOU (another writer could create the + // blob between this check and `close`) is acceptable: it is strictly better + // than today's silent clobber, and the writer-level `if_not_exists` still + // guards the small-file and racing-writer cases. + if !force { + match op.stat(&key).await { + Ok(_) => { + miette::bail!("Package {blob_url} already exists. Use --force to overwrite."); + } + Err(e) if e.kind() == ErrorKind::NotFound => {} + Err(e) => return Err(e).into_diagnostic(), + } + } - // Rewind the file to the beginning. - file.rewind().await.into_diagnostic()?; + let (mut file, size, sha256_hex, md5_hex) = package_digests(package_file).await?; + + // opendal 0.57 attaches `user_metadata` only on the single-shot Put Blob path + // (used when the whole package fits in one `DESIRED_CHUNK_SIZE` write), not on + // the multi-block Put Block List path taken by larger packages. There is no + // post-write set-metadata operation on the azblob backend to compensate, so + // packages above `DESIRED_CHUNK_SIZE` land without `package-sha256` / + // `package-md5`. Warn so the missing metadata is not silent. + if size as usize > DESIRED_CHUNK_SIZE { + tracing::warn!( + "Package {blob_url} is larger than {DESIRED_CHUNK_SIZE} bytes and is uploaded via \ + Azure's multi-block path, which opendal 0.57 does not attach blob metadata to; \ + package-sha256/package-md5 will be missing on this blob." + ); + } // Construct a writer for the package. Setting `chunk` and `concurrent` // enables opendal's concurrent block upload: data is buffered into @@ -118,8 +136,8 @@ async fn upload_single_package( .concurrent(PART_CONCURRENCY) .if_not_exists(!force) .user_metadata([ - (String::from("package-sha256"), hex::encode(sha256hash)), - (String::from("package-md5"), hex::encode(md5hash)), + (String::from("package-sha256"), sha256_hex), + (String::from("package-md5"), md5_hex), ]) .await { @@ -171,6 +189,38 @@ async fn upload_single_package( Ok(()) } +/// Streams `package_file` once to compute its sha256 and md5 digests. +/// +/// Returns the file handle rewound to the start, the total size in bytes, and +/// the hex-encoded sha256 and md5 digests (in that order) that are attached as +/// blob metadata. +async fn package_digests( + package_file: &Path, +) -> miette::Result<( + tokio::io::BufReader, + u64, + String, + String, +)> { + let file = tokio::io::BufReader::new( + fs_err::tokio::File::open(package_file) + .await + .into_diagnostic()?, + ); + let sha256_reader = HashingReader::<_, Sha256>::new(file); + let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); + let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) + .await + .into_diagnostic()?; + let (sha256_reader, md5hash) = md5_reader.finalize(); + let (mut file, sha256hash) = sha256_reader.finalize(); + + // Rewind the file to the beginning. + file.rewind().await.into_diagnostic()?; + + Ok((file, size, hex::encode(sha256hash), hex::encode(md5hash))) +} + /// Build an opendal [`AzblobConfig`] from a channel URL and credentials. /// /// The account name, endpoint, container, and root prefix are all derived from @@ -226,3 +276,93 @@ fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Resul ..Default::default() }) } + +#[cfg(test)] +mod test { + use opendal::{Operator, services::Memory}; + use rattler_digest::{Md5, compute_file_digest}; + use url::Url; + + use super::{package_digests, upload_single_package}; + use crate::upload::package::ExtractedPackage; + use crate::upload::test_utils::test_package_path; + + fn memory_operator() -> Operator { + Operator::new(Memory::default()).unwrap().finish() + } + + fn test_channel() -> Url { + Url::parse("https://account.blob.core.windows.net/container/prefix").unwrap() + } + + fn package_key() -> String { + let path = test_package_path(); + let package = ExtractedPackage::from_package_file(&path).unwrap(); + format!( + "{}/{}", + package.subdir().unwrap(), + package.filename().unwrap() + ) + } + + /// C2: without `--force`, uploading over an existing blob must error rather + /// than silently overwrite it, at all package sizes. This exercises the + /// explicit pre-write `stat` guard that backstops opendal's + /// `if_not_exists`, which is dropped on the multi-block upload path. + #[tokio::test] + async fn test_existing_blob_without_force_errors() { + let op = memory_operator(); + let channel = test_channel(); + let package = test_package_path(); + + // Seed the target blob so the next upload finds it already present. + upload_single_package(&op, &channel, &package, true) + .await + .expect("initial force upload should succeed"); + + let err = upload_single_package(&op, &channel, &package, false) + .await + .expect_err("upload over an existing blob without --force must fail"); + assert!( + err.to_string().contains("already exists"), + "unexpected error: {err}" + ); + } + + /// A non-forced upload into an empty container succeeds. + #[tokio::test] + async fn test_upload_into_empty_container_succeeds() { + let op = memory_operator(); + upload_single_package(&op, &test_channel(), &test_package_path(), false) + .await + .expect("upload into an empty container should succeed"); + + let meta = op.stat(&package_key()).await.unwrap(); + let expected_size = std::fs::metadata(test_package_path()).unwrap().len(); + assert_eq!(meta.content_length(), expected_size); + } + + /// H1: the sha256/md5 attached as blob metadata are the canonical digests of + /// the package. This guards the values and encoding fed into + /// `user_metadata`. + /// + /// Note: the in-memory opendal service does not persist `user_metadata`, and + /// Azure's multi-block path drops it entirely (see `upload_single_package`), + /// so asserting the metadata is actually present on the stored blob requires + /// an Azurite-backed integration test that this crate does not yet have. + #[tokio::test] + async fn test_package_digests_match_canonical_hashes() { + let package = test_package_path(); + let (_file, size, sha256_hex, md5_hex) = package_digests(&package).await.unwrap(); + + assert_eq!(size, std::fs::metadata(&package).unwrap().len()); + assert_eq!( + sha256_hex, + crate::upload::package::sha256_sum(&package).unwrap() + ); + assert_eq!( + md5_hex, + hex::encode(compute_file_digest::(&package).unwrap()) + ); + } +} From 853415990022877c8745c8ec02d5f1ee6cda53e7 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:04:15 +1000 Subject: [PATCH 44/98] fix(gateway): carry forward storable policy on shard index 304 --- .../src/gateway/sharded_subdir/tokio/index.rs | 145 ++++++++++++------ 1 file changed, 96 insertions(+), 49 deletions(-) diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs index e99d67feb2..b4bcf57c68 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs @@ -297,64 +297,38 @@ pub async fn fetch_index( return Err(create_subdir_not_found_error(channel_base_url)); } - // A `304 Not Modified` means the cached shard index is - // still valid, so serve it directly. Servers that omit - // `Cache-Control` on the shard index (e.g. Azure Blob - // Storage) leave the cached policy with no freshness - // lifetime, so without this guard the 304 falls through to - // `from_response`, which rejects it as an unexpected status. - // - // Persist a refreshed policy from this revalidation so the - // next fetch is a local cache hit rather than yet another - // conditional round-trip: `shard_index_cache_policy` - // synthesizes a small `max-age` for the Cache-Control-less - // Azure case, so freshness accumulates. - if response.status() == reqwest::StatusCode::NOT_MODIFIED { - match read_cached_body(&mut cache_reader).await { - Ok(body) => { - tracing::debug!("shard index revalidated (304 Not Modified)"); - let refreshed = - shard_index_cache_policy(&canonical_request, &response); - let mut guard = cache_reader.into_inner(); - if let Err(e) = write_shard_index_cache( - guard.inner_mut(), - refreshed, - Bytes::from(body.clone()), - ) - .await - { - tracing::warn!( - "failed to persist refreshed shard index cache policy: {e}" - ); - } - if let Some((reporter, index)) = download_reporter { - reporter.on_download_complete(response.url(), index); - } - return parse_shard_index(body).await; - } - Err(e) => { - // Cache unreadable; fall through to a fresh fetch. - tracing::warn!("the cached shard index has been corrupted: {e}"); - } - } - } - match cache_header.policy.after_response( &state_request, &response, SystemTime::now(), ) { - AfterResponse::NotModified(_policy, _) => { - // The cached file is still valid - match read_shard_index_from_reader(&mut cache_reader).await { - Ok(shard_index) => { + AfterResponse::NotModified(refreshed, _) => { + // The cached file is still valid. `after_response` + // returns a refreshed policy derived from the stored + // 200 response with the 304's headers merged, so it + // stays storable and retains the original (or + // synthesized) freshness window. Persist it so the + // next fetch inside the window is a local cache hit + // instead of yet another conditional round-trip. + match read_cached_body(&mut cache_reader).await { + Ok(body) => { tracing::debug!("shard index cache was not modified"); + let mut guard = cache_reader.into_inner(); + if let Err(e) = write_shard_index_cache( + guard.inner_mut(), + refreshed, + Bytes::from(body.clone()), + ) + .await + { + tracing::warn!( + "failed to persist refreshed shard index cache policy: {e}" + ); + } if let Some((reporter, index)) = download_reporter { reporter.on_download_complete(response.url(), index); } - // If reading the file failed for some reason we'll just - // fetch it again. - return Ok(shard_index); + return parse_shard_index(body).await; } Err(e) => { tracing::warn!( @@ -642,3 +616,76 @@ impl RequestLike for SimpleRequest { &self.uri() == other } } + +#[cfg(test)] +mod tests { + use std::time::{Duration, SystemTime}; + + use http::{StatusCode, header}; + use http_cache_semantics::{AfterResponse, BeforeRequest}; + use url::Url; + + use super::{SimpleRequest, shard_index_cache_policy}; + + /// Builds a bodyless `reqwest::Response`, mirroring how an origin (e.g. + /// Azure Blob Storage) answers for a shard index. + fn response(status: StatusCode, etag: &str, cache_control: Option<&str>) -> reqwest::Response { + let mut builder = http::Response::builder() + .status(status) + .header(header::ETAG, etag); + if let Some(cache_control) = cache_control { + builder = builder.header(header::CACHE_CONTROL, cache_control); + } + reqwest::Response::from(builder.body(Vec::new()).unwrap()) + } + + /// A 304 revalidation must persist a policy that keeps a non-zero freshness + /// window, so a fetch within that window is a local cache hit instead of yet + /// another conditional round-trip. + #[test] + fn revalidation_policy_retains_freshness() { + let url = Url::parse( + "https://example.blob.core.windows.net/channel/noarch/repodata_shards.msgpack.zst", + ) + .unwrap(); + let request = SimpleRequest::get(&url); + let now = SystemTime::now(); + + // The original 200 carried no `Cache-Control`, so `shard_index_cache_policy` + // synthesizes a small `max-age` and the stored policy starts fresh. + let original = response(StatusCode::OK, "\"v1\"", None); + let stored = shard_index_cache_policy(&request, &original); + assert!(matches!( + stored.before_request(&request, now), + BeforeRequest::Fresh(_) + )); + + // A later revalidation is answered with a 304 that, like Azure, carries no + // `Cache-Control`. Routing it through `after_response` yields the policy + // that is persisted for the next fetch. + let not_modified = response(StatusCode::NOT_MODIFIED, "\"v1\"", None); + let refreshed = match stored.after_response(&request, ¬_modified, now) { + AfterResponse::NotModified(policy, _) => policy, + AfterResponse::Modified(_, _) => { + panic!("a matching ETag must revalidate as NotModified") + } + }; + + // The refreshed policy must retain a non-zero freshness window so the next + // fetch is served straight from the cache. + assert!( + refreshed.time_to_live(now) > Duration::ZERO, + "refreshed policy lost its freshness window" + ); + assert!(matches!( + refreshed.before_request(&request, now), + BeforeRequest::Fresh(_) + )); + + // Regression guard: building a policy directly from the 304 (a status that + // is not storable) yields zero freshness, which would force perpetual + // revalidation. + let from_304 = shard_index_cache_policy(&request, ¬_modified); + assert_eq!(from_304.time_to_live(now), Duration::ZERO); + } +} From e3345331195ff277d6038e2179f3a0907cd333dd Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:19:18 +1000 Subject: [PATCH 45/98] fix(py): register AzureMiddleware in default client --- py-rattler/rattler/networking/client.py | 3 +- py-rattler/tests/unit/test_client.py | 47 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 py-rattler/tests/unit/test_client.py diff --git a/py-rattler/rattler/networking/client.py b/py-rattler/rattler/networking/client.py index 5ef7d84b82..d6e98b0e6e 100644 --- a/py-rattler/rattler/networking/client.py +++ b/py-rattler/rattler/networking/client.py @@ -76,7 +76,7 @@ def default_client( ) -> Client: """ Returns a client with the standard middleware stack: retry, - authentication, OCI, GCS and S3. + authentication, OCI, GCS, Azure and S3. Args: max_retries: Maximum retry attempts for transient errors (default 3). @@ -98,6 +98,7 @@ def default_client( AuthenticationMiddleware(), OciMiddleware(), GCSMiddleware(), + AzureMiddleware(), S3Middleware(), ], headers=headers, diff --git a/py-rattler/tests/unit/test_client.py b/py-rattler/tests/unit/test_client.py new file mode 100644 index 0000000000..13aa19e888 --- /dev/null +++ b/py-rattler/tests/unit/test_client.py @@ -0,0 +1,47 @@ +# type: ignore +from __future__ import annotations + +import rattler.networking.client as client_module +from rattler.networking import Client +from rattler.networking.middleware import ( + AuthenticationMiddleware, + AzureMiddleware, + GCSMiddleware, + OciMiddleware, + RetryMiddleware, + S3Middleware, +) + + +def test_default_client_stack_includes_azure(monkeypatch) -> None: + """The default client's middleware stack must include every cloud backend.""" + constructed: list[type] = [] + + for name in ( + "RetryMiddleware", + "AuthenticationMiddleware", + "OciMiddleware", + "GCSMiddleware", + "AzureMiddleware", + "S3Middleware", + ): + original = getattr(client_module, name) + + def record(*args, _original=original, **kwargs): + constructed.append(_original) + return _original(*args, **kwargs) + + monkeypatch.setattr(client_module, name, record) + + client = Client.default_client() + + assert isinstance(client, Client) + for middleware in ( + RetryMiddleware, + AuthenticationMiddleware, + OciMiddleware, + GCSMiddleware, + AzureMiddleware, + S3Middleware, + ): + assert middleware in constructed From 0a6b504ebe9b4e608cd596e021b3f61fa1d6d753 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:24:05 +1000 Subject: [PATCH 46/98] refactor(azure): async SAS mint via tokio::process, gate s3-only HashMap import --- Cargo.lock | 1 + crates/rattler-bin/src/commands/client.rs | 4 +++- crates/rattler_azure/Cargo.toml | 9 +++++-- crates/rattler_azure/src/clap.rs | 29 +++++++++++++---------- crates/rattler_azure/src/lib.rs | 21 ++++++++-------- crates/rattler_index/src/main.rs | 8 ++++--- crates/rattler_upload/src/lib.rs | 1 + 7 files changed, 44 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2a2b7e735..ea40207aca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5052,6 +5052,7 @@ dependencies = [ "clap", "jiff", "thiserror 2.0.18", + "tokio", "url", "which", ] diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index dc7a7d69ca..9271e2fcc9 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -1,4 +1,6 @@ -use std::{collections::HashMap, sync::Arc}; +#[cfg(feature = "s3")] +use std::collections::HashMap; +use std::sync::Arc; use miette::{Context, IntoDiagnostic}; use rattler_networking::{ diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index be57685f93..f097666a65 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -12,12 +12,17 @@ readme.workspace = true [features] default = [] # CLI credential options; resolving them may mint a short-lived user-delegation -# SAS by shelling out to the Azure CLI (which needs `jiff` for the expiry). -clap = ["dep:clap", "dep:jiff", "dep:which"] +# SAS by shelling out to the Azure CLI (which needs `jiff` for the expiry and +# `tokio` to run `az` without blocking the async runtime). +clap = ["dep:clap", "dep:jiff", "dep:which", "dep:tokio"] [dependencies] clap = { workspace = true, optional = true } jiff = { workspace = true, optional = true } thiserror = { workspace = true } +tokio = { workspace = true, features = ["process"], optional = true } url = { workspace = true } which = { workspace = true, optional = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index 93957dd457..f9110a6707 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -69,7 +69,7 @@ impl AzureAuthSource { /// container returned by `cli_context` with those permissions. The account /// key and SAS token arms never invoke `cli_context`, so callers pay for /// account/container derivation only on the minting path. - pub fn resolve( + pub async fn resolve( self, permissions: &str, cli_context: impl FnOnce() -> Result, @@ -79,7 +79,8 @@ impl AzureAuthSource { AzureAuthSource::SasToken(token) => Ok(AzureCredentials::SasToken(token)), AzureAuthSource::AzureCli { ttl } => { let AzureCoordinates { account, container } = cli_context()?; - let token = mint_user_delegation_sas(&account, &container, permissions, ttl)?; + let token = + mint_user_delegation_sas(&account, &container, permissions, ttl).await?; Ok(AzureCredentials::SasToken(token)) } } @@ -172,12 +173,12 @@ impl AzureCredentialsOpts { /// Precedence is applied by [`AzureCredentialsOpts::source`]. `permissions` /// and `cli_context` are consulted only when the winning source is /// `--azure-cli`; see [`AzureAuthSource::resolve`]. - pub fn resolve( + pub async fn resolve( self, permissions: &str, cli_context: impl FnOnce() -> Result, ) -> Result { - self.source()?.resolve(permissions, cli_context) + self.source()?.resolve(permissions, cli_context).await } } @@ -203,26 +204,28 @@ mod tests { panic!("cli_context should not be called for non-azure-cli sources"); } - #[test] - fn account_key_resolves() { + #[tokio::test] + async fn account_key_resolves() { assert!(matches!( - opts(Some("key"), None, false).resolve("cw", unreachable_context), + opts(Some("key"), None, false).resolve("cw", unreachable_context).await, Ok(AzureCredentials::AccountKey(k)) if k == "key" )); } - #[test] - fn sas_token_resolves() { + #[tokio::test] + async fn sas_token_resolves() { assert!(matches!( - opts(None, Some("sv=..."), false).resolve("cw", unreachable_context), + opts(None, Some("sv=..."), false).resolve("cw", unreachable_context).await, Ok(AzureCredentials::SasToken(t)) if t == "sv=..." )); } - #[test] - fn none_is_rejected() { + #[tokio::test] + async fn none_is_rejected() { assert!(matches!( - opts(None, None, false).resolve("cw", unreachable_context), + opts(None, None, false) + .resolve("cw", unreachable_context) + .await, Err(AzureCredentialsError::Missing) )); } diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 151b04a1e8..5009bb3a09 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -187,8 +187,8 @@ pub enum AzureCliSasError { /// `permissions` is the Azure SAS permission string (e.g. `"cw"`). The returned /// token has no leading `?`. Requires `az` on `PATH` and a prior `az login`. /// -/// This blocks the calling thread while the `az` process runs; it is meant to be -/// called once at setup time. +/// Runs the `az` process on the tokio runtime; it is meant to be called once at +/// setup time. /// /// # Container-scope limitation /// @@ -199,7 +199,7 @@ pub enum AzureCliSasError { /// prefix-scoping a flat container is not possible without a stored access /// policy, which this path deliberately does not create. #[cfg(feature = "clap")] -pub fn mint_user_delegation_sas( +pub async fn mint_user_delegation_sas( account: &str, container: &str, permissions: &str, @@ -245,6 +245,7 @@ pub fn mint_user_delegation_sas( "tsv", ]) .output() + .await .map_err(|err| { if err.kind() == std::io::ErrorKind::NotFound { AzureCliSasError::AzNotFound(err) @@ -265,23 +266,23 @@ pub fn mint_user_delegation_sas( Ok(token) } -/// Build the [`std::process::Command`] used to invoke the Azure CLI. +/// Build the [`tokio::process::Command`] used to invoke the Azure CLI. /// -/// On Windows the Azure CLI is a `az.cmd` batch shim; `std::process` does not -/// honor `PATHEXT`, so a bare `az` fails to resolve it. `which` applies `PATHEXT` +/// On Windows the Azure CLI is a `az.cmd` batch shim; the process spawner does +/// not honor `PATHEXT`, so a bare `az` fails to resolve it. `which` applies `PATHEXT` /// to find the real `az`/`az.cmd` path, which is then invoked directly. Routing /// through the command interpreter (`cmd /C az ...`) is deliberately avoided: it /// would expose the command line to `cmd` metacharacter interpretation, an /// argument-injection vector. #[cfg(all(feature = "clap", windows))] -fn az_command() -> Result { +fn az_command() -> Result { let path = which::which("az").map_err(AzureCliSasError::AzResolve)?; - Ok(std::process::Command::new(path)) + Ok(tokio::process::Command::new(path)) } #[cfg(all(feature = "clap", not(windows)))] -fn az_command() -> std::process::Command { - std::process::Command::new("az") +fn az_command() -> tokio::process::Command { + tokio::process::Command::new("az") } #[cfg(test)] diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index f3557b90ce..142f6ade4b 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -266,9 +266,11 @@ async fn main() -> anyhow::Result<()> { effective_index_options(&resolved); let channel_metadata = ChannelMetadata::from_index_config(&resolved); - let credentials = credentials.resolve(AZURE_INDEX_SAS_PERMISSIONS, || { - Ok(rattler_azure::account_and_container(&channel)?) - })?; + let credentials = credentials + .resolve(AZURE_INDEX_SAS_PERMISSIONS, || { + Ok(rattler_azure::account_and_container(&channel)?) + }) + .await?; index_azure_with_channel_metadata( IndexAzureConfig { diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index 4d806235e4..8da7cfcc9a 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -89,6 +89,7 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, || { Ok(rattler_azure::account_and_container(&channel)?) }) + .await .into_diagnostic()?; upload::upload_package_to_azure( channel, From b6c3441ee3350f91e6a1eeb66483a08c756577fc Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:33:39 +1000 Subject: [PATCH 47/98] feat(azure): treat persisted az login profile as a credential source --- .../src/azure_middleware.rs | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 3ff947d063..a654d92566 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -20,11 +20,8 @@ const X_MS_VERSION: &str = "2021-12-02"; /// (a hard error). It is a presence check only — it does not validate that the /// values are usable; reqsign does that when it actually signs. /// -/// NOTE: an interactive `az login` CLI session is intentionally NOT detected -/// here. Doing so would require shelling out to `az` (or parsing its token -/// cache), which is more than a cheap env probe. The residual gap: a machine -/// authenticated only via `az login` whose session is broken will still fall -/// back to an unsigned request rather than erroring. +/// A persisted `az login` session (a profile on disk) counts as a source too; +/// see [`azure_cli_session_present`]. fn azure_credential_source_present() -> bool { // Explicit Shared Key or SAS token. if std::env::var_os("AZURE_STORAGE_ACCOUNT_KEY").is_some() @@ -48,7 +45,27 @@ fn azure_credential_source_present() -> bool { { return true; } - false + // Persisted `az login` session. + azure_cli_session_present() +} + +/// Whether a persisted `az login` session exists on disk. +/// +/// `az login` writes an `azureProfile.json` into the Azure CLI config dir +/// (`$AZURE_CONFIG_DIR`, else `~/.azure`). Its presence means a login was +/// performed at some point, so a *broken* az-login credential hard-errors +/// rather than silently downgrading to an unsigned request. This is a presence +/// check only — reqsign validates the token when it actually signs, so a stale +/// profile with no usable token still yields a hard error, never anonymous. +fn azure_cli_session_present() -> bool { + let config_dir = match std::env::var_os("AZURE_CONFIG_DIR") { + Some(dir) => std::path::PathBuf::from(dir), + None => match std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { + Some(home) => std::path::PathBuf::from(home).join(".azure"), + None => return false, + }, + }; + config_dir.join("azureProfile.json").exists() } /// Middleware that rewrites `az://` URLs to HTTPS Azure Blob Storage URLs and @@ -403,6 +420,25 @@ mod tests { )); } + /// A persisted `az login` profile counts as a credential source (so a + /// broken az-login session hard-errors instead of going anonymous), while + /// an empty config dir does not. + #[test] + fn detects_az_login_profile_on_disk() { + let dir = tempfile::tempdir().unwrap(); + temp_env::with_var("AZURE_CONFIG_DIR", Some(dir.path().as_os_str()), || { + assert!( + !azure_cli_session_present(), + "no profile file yet ⇒ not a credential source" + ); + std::fs::write(dir.path().join("azureProfile.json"), "{}").unwrap(); + assert!( + azure_cli_session_present(), + "azureProfile.json present ⇒ credential source" + ); + }); + } + /// When a credential source is detected but signing fails /// (`CredentialInvalid`), the failure must be a hard error rather than an /// unsigned fallback: a broken credential must not silently go anonymous. An From c87d826de2e2f32a405ebaed63afcc48f86043ef Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:23:04 +1000 Subject: [PATCH 48/98] perf(azure): skip credential resolution when no source detected --- .../src/azure_middleware.rs | 111 ++++++++++++++---- 1 file changed, 86 insertions(+), 25 deletions(-) diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index a654d92566..b979a948a3 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -2,7 +2,7 @@ use async_trait::async_trait; use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; use reqsign_command_execute_tokio::TokioCommandExecute; -use reqsign_core::{Context, ErrorKind, OsEnv, ProvideCredential, Signer}; +use reqsign_core::{Context, OsEnv, ProvideCredential, Signer}; use reqsign_file_read_tokio::TokioFileRead; use reqsign_http_send_reqwest::ReqwestHttpSend; use reqwest::{Client, Request, Response}; @@ -22,6 +22,17 @@ const X_MS_VERSION: &str = "2021-12-02"; /// /// A persisted `az login` session (a profile on disk) counts as a source too; /// see [`azure_cli_session_present`]. +/// +/// This result gates whether credentials are resolved at all: when it is +/// `false`, [`AzureMiddleware::sign`] sends the request unsigned *without* +/// probing, so anonymous public-container reads do not block on the +/// managed-identity / IMDS timeout. The tradeoff is that a *bare* system-assigned +/// managed identity — one reached only via the IMDS endpoint with no +/// `IDENTITY_ENDPOINT` / `MSI_ENDPOINT` / federated-token env var set — is not +/// detected here, so its requests go unsigned. App Service, Functions, Cloud +/// Shell and AKS workload identity all set one of those env vars and are +/// unaffected; a bare `IaaS` VM identity must export `MSI_ENDPOINT` (or supply an +/// explicit credential) to be used. fn azure_credential_source_present() -> bool { // Explicit Shared Key or SAS token. if std::env::var_os("AZURE_STORAGE_ACCOUNT_KEY").is_some() @@ -100,7 +111,7 @@ fn azure_cli_session_present() -> bool { /// zero ambient credentials. When a credential source *is* configured /// (see [`azure_credential_source_present`]) but signing fails, that is a /// **hard error** — reqsign collapses "no credential" and "broken credential" -/// into the same [`ErrorKind::CredentialInvalid`], so a broken credential must +/// into the same [`reqsign_core::ErrorKind::CredentialInvalid`], so a broken credential must /// not be silently downgraded to an anonymous request. #[derive(Clone)] pub struct AzureMiddleware { @@ -178,17 +189,22 @@ impl AzureMiddleware { /// Sign a reqwest `Request` in place using reqsign. /// - /// Two cases short-circuit without touching the request: + /// Two cases short-circuit without invoking reqsign at all: /// - The URL already carries an explicit SAS (`?...&sig=...`). Signing would /// add an `Authorization` header that Azure prefers over the SAS, silently /// overriding the caller's explicit token. - /// - No credential source is configured. reqsign surfaces this as - /// [`ErrorKind::CredentialInvalid`]; the request is then sent unsigned so - /// public/anonymous containers stay reachable. + /// - No credential source was detected (`credential_source_present` is + /// `false`). The request is sent unsigned so public/anonymous containers + /// stay reachable — and crucially, credential *resolution* is skipped + /// entirely. Otherwise reqsign would probe the managed-identity / IMDS + /// endpoint and block until it times out (~30s on a machine with no + /// metadata service) before we could fall back, making every anonymous + /// public-channel read pay that timeout. /// - /// If a credential source *is* configured but signing still fails with - /// [`ErrorKind::CredentialInvalid`] (a broken key/token, not an absent one), - /// the error is propagated rather than downgraded to an unsigned request. + /// If a credential source *is* configured but signing fails with + /// [`reqsign_core::ErrorKind::CredentialInvalid`] (a broken key/token, not + /// an absent one), the error is propagated rather than downgraded to an + /// unsigned request. async fn sign(&self, req: &mut Request) -> MiddlewareResult<()> { if Self::has_sas_token(req.url()) { return Ok(()); @@ -199,6 +215,15 @@ impl AzureMiddleware { .insert("x-ms-version", http::HeaderValue::from_static(X_MS_VERSION)); } + // No credential source detected: send unsigned without probing. See the + // doc comment above — probing here would block on the IMDS timeout. + if !self.credential_source_present { + tracing::debug!( + "no Azure credential source detected; sending `az://` request unsigned" + ); + return Ok(()); + } + let mut builder = http::Request::builder() .method(req.method().clone()) .uri(req.url().as_str()); @@ -212,22 +237,14 @@ impl AzureMiddleware { })?; let (mut parts, ()) = http_req.into_parts(); - match self.signer.sign(&mut parts, None).await { - Ok(()) => {} - // reqsign reports both "no credential configured" and "credential is - // broken" as `CredentialInvalid`. Only fall back to unsigned when no - // credential source was detected; otherwise a broken credential must - // surface as a hard error instead of silently going anonymous. - Err(e) - if e.kind() == ErrorKind::CredentialInvalid && !self.credential_source_present => - { - tracing::debug!( - "no Azure credential source detected; sending `az://` request unsigned" - ); - return Ok(()); - } - Err(e) => return Err(reqwest_middleware::Error::Middleware(anyhow::anyhow!(e))), - } + // A credential source is present (the absent case short-circuited to + // unsigned above). reqsign reports a broken key/token as + // `CredentialInvalid`; that must surface as a hard error rather than + // silently going anonymous, so any signing failure is propagated. + self.signer + .sign(&mut parts, None) + .await + .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e)))?; *req.headers_mut() = parts.headers; let signed_url = Url::parse(&parts.uri.to_string()).map_err(|e| { @@ -420,6 +437,50 @@ mod tests { )); } + /// With no credential source detected, `sign` must NOT invoke the credential + /// provider at all — resolution is skipped so anonymous reads don't block on + /// the IMDS timeout. Uses a provider that flips a flag if it is ever asked. + #[tokio::test] + async fn skips_credential_resolution_when_no_source() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + #[derive(Debug)] + struct RecordingProvider(Arc); + impl ProvideCredential for RecordingProvider { + type Credential = Credential; + async fn provide_credential( + &self, + _ctx: &Context, + ) -> reqsign_core::Result> { + self.0.store(true, Ordering::SeqCst); + Ok(None) + } + } + + let probed = Arc::new(AtomicBool::new(false)); + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + RecordingProvider(probed.clone()), + false, + ); + let mut req = Client::new() + .get("https://acct.blob.core.windows.net/pub/noarch/repodata.json") + .build() + .unwrap(); + + middleware.sign(&mut req).await.unwrap(); + + assert!( + !probed.load(Ordering::SeqCst), + "credential provider must not be probed when no source is detected" + ); + assert!( + req.headers().get(http::header::AUTHORIZATION).is_none(), + "unsigned request must not carry an Authorization header" + ); + } + /// A persisted `az login` profile counts as a credential source (so a /// broken az-login session hard-errors instead of going anonymous), while /// an empty config dir does not. From e74785200746e0056d2428ebbfe25bfddcd4b72e Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:34:22 +1000 Subject: [PATCH 49/98] refactor(azure): drop unused blob metadata write from upload path --- crates/rattler_upload/src/upload/azure.rs | 95 ++++------------------- 1 file changed, 14 insertions(+), 81 deletions(-) diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index f71ab4d12f..01c72efa8d 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -4,8 +4,7 @@ use futures::StreamExt; use miette::IntoDiagnostic; use opendal::{Configurator, ErrorKind, Operator, services::AzblobConfig}; use rattler_azure::AzureCredentials; -use rattler_digest::{HashingReader, Md5, Sha256}; -use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use tokio::io::AsyncReadExt; use tokio_util::bytes::BytesMut; use url::Url; @@ -107,38 +106,29 @@ async fn upload_single_package( } } - let (mut file, size, sha256_hex, md5_hex) = package_digests(package_file).await?; - - // opendal 0.57 attaches `user_metadata` only on the single-shot Put Blob path - // (used when the whole package fits in one `DESIRED_CHUNK_SIZE` write), not on - // the multi-block Put Block List path taken by larger packages. There is no - // post-write set-metadata operation on the azblob backend to compensate, so - // packages above `DESIRED_CHUNK_SIZE` land without `package-sha256` / - // `package-md5`. Warn so the missing metadata is not silent. - if size as usize > DESIRED_CHUNK_SIZE { - tracing::warn!( - "Package {blob_url} is larger than {DESIRED_CHUNK_SIZE} bytes and is uploaded via \ - Azure's multi-block path, which opendal 0.57 does not attach blob metadata to; \ - package-sha256/package-md5 will be missing on this blob." - ); - } + // The streaming loop below needs the package size to chunk the upload. + let size = fs_err::tokio::metadata(package_file) + .await + .into_diagnostic()? + .len(); + let mut file = tokio::io::BufReader::new( + fs_err::tokio::File::open(package_file) + .await + .into_diagnostic()?, + ); // Construct a writer for the package. Setting `chunk` and `concurrent` // enables opendal's concurrent block upload: data is buffered into // `DESIRED_CHUNK_SIZE` blocks and up to `PART_CONCURRENCY` blocks are // uploaded in parallel. `if_not_exists(!force)` maps to an `If-None-Match: *` // precondition so an existing blob is not silently overwritten unless - // `--force` was passed. (opendal's azblob backend does not support setting - // `content_disposition` on write, so it is not set here.) + // `--force` was passed; note opendal 0.57 only honours it on the single-shot + // Put Blob path, so the pre-write `stat` above is what guards large uploads. let mut writer = match op .writer_with(&key) .chunk(DESIRED_CHUNK_SIZE) .concurrent(PART_CONCURRENCY) .if_not_exists(!force) - .user_metadata([ - (String::from("package-sha256"), sha256_hex), - (String::from("package-md5"), md5_hex), - ]) .await { Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { @@ -189,38 +179,6 @@ async fn upload_single_package( Ok(()) } -/// Streams `package_file` once to compute its sha256 and md5 digests. -/// -/// Returns the file handle rewound to the start, the total size in bytes, and -/// the hex-encoded sha256 and md5 digests (in that order) that are attached as -/// blob metadata. -async fn package_digests( - package_file: &Path, -) -> miette::Result<( - tokio::io::BufReader, - u64, - String, - String, -)> { - let file = tokio::io::BufReader::new( - fs_err::tokio::File::open(package_file) - .await - .into_diagnostic()?, - ); - let sha256_reader = HashingReader::<_, Sha256>::new(file); - let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); - let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) - .await - .into_diagnostic()?; - let (sha256_reader, md5hash) = md5_reader.finalize(); - let (mut file, sha256hash) = sha256_reader.finalize(); - - // Rewind the file to the beginning. - file.rewind().await.into_diagnostic()?; - - Ok((file, size, hex::encode(sha256hash), hex::encode(md5hash))) -} - /// Build an opendal [`AzblobConfig`] from a channel URL and credentials. /// /// The account name, endpoint, container, and root prefix are all derived from @@ -280,10 +238,9 @@ fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Resul #[cfg(test)] mod test { use opendal::{Operator, services::Memory}; - use rattler_digest::{Md5, compute_file_digest}; use url::Url; - use super::{package_digests, upload_single_package}; + use super::upload_single_package; use crate::upload::package::ExtractedPackage; use crate::upload::test_utils::test_package_path; @@ -341,28 +298,4 @@ mod test { let expected_size = std::fs::metadata(test_package_path()).unwrap().len(); assert_eq!(meta.content_length(), expected_size); } - - /// H1: the sha256/md5 attached as blob metadata are the canonical digests of - /// the package. This guards the values and encoding fed into - /// `user_metadata`. - /// - /// Note: the in-memory opendal service does not persist `user_metadata`, and - /// Azure's multi-block path drops it entirely (see `upload_single_package`), - /// so asserting the metadata is actually present on the stored blob requires - /// an Azurite-backed integration test that this crate does not yet have. - #[tokio::test] - async fn test_package_digests_match_canonical_hashes() { - let package = test_package_path(); - let (_file, size, sha256_hex, md5_hex) = package_digests(&package).await.unwrap(); - - assert_eq!(size, std::fs::metadata(&package).unwrap().len()); - assert_eq!( - sha256_hex, - crate::upload::package::sha256_sum(&package).unwrap() - ); - assert_eq!( - md5_hex, - hex::encode(compute_file_digest::(&package).unwrap()) - ); - } } From 401a71aedb80e52ae7757912175496643048cf5c Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:43:50 +1000 Subject: [PATCH 50/98] refactor(index): drop staleness re-index, restore upstream reconciliation --- Cargo.lock | 1 - crates/rattler_index/Cargo.toml | 1 - crates/rattler_index/src/lib.rs | 59 ++---------- crates/rattler_index/tests/azure_md5_probe.rs | 80 ---------------- .../tests/integration/basic_indexing.rs | 92 ------------------- 5 files changed, 7 insertions(+), 226 deletions(-) delete mode 100644 crates/rattler_index/tests/azure_md5_probe.rs diff --git a/Cargo.lock b/Cargo.lock index ea40207aca..da773cfa8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5218,7 +5218,6 @@ version = "0.30.9" dependencies = [ "ahash", "anyhow", - "base64 0.22.1", "bytes", "clap", "clap-verbosity-flag", diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index 7dd92cd6f2..6b6415f183 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -38,7 +38,6 @@ path = "src/main.rs" thiserror = { workspace = true } ahash = { workspace = true } anyhow = { workspace = true } -base64 = { workspace = true } bytes = { workspace = true } indexmap = { workspace = true } jiff = { workspace = true } diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 86ae4cb4f1..591f733d8f 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -18,7 +18,6 @@ use std::{ }; use anyhow::{Context, Result}; -use base64::Engine; use bytes::buf::Buf; use fs_err::{self as fs}; use futures::{StreamExt, stream::FuturesUnordered}; @@ -772,28 +771,21 @@ async fn index_subdir_inner( } }; - let existing = op.list_with(&format!("{}/", subdir.as_str())).await?; - // get the md5 hashes of each uploaded .conda, storing an optional md5 hash and a archive size - // for each, so that we can later check if the contents has changed - let uploaded_hashes: HashMap> = existing + // List all the packages in the subdirectory. + let uploaded_packages: HashSet = op + .list_with(&format!("{}/", subdir.as_str())) + .await? .iter() .filter_map(|entry| { - let meta = entry.metadata(); - if meta.mode().is_file() { + if entry.metadata().mode().is_file() { + let filename = entry.name().to_string(); // Check if the file is an archive package file. - DistArchiveIdentifier::try_from_filename(entry.name()) - // opendal populates content_md5 from the backend's Content-MD5 - // header on list (verified for azure/azblob via the azure_md5_probe - // test). md5 is documented best-effort, so backends that omit it - // yield None here and those packages get re-indexed. - .map(|id| (id, meta.content_md5())) + DistArchiveIdentifier::try_from_filename(&filename) } else { None } }) .collect(); - let uploaded_packages: HashSet = - uploaded_hashes.keys().cloned().collect(); tracing::debug!( "Found {} already uploaded packages in subdir {}.", @@ -821,43 +813,6 @@ async fn index_subdir_inner( registered_packages.remove(filename); } - // Re-index packages whose file no longer matches the md5 recorded in the - // previous repodata. `.conda`/`.tar.bz2` archives aren't reproducible, so a - // package rebuilt and republished under the same filename has different - // bytes; without this the stale record's sha256/size are kept and clients - // hit a hash mismatch on download. Dropping the mismatched entry here moves - // it into `packages_to_add` below, which re-reads and re-hashes it. - let stale = registered_packages - .iter() - .filter(|(id, pkg)| { - // Not stale only if the backend's md5 matches the one in the previous - // repodata. opendal exposes content_md5 as the base64 Content-MD5 header - // (e.g. azure), so decode it to raw bytes before comparing to the record's - // 16-byte digest. Missing/undecodable md5 => treat as stale and re-index. - if let Some(Some(new_md5)) = uploaded_hashes.get(id) - && let Some(old_md5) = pkg.record.md5 - && let Ok(new_md5) = base64::engine::general_purpose::STANDARD.decode(new_md5) - && old_md5.as_slice() == new_md5.as_slice() - { - false - } else { - true - } - }) - .map(|(id, _)| id.clone()) - .collect::>(); - - if !stale.is_empty() { - tracing::warn!( - "Re-indexing {} packages in subdir {} whose md5 changed since the last index.", - stale.len(), - subdir - ); - } - for filename in &stale { - registered_packages.remove(filename); - } - let packages_to_add = uploaded_packages .difference(®istered_packages.keys().cloned().collect::>()) .cloned() diff --git a/crates/rattler_index/tests/azure_md5_probe.rs b/crates/rattler_index/tests/azure_md5_probe.rs deleted file mode 100644 index e32f50db62..0000000000 --- a/crates/rattler_index/tests/azure_md5_probe.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! Probe: does opendal's azblob backend populate `content_md5` on list? -//! -//! This is exactly the signal the md5-based re-index check in `index_subdir_inner` -//! relies on (`entry.metadata().content_md5()`). If azure doesn't set it on a plain -//! `list_with`, that check silently forces a full re-index of the channel. -//! -//! Run with: -//! ```text -//! docker run -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ -//! azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck -//! AZURE_STORAGE_ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== \ -//! cargo test -p rattler_index --features azure --test azure_md5_probe -- --ignored --nocapture -//! ``` -#![cfg(feature = "azure")] - -use base64::Engine; -use opendal::{Configurator, Operator, services::AzblobConfig}; - -const ACCOUNT: &str = "devstoreaccount1"; -const CONTAINER: &str = "md5-probe"; -const ENDPOINT: &str = "http://127.0.0.1:10000/devstoreaccount1"; - -fn azurite_operator() -> Operator { - let key = std::env::var("AZURE_STORAGE_ACCOUNT_KEY").expect("AZURE_STORAGE_ACCOUNT_KEY"); - let cfg = AzblobConfig { - container: CONTAINER.to_string(), - account_name: Some(ACCOUNT.to_string()), - account_key: Some(key), - endpoint: Some(ENDPOINT.to_string()), - ..Default::default() - }; - Operator::new(cfg.into_builder()).unwrap().finish() -} - -#[tokio::test] -#[ignore = "requires a running Azurite emulator"] -async fn azure_populates_content_md5_on_list() { - let op = azurite_operator(); - let _ = op.create_dir("noarch/").await; - - let name = "noarch/empty-0.1.0-h4616a5c_0.conda"; - let bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("../../test-data/packages/empty-0.1.0-h4616a5c_0.conda"), - ) - .unwrap(); - op.write(name, bytes).await.unwrap(); - - // Exactly what index_subdir_inner does: plain list, then read content_md5 off - // each entry's listing metadata. - let entries = op.list_with("noarch/").await.unwrap(); - let mut saw_file = false; - for entry in &entries { - let meta = entry.metadata(); - if !meta.mode().is_file() { - continue; - } - saw_file = true; - println!( - "LIST {} -> content_md5 = {:?}", - entry.name(), - meta.content_md5() - ); - } - assert!(saw_file, "no file entry found in listing"); - - // For contrast: does an explicit stat populate it? - let stat = op.stat(name).await.unwrap(); - println!("STAT {name} -> content_md5 = {:?}", stat.content_md5()); - - // The re-index check decodes this base64 header to compare against the record's - // raw 16-byte md5, so lock in that it is standard-base64 of exactly 16 bytes. - let md5 = stat - .content_md5() - .expect("azure did not populate content_md5"); - let raw = base64::engine::general_purpose::STANDARD - .decode(md5) - .expect("content_md5 was not valid base64"); - assert_eq!(raw.len(), 16, "decoded md5 was not 16 bytes: {raw:?}"); -} diff --git a/crates/rattler_index/tests/integration/basic_indexing.rs b/crates/rattler_index/tests/integration/basic_indexing.rs index bf8b9a7e97..c2062b6b0a 100644 --- a/crates/rattler_index/tests/integration/basic_indexing.rs +++ b/crates/rattler_index/tests/integration/basic_indexing.rs @@ -378,98 +378,6 @@ async fn test_index_repodata_revision_from_index_json() { assert_eq!(revision["newest"], 1710000000000i64); } -/// Regression: a package rebuilt and republished under the same filename (new -/// bytes) must be re-hashed on an incremental index. Keying only on filename -/// left the previous build's sha256/size in repodata, so clients downloading the -/// current blob hit a hash mismatch. -#[tokio::test] -async fn test_incremental_reindexes_replaced_package() { - // Incremental index (force: false) — the path that carried the bug. - async fn index(channel: &Path) { - index_fs(IndexFsConfig { - channel: channel.into(), - target_platform: Some(Platform::NoArch), - repodata_patch: None, - write_zst: false, - write_shards: false, - repodata_revisions: Vec::new(), - package_revision_assignment: PackageRevisionAssignment::default(), - force: false, - max_parallel: 1, - multi_progress: None, - }) - .await - .unwrap(); - } - - let temp_dir = tempfile::tempdir().unwrap(); - let subdir_path = temp_dir.path().join("noarch"); - fs::create_dir(&subdir_path).unwrap(); - let package_name = "stale-demo-1.0.0-h123_0.tar.bz2"; - - // Build a tar.bz2 package with a payload of the given size and write it into - // the channel under a fixed filename, overwriting any previous build. - let build = |payload_len: usize| { - let build_dir = temp_dir.path().join(format!("build-{payload_len}")); - let info_dir = build_dir.join("info"); - fs::create_dir_all(&info_dir).unwrap(); - fs::write( - info_dir.join("index.json"), - r#"{"build":"h123_0","build_number":0,"name":"stale-demo","noarch":"generic","subdir":"noarch","timestamp":1710000000000,"version":"1.0.0"}"#, - ) - .unwrap(); - fs::write(build_dir.join("payload.txt"), "x".repeat(payload_len)).unwrap(); - let writer = File::create(subdir_path.join(package_name)).unwrap(); - write_tar_bz2_package( - writer, - &build_dir, - &[info_dir.join("index.json"), build_dir.join("payload.txt")], - CompressionLevel::Default, - None, - None, - ) - .unwrap(); - }; - - let recorded = |field: &str| -> Value { - let repodata: Value = - serde_json::from_reader(File::open(subdir_path.join("repodata.json")).unwrap()) - .unwrap(); - repodata - .pointer(&format!("/packages/{package_name}/{field}")) - .expect("package present in repodata") - .clone() - }; - - // Baseline: first build, indexed. - build(64); - index(temp_dir.path()).await; - let disk1 = fs::metadata(subdir_path.join(package_name)).unwrap().len(); - assert_eq!(recorded("size").as_u64(), Some(disk1)); - let sha1 = recorded("sha256"); - - // Rebuild under the same filename with very different bytes, then reindex - // incrementally. - build(65536); - index(temp_dir.path()).await; - let disk2 = fs::metadata(subdir_path.join(package_name)).unwrap().len(); - - assert_ne!( - disk1, disk2, - "test setup: rebuild must change the file size" - ); - assert_eq!( - recorded("size").as_u64(), - Some(disk2), - "repodata size must match the rebuilt file" - ); - assert_ne!( - recorded("sha256"), - sha1, - "repodata sha256 must reflect the rebuilt bytes" - ); -} - #[tokio::test] async fn test_index_writes_channel_metadata() { let temp_dir = tempfile::tempdir().unwrap(); From eba1547102aa9beeafb809b286c430e382fdf609 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:49:45 +1000 Subject: [PATCH 51/98] fix(ci): allowlist AKS in typos, refresh py-rattler lock --- py-rattler/Cargo.lock | 3 ++- typos.toml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index 586b761b2d..5cc44bdbcd 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -4119,7 +4119,9 @@ dependencies = [ "clap", "jiff", "thiserror 2.0.18", + "tokio", "url", + "which", ] [[package]] @@ -4235,7 +4237,6 @@ version = "0.30.9" dependencies = [ "ahash", "anyhow", - "base64", "bytes", "clap", "clap-verbosity-flag", diff --git a/typos.toml b/typos.toml index 0fd1f25226..bffe0904e9 100644 --- a/typos.toml +++ b/typos.toml @@ -19,3 +19,4 @@ strat = "strat" haa = "haa" intoto = "intoto" certifi = "certifi" +AKS = "AKS" From 6acc02fa1afa6bf408128c6a4db4b3f3e7fc33b1 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:53:04 +1000 Subject: [PATCH 52/98] fix(docs): unlink public->private intra-doc refs in azure crates --- crates/rattler_networking/src/azure_middleware.rs | 2 +- crates/rattler_upload/src/upload/azure.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index b979a948a3..6c3fddee7a 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -109,7 +109,7 @@ fn azure_cli_session_present() -> bool { /// When **no credential source is detected**, the request is sent **unsigned** /// rather than failing, so public/anonymous containers remain reachable with /// zero ambient credentials. When a credential source *is* configured -/// (see [`azure_credential_source_present`]) but signing fails, that is a +/// (see `azure_credential_source_present`) but signing fails, that is a /// **hard error** — reqsign collapses "no credential" and "broken credential" /// into the same [`reqsign_core::ErrorKind::CredentialInvalid`], so a broken credential must /// not be silently downgraded to an anonymous request. diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 01c72efa8d..7efd251339 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -29,7 +29,7 @@ pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "cw"; /// The channel URL must be of the form /// `https://.blob.core.windows.net//`; the account /// name, endpoint, container, and root prefix are all derived from it (see -/// [`azblob_config`]). Because the account is derived from the host, upload +/// `azblob_config`). Because the account is derived from the host, upload /// requires this dotted `.blob...` form and does not support /// path-style or emulator (Azurite) endpoints. The full blob host lives in the /// channel URL itself, so no separate account/endpoint configuration is needed. From 03aafcc85439fbf1609759bb4186d5f05082b165 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:35:42 +1000 Subject: [PATCH 53/98] feat(azure): accept az:// channel URLs in index/upload via shared parse_channel_url --- crates/rattler_azure/src/lib.rs | 156 ++++++++++++++++++++---- crates/rattler_index/src/main.rs | 24 +--- crates/rattler_upload/src/upload/opt.rs | 25 +--- 3 files changed, 137 insertions(+), 68 deletions(-) diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 5009bb3a09..9501507382 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -24,8 +24,10 @@ use url::Url; /// and derived by the consumer. /// /// The type deliberately has no `Serialize`/`Deserialize`: it holds raw account -/// keys and SAS tokens, so serialization would risk leaking secrets to disk. -#[derive(Debug, Clone)] +/// keys and SAS tokens, so serialization would risk leaking secrets to disk. For +/// the same reason `Debug` is implemented by hand to redact the secret values +/// rather than derived. +#[derive(Clone)] pub enum AzureCredentials { /// A shared storage account key. AccountKey(String), @@ -34,6 +36,17 @@ pub enum AzureCredentials { SasToken(String), } +impl std::fmt::Debug for AzureCredentials { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Print only the variant, never the secret it carries. + let variant = match self { + AzureCredentials::AccountKey(_) => "AccountKey", + AzureCredentials::SasToken(_) => "SasToken", + }; + f.debug_tuple(variant).field(&"").finish() + } +} + /// Strip a single leading `?` from a SAS token. /// /// `--sas-token` may be supplied with or without a leading `?`, but a SAS minted @@ -54,7 +67,8 @@ pub enum AzureUrlError { /// The URL carries userinfo (`user:pass@host`). #[error( "Azure blob URL must not contain userinfo (`user:pass@host`): the `user@host` form is a \ - host-spoofing vector that can disguise the real target host" + host-spoofing vector that can disguise the real target host, and userinfo is invalid in \ + blob URLs" )] UserInfoNotAllowed, @@ -79,12 +93,26 @@ pub enum AzureUrlError { container names are restricted to that set" )] InvalidCharacters(String), + + /// The channel URL string could not be parsed. + #[error("`{value}` is not a valid URL")] + InvalidUrl { + /// The offending input. + value: String, + /// The underlying parse error. + #[source] + source: url::ParseError, + }, + + /// The channel URL does not use the `az://` scheme. + #[error( + "Azure blob channel URL must use the `az://` scheme, e.g. \ + `az://.blob.core.windows.net//...`: got `{0}`" + )] + InvalidScheme(String), } /// The storage account and container an Azure Blob channel URL resolves to. -/// -/// A named pair rather than a bare `(String, String)` so the two cannot be -/// silently transposed at a call site. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AzureCoordinates { /// The storage account name (first label of the host). @@ -117,6 +145,7 @@ pub fn account_and_container(url: &Url) -> Result Result bool { + name.chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') +} + +/// Parse and validate an Azure Blob **channel** URL. +/// +/// The only accepted form is the `az://` channel scheme — +/// `az://.blob.//` — which is rewritten to +/// `https://` here so every consumer downstream works with a real wire URL and +/// never sees the `az` scheme. A bare `http(s)://` URL is deliberately *not* +/// accepted: `az://` is the single canonical spelling for an Azure channel +/// (matching how it is written in configuration and used on the fetch path), and +/// accepting the wire URL as a second spelling would only invite confusion. The +/// host and container are validated via [`account_and_container`]. +pub fn parse_channel_url(value: &str) -> Result { + // Require the `az://` scheme, then rewrite to `https://` before parsing so + // the host is parsed by the URL crate's special-scheme host parser and the + // `az` scheme never leaks downstream. + let rest = value + .strip_prefix("az://") + .ok_or_else(|| AzureUrlError::InvalidScheme(value.to_string()))?; + let url = + Url::parse(&format!("https://{rest}")).map_err(|source| AzureUrlError::InvalidUrl { + value: value.to_string(), + source, + })?; + account_and_container(&url)?; + Ok(url) +} + /// Errors that can occur while minting a user-delegation SAS via the Azure CLI. #[cfg(feature = "clap")] #[derive(Debug, thiserror::Error)] @@ -219,10 +278,7 @@ pub async fn mint_user_delegation_sas( // is not floored down to the enclosing whole minute. let expiry = expiry.strftime("%Y-%m-%dT%H:%M:%SZ").to_string(); - #[cfg(windows)] let mut command = az_command()?; - #[cfg(not(windows))] - let mut command = az_command(); let output = command .args([ @@ -268,23 +324,18 @@ pub async fn mint_user_delegation_sas( /// Build the [`tokio::process::Command`] used to invoke the Azure CLI. /// -/// On Windows the Azure CLI is a `az.cmd` batch shim; the process spawner does -/// not honor `PATHEXT`, so a bare `az` fails to resolve it. `which` applies `PATHEXT` -/// to find the real `az`/`az.cmd` path, which is then invoked directly. Routing -/// through the command interpreter (`cmd /C az ...`) is deliberately avoided: it -/// would expose the command line to `cmd` metacharacter interpretation, an -/// argument-injection vector. -#[cfg(all(feature = "clap", windows))] +/// `which` resolves `az` up front so a missing CLI surfaces as [`AzureCliSasError::AzResolve`] +/// rather than an opaque spawn failure. It also matters on Windows, where the CLI +/// is an `az.cmd` batch shim: the process spawner does not honor `PATHEXT`, so a +/// bare `az` fails to resolve, but `which` applies `PATHEXT` to find the real path. +/// The resolved path is invoked directly; routing through the command interpreter +/// (`cmd /C az ...`) is deliberately avoided as an argument-injection vector. +#[cfg(feature = "clap")] fn az_command() -> Result { let path = which::which("az").map_err(AzureCliSasError::AzResolve)?; Ok(tokio::process::Command::new(path)) } -#[cfg(all(feature = "clap", not(windows)))] -fn az_command() -> tokio::process::Command { - tokio::process::Command::new("az") -} - #[cfg(test)] mod tests { use super::*; @@ -318,4 +369,59 @@ mod tests { Err(AzureUrlError::InvalidCharacters(_)) )); } + + #[test] + fn parse_channel_url_normalizes_az_to_https() { + let url = parse_channel_url("az://acct.blob.core.windows.net/general/noarch").unwrap(); + assert_eq!(url.scheme(), "https"); + assert_eq!( + url.as_str(), + "https://acct.blob.core.windows.net/general/noarch" + ); + } + + #[test] + fn parse_channel_url_rejects_bare_http_and_https() { + for input in [ + "https://acct.blob.core.windows.net/general", + "http://acct.blob.core.windows.net/general", + "ftp://acct.blob.core.windows.net/general", + "AZ://acct.blob.core.windows.net/general", + "acct.blob.core.windows.net/general", + ] { + assert!( + matches!( + parse_channel_url(input), + Err(AzureUrlError::InvalidScheme(_)) + ), + "expected InvalidScheme for {input}" + ); + } + } + + #[test] + fn parse_channel_url_propagates_validation_errors() { + assert!(matches!( + parse_channel_url("az://acct.blob.core.windows.net@evil.example/general"), + Err(AzureUrlError::UserInfoNotAllowed) + )); + } +} + +#[cfg(test)] +mod debug_redaction_tests { + use super::*; + + #[test] + fn debug_never_prints_secret() { + for creds in [ + AzureCredentials::AccountKey("supersecretkey".into()), + AzureCredentials::SasToken("sig=deadbeef".into()), + ] { + let out = format!("{creds:?}"); + assert!(out.contains(""), "not redacted: {out}"); + assert!(!out.contains("supersecret")); + assert!(!out.contains("deadbeef")); + } + } } diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 142f6ade4b..7b71249448 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -36,25 +36,6 @@ fn parse_s3_url(value: &str) -> Result { } } -#[cfg(feature = "azure")] -fn parse_azure_url(value: &str) -> Result { - let url: Url = Url::parse(value).map_err(|e| format!("`{value}` isn't a valid URL: {e}"))?; - // Require an `.blob.` host and a container segment, e.g. - // https://.blob.core.windows.net//. The account - // and container are derived exactly as the index path derives them, via - // `rattler_azure::account_and_container`, which rejects IP literals and - // single-label hosts (localhost, the Azurite emulator) and container-less - // URLs. - if !matches!(url.scheme(), "http" | "https") - || rattler_azure::account_and_container(&url).is_err() - { - return Err(format!( - "Only Azure Blob URLs of format https://.blob.core.windows.net//... can be used, not `{value}`" - )); - } - Ok(url) -} - /// SAS permissions requested when minting a user-delegation SAS for indexing. /// Indexing does a read-modify-write of repodata and lists/reads packages, so it /// needs read, write, list, and create (`r` + `w` + `l` + `c`). @@ -134,8 +115,9 @@ enum Commands { #[cfg(feature = "azure")] Azblob { /// The Azure Blob channel URL, e.g. - /// `https://.blob.core.windows.net//`. - #[arg(value_parser = parse_azure_url)] + /// `az://.blob.core.windows.net//` + /// (or the equivalent `https://` form). + #[arg(value_parser = rattler_azure::parse_channel_url)] channel: Url, #[clap(flatten)] diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 4686fa21e4..521f7a2fb1 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -418,26 +418,6 @@ pub struct S3Opts { pub force: bool, } -#[cfg(feature = "azure")] -fn parse_azure_url(value: &str) -> Result { - let url: Url = - Url::parse(value).map_err(|err| format!("`{value}` isn't a valid URL: {err}"))?; - // Require an `.blob.` host and a container segment, e.g. - // https://.blob.core.windows.net//. The account - // and container are derived exactly as the upload path derives them, via - // `rattler_azure::account_and_container`, which rejects IP literals and - // single-label hosts (localhost, the Azurite emulator) and container-less - // URLs. - if !matches!(url.scheme(), "http" | "https") - || rattler_azure::account_and_container(&url).is_err() - { - return Err(format!( - "Only Azure Blob URLs of format https://.blob.core.windows.net//... can be used, not `{value}`" - )); - } - Ok(url) -} - /// Options for uploading to Azure Blob Storage. /// /// Authentication is supplied with either an account key or a shared access @@ -446,8 +426,9 @@ fn parse_azure_url(value: &str) -> Result { #[derive(Clone, Debug, PartialEq, Parser)] pub struct AzureOpts { /// The channel URL in the Azure Blob container to upload the package to, - /// e.g., `https://myaccount.blob.core.windows.net/my-container/my-channel` - #[arg(short, long, env = "AZURE_CHANNEL", value_parser = parse_azure_url)] + /// e.g., `az://myaccount.blob.core.windows.net/my-container/my-channel` + /// (or the equivalent `https://` form) + #[arg(short, long, env = "AZURE_CHANNEL", value_parser = rattler_azure::parse_channel_url)] pub channel: Url, #[clap(flatten)] From f5bf06b4aac69e1ad01928058df8a31970299969 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:39:25 +1000 Subject: [PATCH 54/98] docs(index): use az:// scheme in azure examples, note https rejected --- crates/rattler_index/README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index 3de5fecc41..784c2db08a 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -25,7 +25,7 @@ Index an Azure Blob Storage channel: ```shell rattler-index --config ./rattler-config.toml azblob \ - https://my-storage-account.blob.core.windows.net/my-container/my-channel \ + az://my-storage-account.blob.core.windows.net/my-container/my-channel \ --azure-cli ``` @@ -51,7 +51,7 @@ export AZURE_STORAGE_SAS_TOKEN=$(az storage container generate-sas \ --permissions rwlc --expiry 2026-01-01T00:00Z \ --auth-mode login --as-user --https-only -o tsv) rattler-index --config ./rattler-config.toml azblob \ - https://my-storage-account.blob.core.windows.net/my-container/my-channel + az://my-storage-account.blob.core.windows.net/my-container/my-channel ``` The `--config` flag points at the same TOML configuration file used by pixi. It @@ -76,11 +76,15 @@ force-path-style = false Azure Blob Storage needs no such block: the storage account and blob endpoint are read directly from the channel URL -(`https://.blob.core.windows.net//`), so the account, +(`az://.blob.core.windows.net//`), so the account, container, and endpoint (including sovereign clouds) are fully determined by the -URL you pass. Credentials are never stored in the config — they are resolved at -runtime from `--account-key` / `--sas-token`, an `az login` session -(`--azure-cli`), or the `DefaultCredentialProvider` chain. +URL you pass. The `az://` scheme is required — it is the single canonical +spelling for an Azure channel and is rewritten +to `https://` internally; a bare `https://` URL is rejected. The host must be a +dotted `.blob.` domain, so IP-literal / single-label hosts (and +hence the Azurite emulator) are not supported. Credentials are never stored in +the config — they are resolved at runtime from `--account-key` / `--sas-token`, +an `az login` session (`--azure-cli`), or the `DefaultCredentialProvider` chain. ## Per-channel index configuration From f1e9b188ff555ed5c7f2b45941b554cf462716a3 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 24 Jul 2026 11:40:06 +1000 Subject: [PATCH 55/98] docs(index): reflow azure channel URL paragraph --- crates/rattler_index/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index 784c2db08a..bdfb8e530f 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -78,13 +78,12 @@ Azure Blob Storage needs no such block: the storage account and blob endpoint are read directly from the channel URL (`az://.blob.core.windows.net//`), so the account, container, and endpoint (including sovereign clouds) are fully determined by the -URL you pass. The `az://` scheme is required — it is the single canonical -spelling for an Azure channel and is rewritten -to `https://` internally; a bare `https://` URL is rejected. The host must be a -dotted `.blob.` domain, so IP-literal / single-label hosts (and -hence the Azurite emulator) are not supported. Credentials are never stored in -the config — they are resolved at runtime from `--account-key` / `--sas-token`, -an `az login` session (`--azure-cli`), or the `DefaultCredentialProvider` chain. +URL you pass. The `az://` scheme is required and is rewritten to `https://` +internally; a bare `https://` URL is rejected. The host must be a dotted +`.blob.` domain, so IP-literal / single-label hosts (and hence +the Azurite emulator) are not supported. Credentials are never stored in the +config — they are resolved at runtime from `--account-key` / `--sas-token`, an +`az login` session (`--azure-cli`), or the `DefaultCredentialProvider` chain. ## Per-channel index configuration From 4c6c6245115b4debd96e7c6b58c458d95dec634e Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 24 Jul 2026 12:01:35 +1000 Subject: [PATCH 56/98] test(azure): use standard st-prefixed account name in fixtures --- crates/rattler_index/src/lib.rs | 8 ++++---- crates/rattler_networking/tests/azure_real_fetch.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 591f733d8f..3a75044222 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -1965,16 +1965,16 @@ mod tests { #[test] fn azblob_config_derives_fields_from_url() { let channel = - Url::parse("https://stgrcondachannel.blob.core.windows.net/general/sub/dir").unwrap(); + Url::parse("https://stcondachannel.blob.core.windows.net/general/sub/dir").unwrap(); let credentials = AzureCredentials::SasToken("sv=token".to_string()); let config = azblob_config(&credentials, &channel).unwrap(); assert_eq!( config.endpoint.as_deref(), - Some("https://stgrcondachannel.blob.core.windows.net") + Some("https://stcondachannel.blob.core.windows.net") ); - assert_eq!(config.account_name.as_deref(), Some("stgrcondachannel")); + assert_eq!(config.account_name.as_deref(), Some("stcondachannel")); assert_eq!(config.container, "general"); assert_eq!(config.root.as_deref(), Some("/sub/dir")); assert_eq!(config.sas_token.as_deref(), Some("sv=token")); @@ -1984,7 +1984,7 @@ mod tests { #[cfg(feature = "azure")] #[test] fn azblob_config_container_only_url() { - let channel = Url::parse("https://stgrcondachannel.blob.core.windows.net/general").unwrap(); + let channel = Url::parse("https://stcondachannel.blob.core.windows.net/general").unwrap(); let credentials = AzureCredentials::AccountKey("key".to_string()); let config = azblob_config(&credentials, &channel).unwrap(); diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs index 3f0a4114c9..b5f5ba40f3 100644 --- a/crates/rattler_networking/tests/azure_real_fetch.rs +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -8,7 +8,7 @@ //! `Storage Blob Data Reader` on the container): //! //! ```text -//! AZURE_TEST_ACCOUNT=stgrcondachannel \ +//! AZURE_TEST_ACCOUNT=stcondachannel \ //! AZURE_TEST_CONTAINER=general \ //! AZURE_TEST_PATH=noarch/repodata.json \ //! cargo test -p rattler_networking --features azure --test azure_real_fetch -- --ignored --nocapture @@ -36,7 +36,7 @@ async fn azure_middleware_fetches_real_repodata() { .build(); // The `az://` host carries the full blob endpoint — same form used in a - // channel URL, e.g. `az://stgrcondachannel.blob.core.windows.net/general`. + // channel URL, e.g. `az://stcondachannel.blob.core.windows.net/general`. let url = format!("az://{host}/{container}/{path}"); println!("fetching {url}"); let resp = client From 55c0cae1610bb09ae850e7c3d96b3af9fd024719 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:05:44 +1000 Subject: [PATCH 57/98] refactor(azure): extract shared azblob_config into rattler_azure --- Cargo.lock | 4 +- crates/rattler_azure/Cargo.toml | 6 ++ crates/rattler_azure/src/lib.rs | 55 ++++++++++++++++++ crates/rattler_index/Cargo.toml | 3 +- crates/rattler_index/src/lib.rs | 71 ++--------------------- crates/rattler_index/src/main.rs | 3 +- crates/rattler_upload/Cargo.toml | 3 +- crates/rattler_upload/src/upload/azure.rs | 62 +------------------- crates/rattler_upload/src/upload/opt.rs | 1 - 9 files changed, 73 insertions(+), 135 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da773cfa8e..1ea5f79ba4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5051,6 +5051,8 @@ version = "0.1.0" dependencies = [ "clap", "jiff", + "opendal", + "percent-encoding", "thiserror 2.0.18", "tokio", "url", @@ -5229,7 +5231,6 @@ dependencies = [ "indicatif", "jiff", "opendal", - "percent-encoding", "rattler_azure", "rattler_conda_types", "rattler_config", @@ -5632,7 +5633,6 @@ dependencies = [ "indicatif", "miette", "opendal", - "percent-encoding", "rattler_azure", "rattler_conda_types", "rattler_config", diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index f097666a65..a4e5ce08a8 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -15,10 +15,16 @@ default = [] # SAS by shelling out to the Azure CLI (which needs `jiff` for the expiry and # `tokio` to run `az` without blocking the async runtime). clap = ["dep:clap", "dep:jiff", "dep:which", "dep:tokio"] +# `azblob_config`: build an opendal `AzblobConfig` from a channel URL. +opendal = ["dep:opendal", "dep:percent-encoding"] [dependencies] clap = { workspace = true, optional = true } jiff = { workspace = true, optional = true } +opendal = { workspace = true, default-features = false, features = [ + "services-azblob", +], optional = true } +percent-encoding = { workspace = true, optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["process"], optional = true } url = { workspace = true } diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 9501507382..ded950fd24 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -200,6 +200,61 @@ pub fn parse_channel_url(value: &str) -> Result { Ok(url) } +/// Build an opendal [`AzblobConfig`](opendal::services::AzblobConfig) from a +/// channel URL and credentials. +/// +/// The account name, endpoint, container, and root prefix are all derived from +/// the URL (`https://.blob.core.windows.net//`); the +/// credentials supply only the account key or SAS token. The URL is expected to +/// already be validated and normalized to `https://` (see [`parse_channel_url`]). +#[cfg(feature = "opendal")] +pub fn azblob_config( + credentials: &AzureCredentials, + channel: &Url, +) -> Result { + let AzureCoordinates { account, container } = account_and_container(channel)?; + + // Preserve a non-default port if one is present; real Azure uses the scheme + // default (443). + let host = channel.host_str().ok_or(AzureUrlError::NoHost)?; + let authority = match channel.port() { + Some(port) => format!("{host}:{port}"), + None => host.to_string(), + }; + + // Root prefix = the path after the container segment. Percent-decode each + // segment: `path_segments()` yields still-encoded segments, and opendal + // percent-encodes the root again, so passing them through verbatim would + // double-encode prefixes containing spaces or `+`. `account_and_container` + // has already confirmed there is at least the container segment. + let root = format!( + "/{}", + channel + .path_segments() + .into_iter() + .flatten() + .skip(1) + .map(|segment| percent_encoding::percent_decode_str(segment).decode_utf8_lossy()) + .collect::>() + .join("/") + ); + + let (account_key, sas_token) = match credentials { + AzureCredentials::AccountKey(key) => (Some(key.clone()), None), + AzureCredentials::SasToken(token) => (None, Some(normalize_sas_token(token).to_string())), + }; + + Ok(opendal::services::AzblobConfig { + endpoint: Some(format!("{}://{}", channel.scheme(), authority)), + account_name: Some(account), + container, + root: Some(root), + account_key, + sas_token, + ..Default::default() + }) +} + /// Errors that can occur while minting a user-delegation SAS via the Azure CLI. #[cfg(feature = "clap")] #[derive(Debug, thiserror::Error)] diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index 6b6415f183..722ebf95c3 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -28,7 +28,7 @@ rustls = [ "opendal/reqwest-rustls-tls", ] s3 = ["opendal/services-s3", "dep:rattler_s3"] -azure = ["opendal/services-azblob", "dep:rattler_azure", "dep:percent-encoding"] +azure = ["opendal/services-azblob", "dep:rattler_azure", "rattler_azure/opendal"] [[bin]] name = "rattler-index" @@ -75,7 +75,6 @@ tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"] } url = { workspace = true } -percent-encoding = { workspace = true, optional = true } zstd = { workspace = true } retry-policies = { workspace = true } diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 3a75044222..cb62c7e3fd 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -25,8 +25,6 @@ use indexmap::IndexMap; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; #[cfg(any(feature = "s3", feature = "azure"))] use opendal::layers::RetryLayer; -#[cfg(feature = "azure")] -use opendal::services::AzblobConfig; #[cfg(feature = "s3")] use opendal::services::S3Config; use opendal::{Configurator, Operator, services::FsConfig}; @@ -1549,67 +1547,6 @@ pub struct IndexAzureConfig { // exposing a knob whose enabled state always fails. } -/// Build an opendal `AzblobConfig` from a channel URL and credentials. -/// -/// The account name, endpoint, container, and root prefix are all derived from -/// the URL (`https://.blob.core.windows.net//`); the -/// credentials supply only the account key or SAS token. -#[cfg(feature = "azure")] -fn azblob_config( - credentials: &AzureCredentials, - channel: &Url, -) -> Result { - let rattler_azure::AzureCoordinates { - account: account_name, - container, - } = rattler_azure::account_and_container(channel)?; - - let host = channel - .host_str() - .ok_or_else(|| anyhow::anyhow!("No host in Azure blob URL"))?; - - let mut segments = channel - .path_segments() - .ok_or_else(|| anyhow::anyhow!("No path in Azure blob URL"))?; - // Skip the container segment; the remainder is the root prefix. Percent-decode - // each segment before joining: `path_segments()` yields still-encoded segments, - // and opendal percent-encodes the root again, so passing them through verbatim - // would double-encode prefixes containing spaces or `+`. - segments.next(); - let root = format!( - "/{}", - segments - .map(|segment| percent_encoding::percent_decode_str(segment).decode_utf8_lossy()) - .collect::>() - .join("/") - ); - - // Preserve a non-default port if one is present; real Azure uses the scheme - // default (443). - let authority = match channel.port() { - Some(port) => format!("{host}:{port}"), - None => host.to_string(), - }; - - let (account_key, sas_token) = match credentials { - AzureCredentials::AccountKey(key) => (Some(key.clone()), None), - AzureCredentials::SasToken(token) => ( - None, - Some(rattler_azure::normalize_sas_token(token).to_string()), - ), - }; - - Ok(AzblobConfig { - endpoint: Some(format!("{}://{}", channel.scheme(), authority)), - account_name: Some(account_name), - container, - root: Some(root), - account_key, - sas_token, - ..Default::default() - }) -} - /// Create a new `repodata.json` for all packages in the channel at the given /// Azure Blob URL. #[cfg(feature = "azure")] @@ -1636,7 +1573,7 @@ pub async fn index_azure_with_channel_metadata( }: IndexAzureConfig, channel_metadata: ChannelMetadata, ) -> anyhow::Result<()> { - let azblob_config = azblob_config(&credentials, &channel)?; + let azblob_config = rattler_azure::azblob_config(&credentials, &channel)?; let builder = azblob_config.into_builder(); let op = Operator::new(builder)?.layer(RetryLayer::new()).finish(); @@ -1968,7 +1905,7 @@ mod tests { Url::parse("https://stcondachannel.blob.core.windows.net/general/sub/dir").unwrap(); let credentials = AzureCredentials::SasToken("sv=token".to_string()); - let config = azblob_config(&credentials, &channel).unwrap(); + let config = rattler_azure::azblob_config(&credentials, &channel).unwrap(); assert_eq!( config.endpoint.as_deref(), @@ -1987,7 +1924,7 @@ mod tests { let channel = Url::parse("https://stcondachannel.blob.core.windows.net/general").unwrap(); let credentials = AzureCredentials::AccountKey("key".to_string()); - let config = azblob_config(&credentials, &channel).unwrap(); + let config = rattler_azure::azblob_config(&credentials, &channel).unwrap(); assert_eq!(config.container, "general"); assert_eq!(config.root.as_deref(), Some("/")); @@ -2002,7 +1939,7 @@ mod tests { Url::parse("http://devstoreaccount1.blob.localhost:10000/testcontainer/ch").unwrap(); let credentials = AzureCredentials::AccountKey("key".to_string()); - let config = azblob_config(&credentials, &channel).unwrap(); + let config = rattler_azure::azblob_config(&credentials, &channel).unwrap(); assert_eq!( config.endpoint.as_deref(), diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 7b71249448..93ad6d16de 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -115,8 +115,7 @@ enum Commands { #[cfg(feature = "azure")] Azblob { /// The Azure Blob channel URL, e.g. - /// `az://.blob.core.windows.net//` - /// (or the equivalent `https://` form). + /// `az://.blob.core.windows.net//`. #[arg(value_parser = rattler_azure::parse_channel_url)] channel: Url, diff --git a/crates/rattler_upload/Cargo.toml b/crates/rattler_upload/Cargo.toml index 98f2e93af5..18903cdf39 100644 --- a/crates/rattler_upload/Cargo.toml +++ b/crates/rattler_upload/Cargo.toml @@ -29,7 +29,7 @@ native-tls = [ "sigstore-sign?/native-tls", ] s3 = ["rattler_networking/s3", "rattler_s3", "dep:opendal", "opendal/services-s3"] -azure = ["dep:rattler_azure", "dep:opendal", "opendal/services-azblob", "dep:percent-encoding"] +azure = ["dep:rattler_azure", "rattler_azure/opendal", "dep:opendal", "opendal/services-azblob"] sigstore-sign = ["dep:sigstore-sign", "dep:sigstore-trust-root"] [package.metadata.docs.rs] @@ -62,7 +62,6 @@ reqwest = { workspace = true, default-features = false, features = [ "stream", ] } url = { workspace = true } -percent-encoding = { workspace = true, optional = true } tracing = { workspace = true } reqwest-middleware = { workspace = true, features = ["json"] } serde_yaml = { workspace = true } diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 7efd251339..62f1108aa3 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use futures::StreamExt; use miette::IntoDiagnostic; -use opendal::{Configurator, ErrorKind, Operator, services::AzblobConfig}; +use opendal::{Configurator, ErrorKind, Operator}; use rattler_azure::AzureCredentials; use tokio::io::AsyncReadExt; use tokio_util::bytes::BytesMut; @@ -40,7 +40,7 @@ pub async fn upload_package_to_azure( package_files: &[PathBuf], force: bool, ) -> miette::Result<()> { - let config = azblob_config(&credentials, &channel)?; + let config = rattler_azure::azblob_config(&credentials, &channel).into_diagnostic()?; let builder = config.into_builder(); let op = Operator::new(builder).into_diagnostic()?.finish(); @@ -94,7 +94,7 @@ async fn upload_single_package( // does nothing for large uploads. An explicit `stat` closes that gap at all // sizes. The residual stat->write TOCTOU (another writer could create the // blob between this check and `close`) is acceptable: it is strictly better - // than today's silent clobber, and the writer-level `if_not_exists` still + // than a silent clobber, and the writer-level `if_not_exists` still // guards the small-file and racing-writer cases. if !force { match op.stat(&key).await { @@ -179,62 +179,6 @@ async fn upload_single_package( Ok(()) } -/// Build an opendal [`AzblobConfig`] from a channel URL and credentials. -/// -/// The account name, endpoint, container, and root prefix are all derived from -/// the URL (`https://.blob.core.windows.net//`); the -/// credentials supply only the account key or SAS token. -fn azblob_config(credentials: &AzureCredentials, channel: &Url) -> miette::Result { - let rattler_azure::AzureCoordinates { - account: account_name, - container, - } = rattler_azure::account_and_container(channel).into_diagnostic()?; - - let mut segments = channel - .path_segments() - .ok_or_else(|| miette::miette!("No path in Azure blob URL"))?; - // Skip the container segment; the remainder is the root prefix. Percent-decode - // each segment before joining: `path_segments()` yields still-encoded segments, - // and opendal percent-encodes the root again, so passing them through verbatim - // would double-encode prefixes containing spaces or `+`. - segments.next(); - let root = format!( - "/{}", - segments - .map(|segment| percent_encoding::percent_decode_str(segment).decode_utf8_lossy()) - .collect::>() - .join("/") - ); - - // Preserve a non-default port if one is present; real Azure uses the scheme - // default (443). - let host = channel - .host_str() - .ok_or_else(|| miette::miette!("No host in Azure blob URL"))?; - let authority = match channel.port() { - Some(port) => format!("{host}:{port}"), - None => host.to_string(), - }; - - let (account_key, sas_token) = match credentials { - AzureCredentials::AccountKey(key) => (Some(key.clone()), None), - AzureCredentials::SasToken(token) => ( - None, - Some(rattler_azure::normalize_sas_token(token).to_string()), - ), - }; - - Ok(AzblobConfig { - endpoint: Some(format!("{}://{}", channel.scheme(), authority)), - account_name: Some(account_name), - container, - root: Some(root), - account_key, - sas_token, - ..Default::default() - }) -} - #[cfg(test)] mod test { use opendal::{Operator, services::Memory}; diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 521f7a2fb1..2623df65fe 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -427,7 +427,6 @@ pub struct S3Opts { pub struct AzureOpts { /// The channel URL in the Azure Blob container to upload the package to, /// e.g., `az://myaccount.blob.core.windows.net/my-container/my-channel` - /// (or the equivalent `https://` form) #[arg(short, long, env = "AZURE_CHANNEL", value_parser = rattler_azure::parse_channel_url)] pub channel: Url, From fd2b1e814c46f30f66d22d93bfcb916d645a23ce Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:15:17 +1000 Subject: [PATCH 58/98] chore: update lock --- py-rattler/Cargo.lock | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index 5cc44bdbcd..cd25592d05 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -4118,6 +4118,8 @@ version = "0.1.0" dependencies = [ "clap", "jiff", + "opendal", + "percent-encoding", "thiserror 2.0.18", "tokio", "url", @@ -4248,7 +4250,6 @@ dependencies = [ "indicatif", "jiff", "opendal", - "percent-encoding", "rattler_azure", "rattler_conda_types", "rattler_config", From 2b29682f79bda71a92440842c5c4a94dbe087999 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:15:22 +1000 Subject: [PATCH 59/98] chore: clean comment --- crates/rattler_networking/src/azure_middleware.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 6c3fddee7a..14de8fc711 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -182,7 +182,8 @@ impl AzureMiddleware { /// Whether the URL carries userinfo (`user` and/or `:pass` before the host). /// Because the host is trusted verbatim, a `user:pass@host` authority is a - /// host-spoofing vector and must be refused. + /// host-spoofing vector and must be refused. Userinfo in a blob URL is invalid + /// regardless and safe to ignore. fn has_userinfo(url: &Url) -> bool { !url.username().is_empty() || url.password().is_some() } From fe0dd807682e23c6a18df010899baeb3af7d2144 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:16:22 +1000 Subject: [PATCH 60/98] feat: use `az` subcommand --- crates/rattler_index/README.md | 4 ++-- crates/rattler_index/src/main.rs | 1 + crates/rattler_upload/src/upload/opt.rs | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index bdfb8e530f..d3eb1f292f 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -24,7 +24,7 @@ rattler-index --config ./rattler-config.toml s3 s3://my-bucket/my-channel Index an Azure Blob Storage channel: ```shell -rattler-index --config ./rattler-config.toml azblob \ +rattler-index --config ./rattler-config.toml az \ az://my-storage-account.blob.core.windows.net/my-container/my-channel \ --azure-cli ``` @@ -50,7 +50,7 @@ export AZURE_STORAGE_SAS_TOKEN=$(az storage container generate-sas \ --account-name my-storage-account --name my-container \ --permissions rwlc --expiry 2026-01-01T00:00Z \ --auth-mode login --as-user --https-only -o tsv) -rattler-index --config ./rattler-config.toml azblob \ +rattler-index --config ./rattler-config.toml az \ az://my-storage-account.blob.core.windows.net/my-container/my-channel ``` diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 93ad6d16de..6d8d81a6e6 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -113,6 +113,7 @@ enum Commands { /// Index a channel stored in an Azure Blob container. #[cfg(feature = "azure")] + #[command(name = "az")] Azblob { /// The Azure Blob channel URL, e.g. /// `az://.blob.core.windows.net//`. diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 2623df65fe..77b44d0e5d 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -124,6 +124,7 @@ pub enum ServerType { #[cfg(feature = "s3")] S3(S3Opts), #[cfg(feature = "azure")] + #[command(name = "az")] Azure(AzureOpts), #[clap(hide = true)] CondaForge(CondaForgeOpts), From 0627c7c5647b7ff420a0f0d84ac139c720255af1 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 27 Jul 2026 09:46:19 +1000 Subject: [PATCH 61/98] chore: update lockfile --- Cargo.lock | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ea5f79ba4..bc1bbcfe82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4939,7 +4939,7 @@ dependencies = [ [[package]] name = "rattler" -version = "0.47.1" +version = "0.48.0" dependencies = [ "anyhow", "assert_matches", @@ -5005,7 +5005,7 @@ dependencies = [ [[package]] name = "rattler-bin" -version = "0.2.4" +version = "0.2.5" dependencies = [ "anyhow", "astral-reqwest-middleware", @@ -5061,7 +5061,7 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.10.3" +version = "0.10.4" dependencies = [ "ahash", "anyhow", @@ -5087,6 +5087,7 @@ dependencies = [ "rayon", "reqwest", "rstest", + "rustix 1.1.4", "serde_json", "simple_spawn_blocking", "tempfile", @@ -5101,7 +5102,7 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.48.1" +version = "0.49.0" dependencies = [ "ahash", "assert_matches", @@ -5153,7 +5154,7 @@ dependencies = [ [[package]] name = "rattler_config" -version = "0.6.1" +version = "0.6.2" dependencies = [ "dirs", "fs-err", @@ -5195,7 +5196,7 @@ dependencies = [ [[package]] name = "rattler_git" -version = "0.2.0" +version = "0.2.1" dependencies = [ "astral-reqwest-middleware", "dashmap", @@ -5216,7 +5217,7 @@ dependencies = [ [[package]] name = "rattler_index" -version = "0.30.9" +version = "0.30.10" dependencies = [ "ahash", "anyhow", @@ -5270,7 +5271,7 @@ dependencies = [ [[package]] name = "rattler_lock" -version = "0.31.5" +version = "0.31.6" dependencies = [ "ahash", "file_url", @@ -5310,7 +5311,7 @@ dependencies = [ [[package]] name = "rattler_menuinst" -version = "0.2.70" +version = "0.2.71" dependencies = [ "configparser", "dirs", @@ -5340,7 +5341,7 @@ dependencies = [ [[package]] name = "rattler_networking" -version = "0.30.2" +version = "0.30.3" dependencies = [ "ambient-id", "anyhow", @@ -5392,7 +5393,7 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.26.8" +version = "0.26.9" dependencies = [ "assert_matches", "astral-reqwest-middleware", @@ -5470,7 +5471,7 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" -version = "0.30.1" +version = "0.31.0" dependencies = [ "ahash", "anyhow", @@ -5544,7 +5545,7 @@ dependencies = [ [[package]] name = "rattler_s3" -version = "0.2.8" +version = "0.2.9" dependencies = [ "aws-config", "aws-credential-types", @@ -5571,7 +5572,7 @@ dependencies = [ [[package]] name = "rattler_shell" -version = "0.27.10" +version = "0.27.11" dependencies = [ "anyhow", "enum_dispatch", @@ -5592,7 +5593,7 @@ dependencies = [ [[package]] name = "rattler_solve" -version = "7.2.2" +version = "8.0.0" dependencies = [ "criterion", "futures", @@ -5602,6 +5603,7 @@ dependencies = [ "jiff", "libc", "once_cell", + "rattler_cache", "rattler_conda_types", "rattler_digest", "rattler_libsolv_c", @@ -5613,6 +5615,7 @@ dependencies = [ "similar-asserts", "tempfile", "thiserror 2.0.18", + "tokio", "tools", "tracing", "url", @@ -5620,7 +5623,7 @@ dependencies = [ [[package]] name = "rattler_upload" -version = "0.9.0" +version = "0.10.0" dependencies = [ "astral-reqwest-middleware", "astral-reqwest-retry", @@ -5659,7 +5662,7 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" -version = "3.0.4" +version = "4.0.0" dependencies = [ "archspec", "libloading 0.9.0", From 5ba93b607cbc5b4db98179262d57f0996ded5a7c Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 27 Jul 2026 10:13:14 +1000 Subject: [PATCH 62/98] fix: relock against upstream instead of downgrading deps --- Cargo.lock | 564 ++++++++++++++++++++++-------------------- py-rattler/Cargo.lock | 34 +-- 2 files changed, 313 insertions(+), 285 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc1bbcfe82..4e37d3cbc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,7 +76,7 @@ dependencies = [ "secrecy", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -146,9 +146,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "apple-native-keyring-store" @@ -211,7 +211,7 @@ dependencies = [ "http 1.4.2", "reqwest", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "tower-service", ] @@ -230,7 +230,7 @@ dependencies = [ "hyper", "reqwest", "retry-policies", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "wasmtimer", @@ -238,15 +238,15 @@ dependencies = [ [[package]] name = "astral-tokio-tar" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08648fef353ab39a9d26f909ad53fc4f071be4c91853b78523f5cc3d9e5ebffd" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" dependencies = [ "futures-core", "libc", "portable-atomic", "rustc-hash", - "rustix 0.38.44", + "rustix 1.1.4", "tokio", "tokio-stream", "xattr", @@ -264,7 +264,7 @@ dependencies = [ "itertools 0.14.0", "memmap2", "reqwest", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tokio-util", @@ -281,7 +281,7 @@ dependencies = [ "crc32fast", "futures-lite", "pin-project", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", ] @@ -332,13 +332,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -355,9 +355,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319" +checksum = "701418aa459dac33e50a0f8e818e5662a16bc018a6ac7423659b70f3799d67a8" dependencies = [ "aws-credential-types", "aws-runtime", @@ -404,9 +404,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.1" +version = "1.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -415,9 +415,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.42.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" dependencies = [ "cc", "cmake", @@ -428,9 +428,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7816e98ee912159f45d307e5ee6bfea4a335a55aee15f7f3e32f81a6f3000f1d" +checksum = "a6b50a43f3ccdf331521c6d6c68b7cc9668b6e09d439ebda9569df5722324d76" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -456,9 +456,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.138.0" +version = "1.139.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c29be98554a0deea25d4eaca131240a224dcbcaf20357e35cc432e17b721ab5" +checksum = "a159b9721a6a41468f967d1029bece78f410b0beb0594498435deb6ff72bfe48" dependencies = [ "arc-swap", "aws-credential-types", @@ -493,9 +493,9 @@ dependencies = [ [[package]] name = "aws-sdk-signin" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fff6102ab555e20bda506f8878fce0e786a936f2821fd32840ea89c3120baf6" +checksum = "4cb1aef0872a7ab9e035a8c14d9ee1c3bd7f86f741b3620df8e8ecfe7b0d14dd" dependencies = [ "arc-swap", "aws-credential-types", @@ -519,9 +519,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.103.0" +version = "1.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0469f435f645ad2162cfb463b15bde37115966ee3acf2d87fb4871ee309b8401" +checksum = "b53416d16c278234845392e38d93bd4481d2f09daa0f005a2277f0aa91f59c22" dependencies = [ "arc-swap", "aws-credential-types", @@ -545,9 +545,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.105.0" +version = "1.106.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "085faefb253f770655e162b9304321e62a1e71adf7f019ee1f4454228a377b3a" +checksum = "cc9b706c3305ed0285d5b1b696c747aa34950f830fb03e3e6c76890f99b9f188" dependencies = [ "arc-swap", "aws-credential-types", @@ -571,9 +571,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.108.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c72b08911d8128dd360fe1b22a9fec0fa8b552dde8ec828dcf20ef5ec974e9f" +checksum = "32d214cdfa5bbe17f117e76a7643fadf32a5234fb597322ef8b1fb4b2f17dbbd" dependencies = [ "arc-swap", "aws-credential-types", @@ -734,19 +734,22 @@ dependencies = [ [[package]] name = "aws-smithy-query" -version = "0.61.1" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd22a6ba36e3f113cb8d5b3d1fe0ed31c76ee608ef63322d753bb8d2c9479e77" +checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea94a9ff8464016338c851e24b472d7131c388c88898a502e781815b2ee6045" +checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -770,9 +773,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ed1ebe6e0a95ea84570225f5a8208dec4b8f77e61a9b0d6f51773fcb4612f0" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -794,7 +797,7 @@ checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -836,9 +839,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.61.1" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea3f68eec3607f02acd24067969ce2abc6ba16aa7d5ce59ca450ed2fb5f78957" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", @@ -848,9 +851,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e957a6c6dbce82b7a91f44231c09273159703769f447cbe85e854dfe9cf67f86" +checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -986,7 +989,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -997,7 +1000,7 @@ dependencies = [ "regex", "rustc-hash", "shlex 1.3.0", - "syn", + "syn 2.0.119", ] [[package]] @@ -1006,7 +1009,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "848df95320021558dd6bb4c26de3fe66724cdcbdbbf3fa720150b52b086ae568" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "libc", "log", "rustix 0.38.44", @@ -1036,9 +1039,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "bitvec" @@ -1105,9 +1108,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.12.3" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cee35f73844aa3014bb606320a6c1f010249dbdf43342fe54b5a4f6a8ed4b79" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", "regex-automata", @@ -1177,9 +1180,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.67" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -1204,9 +1207,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" @@ -1283,9 +1286,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "d91e0c145792ef73a6ad36d27c75ac09f1832222a3c209689d90f534685ee5b7" dependencies = [ "clap_builder", "clap_derive", @@ -1304,9 +1307,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" dependencies = [ "anstream", "anstyle", @@ -1326,9 +1329,9 @@ dependencies = [ [[package]] name = "clap_complete_nushell" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb9e9715d29a754b468591be588f6b926f5b0a1eb6a8b62acabeb66ff84d897" +checksum = "933b05d5d83ff65fd7eaf5d106c792f2264908790a2642aca57429767b762ce2" dependencies = [ "clap", "clap_complete", @@ -1336,14 +1339,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -1742,7 +1745,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1765,7 +1768,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] @@ -1776,7 +1779,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1852,7 +1855,7 @@ dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1861,7 +1864,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -1885,7 +1888,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1973,7 +1976,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "objc2", ] @@ -1985,7 +1988,7 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2016,7 +2019,7 @@ checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2138,7 +2141,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2150,7 +2153,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2210,9 +2213,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "ff" @@ -2237,7 +2240,7 @@ dependencies = [ "itertools 0.15.0", "percent-encoding", "rstest", - "thiserror 2.0.18", + "thiserror 2.0.19", "typed-path", "url", ] @@ -2378,9 +2381,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -2393,9 +2396,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -2403,15 +2406,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -2420,9 +2423,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-lite" @@ -2439,26 +2442,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-timer" @@ -2468,9 +2471,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -2542,15 +2545,15 @@ checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "glob" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" [[package]] name = "globset" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" dependencies = [ "aho-corasick", "bstr", @@ -2592,7 +2595,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", - "thiserror 2.0.18", + "thiserror 2.0.19", "time", "tokio", "url", @@ -2613,7 +2616,7 @@ dependencies = [ "rand 0.10.2", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", ] @@ -2641,7 +2644,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.18", + "thiserror 2.0.19", "time", "url", ] @@ -2937,9 +2940,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -3295,11 +3298,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.32" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "961d16382652bfdd8c6f68b223b26a8c93e0d475c672f414411db31c6c5c900e" +checksum = "e184d09547b80eb7e20d141ba2fb1fbac843ca53f4cf1b31210adc4c1adc6e16" dependencies = [ "defmt", + "jiff-core", "jiff-static", "jiff-tzdb-platform", "js-sys", @@ -3311,15 +3315,25 @@ dependencies = [ "windows-link", ] +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + [[package]] name = "jiff-static" -version = "0.2.32" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0879bd39df99c4c5e2c6615ccc026391a423dde10532c573e6086eb94a802cc" +checksum = "323da076b7a6faf914dc677cb05a4b907742ff7375c8322c9e7f5061e5e0e9de" dependencies = [ + "jiff-core", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3349,7 +3363,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.18", + "thiserror 2.0.19", "walkdir", "windows-link", ] @@ -3364,7 +3378,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.119", ] [[package]] @@ -3383,7 +3397,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3416,7 +3430,7 @@ dependencies = [ "jsonptr", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -3467,7 +3481,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn", + "syn 2.0.119", ] [[package]] @@ -3487,9 +3501,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libdbus-sys" @@ -3683,7 +3697,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3783,7 +3797,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "cfg_aliases", "libc", @@ -3984,7 +3998,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "dispatch2", "objc2", ] @@ -4001,7 +4015,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "objc2", ] @@ -4218,7 +4232,7 @@ version = "0.10.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "cfg-if", "foreign-types", "libc", @@ -4234,7 +4248,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4457,9 +4471,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" +checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" dependencies = [ "memchr", "ucd-trie", @@ -4467,9 +4481,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" +checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" dependencies = [ "pest", "pest_generator", @@ -4477,22 +4491,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" +checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "pest_meta" -version = "2.8.7" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" +checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" dependencies = [ "pest", ] @@ -4526,7 +4540,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4628,9 +4642,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "portable-atomic-util" @@ -4672,7 +4686,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.119", ] [[package]] @@ -4695,9 +4709,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -4710,7 +4724,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.13.0", + "bitflags 2.13.1", "num-traits", "rand 0.9.5", "rand_chacha 0.9.0", @@ -4730,7 +4744,7 @@ dependencies = [ "hex", "percent-encoding", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -4773,7 +4787,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "web-time", @@ -4796,7 +4810,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.19", "tinyvec", "tracing", "web-time", @@ -4818,9 +4832,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -4991,7 +5005,7 @@ dependencies = [ "smallvec", "temp-env", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tools", "tower", @@ -5053,7 +5067,7 @@ dependencies = [ "jiff", "opendal", "percent-encoding", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "url", "which", @@ -5091,7 +5105,7 @@ dependencies = [ "serde_json", "simple_spawn_blocking", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-stream", "tools", @@ -5145,7 +5159,7 @@ dependencies = [ "smallvec", "strum", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tools", "tracing", "typed-path", @@ -5165,7 +5179,7 @@ dependencies = [ "serde_ignored", "serde_json", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "toml", "tracing", "url", @@ -5207,7 +5221,7 @@ dependencies = [ "reqwest", "serde", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "url", @@ -5248,7 +5262,7 @@ dependencies = [ "sha2 0.11.0", "tar", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tools", "tracing", @@ -5293,7 +5307,7 @@ dependencies = [ "serde_with", "serde_yaml", "similar-asserts", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "typed-path", "url", @@ -5305,7 +5319,7 @@ name = "rattler_macros" version = "1.1.2" dependencies = [ "quote", - "syn", + "syn 2.0.119", "trybuild", ] @@ -5331,7 +5345,7 @@ dependencies = [ "sha2 0.11.0", "shlex 2.0.1", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "unicode-normalization", "which", @@ -5383,7 +5397,7 @@ dependencies = [ "sha2 0.11.0", "temp-env", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "tracing-test", @@ -5426,7 +5440,7 @@ dependencies = [ "simple_spawn_blocking", "tar", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tools", @@ -5530,7 +5544,7 @@ dependencies = [ "strum", "superslice", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tools", @@ -5554,7 +5568,7 @@ dependencies = [ "clap", "rattler_networking", "serde", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "url", ] @@ -5586,7 +5600,7 @@ dependencies = [ "shlex 2.0.1", "sysinfo", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", ] @@ -5614,7 +5628,7 @@ dependencies = [ "serde_json", "similar-asserts", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tools", "tracing", @@ -5653,7 +5667,7 @@ dependencies = [ "sigstore-sign", "sigstore-trust-root", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tokio-util", "tracing", @@ -5673,7 +5687,7 @@ dependencies = [ "regex", "serde", "temp-env", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "winver", ] @@ -5704,7 +5718,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", ] [[package]] @@ -5715,27 +5729,27 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -5752,9 +5766,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -5764,9 +5778,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -5947,9 +5961,9 @@ dependencies = [ [[package]] name = "resolvo" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc9b6c092a52fadb381b74d0f859b321ea7b9f4f8365acd7e4947f3a26517aa" +checksum = "e1560db4f7a060680c14a93a927e049677bcc9a5a01fc0a2a57170dbee81e33a" dependencies = [ "ahash", "bitvec", @@ -6063,7 +6077,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn", + "syn 2.0.119", "unicode-ident", ] @@ -6075,7 +6089,7 @@ checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14" dependencies = [ "quote", "rand 0.8.7", - "syn", + "syn 2.0.119", ] [[package]] @@ -6115,7 +6129,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -6128,7 +6142,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys 0.12.1", @@ -6164,9 +6178,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -6341,7 +6355,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -6360,9 +6374,9 @@ dependencies = [ [[package]] name = "self_cell" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b12e76d157a900eb52e81bc6e9f3069344290341720e9178cde2407113ac8d89" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" [[package]] name = "semver" @@ -6372,9 +6386,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -6414,22 +6428,22 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -6444,9 +6458,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "indexmap 2.14.0", "itoa", @@ -6478,13 +6492,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -6537,7 +6551,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -6668,7 +6682,7 @@ dependencies = [ "sigstore-rekor", "sigstore-tsa", "sigstore-types", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -6688,7 +6702,7 @@ dependencies = [ "signature", "sigstore-types", "spki", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "x509-cert", ] @@ -6706,7 +6720,7 @@ dependencies = [ "sigstore-crypto", "sigstore-oidc", "sigstore-types", - "thiserror 2.0.18", + "thiserror 2.0.19", "url", ] @@ -6720,7 +6734,7 @@ dependencies = [ "hex", "sigstore-crypto", "sigstore-types", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -6737,7 +6751,7 @@ dependencies = [ "serde_json", "sigstore-crypto", "sigstore-types", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "url", ] @@ -6756,7 +6770,7 @@ dependencies = [ "sigstore-crypto", "sigstore-merkle", "sigstore-types", - "thiserror 2.0.18", + "thiserror 2.0.19", "url", ] @@ -6776,7 +6790,7 @@ dependencies = [ "sigstore-trust-root", "sigstore-tsa", "sigstore-types", - "thiserror 2.0.18", + "thiserror 2.0.19", "x509-cert", ] @@ -6796,7 +6810,7 @@ dependencies = [ "sigstore-crypto", "sigstore-tuf", "sigstore-types", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "x509-cert", @@ -6822,7 +6836,7 @@ dependencies = [ "rustls-webpki", "sigstore-crypto", "sigstore-types", - "thiserror 2.0.18", + "thiserror 2.0.19", "tracing", "x509-cert", "x509-tsp", @@ -6844,7 +6858,7 @@ dependencies = [ "sigstore-crypto", "sigstore-types", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "url", @@ -6861,14 +6875,14 @@ dependencies = [ "pem", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "simd-json" @@ -7019,7 +7033,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7057,9 +7071,20 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -7083,7 +7108,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7092,7 +7117,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "byteorder", "enum-as-inner", "libc", @@ -7121,7 +7146,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.13.0", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -7164,9 +7189,9 @@ dependencies = [ [[package]] name = "target-triple" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "591ef38edfb78ca4771ee32cf494cb8771944bee237a9b91fc9c1424ac4b777b" +checksum = "c3a6bfce3d99adfa72d24750a61f782f3036a81e7f86d8841ee1326deaebd171" [[package]] name = "temp-env" @@ -7231,11 +7256,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.19", ] [[package]] @@ -7246,18 +7271,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -7271,9 +7296,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.53" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", "js-sys", @@ -7292,9 +7317,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -7362,14 +7387,14 @@ checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -7384,13 +7409,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7415,9 +7440,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -7427,23 +7452,24 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-io", "futures-sink", + "libc", "pin-project-lite", "tokio", ] [[package]] name = "toml" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ "indexmap 2.14.0", "serde_core", @@ -7465,9 +7491,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.12+spec-1.1.0" +version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap 2.14.0", "serde_core", @@ -7489,9 +7515,9 @@ dependencies = [ [[package]] name = "toml_writer" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "tools" @@ -7508,7 +7534,7 @@ dependencies = [ "rattler_digest", "reqwest", "tempfile", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "url", ] @@ -7535,7 +7561,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "async-compression", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-core", "futures-util", @@ -7558,7 +7584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ "async-compression", - "bitflags 2.13.0", + "bitflags 2.13.1", "bytes", "futures-core", "futures-util", @@ -7609,7 +7635,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7669,7 +7695,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" dependencies = [ "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -7832,9 +7858,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.5" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -7982,7 +8008,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.119", "wasm-bindgen-shared", ] @@ -8044,18 +8070,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" dependencies = [ "rustls-pki-types", ] [[package]] name = "which" -version = "8.0.4" +version = "8.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d7cd18d4acb58fb3cdfe9ea54e6cd96a4e7d4cc45c56338b236e82dad47248" +checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c" dependencies = [ "libc", ] @@ -8153,7 +8179,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8164,7 +8190,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8464,9 +8490,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.16" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d93c89cdc2d3a63c3ec48ffe926931bdc069eafa8e4402fe6d8f790c9d1e576" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yoke" @@ -8487,28 +8513,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.54" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8528,7 +8554,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -8549,7 +8575,7 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -8582,7 +8608,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index cd25592d05..ca434c3106 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -4067,7 +4067,7 @@ dependencies = [ [[package]] name = "rattler" -version = "0.47.1" +version = "0.48.0" dependencies = [ "anyhow", "astral-reqwest-middleware", @@ -4128,7 +4128,7 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.10.3" +version = "0.10.4" dependencies = [ "ahash", "anyhow", @@ -4149,6 +4149,7 @@ dependencies = [ "rattler_redaction", "rayon", "reqwest", + "rustix 1.1.4", "serde_json", "simple_spawn_blocking", "tempfile", @@ -4160,7 +4161,7 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.48.1" +version = "0.49.0" dependencies = [ "ahash", "core-foundation 0.10.1", @@ -4200,7 +4201,7 @@ dependencies = [ [[package]] name = "rattler_config" -version = "0.6.1" +version = "0.6.2" dependencies = [ "dirs", "fs-err", @@ -4235,7 +4236,7 @@ dependencies = [ [[package]] name = "rattler_index" -version = "0.30.9" +version = "0.30.10" dependencies = [ "ahash", "anyhow", @@ -4274,7 +4275,7 @@ dependencies = [ [[package]] name = "rattler_lock" -version = "0.31.5" +version = "0.31.6" dependencies = [ "ahash", "file_url", @@ -4309,7 +4310,7 @@ dependencies = [ [[package]] name = "rattler_menuinst" -version = "0.2.70" +version = "0.2.71" dependencies = [ "configparser", "dirs", @@ -4338,7 +4339,7 @@ dependencies = [ [[package]] name = "rattler_networking" -version = "0.30.2" +version = "0.30.3" dependencies = [ "ambient-id", "anyhow", @@ -4381,7 +4382,7 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.26.8" +version = "0.26.9" dependencies = [ "astral-reqwest-middleware", "astral-tokio-tar", @@ -4439,7 +4440,7 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" -version = "0.30.1" +version = "0.31.0" dependencies = [ "ahash", "anyhow", @@ -4501,7 +4502,7 @@ dependencies = [ [[package]] name = "rattler_s3" -version = "0.2.8" +version = "0.2.9" dependencies = [ "aws-config", "aws-credential-types", @@ -4517,7 +4518,7 @@ dependencies = [ [[package]] name = "rattler_shell" -version = "0.27.10" +version = "0.27.11" dependencies = [ "anyhow", "enum_dispatch", @@ -4536,7 +4537,7 @@ dependencies = [ [[package]] name = "rattler_solve" -version = "7.2.2" +version = "8.0.0" dependencies = [ "futures", "hex", @@ -4549,11 +4550,12 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "tracing", + "url", ] [[package]] name = "rattler_virtual_packages" -version = "3.0.4" +version = "4.0.0" dependencies = [ "archspec", "libloading", @@ -4829,9 +4831,9 @@ dependencies = [ [[package]] name = "resolvo" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc9b6c092a52fadb381b74d0f859b321ea7b9f4f8365acd7e4947f3a26517aa" +checksum = "e1560db4f7a060680c14a93a927e049677bcc9a5a01fc0a2a57170dbee81e33a" dependencies = [ "ahash", "bitvec", From ed92ba608f374477cb79d177138f26fc6508cd1c Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:56:42 +1000 Subject: [PATCH 63/98] fix(upload): request SAS read permission and fail fast on azure upload --- crates/rattler_upload/src/upload/azure.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 62f1108aa3..05a1305867 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -1,6 +1,6 @@ use std::path::{Path, PathBuf}; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use miette::IntoDiagnostic; use opendal::{Configurator, ErrorKind, Operator}; use rattler_azure::AzureCredentials; @@ -21,8 +21,10 @@ const PART_CONCURRENCY: usize = 4; const PACKAGE_CONCURRENCY: usize = 4; /// SAS permissions requested when minting a user-delegation SAS for uploads. -/// Uploading only needs to create and write blobs (`c` + `w`). -pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "cw"; +/// Creating and writing blobs needs `c` + `w`; `r` is required on top of those +/// because the overwrite guard `stat`s each blob before writing it, and a +/// `stat` (HEAD Blob) is a read. The SAS stays container-scoped and short-lived. +pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "rcw"; /// Uploads packages to a channel in an Azure Blob Storage container. /// @@ -46,18 +48,16 @@ pub async fn upload_package_to_azure( let op = Operator::new(builder).into_diagnostic()?.finish(); // Upload multiple packages concurrently. Each package is written to its own - // key, so the individual uploads are independent. + // key, so the individual uploads are independent. The first failure aborts + // the remaining uploads rather than letting them run to completion. futures::stream::iter(package_files.iter()) - .map(|package_file| { + .map(Ok) + .try_for_each_concurrent(PACKAGE_CONCURRENCY, |package_file| { let op = op.clone(); let channel = &channel; async move { upload_single_package(&op, channel, package_file, force).await } }) - .buffer_unordered(PACKAGE_CONCURRENCY) - .collect::>() - .await - .into_iter() - .collect::>>()?; + .await?; Ok(()) } From 8ba19c76717a4dabbacb858fd7787c57cba7791b Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:58:05 +1000 Subject: [PATCH 64/98] fix(azure): redact secrets in AzureAuthSource and AzureCredentialsOpts Debug --- crates/rattler_azure/src/clap.rs | 79 +++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index f9110a6707..eea35c5c76 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -45,7 +45,10 @@ pub enum AzureCredentialsError { /// reason about combinations. Only [`AzureAuthSource::AzureCli`] carries state /// (the minting TTL), which is why account/container derivation is needed for /// that arm alone. -#[derive(Clone, Debug, PartialEq, Eq)] +/// +/// Two of the three arms carry a secret, so `Debug` is implemented by hand to +/// redact them rather than derived (see [`AzureCredentials`]). +#[derive(Clone, PartialEq, Eq)] pub enum AzureAuthSource { /// Use a shared storage account key verbatim. AccountKey(String), @@ -61,6 +64,22 @@ pub enum AzureAuthSource { }, } +impl std::fmt::Debug for AzureAuthSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + // Print only the variant, never the secret it carries. + AzureAuthSource::AccountKey(_) => { + f.debug_tuple("AccountKey").field(&"").finish() + } + AzureAuthSource::SasToken(_) => f.debug_tuple("SasToken").field(&"").finish(), + // The TTL is not a secret. + AzureAuthSource::AzureCli { ttl } => { + f.debug_struct("AzureCli").field("ttl", ttl).finish() + } + } + } +} + impl AzureAuthSource { /// Resolve this source into concrete [`AzureCredentials`]. /// @@ -94,7 +113,12 @@ impl AzureAuthSource { /// exported *and* `--azure-cli` is passed), so [`AzureCredentialsOpts::source`] /// applies an explicit precedence rather than treating the combination as an /// error — see that method for the exact ordering. -#[derive(Clone, Debug, PartialEq, Parser)] +/// +/// `account_key` and `sas_token` are secrets, so `Debug` is implemented by hand +/// to redact them rather than derived (see [`AzureCredentials`]). This matters +/// more here than elsewhere: these options are typically part of a larger clap +/// struct that gets logged wholesale on a parse or startup failure. +#[derive(Clone, PartialEq, Parser)] pub struct AzureCredentialsOpts { /// The Azure Storage account key. /// @@ -141,6 +165,20 @@ pub struct AzureCredentialsOpts { pub azure_cli_sas_ttl_minutes: u64, } +impl std::fmt::Debug for AzureCredentialsOpts { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Whether a secret was supplied is useful for diagnosing precedence; the + // secret itself never is. + let redact = |value: &Option| value.as_ref().map(|_| ""); + f.debug_struct("AzureCredentialsOpts") + .field("account_key", &redact(&self.account_key)) + .field("sas_token", &redact(&self.sas_token)) + .field("azure_cli", &self.azure_cli) + .field("azure_cli_sas_ttl_minutes", &self.azure_cli_sas_ttl_minutes) + .finish() + } +} + impl AzureCredentialsOpts { /// Collapse the supplied options into a single, unambiguous /// [`AzureAuthSource`]. @@ -305,6 +343,43 @@ mod tests { assert_eq!(err.kind(), clap::error::ErrorKind::ArgumentConflict); } + /// Neither the resolved source nor the raw options may print a secret. Both + /// types hand-write `Debug` precisely so that a wholesale `{:?}` of an + /// enclosing CLI struct cannot leak a key or token into a log. + #[test] + fn debug_never_prints_secrets() { + let sources = [ + AzureAuthSource::AccountKey("supersecretkey".into()), + AzureAuthSource::SasToken("sig=deadbeef".into()), + ]; + for source in &sources { + let out = format!("{source:?}"); + assert!(out.contains(""), "not redacted: {out}"); + assert!(!out.contains("supersecret"), "leaked key: {out}"); + assert!(!out.contains("deadbeef"), "leaked token: {out}"); + } + + // The TTL arm carries no secret, so it stays fully printable. + let cli = AzureAuthSource::AzureCli { + ttl: Duration::from_secs(60), + }; + assert!(format!("{cli:?}").contains("60")); + + let out = format!("{:?}", opts(Some("supersecretkey"), None, false)); + assert!(out.contains(""), "not redacted: {out}"); + assert!(!out.contains("supersecret"), "leaked key: {out}"); + + let out = format!("{:?}", opts(None, Some("sig=deadbeef"), false)); + assert!(out.contains(""), "not redacted: {out}"); + assert!(!out.contains("deadbeef"), "leaked token: {out}"); + + // Absent secrets print as `None`, so the redaction cannot be mistaken for + // a supplied-but-hidden value. + let out = format!("{:?}", opts(None, None, true)); + assert!(out.contains("account_key: None"), "unexpected: {out}"); + assert!(out.contains("sas_token: None"), "unexpected: {out}"); + } + /// A zero TTL is rejected, and the maximum is capped so `minutes * 60` /// cannot overflow. #[test] From 95ffc0aa625bc1e12bfdf462c049a62473e6f0bd Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:02:25 +1000 Subject: [PATCH 65/98] feat(config): add azure-options table for per-host endpoint grants --- Cargo.lock | 3 + crates/rattler_azure/Cargo.toml | 5 + crates/rattler_azure/src/lib.rs | 6 + crates/rattler_azure/src/options.rs | 203 ++++++++++++++++++ crates/rattler_config/Cargo.toml | 5 + crates/rattler_config/src/config.rs | 14 ++ crates/rattler_config/src/config/azure.rs | 123 +++++++++++ .../test-data/compat/kitchen-sink.toml | 10 + ...at__merge__kitchen_sink_plus_override.snap | 10 + ...t__parse__deprecated-and-unknown.toml.snap | 3 + .../compat__parse__kitchen-sink.toml.snap | 14 ++ .../compat__parse__override-layer.toml.snap | 3 + ...ompat__parse__snake-case-aliases.toml.snap | 3 + 13 files changed, 402 insertions(+) create mode 100644 crates/rattler_azure/src/options.rs create mode 100644 crates/rattler_config/src/config/azure.rs diff --git a/Cargo.lock b/Cargo.lock index f59b23d111..ac812889b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5057,8 +5057,10 @@ dependencies = [ "jiff", "opendal", "percent-encoding", + "serde", "thiserror 2.0.19", "tokio", + "toml", "url", "which", ] @@ -5164,6 +5166,7 @@ dependencies = [ "fs-err", "indexmap 2.14.0", "insta", + "rattler_azure", "rattler_conda_types", "serde", "serde_ignored", diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index a4e5ce08a8..aeccccf3bd 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -17,6 +17,9 @@ default = [] clap = ["dep:clap", "dep:jiff", "dep:which", "dep:tokio"] # `azblob_config`: build an opendal `AzblobConfig` from a channel URL. opendal = ["dep:opendal", "dep:percent-encoding"] +# Per-host endpoint options (the `azure-options` config table). Pulls in `serde` +# only, so config crates can depend on this crate with default features off. +serde = ["dep:serde"] [dependencies] clap = { workspace = true, optional = true } @@ -25,6 +28,7 @@ opendal = { workspace = true, default-features = false, features = [ "services-azblob", ], optional = true } percent-encoding = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"], optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["process"], optional = true } url = { workspace = true } @@ -32,3 +36,4 @@ which = { workspace = true, optional = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } +toml = { workspace = true } diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index ded950fd24..807c66cdd5 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -13,6 +13,12 @@ #[cfg(feature = "clap")] pub mod clap; +#[cfg(feature = "serde")] +pub mod options; + +#[cfg(feature = "serde")] +pub use options::{Addressing, Auth, AzureEndpointOptions, Scheme}; + use url::Url; /// Credentials for authenticating to Azure Blob storage. diff --git a/crates/rattler_azure/src/options.rs b/crates/rattler_azure/src/options.rs new file mode 100644 index 0000000000..9b06ad2603 --- /dev/null +++ b/crates/rattler_azure/src/options.rs @@ -0,0 +1,203 @@ +//! Per-host endpoint options for Azure Blob channels. +//! +//! An entry in the `azure-options` config table is the *only* thing that grants +//! a host anything: without one, a channel on that host is fetched anonymously +//! over https in host-style addressing. There is deliberately no hardcoded list +//! of "official" Azure suffixes — since a grant must be written per host, +//! suffix classification carries no security weight, and the absence of the list +//! is what lets custom endpoints and the Azurite emulator work at all. +//! +//! # Why enums for what the config spells as bools +//! +//! The TOML surface stays `auth = true` / `path-style = true`, because that is +//! the ergonomic spelling and it mirrors `s3-options`. Internally each is an +//! enum, so no call site can mix up two unrelated booleans, and the meaning of a +//! value is legible without chasing the field name. The bridge is +//! `#[serde(from = "bool", into = "bool")]` plus a `From` impl each way. + +use serde::{Deserialize, Serialize}; + +/// Whether credentials may attach to requests for a host. +/// +/// Defaults to [`Auth::Anonymous`]: a host gets no credentials until a config +/// entry says otherwise. Serializes as the bool `auth` in `azure-options`. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(from = "bool", into = "bool")] +pub enum Auth { + /// Send requests unsigned. No credential is resolved, so no ambient + /// credential can be exfiltrated to this host, and nothing blocks on the + /// managed-identity / IMDS probe. + #[default] + Anonymous, + + /// Run the standard Azure credential chain for this host and sign with what + /// it returns. Because this is an explicit grant, a broken or unusable + /// credential is a hard error — never a silent downgrade to anonymous. + DefaultChain, +} + +impl From for Auth { + fn from(value: bool) -> Self { + if value { + Auth::DefaultChain + } else { + Auth::Anonymous + } + } +} + +impl From for bool { + fn from(value: Auth) -> Self { + matches!(value, Auth::DefaultChain) + } +} + +impl Auth { + /// Whether this grant permits a credential to be sent. + pub fn is_granted(self) -> bool { + matches!(self, Auth::DefaultChain) + } +} + +/// The wire scheme an `az://` channel URL is rewritten to when a request is sent. +/// +/// Defaults to [`Scheme::Https`]. `Http` exists for local emulators such as +/// Azurite; choosing it is an explicit, per-host decision in config, so a plain +/// `az://` URL can never be silently downgraded to cleartext. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Scheme { + /// Send requests over TLS. + #[default] + Https, + + /// Send requests in cleartext. For local emulators only. + Http, +} + +impl Scheme { + /// The scheme as it appears in a URL, without the `://`. + pub fn as_str(self) -> &'static str { + match self { + Scheme::Https => "https", + Scheme::Http => "http", + } + } +} + +impl std::fmt::Display for Scheme { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Where the storage account name is found in a blob URL. +/// +/// Defaults to [`Addressing::HostStyle`], which is how real Azure addresses +/// accounts. Serializes as the bool `path-style` in `azure-options`. Mirrors +/// `rattler_s3::S3AddressingStyle`. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(from = "bool", into = "bool")] +pub enum Addressing { + /// The account is the first label of the host, as in + /// `.blob.core.windows.net/`. Requires a dotted host, so + /// IP literals and single-label hosts cannot be addressed this way. + #[default] + HostStyle, + + /// The account is the first path segment, as in + /// `//`. This is what Azurite and other emulators + /// use, and it is the only form that works for an IP or single-label host. + PathStyle, +} + +impl From for Addressing { + fn from(value: bool) -> Self { + if value { + Addressing::PathStyle + } else { + Addressing::HostStyle + } + } +} + +impl From for bool { + fn from(value: Addressing) -> Self { + matches!(value, Addressing::PathStyle) + } +} + +/// How to reach, and whether to authenticate to, one Azure Blob host. +/// +/// The default value is the no-entry behaviour: anonymous, https, host-style. A +/// host with no config entry behaves exactly as if it had a defaulted entry, so +/// callers can look up an absent host and fall back to `default()` rather than +/// branching on presence. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct AzureEndpointOptions { + /// Whether credentials may be sent to this host. + pub auth: Auth, + + /// The scheme `az://` is rewritten to for this host. + pub scheme: Scheme, + + /// Where the account name is found in the URL for this host. + pub path_style: Addressing, +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The whole point of the bool bridge: the TOML stays boolean while the code + /// sees enums, and an absent field takes the safe default. + #[test] + fn toml_bools_map_to_enums() { + let opts: AzureEndpointOptions = toml::from_str( + r#" + auth = true + scheme = "http" + path-style = true + "#, + ) + .unwrap(); + assert_eq!( + opts, + AzureEndpointOptions { + auth: Auth::DefaultChain, + scheme: Scheme::Http, + path_style: Addressing::PathStyle, + } + ); + + // An empty entry is the same as no entry: anonymous, https, host-style. + let empty: AzureEndpointOptions = toml::from_str("").unwrap(); + assert_eq!(empty, AzureEndpointOptions::default()); + assert_eq!(empty.auth, Auth::Anonymous); + assert!(!empty.auth.is_granted()); + assert_eq!(empty.scheme, Scheme::Https); + assert_eq!(empty.path_style, Addressing::HostStyle); + + // `auth = false` is spelled out explicitly by some users; it must not be + // mistaken for a grant. + let denied: AzureEndpointOptions = toml::from_str("auth = false").unwrap(); + assert!(!denied.auth.is_granted()); + } + + /// Round-tripping must preserve the boolean spelling, not leak the enum + /// variant names into a written config file. + #[test] + fn enums_serialize_back_to_bools() { + let toml = toml::to_string(&AzureEndpointOptions { + auth: Auth::DefaultChain, + scheme: Scheme::Http, + path_style: Addressing::PathStyle, + }) + .unwrap(); + assert!(toml.contains("auth = true"), "{toml}"); + assert!(toml.contains("path-style = true"), "{toml}"); + assert!(toml.contains(r#"scheme = "http""#), "{toml}"); + assert!(!toml.contains("DefaultChain"), "{toml}"); + } +} diff --git a/crates/rattler_config/Cargo.toml b/crates/rattler_config/Cargo.toml index cc115124df..ca7b49ec2d 100644 --- a/crates/rattler_config/Cargo.toml +++ b/crates/rattler_config/Cargo.toml @@ -23,6 +23,11 @@ thiserror = { workspace = true } dirs = { workspace = true } fs-err = { workspace = true } indexmap = { workspace = true } +# Default features off: this pulls in only `serde`, so the config crate stays +# free of the networking/opendal/CLI weight behind the azure feature flags. +rattler_azure = { workspace = true, default-features = false, features = [ + "serde", +] } rattler_conda_types = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_ignored = { workspace = true } diff --git a/crates/rattler_config/src/config.rs b/crates/rattler_config/src/config.rs index d514a3ae6a..6c0421a226 100644 --- a/crates/rattler_config/src/config.rs +++ b/crates/rattler_config/src/config.rs @@ -20,12 +20,14 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use thiserror::Error; use url::Url; +use crate::config::azure::AzureOptionsMap; use crate::config::s3::S3OptionsMap; use crate::config::{ build::BuildConfig, concurrency::ConcurrencyConfig, index::IndexConfig, proxy::ProxyConfig, repodata_config::RepodataConfig, run_post_link_scripts::RunPostLinkScripts, }; +pub mod azure; pub mod build; pub mod channel_config; pub mod concurrency; @@ -194,6 +196,14 @@ pub struct CommonConfig { #[serde(skip_serializing_if = "S3OptionsMap::is_default")] pub s3_options: S3OptionsMap, + /// Configuration for Azure Blob. + /// + /// User-scoped by contract: an entry grants a host credentials, so this must + /// never be read from a project- or workspace-level manifest. + #[serde(default)] + #[serde(skip_serializing_if = "AzureOptionsMap::is_default")] + pub azure_options: AzureOptionsMap, + /// Per-channel configuration for `rattler-index`. #[serde(default, skip_serializing_if = "IndexConfig::is_empty")] pub index_config: IndexConfig, @@ -251,6 +261,7 @@ impl Default for CommonConfig { concurrency: ConcurrencyConfig::default(), proxy_config: ProxyConfig::default(), s3_options: S3OptionsMap::default(), + azure_options: AzureOptionsMap::default(), index_config: IndexConfig::default(), run_post_link_scripts: None, allow_symbolic_links: None, @@ -305,6 +316,7 @@ impl Config for CommonConfig { concurrency: self.concurrency.merge_config(&other.concurrency)?, proxy_config: self.proxy_config.merge_config(&other.proxy_config)?, s3_options: self.s3_options.merge_config(&other.s3_options)?, + azure_options: self.azure_options.merge_config(&other.azure_options)?, index_config: self.index_config.merge_config(&other.index_config)?, run_post_link_scripts: other .run_post_link_scripts @@ -322,6 +334,7 @@ impl Config for CommonConfig { self.concurrency.validate()?; self.proxy_config.validate()?; self.s3_options.validate()?; + self.azure_options.validate()?; self.index_config.validate()?; Ok(()) } @@ -348,6 +361,7 @@ impl Config for CommonConfig { keys.extend(prefixed_keys("concurrency", self.concurrency.keys())); keys.extend(prefixed_keys("proxy-config", self.proxy_config.keys())); keys.extend(prefixed_keys("s3-options", self.s3_options.keys())); + keys.extend(prefixed_keys("azure-options", self.azure_options.keys())); keys } } diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs new file mode 100644 index 0000000000..ba1663a499 --- /dev/null +++ b/crates/rattler_config/src/config/azure.rs @@ -0,0 +1,123 @@ +use indexmap::IndexMap; +use rattler_azure::AzureEndpointOptions; +use serde::{Deserialize, Serialize}; + +use crate::config::Config; + +/// Per-host options for Azure Blob channels, keyed by host (including a port +/// where one is used, e.g. `127.0.0.1:10000`). +/// +/// An entry is a *grant*: it is the only way a host gets credentials, a +/// non-default scheme, or path-style addressing. A host with no entry is fetched +/// anonymously over https in host-style addressing, so an empty map is the safe +/// default and [`AzureOptionsMap::get`] can answer for absent hosts too. +/// +/// # Scope +/// +/// Entries are **user-scoped by contract**. A tool must never read this table +/// from a project- or workspace-level manifest: doing so would let a checked-out +/// repository name a host and have the user's ambient Azure credentials sent to +/// it. Keep it to user- and system-level config files. +#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct AzureOptionsMap(pub IndexMap); + +impl AzureOptionsMap { + /// Returns `true` if no Azure hosts are configured. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// The options for `host`, or the defaults (anonymous, https, host-style) + /// when it has no entry. + /// + /// Callers should prefer this over indexing the map: "no entry" and "a + /// defaulted entry" are defined to behave identically, so branching on + /// presence only invites the two paths to drift apart. + pub fn get(&self, host: &str) -> AzureEndpointOptions { + self.0.get(host).copied().unwrap_or_default() + } +} + +impl Config for AzureOptionsMap { + fn is_default(&self) -> bool { + self.0.is_empty() + } + + fn merge_config(self, other: &Self) -> Result { + // Merge the two maps, with `other`'s entries overwriting existing keys. + // A host is granted or not as a whole; entries are not merged field-wise, + // so a higher-precedence file cannot partially relax a grant. + let mut merged = self.0.clone(); + for (key, value) in &other.0 { + merged.insert(key.clone(), *value); + } + Ok(AzureOptionsMap(merged)) + } + + fn validate(&self) -> Result<(), super::ValidationError> { + Ok(()) + } + + fn keys(&self) -> Vec { + self.0.keys().map(ToString::to_string).collect() + } +} + +#[cfg(test)] +mod tests { + use rattler_azure::{Addressing, Auth, Scheme}; + + use super::*; + + /// The table parses in the shape documented for users, and an absent host + /// answers with the anonymous defaults rather than requiring a presence check. + #[test] + fn table_parses_and_absent_hosts_default() { + let map: AzureOptionsMap = toml::from_str( + r#" + ["mycompany.blob.core.windows.net"] + auth = true + + ["127.0.0.1:10000"] + auth = true + scheme = "http" + path-style = true + "#, + ) + .unwrap(); + + let real = map.get("mycompany.blob.core.windows.net"); + assert_eq!(real.auth, Auth::DefaultChain); + assert_eq!(real.scheme, Scheme::Https); + assert_eq!(real.path_style, Addressing::HostStyle); + + let azurite = map.get("127.0.0.1:10000"); + assert_eq!(azurite.auth, Auth::DefaultChain); + assert_eq!(azurite.scheme, Scheme::Http); + assert_eq!(azurite.path_style, Addressing::PathStyle); + + // An unlisted host gets no grant. + let unlisted = map.get("someoneelse.blob.core.windows.net"); + assert!(!unlisted.auth.is_granted()); + assert_eq!(unlisted, AzureEndpointOptions::default()); + } + + /// A later config file overwrites a host wholesale. It must not be able to + /// keep an earlier `auth = true` while changing only the scheme. + #[test] + fn merge_replaces_entries_wholesale() { + let base: AzureOptionsMap = + toml::from_str("[\"host.example\"]\nauth = true\npath-style = true\n").unwrap(); + let over: AzureOptionsMap = + toml::from_str("[\"host.example\"]\nscheme = \"http\"\n").unwrap(); + + let merged = base.merge_config(&over).unwrap(); + let entry = merged.get("host.example"); + assert_eq!(entry.scheme, Scheme::Http); + assert!( + !entry.auth.is_granted(), + "overwriting an entry must not inherit the previous grant" + ); + assert_eq!(entry.path_style, Addressing::HostStyle); + } +} diff --git a/crates/rattler_config/test-data/compat/kitchen-sink.toml b/crates/rattler_config/test-data/compat/kitchen-sink.toml index 28ab3955b6..31c4416480 100644 --- a/crates/rattler_config/test-data/compat/kitchen-sink.toml +++ b/crates/rattler_config/test-data/compat/kitchen-sink.toml @@ -39,6 +39,16 @@ endpoint-url = "https://s3.example.com" region = "eu-central-1" force-path-style = true +# A private channel on real Azure: the grant is the whole entry. +[azure-options."mycompany.blob.core.windows.net"] +auth = true + +# A local Azurite emulator: the entry is also the endpoint trust declaration. +[azure-options."127.0.0.1:10000"] +auth = true +scheme = "http" +path-style = true + [index-config] write-zst = true write-shards = false diff --git a/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap b/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap index 49b41a3210..5738e529f3 100644 --- a/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap +++ b/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap @@ -52,6 +52,16 @@ endpoint-url = "https://minio.example.com/" region = "auto" force-path-style = false +[azure-options."127.0.0.1:10000"] +auth = true +scheme = "http" +path-style = true + +[azure-options."mycompany.blob.core.windows.net"] +auth = true +scheme = "https" +path-style = false + [index-config] write-zst = true write-shards = false diff --git a/crates/rattler_config/tests/snapshots/compat__parse__deprecated-and-unknown.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__deprecated-and-unknown.toml.snap index e0a41e88ba..1f22ee1fdd 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__deprecated-and-unknown.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__deprecated-and-unknown.toml.snap @@ -88,6 +88,9 @@ expression: "(unused, normalized(config))" s3_options: S3OptionsMap( {}, ), + azure_options: AzureOptionsMap( + {}, + ), index_config: IndexConfig { default: IndexChannelConfig { write_zst: None, diff --git a/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap index c16deeeba4..d0bacbf0ca 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap @@ -206,6 +206,20 @@ expression: "(unused, normalized(config))" }, }, ), + azure_options: AzureOptionsMap( + { + "127.0.0.1:10000": AzureEndpointOptions { + auth: DefaultChain, + scheme: Http, + path_style: PathStyle, + }, + "mycompany.blob.core.windows.net": AzureEndpointOptions { + auth: DefaultChain, + scheme: Https, + path_style: HostStyle, + }, + }, + ), index_config: IndexConfig { default: IndexChannelConfig { write_zst: Some( diff --git a/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap index f0d2caef56..1c889eb8f0 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap @@ -136,6 +136,9 @@ expression: "(unused, normalized(config))" }, }, ), + azure_options: AzureOptionsMap( + {}, + ), index_config: IndexConfig { default: IndexChannelConfig { write_zst: None, diff --git a/crates/rattler_config/tests/snapshots/compat__parse__snake-case-aliases.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__snake-case-aliases.toml.snap index 1d94eb42e0..f5834d1339 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__snake-case-aliases.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__snake-case-aliases.toml.snap @@ -68,6 +68,9 @@ expression: "(unused, normalized(config))" s3_options: S3OptionsMap( {}, ), + azure_options: AzureOptionsMap( + {}, + ), index_config: IndexConfig { default: IndexChannelConfig { write_zst: None, From b411d71f4fd60f2296f8e276e23e325928f0155a Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:54:17 +1000 Subject: [PATCH 66/98] feat(azure): add AzureChannelUrl and AzureHost with normalized option keys --- crates/rattler_azure/Cargo.toml | 6 +- crates/rattler_azure/src/lib.rs | 1060 +++++++++++++++-- crates/rattler_azure/src/options.rs | 102 +- crates/rattler_config/src/config/azure.rs | 105 +- .../compat__parse__kitchen-sink.toml.snap | 8 +- crates/rattler_index/src/main.rs | 22 +- crates/rattler_upload/src/lib.rs | 5 +- crates/rattler_upload/src/upload/opt.rs | 16 +- py-rattler/Cargo.lock | 2 + 9 files changed, 1165 insertions(+), 161 deletions(-) diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index aeccccf3bd..690a1f68b1 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -17,8 +17,10 @@ default = [] clap = ["dep:clap", "dep:jiff", "dep:which", "dep:tokio"] # `azblob_config`: build an opendal `AzblobConfig` from a channel URL. opendal = ["dep:opendal", "dep:percent-encoding"] -# Per-host endpoint options (the `azure-options` config table). Pulls in `serde` -# only, so config crates can depend on this crate with default features off. +# Serde derives for the `azure-options` config table: the per-host endpoint +# options and the `AzureHost` used as their map key. The types themselves are +# always compiled; only the derives are gated. Pulls in `serde` alone, so config +# crates can depend on this crate with default features off. serde = ["dep:serde"] [dependencies] diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 807c66cdd5..5ca4ce3d9c 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -1,23 +1,26 @@ //! Helpers for deriving Azure Blob coordinates from channel URLs and for minting //! short-lived credentials for them. //! -//! # Trusted-host model +//! # Host model //! -//! [`account_and_container`] trusts the URL host verbatim: whatever host is -//! named is taken to be the storage endpoint, and any ambient AAD credentials -//! (an `az login` session) are sent to that host. Userinfo (`user:pass@host`) is -//! rejected because it is a host-spoofing vector, but an honest, arbitrary host -//! is the caller's responsibility — this crate does not police which hosts are -//! legitimate Azure endpoints. +//! This crate does not police which hosts are legitimate Azure endpoints: the +//! host a channel URL names is taken to be the storage endpoint it says it is. +//! What a host is *granted* — credentials, wire scheme, addressing style — is +//! declared per host in [`options`] and never inferred from the host name, and +//! the default grant is [`Auth::Anonymous`], so naming a host in a URL by itself +//! sends nothing to it. Nothing here transmits a credential either: signing lives +//! in `rattler_networking`, and [`account_and_container`] only reads a URL. +//! +//! Userinfo (`user:pass@host`) is rejected wherever a host is parsed, because +//! `az://real.host@evil.example/…` reads as the real host while addressing the +//! attacker's. #[cfg(feature = "clap")] pub mod clap; -#[cfg(feature = "serde")] pub mod options; -#[cfg(feature = "serde")] -pub use options::{Addressing, Auth, AzureEndpointOptions, Scheme}; +pub use options::{Addressing, Auth, AzureEndpointOptions, AzureScheme}; use url::Url; @@ -25,9 +28,9 @@ use url::Url; /// /// Exactly one authentication method is carried, so the ambiguous "both a key /// and a SAS token" and "neither" states are unrepresentable. The storage -/// account name, endpoint, and container are not stored here: they are fully -/// determined by the channel URL (`https://.blob.core.windows.net//...`) -/// and derived by the consumer. +/// account name, endpoint, and container are not stored here: they are derived +/// by the consumer from the channel URL together with the host's addressing +/// style (see [`account_and_container`]). /// /// The type deliberately has no `Serialize`/`Deserialize`: it holds raw account /// keys and SAS tokens, so serialization would risk leaking secrets to disk. For @@ -78,14 +81,36 @@ pub enum AzureUrlError { )] UserInfoNotAllowed, - /// The host is not a dotted domain, so no storage account can be derived. + /// The text handed to [`AzureHost::parse`] is not a usable `host[:port]`. + /// + /// This is what a malformed `azure-options` key produces, so it quotes the + /// text back and says what was expected instead. + #[error("`{authority}` is not a valid Azure host: {reason}; expected `host` or `host:port`")] + InvalidHostAuthority { + /// The offending authority text. + authority: String, + /// Why it was rejected. + reason: String, + }, + + /// Host-style addressing was requested but the host has no account label: it + /// is an IP literal, or a domain with only one label. + /// + /// This is the error an Azurite or custom-endpoint user hits first, and the + /// fix is a config line rather than a URL change, so the message names that + /// line verbatim instead of leaving the user to discover `path-style`. The + /// host is spelled the way [`AzureHost`] spells it, which is the way the + /// config table is keyed — a key copied out of this message matches. #[error( - "Azure blob URL host `{0}` is not a dotted domain of the form `.blob.`; \ - IP literals and single-label hosts have no derivable storage account" + "Azure blob URL host `{0}` is not a dotted domain of the form `.blob.`, \ + so its first label cannot be a storage account. Such a host needs path-style addressing, \ + where the storage account is the first path segment instead; that is not selectable from \ + configuration yet, and will be enabled by `[azure-options.\"{0}\"]` with \ + `path-style = true`" )] InvalidHost(String), - /// The account name (first host label) is empty. + /// The URL has no path segment to read the account from (path-style only). #[error("could not derive account name from Azure blob URL")] NoAccount, @@ -93,12 +118,20 @@ pub enum AzureUrlError { #[error("no container in Azure blob URL")] NoContainer, - /// The derived account or container contains characters outside `[a-z0-9-]`. + /// The derived account name is not a legal Azure storage account name. + #[error( + "`{0}` is not a valid Azure storage account name: account names are 3-24 characters of \ + lowercase letters and digits only" + )] + InvalidAccountName(String), + + /// The derived container name is not a legal Azure blob container name. #[error( - "Azure blob URL component `{0}` contains characters outside [a-z0-9-]; account and \ - container names are restricted to that set" + "`{0}` is not a valid Azure blob container name: container names are 3-63 characters of \ + lowercase letters, digits and hyphens, must start and end with a letter or digit, and \ + must not contain consecutive hyphens" )] - InvalidCharacters(String), + InvalidContainerName(String), /// The channel URL string could not be parsed. #[error("`{value}` is not a valid URL")] @@ -121,53 +154,77 @@ pub enum AzureUrlError { /// The storage account and container an Azure Blob channel URL resolves to. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AzureCoordinates { - /// The storage account name (first label of the host). + /// The storage account name — the first host label under + /// [`Addressing::HostStyle`], the first path segment under + /// [`Addressing::PathStyle`]. pub account: String, - /// The blob container name (first path segment). + /// The blob container name — the first path segment under + /// [`Addressing::HostStyle`], the second under [`Addressing::PathStyle`]. pub container: String, } -/// Derive the storage account name and container from an Azure Blob channel URL -/// of the form `https://.blob.core.windows.net//`. +/// Derive the storage account name and container from an Azure Blob channel URL. /// -/// The account name is the first label of the host, so the host must be a dotted -/// domain; IP-literal and single-label hosts (e.g. `localhost` or the Azurite -/// emulator) are rejected because no account can be derived from them. +/// Where the account name lives is decided by `addressing`, which comes from the +/// host's `azure-options` entry — it is not guessable from the URL, because +/// `https://host/a/b` is a valid reading under both styles: +/// +/// - [`Addressing::HostStyle`] (real Azure, the default): account = first label +/// of the host, container = first path segment. The host must be a domain with +/// at least two labels, so IP literals and single-label hosts fail with +/// [`AzureUrlError::InvalidHost`], whose message names the config line that +/// switches to path-style. +/// - [`Addressing::PathStyle`] (Azurite and other emulators): account = first +/// path segment, container = second. Any host shape is accepted, since the host +/// carries no account information at all. /// /// The host is otherwise trusted verbatim (see the [crate-level docs] for the -/// trusted-host model): userinfo (`user:pass@host`) is rejected as a -/// host-spoofing vector, but an honest, arbitrary host is the caller's -/// responsibility. The derived account and container are additionally restricted -/// to `[a-z0-9-]` so that argument-injection-shaped values can never reach the -/// `az` subprocess. +/// host model): userinfo (`user:pass@host`) is rejected as a host-spoofing +/// vector, but an honest, arbitrary host is the caller's responsibility. The +/// derived account and container are additionally held to Azure's own naming +/// rules — under *both* addressing styles, since path-style takes the account +/// from user-controlled path text. Those rules reject an empty name, any +/// character outside `[a-z0-9-]`, and a leading `-`, which together are what keep +/// an option-shaped value such as `--as-user` from ever reaching the `az` argv in +/// [`mint_user_delegation_sas`]. /// /// [crate-level docs]: crate -pub fn account_and_container(url: &Url) -> Result { +pub fn account_and_container( + url: &Url, + addressing: Addressing, +) -> Result { if !url.username().is_empty() || url.password().is_some() { return Err(AzureUrlError::UserInfoNotAllowed); } - let host = url.host_str().ok_or(AzureUrlError::NoHost)?; - if !matches!(url.host(), Some(url::Host::Domain(domain)) if domain.contains('.')) { - return Err(AzureUrlError::InvalidHost(host.to_string())); - } - - let account = host - .split('.') - .next() - .filter(|name| !name.is_empty()) - .ok_or(AzureUrlError::NoAccount)?; - let container = url + // Re-normalize through `AzureHost` rather than reading `host_str()` directly, + // so the account label is taken from the same spelling the config table is + // keyed by — trailing dot stripped, empty labels rejected, IP literals + // discriminated by type rather than by counting dots. + let host = AzureHost::from_url(url)?; + // Empty segments are never a valid name, so an empty one is a missing one. + let mut segments = url .path_segments() - .and_then(|mut segments| segments.next()) - .filter(|segment| !segment.is_empty()) - .ok_or(AzureUrlError::NoContainer)?; + .into_iter() + .flatten() + .map(|segment| (!segment.is_empty()).then_some(segment)); + let mut next_segment = || segments.next().flatten(); - for component in [account, container] { - if !is_valid_component(component) { - return Err(AzureUrlError::InvalidCharacters(component.to_string())); + let (account, container) = match addressing { + Addressing::HostStyle => { + let account = host + .account_label() + .ok_or_else(|| AzureUrlError::InvalidHost(host.to_string()))?; + (account, next_segment().ok_or(AzureUrlError::NoContainer)?) } - } + Addressing::PathStyle => ( + next_segment().ok_or(AzureUrlError::NoAccount)?, + next_segment().ok_or(AzureUrlError::NoContainer)?, + ), + }; + + validate_account(account)?; + validate_container(container)?; Ok(AzureCoordinates { account: account.to_string(), @@ -175,50 +232,466 @@ pub fn account_and_container(url: &Url) -> Result bool { - name.chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') +/// Check a derived name against Azure's storage account naming rules. +/// +/// Azure's rules are stricter than "whatever a URL host label may contain": 3-24 +/// characters of lowercase letters and digits, no hyphens at all. Enforcing the +/// real rules rather than a permissive charset is what makes the guarantee in +/// [`account_and_container`] true — a name that cannot contain `-` can never be +/// read as an option by a subprocess — and it rejects names Azure would reject +/// anyway, while the error can still name the URL that produced them. The length +/// bound is also what rejects an empty name, so emptiness is not left to a +/// caller's `filter`. +fn validate_account(name: &str) -> Result<(), AzureUrlError> { + let valid = (3..=24).contains(&name.len()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()); + valid + .then_some(()) + .ok_or_else(|| AzureUrlError::InvalidAccountName(name.to_string())) +} + +/// Check a derived name against Azure's blob container naming rules. +/// +/// 3-63 characters of lowercase letters, digits and hyphens, with no leading or +/// trailing hyphen and no consecutive hyphens. The leading-hyphen rule is the one +/// carrying security weight: `--https-only` is a perfectly good `[a-z0-9-]` +/// string, so a charset check alone would hand it to the `az` argv as an option. +fn validate_container(name: &str) -> Result<(), AzureUrlError> { + let valid = (3..=63).contains(&name.len()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + && !name.starts_with('-') + && !name.ends_with('-') + && !name.contains("--"); + valid + .then_some(()) + .ok_or_else(|| AzureUrlError::InvalidContainerName(name.to_string())) +} + +/// A normalized Azure Blob endpoint authority: a host, and its port when one is +/// written. +/// +/// This is the identity of an endpoint everywhere on the Azure path — the +/// `azure-options` table key, the authority of an [`AzureChannelUrl`], and the +/// text the guided errors tell a user to write. Because the *same* type +/// deserializes a config key and is handed out by [`AzureChannelUrl::host`], a +/// written key and a lookup cannot disagree about spelling: both are the output of +/// [`AzureHost::parse`]. +/// +/// # Why not a `Url` +/// +/// A `Url`'s port is scheme-relative: `https://host:443` serializes back as +/// `https://host`, because 443 is the https default. Keeping the authority in a +/// `Url` with a fixed scheme therefore *loses* a user-written `:443`, and makes +/// equality scheme-relative along with it (`host:443` == `host`). Holding the host +/// and the port as separate values keeps a written port a written port, whatever +/// scheme the request eventually uses. +/// +/// # Normalization +/// +/// [`parse`](Self::parse) is the only way in. It applies the URL Standard's host +/// parser (via [`url::Host`]) — lowercasing, IDNA (`ünï.example` → +/// `xn--n-nga1b.example`), and canonical, *typed* IP literals (`0x7f.1` → +/// [`url::Host::Ipv4`], `[0:0:0:0:0:0:0:1]` → [`url::Host::Ipv6`]) — plus two +/// rules of its own: +/// +/// - a single trailing dot, the DNS root label, is stripped, so +/// `acct.blob.example.` and `acct.blob.example` are one host; +/// - no label may be empty, so `acct..blob.example` is rejected rather than +/// silently handing an empty label to account derivation. +/// +/// The port is deliberately *not* normalized: which scheme this host is reached +/// over is decided later, by its options entry, so no port here can be known to +/// be a default. A written port is kept verbatim and an absent one stays absent. +/// +/// [`Display`](std::fmt::Display) writes the canonical `host[:port]` text with an +/// IPv6 literal bracketed, and round-trips through [`parse`](Self::parse). +#[derive(Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(try_from = "String", into = "String") +)] +pub struct AzureHost { + host: url::Host, + port: Option, +} + +impl AzureHost { + /// Parse and normalize a bare `host[:port]` authority. + pub fn parse(authority: &str) -> Result { + if authority.contains('@') { + return Err(AzureUrlError::UserInfoNotAllowed); + } + if authority.contains(['/', '\\', '?', '#']) { + return Err(AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: "it carries a path, query or fragment".to_string(), + }); + } + + // Two parses, each for the one thing it is authoritative about, because no + // single scheme gives both. `https` is a special scheme, so it runs the URL + // Standard's host parser: lowercasing, IDNA, and IP literals as typed + // `Ipv4`/`Ipv6` hosts — but it also drops `:443`. `az` is not special, so + // it has no default port to drop, but its opaque-host parsing leaves the + // host unnormalized (`MyCompany.X` stays mixed case, `127.0.0.1` arrives as + // a `Domain`). Host from the first, port from the second. + let normalized = Self::parse_as(authority, "https")?; + let verbatim = Self::parse_as(authority, "az")?; + + let host = normalized.host().ok_or(AzureUrlError::NoHost)?.to_owned(); + Self::normalized(host, verbatim.port(), authority) + } + + /// Parse `://`, reporting a failure against the authority + /// text the caller actually wrote. + fn parse_as(authority: &str, scheme: &str) -> Result { + Url::parse(&format!("{scheme}://{authority}")).map_err(|err| { + AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: err.to_string(), + } + }) + } + + /// Apply the two rules the URL host parser does not: strip the DNS root + /// label, reject empty labels. + /// + /// Private, so every route in goes through [`parse`](Self::parse) and neither + /// rule can be skipped. + fn normalized( + host: url::Host, + port: Option, + authority: &str, + ) -> Result { + let url::Host::Domain(domain) = &host else { + // An IP literal is already fully canonical, and has no labels. + return Ok(Self { host, port }); + }; + + // Re-run the host parser on the trimmed name so there is exactly one + // normalization path rather than a second, hand-rolled one. + let host = url::Host::parse(domain.strip_suffix('.').unwrap_or(domain)).map_err(|err| { + AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: err.to_string(), + } + })?; + if let url::Host::Domain(domain) = &host { + // Only one trailing dot is stripped, so `acct.example..` still has an + // empty label here — as does `acct..example`. Rejecting both is what + // lets `Display` round-trip, and what stops account derivation from + // handing out an empty account name. + if domain.split('.').any(str::is_empty) { + return Err(AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: "one of its labels is empty".to_string(), + }); + } + } + Ok(Self { host, port }) + } + + /// The authority of an already-parsed URL, re-normalized. + /// + /// Note the asymmetry with [`AzureChannelUrl`], which stores an `AzureHost` + /// rather than a `Url` for exactly this reason: a `Url` has *already* dropped + /// a port equal to its scheme's default, so `https://host:443/…` arrives here + /// as `host` and cannot be told apart from `https://host/…`. Prefer + /// [`AzureChannelUrl::host`] wherever the channel URL itself is in reach. + pub fn from_url(url: &Url) -> Result { + let host = url.host_str().ok_or(AzureUrlError::NoHost)?; + match url.port() { + Some(port) => Self::parse(&format!("{host}:{port}")), + None => Self::parse(host), + } + } + + /// The parsed host, without the port. + pub fn host(&self) -> &url::Host { + &self.host + } + + /// The port, when the authority names one. + pub fn port(&self) -> Option { + self.port + } + + /// The storage account label under host-style addressing. + /// + /// `None` whenever the host cannot carry an account name, which the stored + /// [`url::Host`] answers by construction rather than by inspecting text: + /// + /// - an IP literal is an [`url::Host::Ipv4`] or [`url::Host::Ipv6`], so it can + /// never be read as a label — including `127.0.0.1`, which a "does it + /// contain a dot" test would happily split into an account named `127`; + /// - a domain must have at least two labels, so `localhost` is rejected. + /// + /// [`parse`](Self::parse) has already guaranteed no label is empty and there + /// is no trailing dot, so "at least two labels" here means "at least two + /// non-empty labels". + fn account_label(&self) -> Option<&str> { + match &self.host { + url::Host::Domain(domain) => { + let mut labels = domain.split('.'); + let first = labels.next()?; + labels.next().is_some().then_some(first) + } + url::Host::Ipv4(_) | url::Host::Ipv6(_) => None, + } + } +} + +impl std::fmt::Display for AzureHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // `url::Host`'s own `Display` brackets an IPv6 literal, which is what an + // authority needs. + write!(f, "{}", self.host)?; + if let Some(port) = self.port { + write!(f, ":{port}")?; + } + Ok(()) + } +} + +impl std::fmt::Debug for AzureHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // The canonical text *is* the identity; the host/port split is an + // implementation detail, and printing it would only make a config dump + // harder to read. + write!(f, "AzureHost({:?})", self.to_string()) + } +} + +impl std::str::FromStr for AzureHost { + type Err = AzureUrlError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } +} + +/// The serde bridge for using an `AzureHost` as a map key: serde hands map keys +/// over as owned strings, so `serde(try_from = "String")` is what routes a written +/// `azure-options` key through [`AzureHost::parse`] instead of storing it raw. +impl TryFrom for AzureHost { + type Error = AzureUrlError; + + fn try_from(value: String) -> Result { + Self::parse(&value) + } +} + +impl From for String { + fn from(host: AzureHost) -> Self { + host.to_string() + } } -/// Parse and validate an Azure Blob **channel** URL. +/// A validated Azure Blob **channel** URL, which has two spellings: `az://…` as +/// the user writes it and in configuration, and `http(s)://…` on the wire. +/// +/// # Why the parts are stored, and not a URL +/// +/// The obvious shape is a struct holding both spellings, which can hold a pair +/// that disagrees — a canonical URL for one host and a wire URL for another — and +/// nothing but discipline stops it. The next-obvious shape is one `Url` in the +/// wire form with a fixed scheme, deriving the other spelling from it. That is +/// worse than it looks: a `Url`'s port is scheme-relative, so storing +/// `az://host:443/…` as `https` drops the port on the way in, and +/// [`wire`](Self::wire) then hands out `http://host/…` — port 80, a different +/// endpoint. +/// +/// So the authority is stored as an [`AzureHost`], which holds host and port +/// explicitly and normalizes both without reference to any scheme, next to the +/// already-normalized path, query and fragment. Every spelling is built from those +/// same parts by one private helper, so no two spellings can disagree about host, +/// port, path or query. /// -/// The only accepted form is the `az://` channel scheme — -/// `az://.blob.//` — which is rewritten to -/// `https://` here so every consumer downstream works with a real wire URL and -/// never sees the `az` scheme. A bare `http(s)://` URL is deliberately *not* -/// accepted: `az://` is the single canonical spelling for an Azure channel -/// (matching how it is written in configuration and used on the fetch path), and -/// accepting the wire URL as a second spelling would only invite confusion. The -/// host and container are validated via [`account_and_container`]. -pub fn parse_channel_url(value: &str) -> Result { - // Require the `az://` scheme, then rewrite to `https://` before parsing so - // the host is parsed by the URL crate's special-scheme host parser and the - // `az` scheme never leaks downstream. - let rest = value - .strip_prefix("az://") - .ok_or_else(|| AzureUrlError::InvalidScheme(value.to_string()))?; - let url = - Url::parse(&format!("https://{rest}")).map_err(|source| AzureUrlError::InvalidUrl { - value: value.to_string(), - source, +/// # Why the scheme is a `wire()` argument and not a field +/// +/// Which scheme a host is reached over comes from its `azure-options` entry, and +/// [`parse`](Self::parse) runs as a clap `value_parser` — before any config file +/// is read. A stored scheme would therefore have to be a guess made at parse time +/// and corrected later, which is exactly the drift this type exists to prevent. +/// Passing it in at call time keeps the choice at the site that makes it. Nothing +/// ties the argument to an options entry, and until options are threaded through +/// the CLI both callers pass [`AzureScheme::Https`]. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct AzureChannelUrl { + /// The authority, normalized independently of any scheme. + host: AzureHost, + + /// The path as the URL Standard normalizes it: always a leading `/`, still + /// percent-encoded. + path: String, + + /// The query, when there is one — a SAS token may be written inline. + query: Option, + + /// The fragment, when there is one. Meaningless to a channel and never sent + /// on the wire, but dropping it would silently rewrite what the user typed. + fragment: Option, +} + +impl AzureChannelUrl { + /// Parse and validate an `az://` channel URL. + /// + /// The only accepted spelling is `az:///<…>`. A bare `http(s)://` URL is + /// deliberately *not* accepted: `az://` is the single canonical spelling for an + /// Azure channel, and accepting the wire URL as a second input spelling would + /// only invite confusion about which one is authoritative. + /// + /// Account and container derivation is *not* performed here: it depends on the + /// host's addressing style, which is config that does not exist yet at clap + /// parse time. It happens in [`account_and_container`], which today runs only + /// where an account name is genuinely needed — minting a SAS from an + /// `az login` session, and building the opendal config for a write. The fetch + /// path never calls it. + pub fn parse(value: &str) -> Result { + // URL schemes are case-insensitive and `Url` lowercases them, so `AZ://…` + // reaches every downstream `scheme() == "az"` comparison as `az`. Matching + // case-insensitively here keeps this parser from rejecting what those + // comparisons accept. + let rest = strip_az_scheme(value) + .ok_or_else(|| AzureUrlError::InvalidScheme(value.to_string()))?; + + // The authority runs to the first path, query or fragment delimiter. `\` is + // in the set because the special-scheme parser used below treats it as `/`, + // and splitting on it keeps the authority this type validates equal to the + // authority that parser sees. + let authority_end = rest.find(['/', '\\', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + let host = AzureHost::parse(authority)?; + + // Parse the whole thing as `https` for the path, query and fragment: the + // special-scheme parser is what normalizes them, and `wire()` hands them + // straight to an `http(s)` URL, so they have to be normalized its way. + let url = Url::parse(&format!("https://{authority}{tail}")).map_err(|source| { + AzureUrlError::InvalidUrl { + value: value.to_string(), + source, + } })?; - account_and_container(&url)?; - Ok(url) + + Ok(Self { + host, + path: url.path().to_string(), + query: url.query().map(str::to_string), + fragment: url.fragment().map(str::to_string), + }) + } + + /// The `az://host/path` spelling: the channel's identity. + /// + /// This is what users write and what is shown back to them, and it is the + /// spelling config keys are meant to be matched against. Today only the + /// `azure-options` host key actually goes through this type (via + /// [`Self::host`]); `rattler_index` still matches `[index-config."…"]` against + /// the https wire string, which is reviewer issue 5 and moves onto + /// `canonical()` in the index/upload plumbing step. + pub fn canonical(&self) -> Url { + self.spelled("az") + } + + /// The `http(s)://host/path` spelling used for actual requests, over the + /// scheme the host's options entry asks for. + pub fn wire(&self, scheme: AzureScheme) -> Url { + self.spelled(scheme.as_str()) + } + + /// Build one spelling of this URL. + /// + /// Both public spellings go through here, so they cannot differ in anything + /// but the scheme: the host, port, path, query and fragment they are built + /// from are literally the same values. + fn spelled(&self, scheme: &str) -> Url { + let mut text = format!("{scheme}://{}{}", self.host, self.path); + if let Some(query) = &self.query { + text.push('?'); + text.push_str(query); + } + if let Some(fragment) = &self.fragment { + text.push('#'); + text.push_str(fragment); + } + // Cannot fail: the authority re-serializes to the normalized form it was + // parsed from, and the path, query and fragment are already-encoded output + // of a `Url` parse. Every host shape `AzureHost` can hold (normalized + // domain, IPv4 literal, bracketed IPv6) is valid both to the special-scheme + // host parser and to the opaque-host parser `az://` gets. + Url::parse(&text).expect("a normalized authority, path and query is a valid URL") + } + + /// The host, with its port when the URL carries one. + /// + /// This is the `azure-options` key for the channel, so options can be looked + /// up without a caller re-deriving it from a URL and getting the port handling + /// subtly wrong. + pub fn host(&self) -> &AzureHost { + &self.host + } +} + +impl std::fmt::Display for AzureChannelUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // The canonical spelling is the one users recognize and the one config is + // keyed by, so it is the only sensible thing to print. + write!(f, "{}", self.canonical()) + } +} + +impl std::str::FromStr for AzureChannelUrl { + type Err = AzureUrlError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } +} + +/// Strip a case-insensitive `az://` prefix, or return `None` when it is absent. +fn strip_az_scheme(value: &str) -> Option<&str> { + const PREFIX: &str = "az://"; + // `get` rather than slicing: a multi-byte leading character would panic on a + // non-char-boundary index. + value + .get(..PREFIX.len()) + .filter(|prefix| prefix.eq_ignore_ascii_case(PREFIX)) + .map(|_| &value[PREFIX.len()..]) } /// Build an opendal [`AzblobConfig`](opendal::services::AzblobConfig) from a /// channel URL and credentials. /// -/// The account name, endpoint, container, and root prefix are all derived from -/// the URL (`https://.blob.core.windows.net//`); the -/// credentials supply only the account key or SAS token. The URL is expected to -/// already be validated and normalized to `https://` (see [`parse_channel_url`]). +/// The account name, endpoint, container, and root prefix are all derived from the +/// URL; the credentials supply only the account key or SAS token. The URL is +/// expected to already be validated and in wire form (see +/// [`AzureChannelUrl::wire`]). +/// +/// **Host-style addressing only.** The URL must read as +/// `.blob.//`; an IP or single-label endpoint +/// is rejected by [`account_and_container`]. #[cfg(feature = "opendal")] pub fn azblob_config( credentials: &AzureCredentials, channel: &Url, ) -> Result { - let AzureCoordinates { account, container } = account_and_container(channel)?; + // ponytail: host-style is hardcoded, which matches every caller today — none + // has an options entry to pass in yet. The hazard is not an IP or emulator + // host, which `account_and_container` rejects outright: it is a *dotted* host + // whose options say `path-style = true`. That combination does not error here. + // It reads the account from the host label, the container from the segment that + // actually holds the account, and `root` then skips one segment too few — so + // writes land in the wrong container. Honouring path-style also needs the + // endpoint built differently: opendal appends `container/root` to `endpoint`, + // so the account has to be appended to the *endpoint* (`http://host/`). + // Both are deferred to the plumbing step that threads the options entry here. + let AzureCoordinates { account, container } = + account_and_container(channel, Addressing::HostStyle)?; // Preserve a non-default port if one is present; real Azure uses the scheme // default (443). @@ -405,7 +878,7 @@ mod tests { fn normal_url_resolves() { let url = Url::parse("https://acct.blob.core.windows.net/general/noarch").unwrap(); assert_eq!( - account_and_container(&url).unwrap(), + account_and_container(&url, Addressing::HostStyle).unwrap(), AzureCoordinates { account: "acct".to_string(), container: "general".to_string(), @@ -416,43 +889,202 @@ mod tests { #[test] fn userinfo_is_rejected() { let url = Url::parse("https://acct.blob.core.windows.net@evil.example/general").unwrap(); + for addressing in [Addressing::HostStyle, Addressing::PathStyle] { + assert!(matches!( + account_and_container(&url, addressing), + Err(AzureUrlError::UserInfoNotAllowed) + )); + } assert!(matches!( - account_and_container(&url), + AzureChannelUrl::parse("az://acct.blob.core.windows.net@evil.example/general"), + Err(AzureUrlError::UserInfoNotAllowed) + )); + assert!(matches!( + AzureHost::parse("acct.blob.core.windows.net@evil.example"), Err(AzureUrlError::UserInfoNotAllowed) )); } + /// Azure's naming rules are what keep injection-shaped values out of the `az` + /// subprocess, so they have to hold under path-style too — where the account + /// comes from user-controlled path text rather than a host label. #[test] - fn invalid_charset_container_is_rejected() { - let url = Url::parse("https://acct.blob.core.windows.net/general;evil/noarch").unwrap(); + fn invalid_component_names_are_rejected_under_both_styles() { + let host_style = + Url::parse("https://acct.blob.core.windows.net/general;evil/noarch").unwrap(); assert!(matches!( - account_and_container(&url), - Err(AzureUrlError::InvalidCharacters(_)) + account_and_container(&host_style, Addressing::HostStyle), + Err(AzureUrlError::InvalidContainerName(_)) )); + + for (path, account_at_fault) in [ + ("http://127.0.0.1:10000/devstore;evil/general", true), + ("http://127.0.0.1:10000/DevStoreAccount1/general", true), + // Azure allows no hyphen at all in an account name. + ("http://127.0.0.1:10000/dev-store/general", true), + // Too short for Azure, whatever the charset says. + ("http://127.0.0.1:10000/ab/general", true), + ( + "http://127.0.0.1:10000/devstoreaccount1/general;evil", + false, + ), + ("http://127.0.0.1:10000/devstoreaccount1/ab", false), + ("http://127.0.0.1:10000/devstoreaccount1/a--b", false), + ("http://127.0.0.1:10000/devstoreaccount1/-general", false), + ("http://127.0.0.1:10000/devstoreaccount1/general-", false), + ] { + let url = Url::parse(path).unwrap(); + let Err(err) = account_and_container(&url, Addressing::PathStyle) else { + panic!("expected a rejection for {path}"); + }; + let matched = if account_at_fault { + matches!(err, AzureUrlError::InvalidAccountName(_)) + } else { + matches!(err, AzureUrlError::InvalidContainerName(_)) + }; + assert!(matched, "unexpected error for {path}: {err}"); + } } + /// The docstring on [`account_and_container`] promises option-shaped values can + /// never reach the `az` argv. A charset of `[a-z0-9-]` alone does not deliver + /// that, because a leading `-` is inside it. #[test] - fn parse_channel_url_normalizes_az_to_https() { - let url = parse_channel_url("az://acct.blob.core.windows.net/general/noarch").unwrap(); - assert_eq!(url.scheme(), "https"); - assert_eq!( - url.as_str(), - "https://acct.blob.core.windows.net/general/noarch" - ); + fn option_shaped_components_are_rejected() { + for (url, account_at_fault) in [ + ("https://--as-user.blob.core.windows.net/general", true), + ("https://-o.blob.core.windows.net/general", true), + ( + "https://acct.blob.core.windows.net/--https-only/noarch", + false, + ), + ("https://acct.blob.core.windows.net/-o/noarch", false), + ] { + let parsed = Url::parse(url).unwrap(); + let Err(err) = account_and_container(&parsed, Addressing::HostStyle) else { + panic!("expected a rejection for {url}"); + }; + let matched = if account_at_fault { + matches!(err, AzureUrlError::InvalidAccountName(_)) + } else { + matches!(err, AzureUrlError::InvalidContainerName(_)) + }; + assert!(matched, "unexpected error for {url}: {err}"); + } + } + + /// An empty name reaching validation must be rejected by validation itself, + /// not only by a caller remembering to filter it out first. + #[test] + fn empty_components_are_rejected() { + assert!(validate_account("").is_err()); + assert!(validate_container("").is_err()); + } + + #[test] + fn path_style_derives_account_from_first_segment() { + for host in ["127.0.0.1:10000", "azurite:10000", "localhost"] { + let url = + Url::parse(&format!("http://{host}/devstoreaccount1/general/noarch")).unwrap(); + assert_eq!( + account_and_container(&url, Addressing::PathStyle).unwrap(), + AzureCoordinates { + account: "devstoreaccount1".to_string(), + container: "general".to_string(), + }, + "path-style derivation failed for {host}" + ); + } + } + + #[test] + fn path_style_needs_two_segments() { + let url = Url::parse("http://127.0.0.1:10000/devstoreaccount1").unwrap(); + assert!(matches!( + account_and_container(&url, Addressing::PathStyle), + Err(AzureUrlError::NoContainer) + )); + + let url = Url::parse("http://127.0.0.1:10000/").unwrap(); + assert!(matches!( + account_and_container(&url, Addressing::PathStyle), + Err(AzureUrlError::NoAccount) + )); } + /// Host-style must keep rejecting hosts it cannot derive an account from — and + /// the rejection must hand the user a config key that would actually match, + /// which means the port has to be in it and the host has to be spelled the way + /// [`AzureHost`] spells it. #[test] - fn parse_channel_url_rejects_bare_http_and_https() { + fn host_style_rejects_undottable_hosts_with_a_guided_error() { + for (host, expected_key) in [ + ("127.0.0.1:10000", "127.0.0.1:10000"), + ("azurite:10000", "azurite:10000"), + ("localhost", "localhost"), + ("[::1]:10000", "[::1]:10000"), + // A trailing dot is the DNS root label, not a second label: this host + // must not sneak past the dotted-domain gate. + ("localhost.", "localhost"), + ("LocalHost", "localhost"), + ] { + let url = Url::parse(&format!("http://{host}/devstoreaccount1/general")).unwrap(); + let err = account_and_container(&url, Addressing::HostStyle) + .expect_err("host-style must not accept an undottable host"); + assert!(matches!(err, AzureUrlError::InvalidHost(_)), "{err}"); + + let message = err.to_string(); + assert!(message.contains("path-style = true"), "{message}"); + let key = format!("[azure-options.\"{expected_key}\"]"); + assert!(message.contains(&key), "{message}"); + } + } + + /// An empty host label is never legal, and used to yield an empty first + /// "label" as the account name. + #[test] + fn empty_host_labels_are_rejected() { + for host in [ + "acct..blob.core.windows.net", + "acct.blob.example..", + ".example", + ] { + assert!( + matches!( + AzureHost::parse(host), + Err(AzureUrlError::InvalidHostAuthority { .. }) + ), + "expected a rejection for {host}" + ); + assert!( + matches!( + AzureChannelUrl::parse(&format!("az://{host}/general/noarch")), + Err(AzureUrlError::InvalidHostAuthority { .. }) + ), + "expected a rejection for {host}" + ); + let url = Url::parse(&format!("https://{host}/general/noarch")).unwrap(); + assert!( + matches!( + account_and_container(&url, Addressing::HostStyle), + Err(AzureUrlError::InvalidHostAuthority { .. }) + ), + "expected a rejection for {host}" + ); + } + } + + #[test] + fn parse_requires_the_az_scheme() { for input in [ "https://acct.blob.core.windows.net/general", "http://acct.blob.core.windows.net/general", "ftp://acct.blob.core.windows.net/general", - "AZ://acct.blob.core.windows.net/general", "acct.blob.core.windows.net/general", ] { assert!( matches!( - parse_channel_url(input), + AzureChannelUrl::parse(input), Err(AzureUrlError::InvalidScheme(_)) ), "expected InvalidScheme for {input}" @@ -460,12 +1092,234 @@ mod tests { } } + /// URL schemes are case-insensitive, and the middleware's `scheme() == "az"` + /// test sees an already-lowercased scheme, so it accepts `AZ://`. This parser + /// must not disagree with it. #[test] - fn parse_channel_url_propagates_validation_errors() { - assert!(matches!( - parse_channel_url("az://acct.blob.core.windows.net@evil.example/general"), - Err(AzureUrlError::UserInfoNotAllowed) - )); + fn parse_accepts_a_scheme_in_any_case() { + for input in [ + "AZ://acct.blob.core.windows.net/general", + "Az://acct.blob.core.windows.net/general", + "aZ://acct.blob.core.windows.net/general", + ] { + let channel = AzureChannelUrl::parse(input) + .unwrap_or_else(|err| panic!("{input} should parse: {err}")); + assert_eq!( + channel.canonical().as_str(), + "az://acct.blob.core.windows.net/general" + ); + } + } + + #[test] + fn canonical_and_wire_round_trip() { + let channel = + AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/noarch").unwrap(); + + assert_eq!( + channel.canonical().as_str(), + "az://acct.blob.core.windows.net/general/noarch" + ); + assert_eq!( + channel.wire(AzureScheme::Https).as_str(), + "https://acct.blob.core.windows.net/general/noarch" + ); + assert_eq!( + channel.wire(AzureScheme::Http).as_str(), + "http://acct.blob.core.windows.net/general/noarch" + ); + assert_eq!(channel.to_string(), channel.canonical().to_string()); + // `FromStr` is the same parser, so the canonical spelling parses back to + // the same value — which is what lets a config key round-trip. + assert_eq!( + channel, + channel + .canonical() + .as_str() + .parse::() + .unwrap() + ); + } + + /// The point of storing the parts: no scheme choice can make the two spellings + /// describe different locations. + #[test] + fn spellings_cannot_disagree() { + for input in [ + "az://acct.blob.core.windows.net/general/noarch", + "az://127.0.0.1:10000/devstoreaccount1/general", + "az://acct.blob.core.windows.net/general/with%20space?sv=token", + // An IPv6 literal is the host shape most likely to break the canonical + // rebuild, since it has to survive being re-parsed as an opaque host. + "az://[::1]:10000/devstoreaccount1/general", + // The scheme-default ports: exactly the spellings a `Url` stored with a + // fixed scheme silently drops. + "az://azurite.local:443/devstoreaccount1/general", + "az://azurite.local:80/devstoreaccount1/general", + ] { + let channel = AzureChannelUrl::parse(input).unwrap(); + let canonical = channel.canonical(); + for scheme in [AzureScheme::Https, AzureScheme::Http] { + let wire = channel.wire(scheme); + assert_eq!(wire.scheme(), scheme.as_str()); + assert_eq!(canonical.host_str(), wire.host_str(), "{input}"); + assert_eq!(canonical.path(), wire.path(), "{input}"); + assert_eq!(canonical.query(), wire.query(), "{input}"); + + // Ports are compared semantically, not textually: `az` has no + // default port so the canonical form always spells one out when the + // URL has one, while a wire URL omits a port equal to its scheme's + // default. An omitted port on `http` *is* 80, so those agree. + let default = match scheme { + AzureScheme::Https => 443, + AzureScheme::Http => 80, + }; + assert_eq!( + wire.port_or_known_default(), + Some(canonical.port().unwrap_or(default)), + "{input} over {scheme}" + ); + } + } + } + + /// The `:443` regression: a wire URL stored with the `https` scheme drops this + /// port, and `wire(Http)` then names a completely different endpoint. + #[test] + fn a_written_default_port_survives() { + let channel = + AzureChannelUrl::parse("az://azurite.local:443/devstoreaccount1/general").unwrap(); + + assert_eq!(channel.host().to_string(), "azurite.local:443"); + assert_eq!(channel.host().port(), Some(443)); + assert_eq!( + channel.canonical().as_str(), + "az://azurite.local:443/devstoreaccount1/general" + ); + assert_eq!( + channel.wire(AzureScheme::Http).as_str(), + "http://azurite.local:443/devstoreaccount1/general" + ); + assert_eq!( + channel.wire(AzureScheme::Https).as_str(), + "https://azurite.local/devstoreaccount1/general" + ); + + // Identity must not be scheme-relative either: a host on 443 is not the + // same endpoint as the same host with no port, because the scheme that + // would make them equal is not known here. + let no_port = + AzureChannelUrl::parse("az://azurite.local/devstoreaccount1/general").unwrap(); + assert_ne!(channel, no_port); + assert_ne!(channel.host(), no_port.host()); + } + + #[test] + fn host_keeps_a_non_default_port() { + let emulator = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + assert_eq!(emulator.host().to_string(), "127.0.0.1:10000"); + assert_eq!( + emulator.wire(AzureScheme::Http).as_str(), + "http://127.0.0.1:10000/devstoreaccount1/general" + ); + assert_eq!( + emulator.canonical().as_str(), + "az://127.0.0.1:10000/devstoreaccount1/general" + ); + + // No port written, none invented. + let azure = AzureChannelUrl::parse("az://acct.blob.core.windows.net/general").unwrap(); + assert_eq!(azure.host().to_string(), "acct.blob.core.windows.net"); + assert_eq!(azure.host().port(), None); + } + + /// Every normalization the URL host parser performs is a way for a written + /// config key and a looked-up host to disagree, unless both go through the same + /// parser. They do: this is that parser, and these are the classes it has to + /// collapse. + #[test] + fn host_normalization_collapses_equivalent_spellings() { + for (written, canonical) in [ + ( + "MyCompany.blob.core.windows.net", + "mycompany.blob.core.windows.net", + ), + ( + "mycompany.blob.core.windows.net:443", + "mycompany.blob.core.windows.net:443", + ), + ("ünï.blob.example", "xn--n-nga1b.blob.example"), + ("xn--n-nga1b.blob.example", "xn--n-nga1b.blob.example"), + ("[0:0:0:0:0:0:0:1]:10000", "[::1]:10000"), + ("[::1]:10000", "[::1]:10000"), + ("0x7f.1", "127.0.0.1"), + ("127.0.0.1", "127.0.0.1"), + ("acct.blob.core.windows.net.", "acct.blob.core.windows.net"), + ("acct.blob.core.windows.net", "acct.blob.core.windows.net"), + ] { + let host = AzureHost::parse(written) + .unwrap_or_else(|err| panic!("{written} should parse: {err}")); + assert_eq!(host.to_string(), canonical, "{written}"); + + // Display and parse round-trip, so a key written out of an `AzureHost` + // parses back to the same host… + let reparsed = AzureHost::parse(canonical).unwrap(); + assert_eq!(reparsed, host, "{written}"); + // …and equal hosts hash equally, so they land on the same map entry. + assert_eq!(hash_of(&host), hash_of(&reparsed), "{written}"); + } + } + + /// A written port is part of the endpoint's identity: nothing here knows the + /// scheme, so nothing here can call 443 or 80 redundant. + #[test] + fn host_equality_is_not_scheme_relative() { + let with_port = AzureHost::parse("azurite.local:443").unwrap(); + let without = AzureHost::parse("azurite.local").unwrap(); + assert_ne!(with_port, without); + assert_ne!(with_port, AzureHost::parse("azurite.local:80").unwrap()); + assert_eq!(with_port.to_string(), "azurite.local:443"); + } + + /// A config key is a bare authority; anything else is a mistake worth naming + /// rather than silently reinterpreting. + #[test] + fn host_rejects_anything_that_is_not_a_bare_authority() { + for authority in [ + "acct.blob.core.windows.net/general", + "acct.blob.core.windows.net?sv=token", + "acct.blob.core.windows.net#frag", + "https://acct.blob.core.windows.net", + "", + "acct.blob.core.windows.net:notaport", + ] { + assert!( + AzureHost::parse(authority).is_err(), + "expected a rejection for {authority:?}" + ); + } + } + + fn hash_of(host: &AzureHost) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + host.hash(&mut hasher); + hasher.finish() + } + + /// Account and container derivation is deliberately *not* part of parsing: the + /// addressing style is config that does not exist yet when clap parses the + /// argument, so an emulator URL must survive parsing and be rejected (or not) + /// later, once its options entry is known. + #[test] + fn parse_defers_account_derivation() { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + let wire = channel.wire(AzureScheme::Http); + + assert!(account_and_container(&wire, Addressing::HostStyle).is_err()); + assert!(account_and_container(&wire, Addressing::PathStyle).is_ok()); } } diff --git a/crates/rattler_azure/src/options.rs b/crates/rattler_azure/src/options.rs index 9b06ad2603..041242e5d2 100644 --- a/crates/rattler_azure/src/options.rs +++ b/crates/rattler_azure/src/options.rs @@ -7,22 +7,38 @@ //! suffix classification carries no security weight, and the absence of the list //! is what lets custom endpoints and the Azurite emulator work at all. //! +//! ponytail: that is the intent, not yet the behaviour. `AzureMiddleware` still +//! decides whether to sign a request from whether Azure credentials happen to be +//! present in the environment, and never reads this table at all; the middleware +//! rewrite is what makes the grant model real. +//! //! # Why enums for what the config spells as bools //! //! The TOML surface stays `auth = true` / `path-style = true`, because that is -//! the ergonomic spelling and it mirrors `s3-options`. Internally each is an +//! the ergonomic spelling and it keeps the table skimmable. Internally each is an //! enum, so no call site can mix up two unrelated booleans, and the meaning of a -//! value is legible without chasing the field name. The bridge is -//! `#[serde(from = "bool", into = "bool")]` plus a `From` impl each way. - -use serde::{Deserialize, Serialize}; +//! value is legible without chasing the field name. The bridge is a +//! `serde(from = "bool", into = "bool")` pair with a `From` impl each way, applied +//! through `cfg_attr` so the serde attributes stay behind the `serde` feature +//! along with the derives they configure. +//! +//! The types themselves are always available — [`Addressing`] decides how +//! [`account_and_container`](crate::account_and_container) reads a URL and +//! [`AzureScheme`] is what [`AzureChannelUrl::wire`](crate::AzureChannelUrl::wire) +//! is spelled in, neither of which involves serde. Only the derives are behind +//! the `serde` feature, so a consumer that just wants the URL types does not +//! pull serde in. /// Whether credentials may attach to requests for a host. /// /// Defaults to [`Auth::Anonymous`]: a host gets no credentials until a config /// entry says otherwise. Serializes as the bool `auth` in `azure-options`. -#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(from = "bool", into = "bool")] +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(from = "bool", into = "bool") +)] pub enum Auth { /// Send requests unsigned. No credential is resolved, so no ambient /// credential can be exfiltrated to this host, and nothing blocks on the @@ -61,12 +77,20 @@ impl Auth { /// The wire scheme an `az://` channel URL is rewritten to when a request is sent. /// -/// Defaults to [`Scheme::Https`]. `Http` exists for local emulators such as +/// Named for the crate rather than spelled bare `Scheme`, because this crate also +/// depends on `opendal`, whose own `Scheme` names a storage service — two very +/// different things one import away from each other. +/// +/// Defaults to [`AzureScheme::Https`]. `Http` exists for local emulators such as /// Azurite; choosing it is an explicit, per-host decision in config, so a plain /// `az://` URL can never be silently downgraded to cleartext. -#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "lowercase")] -pub enum Scheme { +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "lowercase") +)] +pub enum AzureScheme { /// Send requests over TLS. #[default] Https, @@ -75,17 +99,17 @@ pub enum Scheme { Http, } -impl Scheme { +impl AzureScheme { /// The scheme as it appears in a URL, without the `://`. pub fn as_str(self) -> &'static str { match self { - Scheme::Https => "https", - Scheme::Http => "http", + AzureScheme::Https => "https", + AzureScheme::Http => "http", } } } -impl std::fmt::Display for Scheme { +impl std::fmt::Display for AzureScheme { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) } @@ -94,14 +118,19 @@ impl std::fmt::Display for Scheme { /// Where the storage account name is found in a blob URL. /// /// Defaults to [`Addressing::HostStyle`], which is how real Azure addresses -/// accounts. Serializes as the bool `path-style` in `azure-options`. Mirrors -/// `rattler_s3::S3AddressingStyle`. -#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(from = "bool", into = "bool")] +/// accounts. Serializes as the bool `path-style` in `azure-options`; `s3-options` +/// spells its equivalent `force-path-style`, also a bool. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(from = "bool", into = "bool") +)] pub enum Addressing { /// The account is the first label of the host, as in - /// `.blob.core.windows.net/`. Requires a dotted host, so - /// IP literals and single-label hosts cannot be addressed this way. + /// `.blob.core.windows.net/`. Requires a domain with at + /// least two labels, so IP literals and single-label hosts cannot be + /// addressed this way. #[default] HostStyle, @@ -133,20 +162,29 @@ impl From for bool { /// host with no config entry behaves exactly as if it had a defaulted entry, so /// callers can look up an absent host and fall back to `default()` rather than /// branching on presence. -#[derive(Default, Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case", default)] +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "kebab-case", default) +)] pub struct AzureEndpointOptions { /// Whether credentials may be sent to this host. pub auth: Auth, /// The scheme `az://` is rewritten to for this host. - pub scheme: Scheme, + pub scheme: AzureScheme, /// Where the account name is found in the URL for this host. - pub path_style: Addressing, + /// + /// The field is named for what it holds, but the config key stays + /// `path-style`: that is the spelling users have written, and the bool bridge + /// is what the key means. + #[cfg_attr(feature = "serde", serde(rename = "path-style"))] + pub addressing: Addressing, } -#[cfg(test)] +#[cfg(all(test, feature = "serde"))] mod tests { use super::*; @@ -166,8 +204,8 @@ mod tests { opts, AzureEndpointOptions { auth: Auth::DefaultChain, - scheme: Scheme::Http, - path_style: Addressing::PathStyle, + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, } ); @@ -176,8 +214,8 @@ mod tests { assert_eq!(empty, AzureEndpointOptions::default()); assert_eq!(empty.auth, Auth::Anonymous); assert!(!empty.auth.is_granted()); - assert_eq!(empty.scheme, Scheme::Https); - assert_eq!(empty.path_style, Addressing::HostStyle); + assert_eq!(empty.scheme, AzureScheme::Https); + assert_eq!(empty.addressing, Addressing::HostStyle); // `auth = false` is spelled out explicitly by some users; it must not be // mistaken for a grant. @@ -191,8 +229,8 @@ mod tests { fn enums_serialize_back_to_bools() { let toml = toml::to_string(&AzureEndpointOptions { auth: Auth::DefaultChain, - scheme: Scheme::Http, - path_style: Addressing::PathStyle, + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, }) .unwrap(); assert!(toml.contains("auth = true"), "{toml}"); diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs index ba1663a499..7155b2d928 100644 --- a/crates/rattler_config/src/config/azure.rs +++ b/crates/rattler_config/src/config/azure.rs @@ -1,17 +1,29 @@ use indexmap::IndexMap; -use rattler_azure::AzureEndpointOptions; +use rattler_azure::{AzureEndpointOptions, AzureHost}; use serde::{Deserialize, Serialize}; use crate::config::Config; -/// Per-host options for Azure Blob channels, keyed by host (including a port -/// where one is used, e.g. `127.0.0.1:10000`). +/// Per-host options for Azure Blob channels, keyed by endpoint authority +/// (including a port where one is used, e.g. `127.0.0.1:10000`). /// /// An entry is a *grant*: it is the only way a host gets credentials, a /// non-default scheme, or path-style addressing. A host with no entry is fetched /// anonymously over https in host-style addressing, so an empty map is the safe /// default and [`AzureOptionsMap::get`] can answer for absent hosts too. /// +/// # Why the key is an [`AzureHost`] and not a `String` +/// +/// A silently-missed grant is the worst failure this table has: Azure answers an +/// unauthorized request for a private container with a 404, so the user is told +/// "not found" rather than "not authorized". Keyed by raw TOML text, every host +/// normalization is such a miss — `MyCompany.blob…` , `host:443`, `ünï.blob…`, +/// `[0:0:0:0:0:0:0:1]:10000` and `0x7f.1` are all spellings a lookup would arrive +/// with in a different form. Keying by [`AzureHost`] deletes the class: the key is +/// deserialized through the same parser that produces the lookup value, so the two +/// cannot disagree. The inner map is private for the same reason — a key that did +/// not go through that parser must be unrepresentable, not merely discouraged. +/// /// # Scope /// /// Entries are **user-scoped by contract**. A tool must never read this table @@ -19,7 +31,7 @@ use crate::config::Config; /// repository name a host and have the user's ambient Azure credentials sent to /// it. Keep it to user- and system-level config files. #[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -pub struct AzureOptionsMap(pub IndexMap); +pub struct AzureOptionsMap(IndexMap); impl AzureOptionsMap { /// Returns `true` if no Azure hosts are configured. @@ -33,9 +45,14 @@ impl AzureOptionsMap { /// Callers should prefer this over indexing the map: "no entry" and "a /// defaulted entry" are defined to behave identically, so branching on /// presence only invites the two paths to drift apart. - pub fn get(&self, host: &str) -> AzureEndpointOptions { + pub fn get(&self, host: &AzureHost) -> AzureEndpointOptions { self.0.get(host).copied().unwrap_or_default() } + + /// The configured hosts, in the order they were written. + pub fn hosts(&self) -> impl Iterator { + self.0.keys() + } } impl Config for AzureOptionsMap { @@ -65,10 +82,14 @@ impl Config for AzureOptionsMap { #[cfg(test)] mod tests { - use rattler_azure::{Addressing, Auth, Scheme}; + use rattler_azure::{Addressing, Auth, AzureScheme}; use super::*; + fn host(authority: &str) -> AzureHost { + AzureHost::parse(authority).expect("test host should parse") + } + /// The table parses in the shape documented for users, and an absent host /// answers with the anonymous defaults rather than requiring a presence check. #[test] @@ -86,18 +107,18 @@ mod tests { ) .unwrap(); - let real = map.get("mycompany.blob.core.windows.net"); + let real = map.get(&host("mycompany.blob.core.windows.net")); assert_eq!(real.auth, Auth::DefaultChain); - assert_eq!(real.scheme, Scheme::Https); - assert_eq!(real.path_style, Addressing::HostStyle); + assert_eq!(real.scheme, AzureScheme::Https); + assert_eq!(real.addressing, Addressing::HostStyle); - let azurite = map.get("127.0.0.1:10000"); + let azurite = map.get(&host("127.0.0.1:10000")); assert_eq!(azurite.auth, Auth::DefaultChain); - assert_eq!(azurite.scheme, Scheme::Http); - assert_eq!(azurite.path_style, Addressing::PathStyle); + assert_eq!(azurite.scheme, AzureScheme::Http); + assert_eq!(azurite.addressing, Addressing::PathStyle); // An unlisted host gets no grant. - let unlisted = map.get("someoneelse.blob.core.windows.net"); + let unlisted = map.get(&host("someoneelse.blob.core.windows.net")); assert!(!unlisted.auth.is_granted()); assert_eq!(unlisted, AzureEndpointOptions::default()); } @@ -112,12 +133,64 @@ mod tests { toml::from_str("[\"host.example\"]\nscheme = \"http\"\n").unwrap(); let merged = base.merge_config(&over).unwrap(); - let entry = merged.get("host.example"); - assert_eq!(entry.scheme, Scheme::Http); + let entry = merged.get(&host("host.example")); + assert_eq!(entry.scheme, AzureScheme::Http); assert!( !entry.auth.is_granted(), "overwriting an entry must not inherit the previous grant" ); - assert_eq!(entry.path_style, Addressing::HostStyle); + assert_eq!(entry.addressing, Addressing::HostStyle); + } + + /// The defect this key type exists to kill: every one of these keys is a + /// spelling a lookup arrives with in normalized form, and with a `String` key + /// each was a silent miss — an anonymous fetch, a 404, and a user told "not + /// found" instead of "not authorized". + #[test] + fn keys_are_normalized_the_same_way_lookups_are() { + for (written, looked_up) in [ + ( + "MyCompany.blob.core.windows.net", + "mycompany.blob.core.windows.net", + ), + ( + "mycompany.blob.core.windows.net:443", + "mycompany.blob.core.windows.net:443", + ), + ("ünï.blob.example", "xn--n-nga1b.blob.example"), + ("[0:0:0:0:0:0:0:1]:10000", "[::1]:10000"), + ("0x7f.1", "127.0.0.1"), + ("acct.blob.core.windows.net.", "acct.blob.core.windows.net"), + ] { + let map: AzureOptionsMap = + toml::from_str(&format!("[\"{written}\"]\nauth = true\n")).unwrap(); + + assert!( + map.get(&host(looked_up)).auth.is_granted(), + "the grant written as `{written}` did not apply to `{looked_up}`" + ); + // The key is stored canonically, so `keys()` reports what a lookup + // would need rather than what happened to be typed, and writing the + // table back out produces a key that parses to the same host. + assert_eq!(map.keys(), vec![looked_up.to_string()], "{written}"); + let written_back = toml::to_string(&map).unwrap(); + assert!( + written_back.contains(&format!("[\"{looked_up}\"]")), + "{written} was written back as {written_back}" + ); + } + } + + /// A key that cannot be a host is a config error worth naming, not an entry + /// that silently never matches. + #[test] + fn an_unparseable_key_is_rejected() { + let err = + toml::from_str::("[\"acct.blob.example/general\"]\nauth = true\n") + .expect_err("a key carrying a path must be rejected"); + assert!( + err.to_string().contains("acct.blob.example/general"), + "{err}" + ); } } diff --git a/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap index d0bacbf0ca..75b8f782b3 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap @@ -208,15 +208,15 @@ expression: "(unused, normalized(config))" ), azure_options: AzureOptionsMap( { - "127.0.0.1:10000": AzureEndpointOptions { + AzureHost("127.0.0.1:10000"): AzureEndpointOptions { auth: DefaultChain, scheme: Http, - path_style: PathStyle, + addressing: PathStyle, }, - "mycompany.blob.core.windows.net": AzureEndpointOptions { + AzureHost("mycompany.blob.core.windows.net"): AzureEndpointOptions { auth: DefaultChain, scheme: Https, - path_style: HostStyle, + addressing: HostStyle, }, }, ), diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 6d8d81a6e6..d8055fd7e5 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -36,6 +36,21 @@ fn parse_s3_url(value: &str) -> Result { } } +/// Parse an `az://` channel URL into the wire URL the rest of this binary still +/// works with. +/// +/// The scheme is fixed to https here because a clap `value_parser` runs before +/// the config file is read, so the host's `azure-options` entry — which decides +/// the scheme — is not available yet. The plumbing step that loads +/// `azure-options` carries the [`rattler_azure::AzureChannelUrl`] through +/// instead, so `--config` can select the scheme and config keys can be matched +/// against its canonical `az://` spelling. +#[cfg(feature = "azure")] +fn parse_azure_channel_url(value: &str) -> Result { + rattler_azure::AzureChannelUrl::parse(value) + .map(|channel| channel.wire(rattler_azure::AzureScheme::Https)) +} + /// SAS permissions requested when minting a user-delegation SAS for indexing. /// Indexing does a read-modify-write of repodata and lists/reads packages, so it /// needs read, write, list, and create (`r` + `w` + `l` + `c`). @@ -117,7 +132,7 @@ enum Commands { Azblob { /// The Azure Blob channel URL, e.g. /// `az://.blob.core.windows.net//`. - #[arg(value_parser = rattler_azure::parse_channel_url)] + #[arg(value_parser = parse_azure_channel_url)] channel: Url, #[clap(flatten)] @@ -250,7 +265,10 @@ async fn main() -> anyhow::Result<()> { let credentials = credentials .resolve(AZURE_INDEX_SAS_PERMISSIONS, || { - Ok(rattler_azure::account_and_container(&channel)?) + Ok(rattler_azure::account_and_container( + &channel, + rattler_azure::Addressing::HostStyle, + )?) }) .await?; diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index 8da7cfcc9a..29e6042b03 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -87,7 +87,10 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { let credentials = azure_opts .credentials .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, || { - Ok(rattler_azure::account_and_container(&channel)?) + Ok(rattler_azure::account_and_container( + &channel, + rattler_azure::Addressing::HostStyle, + )?) }) .await .into_diagnostic()?; diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 77b44d0e5d..b1b3426532 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -389,6 +389,20 @@ pub struct AnacondaOpts { pub force: bool, } +/// Parse an `az://` channel URL into the wire URL the upload path still works +/// with. +/// +/// The scheme is fixed to https because a clap `value_parser` runs before any +/// config is loaded, so the host's `azure-options` entry — which decides the +/// scheme — is not known yet. The plumbing step that loads those options carries +/// the [`rattler_azure::AzureChannelUrl`] itself instead, which is what lets a +/// custom-scheme (emulator) endpoint be uploaded to. +#[cfg(feature = "azure")] +fn parse_azure_channel_url(value: &str) -> Result { + rattler_azure::AzureChannelUrl::parse(value) + .map(|channel| channel.wire(rattler_azure::AzureScheme::Https)) +} + #[cfg(feature = "s3")] fn parse_s3_url(value: &str) -> Result { let url: Url = @@ -428,7 +442,7 @@ pub struct S3Opts { pub struct AzureOpts { /// The channel URL in the Azure Blob container to upload the package to, /// e.g., `az://myaccount.blob.core.windows.net/my-container/my-channel` - #[arg(short, long, env = "AZURE_CHANNEL", value_parser = rattler_azure::parse_channel_url)] + #[arg(short, long, env = "AZURE_CHANNEL", value_parser = parse_azure_channel_url)] pub channel: Url, #[clap(flatten)] diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index cdbfb339af..f379dd0f10 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -4120,6 +4120,7 @@ dependencies = [ "jiff", "opendal", "percent-encoding", + "serde", "thiserror 2.0.18", "tokio", "url", @@ -4206,6 +4207,7 @@ dependencies = [ "dirs", "fs-err", "indexmap 2.14.0", + "rattler_azure", "rattler_conda_types", "serde", "serde_ignored", From 5b0a824cf3b56e428a7686be9cacf266caa310e4 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:26:28 +1000 Subject: [PATCH 67/98] feat(networking): drive azure signing from azure-options grants --- crates/rattler-bin/src/commands/client.rs | 15 +- crates/rattler_azure/src/options.rs | 5 - crates/rattler_networking/Cargo.toml | 2 + .../src/azure_middleware.rs | 694 ++++++++++-------- .../tests/azure_real_fetch.rs | 17 +- py-rattler/rattler/networking/middleware.py | 6 +- py-rattler/src/networking/client.rs | 7 +- py-rattler/src/networking/middleware.rs | 5 + 8 files changed, 435 insertions(+), 316 deletions(-) diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index 9271e2fcc9..8379b32c77 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "s3")] +#[cfg(any(feature = "s3", feature = "azure"))] use std::collections::HashMap; use std::sync::Arc; @@ -62,11 +62,16 @@ pub fn create_client_with_middleware( #[cfg(feature = "gcs")] let client = client.with(rattler_networking::GCSMiddleware::default()); // `az://` URLs carry the full blob endpoint, so the middleware needs no - // configuration — it just swaps the scheme and signs via reqsign. It reuses - // the configured download client for reqsign's credential resolution so - // proxy/CA/TLS settings carry through. + // configuration to *reach* one — it just swaps the scheme. The empty options + // table means every `az://` fetch here is anonymous; this CLI reads no config + // file, so there is no `azure-options` to hand it (the same reason S3 gets an + // empty map above). It reuses the configured download client for reqsign's + // credential resolution so proxy/CA/TLS settings carry through. #[cfg(feature = "azure")] - let client = client.with(rattler_networking::AzureMiddleware::new(download_client)); + let client = client.with(rattler_networking::AzureMiddleware::new( + download_client, + HashMap::new(), + )); Ok(client.build()) } diff --git a/crates/rattler_azure/src/options.rs b/crates/rattler_azure/src/options.rs index 041242e5d2..5540cbd265 100644 --- a/crates/rattler_azure/src/options.rs +++ b/crates/rattler_azure/src/options.rs @@ -7,11 +7,6 @@ //! suffix classification carries no security weight, and the absence of the list //! is what lets custom endpoints and the Azurite emulator work at all. //! -//! ponytail: that is the intent, not yet the behaviour. `AzureMiddleware` still -//! decides whether to sign a request from whether Azure credentials happen to be -//! present in the environment, and never reads this table at all; the middleware -//! rewrite is what makes the grant model real. -//! //! # Why enums for what the config spells as bools //! //! The TOML surface stays `auth = true` / `path-style = true`, because that is diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 29f875300b..3d1d4e8656 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -18,6 +18,7 @@ rustls = ["reqwest/rustls", "ambient-id/rustls"] gcs = ["google-cloud-auth", "tokio/sync"] s3 = ["aws-config", "aws-sdk-s3", "aws-smithy-http-client"] azure = [ + "dep:rattler_azure", "dep:reqsign-core", "dep:reqsign-azure-storage", "dep:reqsign-command-execute-tokio", @@ -75,6 +76,7 @@ thiserror = { workspace = true } tokio = { workspace = true, optional = true } tracing = { workspace = true } url = { workspace = true } +rattler_azure = { workspace = true, optional = true } rattler_config = { workspace = true, optional = true } reqsign-core = { workspace = true, optional = true } reqsign-azure-storage = { workspace = true, optional = true } diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 14de8fc711..13e420519e 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -1,5 +1,8 @@ //! Middleware to handle `az://` URLs to pull artifacts from Azure Blob Storage. +use std::collections::HashMap; + use async_trait::async_trait; +use rattler_azure::{Auth, AzureChannelUrl, AzureEndpointOptions, AzureHost}; use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; use reqsign_command_execute_tokio::TokioCommandExecute; use reqsign_core::{Context, OsEnv, ProvideCredential, Signer}; @@ -12,116 +15,79 @@ use url::Url; /// The Azure Storage REST API version sent on every request. const X_MS_VERSION: &str = "2021-12-02"; -/// Whether an Azure credential source appears configured in the process -/// environment. -/// -/// Used to distinguish "no credential at all" (fall back to an unsigned, -/// anonymous request) from "a credential is configured but signing failed" -/// (a hard error). It is a presence check only — it does not validate that the -/// values are usable; reqsign does that when it actually signs. -/// -/// A persisted `az login` session (a profile on disk) counts as a source too; -/// see [`azure_cli_session_present`]. -/// -/// This result gates whether credentials are resolved at all: when it is -/// `false`, [`AzureMiddleware::sign`] sends the request unsigned *without* -/// probing, so anonymous public-container reads do not block on the -/// managed-identity / IMDS timeout. The tradeoff is that a *bare* system-assigned -/// managed identity — one reached only via the IMDS endpoint with no -/// `IDENTITY_ENDPOINT` / `MSI_ENDPOINT` / federated-token env var set — is not -/// detected here, so its requests go unsigned. App Service, Functions, Cloud -/// Shell and AKS workload identity all set one of those env vars and are -/// unaffected; a bare `IaaS` VM identity must export `MSI_ENDPOINT` (or supply an -/// explicit credential) to be used. -fn azure_credential_source_present() -> bool { - // Explicit Shared Key or SAS token. - if std::env::var_os("AZURE_STORAGE_ACCOUNT_KEY").is_some() - || std::env::var_os("AZURE_STORAGE_SAS_TOKEN").is_some() - { - return true; - } - // Service-principal flows need both a client and a tenant to be meaningful. - if std::env::var_os("AZURE_CLIENT_ID").is_some() - && std::env::var_os("AZURE_TENANT_ID").is_some() - { - return true; - } - // Workload identity federation (e.g. AKS). - if std::env::var_os("AZURE_FEDERATED_TOKEN_FILE").is_some() { - return true; - } - // Managed identity endpoints (App Service / Functions / Cloud Shell / IMDS - // override). - if std::env::var_os("MSI_ENDPOINT").is_some() || std::env::var_os("IDENTITY_ENDPOINT").is_some() - { - return true; - } - // Persisted `az login` session. - azure_cli_session_present() -} - -/// Whether a persisted `az login` session exists on disk. -/// -/// `az login` writes an `azureProfile.json` into the Azure CLI config dir -/// (`$AZURE_CONFIG_DIR`, else `~/.azure`). Its presence means a login was -/// performed at some point, so a *broken* az-login credential hard-errors -/// rather than silently downgrading to an unsigned request. This is a presence -/// check only — reqsign validates the token when it actually signs, so a stale -/// profile with no usable token still yields a hard error, never anonymous. -fn azure_cli_session_present() -> bool { - let config_dir = match std::env::var_os("AZURE_CONFIG_DIR") { - Some(dir) => std::path::PathBuf::from(dir), - None => match std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE")) { - Some(home) => std::path::PathBuf::from(home).join(".azure"), - None => return false, - }, - }; - config_dir.join("azureProfile.json").exists() -} - -/// Middleware that rewrites `az://` URLs to HTTPS Azure Blob Storage URLs and -/// signs them. +/// Middleware that rewrites `az://` URLs to their wire form and, where a host is +/// granted credentials, signs them. /// /// The `az://` URL carries the full blob endpoint in its host, so rewriting is a /// plain scheme swap: `az://{host}/{path}` → `https://{host}/{path}`. A conda /// channel is therefore addressed the same way it is on the wire, e.g. /// `az://myaccount.blob.core.windows.net/mycontainer` — no separate account or -/// endpoint configuration is needed. Sovereign clouds and emulators (Azurite) -/// work automatically because the endpoint is spelled out in the host. +/// endpoint configuration is needed. Sovereign clouds work with no configuration +/// at all; an emulator needs only the `scheme = "http"` line below. /// /// # Trust model /// -/// The URL host is **trusted verbatim**: it becomes the HTTPS request target -/// unchanged, with no allow-list of accounts or endpoints. Whatever ambient -/// AAD / Shared-Key credential resolves below is applied to that host — so a -/// channel author who controls the `az://` URL controls where the request, and -/// any credential material, is sent. Because of this, **userinfo is rejected**: -/// an `az://user:pass@host/...` authority is a host-spoofing vector (the real -/// host can be hidden behind userinfo) and such requests are refused before any -/// rewrite or signing. +/// **Anonymous by default.** With no entry for a host, its requests are sent +/// unsigned and *no credential is resolved at all* — so no ambient Azure +/// credential can leak to a host the user never named, and an anonymous read of a +/// public container does not block on the managed-identity / IMDS probe. +/// +/// A credential attaches to a host only because an [`AzureEndpointOptions`] entry +/// for it says [`Auth::DefaultChain`], which comes from the user's `azure-options` +/// config table: +/// +/// ```toml +/// [azure-options."mycompany.blob.core.windows.net"] +/// auth = true +/// +/// [azure-options."127.0.0.1:10000"] # Azurite +/// auth = true +/// scheme = "http" +/// ``` +/// +/// Two consequences of the grant being explicit: /// -/// Credentials are resolved by reqsign's [`DefaultCredentialProvider`] chain, in -/// its usual order: environment variables, then workload/managed identity, then -/// the Azure CLI (`az login`). rattler's [`crate::AuthenticationStorage`] is not -/// consulted for Azure — there is no `Authentication` Azure variant — so -/// per-host credentials configured there do not apply to `az://` requests. +/// - **Nothing is inferred from the host name.** There is no allow-list of +/// "official" Azure suffixes, and none is needed: a host nobody granted gets +/// nothing regardless of what it is called, and an entry for a custom host *is* +/// the declaration that the user trusts that endpoint. +/// - **A broken credential is a hard error.** Because the user asked for signing, +/// an unusable credential must be reported, not silently downgraded to an +/// anonymous request that Azure will answer with a confusing 404. /// -/// When **no credential source is detected**, the request is sent **unsigned** -/// rather than failing, so public/anonymous containers remain reachable with -/// zero ambient credentials. When a credential source *is* configured -/// (see `azure_credential_source_present`) but signing fails, that is a -/// **hard error** — reqsign collapses "no credential" and "broken credential" -/// into the same [`reqsign_core::ErrorKind::CredentialInvalid`], so a broken credential must -/// not be silently downgraded to an anonymous request. +/// Entries are user-scoped by contract: a project- or workspace-level manifest +/// must never be allowed to write one, since that would let a checked-out +/// repository name a host and receive the user's credentials. +/// +/// `az://user:pass@host/...` is refused outright. The host becomes the request +/// target verbatim, so userinfo is a host-spoofing vector — the real authority can +/// hide behind it — and userinfo is invalid in a blob URL anyway. +/// +/// Granted credentials are resolved by reqsign's [`DefaultCredentialProvider`] +/// chain, in its usual order: environment variables, then workload/managed +/// identity, then the Azure CLI (`az login`). rattler's +/// [`crate::AuthenticationStorage`] is not consulted for Azure — there is no +/// `Authentication` Azure variant — so per-host credentials configured there do +/// not apply to `az://` requests. #[derive(Clone)] pub struct AzureMiddleware { /// reqsign signer; caches the resolved credential internally. signer: Signer, - /// Whether an Azure credential source appears configured in the process - /// environment. Captured at construction. When `true`, a signing failure is - /// propagated as a hard error instead of falling back to an unsigned - /// request. - credential_source_present: bool, + + /// Per-host endpoint options, keyed by the same normalized authority the + /// `azure-options` config table is keyed by. An absent host is *defined* to + /// behave as a defaulted entry (anonymous, https), so a miss is never a + /// separate code path. + /// + /// ponytail: a plain `HashMap` rather than `rattler_config::AzureOptionsMap`, + /// mirroring [`crate::S3Middleware`]. No caller has a `rattler_config::Config` + /// in hand today — every one of them passes an empty table — so taking the + /// config type would buy a mandatory `rattler_config` edge on the `azure` + /// feature for zero saved conversions. When a caller does grow one, add a + /// `#[cfg(feature = "rattler_config")]` helper next to + /// [`crate::s3_middleware::compute_s3_config_from_config`] rather than + /// changing this signature. + options: HashMap, } impl AzureMiddleware { @@ -131,25 +97,23 @@ impl AzureMiddleware { /// identity / AAD token fetches), so it must be the caller's configured /// client — proxy, CA bundle, and TLS settings carry through to those /// requests. - pub fn new(client: Client) -> Self { - Self::with_credential_provider( - client, - DefaultCredentialProvider::new(), - azure_credential_source_present(), - ) + /// + /// `options` is the `azure-options` table: the per-host grants. An empty map + /// means every `az://` request is anonymous. + pub fn new(client: Client, options: HashMap) -> Self { + Self::with_credential_provider(client, DefaultCredentialProvider::new(), options) } /// Build the middleware around an explicit credential provider. /// - /// [`AzureMiddleware::new`] wires up the [`DefaultCredentialProvider`] chain - /// and detects the credential source from the environment; tests use this - /// seam to inject a deterministic provider (e.g. an empty chain, or a static - /// key) and an explicit `credential_source_present` flag without touching the + /// [`AzureMiddleware::new`] wires up the [`DefaultCredentialProvider`] chain; + /// tests use this seam to inject a deterministic provider (an empty chain + /// standing in for a broken credential, or a static key) without touching the /// ambient environment. fn with_credential_provider( client: Client, provider: impl ProvideCredential + 'static, - credential_source_present: bool, + options: HashMap, ) -> Self { let ctx = Context::new() .with_file_read(TokioFileRead) @@ -157,21 +121,34 @@ impl AzureMiddleware { .with_command_execute(TokioCommandExecute) .with_env(OsEnv); let signer = Signer::new(ctx, provider, RequestSigner::new()); - Self { - signer, - credential_source_present, - } + Self { signer, options } } - /// Rewrite an `az://{host}/{path}` URL to its HTTPS equivalent by swapping - /// the scheme. Host, path, query and fragment are preserved verbatim. - fn rewrite_url(az_url: &Url) -> MiddlewareResult { - let https = az_url.as_str().replacen("az://", "https://", 1); - Url::parse(&https).map_err(|e| { - reqwest_middleware::Error::Middleware(anyhow::anyhow!( - "failed to parse constructed azure URL '{https}': {e}" - )) - }) + /// Resolve an `az://` request URL to the channel URL it names and the options + /// configured for its host. + /// + /// Going through [`AzureChannelUrl`] is what keeps this middleware from owning + /// a second copy of rules that live in `rattler_azure`: that parser is what + /// rejects userinfo, and it normalizes the authority into the exact spelling + /// the options table is keyed by, so a grant cannot miss over case, a trailing + /// dot, an IDNA name or an IP literal written oddly. + /// + /// [`AzureEndpointOptions::addressing`] is deliberately unused here: the fetch + /// path never needs an account name, it only forwards a path. Addressing + /// matters to the write path, which derives coordinates via + /// `rattler_azure::account_and_container`. + fn resolve(&self, url: &Url) -> MiddlewareResult<(AzureChannelUrl, AzureEndpointOptions)> { + let channel = AzureChannelUrl::parse(url.as_str()).map_err(|e| { + // The URL is not echoed back: the one rejection a user hits here is + // userinfo, and quoting it would print their password. + reqwest_middleware::Error::Middleware(anyhow::Error::from(e)) + })?; + let options = self + .options + .get(channel.host()) + .copied() + .unwrap_or_default(); + Ok((channel, options)) } /// Whether the URL already carries an explicit SAS token (a `sig` query @@ -180,33 +157,24 @@ impl AzureMiddleware { url.query_pairs().any(|(key, _)| key == "sig") } - /// Whether the URL carries userinfo (`user` and/or `:pass` before the host). - /// Because the host is trusted verbatim, a `user:pass@host` authority is a - /// host-spoofing vector and must be refused. Userinfo in a blob URL is invalid - /// regardless and safe to ignore. - fn has_userinfo(url: &Url) -> bool { - !url.username().is_empty() || url.password().is_some() - } - - /// Sign a reqwest `Request` in place using reqsign. + /// Sign a reqwest `Request` in place using reqsign, when `auth` grants it. /// - /// Two cases short-circuit without invoking reqsign at all: + /// Two cases return without invoking reqsign at all: /// - The URL already carries an explicit SAS (`?...&sig=...`). Signing would /// add an `Authorization` header that Azure prefers over the SAS, silently /// overriding the caller's explicit token. - /// - No credential source was detected (`credential_source_present` is - /// `false`). The request is sent unsigned so public/anonymous containers - /// stay reachable — and crucially, credential *resolution* is skipped - /// entirely. Otherwise reqsign would probe the managed-identity / IMDS - /// endpoint and block until it times out (~30s on a machine with no - /// metadata service) before we could fall back, making every anonymous - /// public-channel read pay that timeout. + /// - [`Auth::Anonymous`] — no grant. Crucially the credential is not *resolved* + /// either: reqsign would otherwise probe the managed-identity / IMDS endpoint + /// and block until it times out (~30s on a machine with no metadata service) + /// before we could decide not to use the result, making every anonymous + /// public-channel read pay that timeout — and it would pull an ambient + /// credential into memory for a host the user never granted. /// - /// If a credential source *is* configured but signing fails with - /// [`reqsign_core::ErrorKind::CredentialInvalid`] (a broken key/token, not - /// an absent one), the error is propagated rather than downgraded to an - /// unsigned request. - async fn sign(&self, req: &mut Request) -> MiddlewareResult<()> { + /// Under [`Auth::DefaultChain`] any signing failure is propagated. reqsign + /// collapses "no credential" and "broken credential" into the same + /// [`reqsign_core::ErrorKind::CredentialInvalid`], and since the user asked for + /// signing there is no case left where going anonymous is the right answer. + async fn sign(&self, req: &mut Request, auth: Auth) -> MiddlewareResult<()> { if Self::has_sas_token(req.url()) { return Ok(()); } @@ -216,13 +184,18 @@ impl AzureMiddleware { .insert("x-ms-version", http::HeaderValue::from_static(X_MS_VERSION)); } - // No credential source detected: send unsigned without probing. See the - // doc comment above — probing here would block on the IMDS timeout. - if !self.credential_source_present { - tracing::debug!( - "no Azure credential source detected; sending `az://` request unsigned" - ); - return Ok(()); + match auth { + Auth::Anonymous => { + // The authority, not `host_str()`: a message naming a host the user + // could act on must carry the port, or it names a host that is not + // the one in their config. + tracing::debug!( + "no `azure-options` auth grant for `{}`; sending `az://` request unsigned", + req.url().authority() + ); + return Ok(()); + } + Auth::DefaultChain => {} } let mut builder = http::Request::builder() @@ -238,10 +211,6 @@ impl AzureMiddleware { })?; let (mut parts, ()) = http_req.into_parts(); - // A credential source is present (the absent case short-circuited to - // unsigned above). reqsign reports a broken key/token as - // `CredentialInvalid`; that must surface as a hard error rather than - // silently going anonymous, so any signing failure is propagated. self.signer .sign(&mut parts, None) .await @@ -272,71 +241,161 @@ impl Middleware for AzureMiddleware { return next.run(req, extensions).await; } - // The host is trusted verbatim as the request target, so userinfo is a - // host-spoofing vector (`az://user:pass@real.host/...` can hide the real - // authority). Reject it before rewriting or signing. This mirrors the - // same rejection in `rattler_azure::account_and_container`; the check is - // inlined here because this middleware does not depend on rattler_azure. - if Self::has_userinfo(req.url()) { - return Err(reqwest_middleware::Error::Middleware(anyhow::anyhow!( - "userinfo is not allowed in `az://` URLs (host-spoofing vector); \ - remove the `user:pass@` component" - ))); + let (channel, options) = self.resolve(req.url())?; + *req.url_mut() = channel.wire(options.scheme); + self.sign(&mut req, options.auth).await?; + + let response = next.run(req, extensions).await?; + + // Azure answers an unauthorized read of a private container with 404, not + // 403, so "no grant" and "no such blob" are the same status on the wire. + // Say so once, naming the config the user would have to write — spelled + // through `AzureHost` so the key printed is the key a lookup arrives with. + if response.status() == http::StatusCode::NOT_FOUND && !options.auth.is_granted() { + // One line, and spelled the way `AzureUrlError::InvalidHost` spells its + // fix: a wrapped multi-line hint is harder to grep out of a log, and + // the two guided messages should read as the same instruction. + tracing::warn!( + "`{}` returned 404 and this host has no `azure-options` auth grant. Azure answers \ + an anonymous read of a *private* container with 404 rather than 403, so a missing \ + grant looks exactly like a missing file. If the container is private, grant it in \ + your user configuration with `[azure-options.\"{}\"]` and `auth = true`.", + channel.canonical(), + channel.host() + ); } - let https_url = Self::rewrite_url(&req.url().clone())?; - *req.url_mut() = https_url; - self.sign(&mut req).await?; - next.run(req, extensions).await + Ok(response) } } #[cfg(test)] mod tests { + use rattler_azure::AzureScheme; + use super::*; + /// The `azure-options` table for one host, as a caller would build it. + fn options( + authority: &str, + options: AzureEndpointOptions, + ) -> HashMap { + HashMap::from([(AzureHost::parse(authority).expect("test host"), options)]) + } + + /// A grant with everything else defaulted: anonymous is the only interesting + /// axis in most of these tests. + fn granted() -> AzureEndpointOptions { + AzureEndpointOptions { + auth: Auth::DefaultChain, + ..Default::default() + } + } + + fn middleware(options: HashMap) -> AzureMiddleware { + AzureMiddleware::new(Client::new(), options) + } + + /// Resolve a URL and hand back the wire spelling its options ask for. + fn wire_of(middleware: &AzureMiddleware, url: &str) -> String { + let (channel, options) = middleware + .resolve(&Url::parse(url).expect("test url")) + .expect("url should resolve"); + channel.wire(options.scheme).to_string() + } + + /// With no entry the scheme defaults to https, and path, query and fragment + /// survive the rewrite untouched. #[test] - fn swaps_scheme_to_https() { - let rewritten = AzureMiddleware::rewrite_url( - &Url::parse("az://myacct.blob.core.windows.net/mychannel/noarch/repodata.json") - .unwrap(), - ) - .unwrap(); + fn rewrites_to_https_without_an_entry() { + let middleware = middleware(HashMap::new()); assert_eq!( - rewritten.as_str(), + wire_of( + &middleware, + "az://myacct.blob.core.windows.net/mychannel/noarch/repodata.json" + ), "https://myacct.blob.core.windows.net/mychannel/noarch/repodata.json" ); - } - - #[test] - fn preserves_query_and_fragment() { - let rewritten = AzureMiddleware::rewrite_url( - &Url::parse("az://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc#frag").unwrap(), - ) - .unwrap(); assert_eq!( - rewritten.as_str(), + wire_of( + &middleware, + "az://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc#frag" + ), "https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc#frag" ); } + /// An emulator entry is the only thing that can send an `az://` URL in + /// cleartext, and the port has to survive — `:10000` is not any scheme's + /// default, but `:443` would be under https and must not be dropped either. #[test] - fn rewrites_azurite_style_host_and_port() { - let rewritten = AzureMiddleware::rewrite_url( - &Url::parse("az://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json").unwrap(), - ) - .unwrap(); + fn rewrites_to_http_for_an_emulator_entry() { + let emulator = middleware(options( + "127.0.0.1:10000", + AzureEndpointOptions { + auth: Auth::DefaultChain, + scheme: AzureScheme::Http, + addressing: rattler_azure::Addressing::PathStyle, + }, + )); assert_eq!( - rewritten.as_str(), + wire_of( + &emulator, + "az://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" + ), + "http://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" + ); + + // The same host with no entry stays on https: an emulator grant must not + // generalize to a scheme downgrade for anyone else. + assert_eq!( + wire_of( + &middleware(HashMap::new()), + "az://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" + ), "https://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" ); } + /// A grant written in any spelling of a host must apply to a request for that + /// host: a silent miss reads as a 404, i.e. "not found" for what is really + /// "not authorized". Delegating to `AzureHost` on both sides is what buys this. + #[test] + fn a_grant_applies_regardless_of_how_the_host_is_spelled() { + let middleware = middleware(options("MyCompany.blob.core.windows.net.", granted())); + let (_, resolved) = middleware + .resolve(&Url::parse("az://mycompany.blob.core.windows.net/c/x.json").unwrap()) + .unwrap(); + assert!(resolved.auth.is_granted()); + } + + /// Userinfo is refused before any rewrite or signing: the host is the request + /// target verbatim, so `user:pass@real.host` can hide the real authority. + /// (Rejection lives in `AzureHost::parse`, so there is one copy of the rule.) + #[test] + fn rejects_userinfo() { + let middleware = middleware(HashMap::new()); + for url in [ + "az://user:pass@acct.blob.core.windows.net/c/x.json", + "az://user@acct.blob.core.windows.net/c/x.json", + ] { + let err = middleware + .resolve(&Url::parse(url).unwrap()) + .expect_err("userinfo must be refused"); + assert!(err.to_string().contains("userinfo"), "{err}"); + } + assert!( + middleware + .resolve(&Url::parse("az://acct.blob.core.windows.net/c/x.json").unwrap()) + .is_ok() + ); + } + #[tokio::test] async fn passes_through_non_az_schemes_unchanged() { use reqwest_middleware::ClientBuilder; let client = ClientBuilder::new(Client::new()) - .with(AzureMiddleware::new(Client::new())) + .with(middleware(HashMap::new())) .build(); // A non-`az` request must not be rewritten; it should be attempted as-is // (and fail on DNS), proving the middleware left it untouched. @@ -347,28 +406,35 @@ mod tests { assert!(result.is_err()); } - #[test] - fn detects_sas_token_in_query() { - assert!(AzureMiddleware::has_sas_token( - &Url::parse("https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc").unwrap() - )); - assert!(!AzureMiddleware::has_sas_token( - &Url::parse("https://acct.blob.core.windows.net/c/x.json?sv=2021").unwrap() - )); - } - - /// With no credential resolvable, a signable `az://` request must be passed - /// through UNSIGNED (not errored), so public/anonymous containers work with - /// zero ambient credentials. An empty provider chain resolves nothing, which - /// reqsign reports as `CredentialInvalid`. + /// A host with no grant is sent unsigned, and its credential is never even + /// resolved — so nothing blocks on the IMDS probe and no ambient credential is + /// pulled into memory for a host the user never named. The provider flips a + /// flag if it is ever asked. #[tokio::test] - async fn passes_request_through_unsigned_when_no_credential() { - use reqsign_core::ProvideCredentialChain; + async fn an_ungranted_host_sends_unsigned_without_resolving_a_credential() { + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; + #[derive(Debug)] + struct RecordingProvider(Arc); + impl ProvideCredential for RecordingProvider { + type Credential = Credential; + async fn provide_credential( + &self, + _ctx: &Context, + ) -> reqsign_core::Result> { + self.0.store(true, Ordering::SeqCst); + Ok(None) + } + } + + let probed = Arc::new(AtomicBool::new(false)); let middleware = AzureMiddleware::with_credential_provider( Client::new(), - ProvideCredentialChain::::new(), - false, + RecordingProvider(probed.clone()), + HashMap::new(), ); let mut req = Client::new() .get("https://acct.blob.core.windows.net/pub/noarch/repodata.json") @@ -376,10 +442,14 @@ mod tests { .unwrap(); middleware - .sign(&mut req) + .sign(&mut req, Auth::Anonymous) .await - .expect("a request with no resolvable credential must pass through unsigned"); + .expect("an ungranted request must pass through unsigned"); + assert!( + !probed.load(Ordering::SeqCst), + "credential provider must not be probed without a grant" + ); assert!( req.headers().get(http::header::AUTHORIZATION).is_none(), "unsigned request must not carry an Authorization header" @@ -390,145 +460,163 @@ mod tests { ); } - /// A URL that already carries a SAS token must not be re-signed even when a - /// credential is available: no `Authorization` header is added. + /// A granted host is actually signed: the credential resolves and the request + /// comes back carrying Shared Key authorization. #[tokio::test] - async fn does_not_sign_url_that_already_has_sas() { + async fn a_granted_host_is_signed() { use reqsign_azure_storage::StaticCredentialProvider; - // A valid base64 account key so the static provider yields a usable - // SharedKey credential that would otherwise sign the request. let middleware = AzureMiddleware::with_credential_provider( Client::new(), + // A valid base64 account key, so the provider yields a usable + // SharedKey credential. StaticCredentialProvider::new_shared_key("acct", "dGVzdF9rZXk="), - true, + options("acct.blob.core.windows.net", granted()), ); let mut req = Client::new() - .get("https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc") + .get("https://acct.blob.core.windows.net/c/noarch/repodata.json") .build() .unwrap(); - middleware.sign(&mut req).await.unwrap(); + middleware.sign(&mut req, Auth::DefaultChain).await.unwrap(); + let authorization = req + .headers() + .get(http::header::AUTHORIZATION) + .expect("a granted host must be signed"); assert!( - req.headers().get(http::header::AUTHORIZATION).is_none(), - "a URL carrying an explicit SAS must not be re-signed" - ); - assert!( - !req.headers().contains_key("x-ms-version"), - "a self-authenticating SAS URL is left untouched" + authorization.to_str().unwrap().starts_with("SharedKey "), + "{authorization:?}" ); } - /// A URL carrying userinfo must be recognised so the fetch path can reject - /// it: the host is trusted verbatim, so `user:pass@host` is a host-spoofing - /// vector. (A request built through reqwest's client strips userinfo into a - /// header before the middleware runs, so the predicate — not the whole - /// client path — is what guards direct `Request` construction.) - #[test] - fn detects_userinfo_in_url() { - assert!(AzureMiddleware::has_userinfo( - &Url::parse("az://user:pass@acct.blob.core.windows.net/c/x.json").unwrap() - )); - assert!(AzureMiddleware::has_userinfo( - &Url::parse("az://user@acct.blob.core.windows.net/c/x.json").unwrap() - )); - assert!(!AzureMiddleware::has_userinfo( - &Url::parse("az://acct.blob.core.windows.net/c/x.json").unwrap() - )); - } - - /// With no credential source detected, `sign` must NOT invoke the credential - /// provider at all — resolution is skipped so anonymous reads don't block on - /// the IMDS timeout. Uses a provider that flips a flag if it is ever asked. + /// The inversion this design turns on: with a grant, an unusable credential is + /// a hard error. It must never degrade to an anonymous request, which Azure + /// would answer with a 404 the user has no way to read as "auth failed". An + /// empty provider chain resolves nothing, which reqsign reports the same way it + /// reports a broken credential. #[tokio::test] - async fn skips_credential_resolution_when_no_source() { - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; - - #[derive(Debug)] - struct RecordingProvider(Arc); - impl ProvideCredential for RecordingProvider { - type Credential = Credential; - async fn provide_credential( - &self, - _ctx: &Context, - ) -> reqsign_core::Result> { - self.0.store(true, Ordering::SeqCst); - Ok(None) - } - } + async fn a_granted_host_with_broken_credentials_is_a_hard_error() { + use reqsign_core::ProvideCredentialChain; - let probed = Arc::new(AtomicBool::new(false)); let middleware = AzureMiddleware::with_credential_provider( Client::new(), - RecordingProvider(probed.clone()), - false, + ProvideCredentialChain::::new(), + options("acct.blob.core.windows.net", granted()), ); let mut req = Client::new() - .get("https://acct.blob.core.windows.net/pub/noarch/repodata.json") + .get("https://acct.blob.core.windows.net/c/noarch/repodata.json") .build() .unwrap(); - middleware.sign(&mut req).await.unwrap(); + let result = middleware.sign(&mut req, Auth::DefaultChain).await; assert!( - !probed.load(Ordering::SeqCst), - "credential provider must not be probed when no source is detected" + result.is_err(), + "a granted-but-failing credential must be a hard error, not unsigned" ); assert!( req.headers().get(http::header::AUTHORIZATION).is_none(), - "unsigned request must not carry an Authorization header" + "a failed signing attempt must not leave a partial Authorization header" ); } - /// A persisted `az login` profile counts as a credential source (so a - /// broken az-login session hard-errors instead of going anonymous), while - /// an empty config dir does not. - #[test] - fn detects_az_login_profile_on_disk() { - let dir = tempfile::tempdir().unwrap(); - temp_env::with_var("AZURE_CONFIG_DIR", Some(dir.path().as_os_str()), || { - assert!( - !azure_cli_session_present(), - "no profile file yet ⇒ not a credential source" - ); - std::fs::write(dir.path().join("azureProfile.json"), "{}").unwrap(); - assert!( - azure_cli_session_present(), - "azureProfile.json present ⇒ credential source" - ); - }); - } - - /// When a credential source is detected but signing fails - /// (`CredentialInvalid`), the failure must be a hard error rather than an - /// unsigned fallback: a broken credential must not silently go anonymous. An - /// empty provider chain yields `CredentialInvalid`, standing in for a broken - /// credential. + /// A URL that already carries a SAS token must not be re-signed even where the + /// host is granted: Azure prefers an `Authorization` header over the SAS, so + /// signing would silently override the caller's explicit token. #[tokio::test] - async fn errors_when_credential_source_present_but_signing_fails() { - use reqsign_core::ProvideCredentialChain; + async fn a_sas_in_the_url_passes_through() { + use reqsign_azure_storage::StaticCredentialProvider; let middleware = AzureMiddleware::with_credential_provider( Client::new(), - ProvideCredentialChain::::new(), - true, + StaticCredentialProvider::new_shared_key("acct", "dGVzdF9rZXk="), + options("acct.blob.core.windows.net", granted()), ); let mut req = Client::new() - .get("https://acct.blob.core.windows.net/c/noarch/repodata.json") + .get("https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc") .build() .unwrap(); - let result = middleware.sign(&mut req).await; + middleware.sign(&mut req, Auth::DefaultChain).await.unwrap(); assert!( - result.is_err(), - "a configured-but-failing credential must be a hard error, not unsigned" + req.headers().get(http::header::AUTHORIZATION).is_none(), + "a URL carrying an explicit SAS must not be re-signed" ); assert!( - req.headers().get(http::header::AUTHORIZATION).is_none(), - "a failed signing attempt must not leave a partial Authorization header" + !req.headers().contains_key("x-ms-version"), + "a self-authenticating SAS URL is left untouched" + ); + assert_eq!( + req.url().as_str(), + "https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc" + ); + } + + /// Serve 404 for everything, over http on localhost, standing in for a private + /// container answering an anonymous read. + async fn spawn_404_server() -> AzureHost { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = axum::Router::new().fallback(axum::http::StatusCode::NOT_FOUND); + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + AzureHost::parse(&addr.to_string()).unwrap() + } + + /// An emulator-shaped entry (http, path-style) with the grant taken from the + /// caller, so one server can exercise both sides of the hint. + fn emulator_entry(auth: Auth) -> AzureEndpointOptions { + AzureEndpointOptions { + auth, + scheme: AzureScheme::Http, + addressing: rattler_azure::Addressing::PathStyle, + } + } + + async fn get_az(middleware: AzureMiddleware, host: &AzureHost) -> reqwest::StatusCode { + reqwest_middleware::ClientBuilder::new(Client::new()) + .with(middleware) + .build() + .get(format!( + "az://{host}/devstoreaccount1/c/noarch/repodata.json" + )) + .send() + .await + .expect("request through azure middleware failed") + .status() + } + + /// The 404 hint must name the config block to add, keyed exactly as the table + /// is keyed — including the port, which an earlier version of this hint + /// dropped, printing a key that could never match. + #[tokio::test] + #[tracing_test::traced_test] + async fn the_404_hint_names_the_config_block_for_an_ungranted_host() { + let host = spawn_404_server().await; + let middleware = middleware(options(&host.to_string(), emulator_entry(Auth::Anonymous))); + + assert_eq!(get_az(middleware, &host).await, 404); + + assert!(logs_contain(&format!("[azure-options.\"{host}\"]"))); + assert!(logs_contain("auth = true")); + } + + /// With a grant in place a 404 means what it says, so the hint would be noise. + #[tokio::test] + #[tracing_test::traced_test] + async fn the_404_hint_is_silent_for_a_granted_host() { + use reqsign_azure_storage::StaticCredentialProvider; + + let host = spawn_404_server().await; + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + StaticCredentialProvider::new_shared_key("devstoreaccount1", "dGVzdF9rZXk="), + options(&host.to_string(), emulator_entry(Auth::DefaultChain)), ); + + assert_eq!(get_az(middleware, &host).await, 404); + + assert!(!logs_contain("azure-options")); } } diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs index b5f5ba40f3..7b3cafff51 100644 --- a/crates/rattler_networking/tests/azure_real_fetch.rs +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -18,6 +18,9 @@ //! (overrides the default `{account}.blob.core.windows.net` host). #![cfg(feature = "azure")] +use std::collections::HashMap; + +use rattler_azure::{Auth, AzureEndpointOptions, AzureHost}; use rattler_networking::AzureMiddleware; use reqwest_middleware::ClientBuilder; @@ -31,8 +34,20 @@ async fn azure_middleware_fetches_real_repodata() { let host = std::env::var("AZURE_TEST_HOST") .unwrap_or_else(|_| format!("{account}.blob.core.windows.net")); + // The grant is what makes this the AAD test rather than an anonymous read: an + // `azure-options` entry for the host is the only thing that lets the + // `az login` credential attach to it, and it makes a broken credential fail + // loudly instead of falling through to an unsigned 404. + let options = HashMap::from([( + AzureHost::parse(&host).expect("AZURE_TEST_HOST is not a valid host[:port]"), + AzureEndpointOptions { + auth: Auth::DefaultChain, + ..Default::default() + }, + )]); + let client = ClientBuilder::new(reqwest::Client::new()) - .with(AzureMiddleware::new(reqwest::Client::new())) + .with(AzureMiddleware::new(reqwest::Client::new(), options)) .build(); // The `az://` host carries the full blob endpoint — same form used in a diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index 4389518483..fc1c1a6047 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -159,7 +159,11 @@ def __repr__(self) -> str: class AzureMiddleware: """ - Middleware to work with az:// URLs + Middleware to work with az:// URLs. + + Fetches are anonymous: the per-host `azure-options` table (credential grants, + `scheme`, `path-style`) is not exposed to Python yet — that mirrors what + `S3Config` does for buckets and is follow-up work. Examples -------- diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index bfc65c5b29..5e8f2d5f37 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -85,7 +85,12 @@ impl PyClientWithMiddleware { client = client.with(GCSMiddleware::from(middleware)); } PyMiddleware::Azure(_middleware) => { - client = client.with(AzureMiddleware::new(reqwest_client.clone())); + // Anonymous only: the options table is not exposed to Python + // yet. See `PyAzureMiddleware`. + client = client.with(AzureMiddleware::new( + reqwest_client.clone(), + std::collections::HashMap::new(), + )); } PyMiddleware::S3(middleware) => { client = client.with(S3Middleware::new( diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index 66428068ef..cbc4ac12b5 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -125,6 +125,11 @@ impl From for GCSMiddleware { } } +/// Azure middleware with no `azure-options` entries, i.e. every `az://` fetch is +/// anonymous over https. +/// +/// Exposing the per-host options table to Python (grants, `scheme`, `path-style`) +/// mirrors what `PyS3Config` does for buckets, and is follow-up work. #[pyclass(from_py_object)] #[repr(transparent)] #[derive(Clone)] From f0fd17bb7dad5e6b5d6008be1862b7803ef3db79 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:26:28 +1000 Subject: [PATCH 68/98] feat(azure): honour path-style addressing in index and upload plumbing --- Cargo.lock | 1 + crates/rattler_azure/src/lib.rs | 231 +++++++++++++++++----- crates/rattler_index/README.md | 45 ++++- crates/rattler_index/src/lib.rs | 41 ++-- crates/rattler_index/src/main.rs | 155 +++++++++++++-- crates/rattler_upload/src/lib.rs | 15 +- crates/rattler_upload/src/upload/azure.rs | 42 ++-- crates/rattler_upload/src/upload/opt.rs | 22 +-- py-rattler/Cargo.lock | 1 + 9 files changed, 423 insertions(+), 130 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac812889b6..f1a16d1fa9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5375,6 +5375,7 @@ dependencies = [ "itertools 0.15.0", "keyring-core", "netrc-rs", + "rattler_azure", "rattler_config", "regex", "reqsign-azure-storage", diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 5ca4ce3d9c..253961c0d5 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -518,9 +518,12 @@ impl From for String { /// [`parse`](Self::parse) runs as a clap `value_parser` — before any config file /// is read. A stored scheme would therefore have to be a guess made at parse time /// and corrected later, which is exactly the drift this type exists to prevent. -/// Passing it in at call time keeps the choice at the site that makes it. Nothing -/// ties the argument to an options entry, and until options are threaded through -/// the CLI both callers pass [`AzureScheme::Https`]. +/// Passing it in at call time keeps the choice at the site that makes it. +/// +/// Nothing in the type ties the argument to an options entry. `rattler-index` +/// takes it from the channel host's entry; `rattler_upload` passes the default, +/// because it reads no config file at all (see the `ponytail:` note in +/// `rattler_upload::upload_from_args`). #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct AzureChannelUrl { /// The authority, normalized independently of any scheme. @@ -588,12 +591,11 @@ impl AzureChannelUrl { /// The `az://host/path` spelling: the channel's identity. /// - /// This is what users write and what is shown back to them, and it is the - /// spelling config keys are meant to be matched against. Today only the - /// `azure-options` host key actually goes through this type (via - /// [`Self::host`]); `rattler_index` still matches `[index-config."…"]` against - /// the https wire string, which is reviewer issue 5 and moves onto - /// `canonical()` in the index/upload plumbing step. + /// This is what users write, what is shown back to them, and what config keys + /// are matched against: `rattler-index` resolves `[index-config."az://…"]` + /// through this spelling, and `[azure-options."…"]` through [`Self::host`]. + /// Matching the wire string instead was reviewer issue 5 — the two spellings + /// exist so a config key never has to guess which one a channel was stored as. pub fn canonical(&self) -> Url { self.spelled("az") } @@ -665,54 +667,77 @@ fn strip_az_scheme(value: &str) -> Option<&str> { } /// Build an opendal [`AzblobConfig`](opendal::services::AzblobConfig) from a -/// channel URL and credentials. +/// channel URL, the endpoint options of its host, and credentials. +/// +/// The account name, endpoint, container and root prefix are all derived from the +/// channel URL, read the way `options.addressing` says to read it and reached over +/// `options.scheme`; the credentials supply only the account key or SAS token. +/// `options.auth` is not consulted — this is the write path, where the credential +/// has already been chosen by the caller. +/// +/// Taking the [`AzureChannelUrl`] rather than a wire `Url` is what keeps the +/// scheme in the config from disagreeing with the scheme in the endpoint: both +/// come from the same `options`. +/// +/// # The two addressing shapes +/// +/// opendal's azblob core builds every request URI as `{endpoint}/{container}/{path}` +/// and its core struct carries no account field at all, so under path-style the +/// account can only reach the URL through `endpoint`: /// -/// The account name, endpoint, container, and root prefix are all derived from the -/// URL; the credentials supply only the account key or SAS token. The URL is -/// expected to already be validated and in wire form (see -/// [`AzureChannelUrl::wire`]). +/// - [`Addressing::HostStyle`]: `endpoint` is `{scheme}://{host}[:{port}]`, the +/// account is the first host label, and `root` is the path after the container. +/// - [`Addressing::PathStyle`]: `endpoint` is +/// `{scheme}://{host}[:{port}]/{account}`, the account is the first path +/// segment, and `root` is the path after *both* the account and the container. /// -/// **Host-style addressing only.** The URL must read as -/// `.blob.//`; an IP or single-label endpoint -/// is rejected by [`account_and_container`]. +/// `account_name` is set under both styles, and is mandatory under both: opendal +/// infers it only from three known Azure suffixes and returns `None` — not an +/// error — for anything else, so omitting it from a path-style config makes +/// shared-key signing quietly never engage, and the failure surfaces as a 403 +/// rather than as a config error. +/// +/// Neither endpoint ends in a slash. `AzblobBuilder::endpoint` trims one, but this +/// builds the config struct literally, where nothing does, and a stray slash would +/// yield `//{container}/…`. #[cfg(feature = "opendal")] pub fn azblob_config( credentials: &AzureCredentials, - channel: &Url, + channel: &AzureChannelUrl, + options: AzureEndpointOptions, ) -> Result { - // ponytail: host-style is hardcoded, which matches every caller today — none - // has an options entry to pass in yet. The hazard is not an IP or emulator - // host, which `account_and_container` rejects outright: it is a *dotted* host - // whose options say `path-style = true`. That combination does not error here. - // It reads the account from the host label, the container from the segment that - // actually holds the account, and `root` then skips one segment too few — so - // writes land in the wrong container. Honouring path-style also needs the - // endpoint built differently: opendal appends `container/root` to `endpoint`, - // so the account has to be appended to the *endpoint* (`http://host/`). - // Both are deferred to the plumbing step that threads the options entry here. - let AzureCoordinates { account, container } = - account_and_container(channel, Addressing::HostStyle)?; - - // Preserve a non-default port if one is present; real Azure uses the scheme - // default (443). - let host = channel.host_str().ok_or(AzureUrlError::NoHost)?; - let authority = match channel.port() { - Some(port) => format!("{host}:{port}"), - None => host.to_string(), + let wire = channel.wire(options.scheme); + let AzureCoordinates { account, container } = account_and_container(&wire, options.addressing)?; + + // The authority comes from `AzureHost`, not from the wire URL: a `Url` has + // already dropped a port equal to its scheme's default, so reading it back + // would turn a written `:443` into no port at all. + let authority = channel.host(); + let endpoint = match options.addressing { + Addressing::HostStyle => format!("{}://{authority}", options.scheme), + Addressing::PathStyle => format!("{}://{authority}/{account}", options.scheme), + }; + + // Root prefix = the path after the segments the coordinates already consumed: + // the container, plus the account when path-style put it in the path. Skipping + // one too few there leaves the account segment inside `root`, so every blob is + // written one directory deeper than the channel actually lives — silently, and + // in the right container, which is what makes it hard to spot. + let consumed = match options.addressing { + Addressing::HostStyle => 1, + Addressing::PathStyle => 2, }; - // Root prefix = the path after the container segment. Percent-decode each - // segment: `path_segments()` yields still-encoded segments, and opendal - // percent-encodes the root again, so passing them through verbatim would - // double-encode prefixes containing spaces or `+`. `account_and_container` - // has already confirmed there is at least the container segment. + // Percent-decode each segment: `path_segments()` yields still-encoded segments + // and opendal percent-encodes `root + path` again, so passing them through + // verbatim would double-encode a prefix containing a space or a `+`. + // `account_and_container` has already confirmed the consumed segments exist. let root = format!( "/{}", - channel - .path_segments() + wire.path_segments() .into_iter() .flatten() - .skip(1) + .skip(consumed) .map(|segment| percent_encoding::percent_decode_str(segment).decode_utf8_lossy()) .collect::>() .join("/") @@ -724,7 +749,7 @@ pub fn azblob_config( }; Ok(opendal::services::AzblobConfig { - endpoint: Some(format!("{}://{}", channel.scheme(), authority)), + endpoint: Some(endpoint), account_name: Some(account), container, root: Some(root), @@ -1308,6 +1333,120 @@ mod tests { hasher.finish() } + /// The path-style recipe, asserted string by string, because every field but + /// `container` differs from host-style and each one fails silently when it is + /// wrong: a missing `account_name` becomes a 403, a trailing slash becomes + /// `//container/…`, and a `root` that skips one segment too few writes the + /// whole channel one directory too deep. + #[cfg(feature = "opendal")] + #[test] + fn azblob_config_under_path_style() { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general/mychannel") + .unwrap(); + let options = AzureEndpointOptions { + auth: Auth::DefaultChain, + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }; + + let config = azblob_config( + &AzureCredentials::AccountKey("key".to_string()), + &channel, + options, + ) + .unwrap(); + + assert_eq!( + config.endpoint.as_deref(), + Some("http://127.0.0.1:10000/devstoreaccount1") + ); + assert_eq!(config.account_name.as_deref(), Some("devstoreaccount1")); + assert_eq!(config.container, "general"); + assert_eq!(config.root.as_deref(), Some("/mychannel")); + assert_eq!(config.account_key.as_deref(), Some("key")); + + let endpoint = config.endpoint.unwrap(); + assert!(!endpoint.ends_with('/'), "{endpoint}"); + let root = config.root.unwrap(); + assert!( + !root.contains("general"), + "the container must not appear in the root: {root}" + ); + assert!( + !root.contains("devstoreaccount1"), + "the account must not appear in the root: {root}" + ); + } + + /// A channel that is a bare `account/container` leaves nothing for the root, + /// which must still be `/` and not the empty string opendal would treat as a + /// relative path. + #[cfg(feature = "opendal")] + #[test] + fn azblob_config_path_style_without_a_prefix() { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + let config = azblob_config( + &AzureCredentials::SasToken("?sv=token".to_string()), + &channel, + AzureEndpointOptions { + auth: Auth::Anonymous, + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) + .unwrap(); + + assert_eq!(config.root.as_deref(), Some("/")); + assert_eq!(config.container, "general"); + // The leading `?` is stripped exactly once, wherever the token came from. + assert_eq!(config.sas_token.as_deref(), Some("sv=token")); + } + + /// Host-style is the shape every existing caller uses, so honouring + /// path-style must not have moved it. + #[cfg(feature = "opendal")] + #[test] + fn azblob_config_under_host_style_is_unchanged() { + let channel = + AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general/sub/dir") + .unwrap(); + let config = azblob_config( + &AzureCredentials::SasToken("sv=token".to_string()), + &channel, + AzureEndpointOptions::default(), + ) + .unwrap(); + + assert_eq!( + config.endpoint.as_deref(), + Some("https://stcondachannel.blob.core.windows.net") + ); + assert_eq!(config.account_name.as_deref(), Some("stcondachannel")); + assert_eq!(config.container, "general"); + assert_eq!(config.root.as_deref(), Some("/sub/dir")); + assert_eq!(config.sas_token.as_deref(), Some("sv=token")); + assert_eq!(config.account_key, None); + } + + /// A prefix with a space arrives here percent-encoded and opendal encodes + /// `root + path` again, so the root has to be handed over decoded. + #[cfg(feature = "opendal")] + #[test] + fn azblob_config_decodes_the_root() { + let channel = + AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/with%20space").unwrap(); + let config = azblob_config( + &AzureCredentials::AccountKey("key".to_string()), + &channel, + AzureEndpointOptions::default(), + ) + .unwrap(); + + assert_eq!(config.root.as_deref(), Some("/with space")); + } + /// Account and container derivation is deliberately *not* part of parsing: the /// addressing style is config that does not exist yet when clap parses the /// argument, so an emulator URL must survive parsing and be rejected (or not) diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index d3eb1f292f..9b73671b41 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -74,16 +74,41 @@ region = "eu-central-1" force-path-style = false ``` -Azure Blob Storage needs no such block: the storage account and blob endpoint -are read directly from the channel URL -(`az://.blob.core.windows.net//`), so the account, -container, and endpoint (including sovereign clouds) are fully determined by the -URL you pass. The `az://` scheme is required and is rewritten to `https://` -internally; a bare `https://` URL is rejected. The host must be a dotted -`.blob.` domain, so IP-literal / single-label hosts (and hence -the Azurite emulator) are not supported. Credentials are never stored in the -config — they are resolved at runtime from `--account-key` / `--sas-token`, an -`az login` session (`--azure-cli`), or the `DefaultCredentialProvider` chain. +Azure Blob channels need no block for the common case: the account, container and +endpoint (including sovereign clouds) are all read from the channel URL +`az://.blob.core.windows.net//`. The `az://` scheme +is required and is rewritten to `https://` for the request; a bare `https://` URL +is rejected. Credentials are never stored in the config — they are resolved at +runtime from `--account-key` / `--sas-token`, an `az login` session +(`--azure-cli`), or the `DefaultCredentialProvider` chain. + +A host that is not a plain `.blob.` domain, or that is not +reached over https, needs an entry under `[azure-options.""]`. The key is +the host with its port when the URL has one: + +| Field | Type | Description | +| --- | --- | --- | +| `auth` | boolean | Whether credentials may be sent to this host. Defaults to `false`, which fetches anonymously. This is the only way a credential attaches to a host, so keep these entries in your user-level config file, never in a checked-in project file. | +| `scheme` | string | The scheme `az://` is rewritten to: `"https"` (default) or `"http"`. Use `http` for local emulators only. | +| `path-style` | boolean | Where the storage account is found. `false` (default) reads it from the first host label. `true` reads it from the first path segment instead, which is the only form that works for an IP-literal or single-label host. | + +Indexing a channel in the Azurite emulator needs all three: + +```toml +[azure-options."127.0.0.1:10000"] +auth = true +scheme = "http" +path-style = true +``` + +```shell +export AZURE_STORAGE_KEY= +rattler-index --config ./rattler-config.toml az \ + az://127.0.0.1:10000/devstoreaccount1/general/my-channel +``` + +Without the entry, that URL fails: host-style addressing cannot read an account +name out of `127.0.0.1`, and the error tells you which line to add. ## Per-channel index configuration diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index cb62c7e3fd..af6d22ef12 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -29,7 +29,7 @@ use opendal::layers::RetryLayer; use opendal::services::S3Config; use opendal::{Configurator, Operator, services::FsConfig}; #[cfg(feature = "azure")] -use rattler_azure::AzureCredentials; +use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpointOptions}; use rattler_conda_types::{ ChannelInfo, ChannelRelations, PackageRecord, PatchInstructions, Platform, RepoData, Shard, ShardedRepodata, ShardedSubdirInfo, UrlOrPath, V3Packages, WhlPackageRecord, @@ -1518,11 +1518,14 @@ pub async fn index_s3_with_channel_metadata( /// Configuration for `index_azure` #[cfg(feature = "azure")] pub struct IndexAzureConfig { - /// The channel to index, as an Azure Blob URL - /// (`https://.blob.core.windows.net//`). - pub channel: Url, + /// The channel to index, as an `az://` Azure Blob channel URL. + pub channel: AzureChannelUrl, /// The credentials to use for Azure Blob access. pub credentials: AzureCredentials, + /// The `azure-options` entry for the channel's host, which decides the wire + /// scheme and whether the account is read from the host or the path. The + /// defaults (https, host-style) describe real Azure. + pub options: AzureEndpointOptions, /// The target platform to index. pub target_platform: Option, /// The path to a repodata patch to apply to the index. @@ -1561,6 +1564,7 @@ pub async fn index_azure_with_channel_metadata( IndexAzureConfig { channel, credentials, + options, target_platform, repodata_patch, write_zst, @@ -1573,7 +1577,7 @@ pub async fn index_azure_with_channel_metadata( }: IndexAzureConfig, channel_metadata: ChannelMetadata, ) -> anyhow::Result<()> { - let azblob_config = rattler_azure::azblob_config(&credentials, &channel)?; + let azblob_config = rattler_azure::azblob_config(&credentials, &channel, options)?; let builder = azblob_config.into_builder(); let op = Operator::new(builder)?.layer(RetryLayer::new()).finish(); @@ -1902,10 +1906,13 @@ mod tests { #[test] fn azblob_config_derives_fields_from_url() { let channel = - Url::parse("https://stcondachannel.blob.core.windows.net/general/sub/dir").unwrap(); + AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general/sub/dir") + .unwrap(); let credentials = AzureCredentials::SasToken("sv=token".to_string()); - let config = rattler_azure::azblob_config(&credentials, &channel).unwrap(); + let config = + rattler_azure::azblob_config(&credentials, &channel, AzureEndpointOptions::default()) + .unwrap(); assert_eq!( config.endpoint.as_deref(), @@ -1921,10 +1928,13 @@ mod tests { #[cfg(feature = "azure")] #[test] fn azblob_config_container_only_url() { - let channel = Url::parse("https://stcondachannel.blob.core.windows.net/general").unwrap(); + let channel = + AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general").unwrap(); let credentials = AzureCredentials::AccountKey("key".to_string()); - let config = rattler_azure::azblob_config(&credentials, &channel).unwrap(); + let config = + rattler_azure::azblob_config(&credentials, &channel, AzureEndpointOptions::default()) + .unwrap(); assert_eq!(config.container, "general"); assert_eq!(config.root.as_deref(), Some("/")); @@ -1936,10 +1946,19 @@ mod tests { #[test] fn azblob_config_preserves_non_default_port() { let channel = - Url::parse("http://devstoreaccount1.blob.localhost:10000/testcontainer/ch").unwrap(); + AzureChannelUrl::parse("az://devstoreaccount1.blob.localhost:10000/testcontainer/ch") + .unwrap(); let credentials = AzureCredentials::AccountKey("key".to_string()); - let config = rattler_azure::azblob_config(&credentials, &channel).unwrap(); + let config = rattler_azure::azblob_config( + &credentials, + &channel, + AzureEndpointOptions { + scheme: rattler_azure::AzureScheme::Http, + ..Default::default() + }, + ) + .unwrap(); assert_eq!( config.endpoint.as_deref(), diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index d8055fd7e5..719573abf5 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -4,6 +4,8 @@ use std::path::PathBuf; use anyhow::Context; use clap::{Parser, Subcommand}; use clap_verbosity_flag::Verbosity; +#[cfg(feature = "azure")] +use rattler_azure::{AzureChannelUrl, AzureEndpointOptions, AzureHost}; use rattler_conda_types::Platform; use rattler_config::config::{ concurrency::default_max_concurrent_solves, index::IndexChannelConfig, @@ -21,7 +23,7 @@ use rattler_index::{IndexS3Config, index_s3_with_channel_metadata}; use rattler_networking::AuthenticationStorage; #[cfg(feature = "s3")] use rattler_s3::S3Credentials; -#[cfg(any(feature = "s3", feature = "azure"))] +#[cfg(feature = "s3")] use url::Url; #[cfg(feature = "s3")] @@ -36,21 +38,6 @@ fn parse_s3_url(value: &str) -> Result { } } -/// Parse an `az://` channel URL into the wire URL the rest of this binary still -/// works with. -/// -/// The scheme is fixed to https here because a clap `value_parser` runs before -/// the config file is read, so the host's `azure-options` entry — which decides -/// the scheme — is not available yet. The plumbing step that loads -/// `azure-options` carries the [`rattler_azure::AzureChannelUrl`] through -/// instead, so `--config` can select the scheme and config keys can be matched -/// against its canonical `az://` spelling. -#[cfg(feature = "azure")] -fn parse_azure_channel_url(value: &str) -> Result { - rattler_azure::AzureChannelUrl::parse(value) - .map(|channel| channel.wire(rattler_azure::AzureScheme::Https)) -} - /// SAS permissions requested when minting a user-delegation SAS for indexing. /// Indexing does a read-modify-write of repodata and lists/reads packages, so it /// needs read, write, list, and create (`r` + `w` + `l` + `c`). @@ -132,8 +119,12 @@ enum Commands { Azblob { /// The Azure Blob channel URL, e.g. /// `az://.blob.core.windows.net//`. - #[arg(value_parser = parse_azure_channel_url)] - channel: Url, + /// + /// Parsed into an [`AzureChannelUrl`] rather than a wire `Url`: the wire + /// scheme comes from the host's `azure-options` entry, which is not read + /// until after clap has run, and the `az://` spelling is what + /// `[index-config."…"]` keys are matched against. + channel: AzureChannelUrl, #[clap(flatten)] credentials: rattler_azure::clap::AzureCredentialsOpts, @@ -257,17 +248,22 @@ async fn main() -> anyhow::Result<()> { channel, credentials, } => { - let target = channel.to_string(); + // `canonical()`, not the wire URL: `[index-config."az://…"]` is how a + // user keys an Azure channel, and matching the https spelling meant + // such a key never applied to anything. + let target = channel.canonical().to_string(); let resolved = resolve_index_channel_config(&config, &target); let (write_zst, write_shards, repodata_revisions, package_revision_assignment) = effective_index_options(&resolved); let channel_metadata = ChannelMetadata::from_index_config(&resolved); + let options = azure_endpoint_options(&config, channel.host()); + let credentials = credentials .resolve(AZURE_INDEX_SAS_PERMISSIONS, || { Ok(rattler_azure::account_and_container( - &channel, - rattler_azure::Addressing::HostStyle, + &channel.wire(options.scheme), + options.addressing, )?) }) .await?; @@ -276,6 +272,7 @@ async fn main() -> anyhow::Result<()> { IndexAzureConfig { channel, credentials, + options, target_platform: cli.target_platform, repodata_patch: cli.repodata_patch, write_zst, @@ -295,6 +292,19 @@ async fn main() -> anyhow::Result<()> { Ok(()) } +/// The `[azure-options.""]` entry for a channel's host, or the anonymous +/// https host-style defaults when there is no config file or no entry. +/// +/// A host without an entry and a host with an empty entry are defined to behave +/// identically, so this never has to report which of the two it found. +#[cfg(feature = "azure")] +fn azure_endpoint_options(config: &Option, host: &AzureHost) -> AzureEndpointOptions { + config + .as_ref() + .map(|config| config.azure_options.get(host)) + .unwrap_or_default() +} + fn resolve_index_channel_config(config: &Option, target: &str) -> IndexChannelConfig { config .as_ref() @@ -321,3 +331,106 @@ fn effective_index_options( package_revision_assignment, ) } + +#[cfg(all(test, feature = "azure"))] +mod tests { + use rattler_azure::{Addressing, Auth, AzureCredentials, AzureScheme}; + + use super::*; + + /// Load a config from TOML the way `--config` does, through a real file, so + /// the test exercises the same deserialization the CLI does. + fn config_from(toml: &str) -> Option { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rattler-config.toml"); + std::fs::write(&path, toml).expect("write config"); + Some(Config::load_from_files(vec![path]).expect("config should load")) + } + + /// Reviewer issue 5: `[index-config."az://…"]` is the only spelling a user + /// would write for an Azure channel, and matching the https wire URL meant it + /// never applied to anything. + #[test] + fn index_config_is_keyed_by_the_canonical_az_url() { + let config = config_from( + r#" + [index-config."az://acct.blob.core.windows.net/general"] + write-shards = false + "#, + ); + let channel = + AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/mychannel").unwrap(); + + let resolved = resolve_index_channel_config(&config, channel.canonical().as_str()); + assert_eq!(resolved.write_shards, Some(false)); + + // The spelling this used to match against, kept as the negative half of + // the proof: had the key been written in wire form it would still be dead. + let wire = channel.wire(AzureScheme::Https).to_string(); + assert_eq!( + resolve_index_channel_config(&config, &wire).write_shards, + None + ); + } + + /// An Azurite entry has to carry all the way to the opendal config, because + /// every one of these four fields is derived differently under path-style and + /// a wrong one fails silently. + #[test] + fn a_path_style_entry_drives_the_azurite_index_config() { + let config = config_from( + r#" + [azure-options."127.0.0.1:10000"] + auth = true + scheme = "http" + path-style = true + "#, + ); + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general/mychannel") + .unwrap(); + + let options = azure_endpoint_options(&config, channel.host()); + assert_eq!(options.auth, Auth::DefaultChain); + assert_eq!(options.scheme, AzureScheme::Http); + assert_eq!(options.addressing, Addressing::PathStyle); + + let azblob = rattler_azure::azblob_config( + &AzureCredentials::AccountKey("key".to_string()), + &channel, + options, + ) + .expect("an Azurite channel must build an opendal config"); + + assert_eq!( + azblob.endpoint.as_deref(), + Some("http://127.0.0.1:10000/devstoreaccount1") + ); + assert_eq!(azblob.account_name.as_deref(), Some("devstoreaccount1")); + assert_eq!(azblob.container, "general"); + assert_eq!(azblob.root.as_deref(), Some("/mychannel")); + } + + /// Without an entry the same URL is an error, not a silently different + /// endpoint: host-style cannot read an account out of an IP literal, and the + /// error names the config line that fixes it. + #[test] + fn an_emulator_host_without_an_entry_is_a_guided_error() { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + let options = azure_endpoint_options(&None, channel.host()); + + let err = rattler_azure::azblob_config( + &AzureCredentials::AccountKey("key".to_string()), + &channel, + options, + ) + .expect_err("host-style cannot address an IP literal"); + let message = err.to_string(); + assert!( + message.contains("[azure-options.\"127.0.0.1:10000\"]"), + "{message}" + ); + assert!(message.contains("path-style = true"), "{message}"); + } +} diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index 29e6042b03..dcae7c800c 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -84,12 +84,22 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { #[cfg(feature = "azure")] ServerType::Azure(azure_opts) => { let channel = azure_opts.channel; + // ponytail: the endpoint options are the anonymous https host-style + // defaults, because `upload_from_args` reads no configuration file at + // all — it only opens the auth store — so there is nowhere for an + // `[azure-options.""]` entry to come from. That ceiling is + // exactly real Azure: an emulator or path-style endpoint cannot be + // uploaded to from this entry point. Lifting it means giving + // `rattler_upload` a `--config` of its own (rattler-build, its only + // caller, already parses one) and passing the host's entry through + // here; the plumbing below it already honours whatever it is handed. + let options = rattler_azure::AzureEndpointOptions::default(); let credentials = azure_opts .credentials .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, || { Ok(rattler_azure::account_and_container( - &channel, - rattler_azure::Addressing::HostStyle, + &channel.wire(options.scheme), + options.addressing, )?) }) .await @@ -97,6 +107,7 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { upload::upload_package_to_azure( channel, credentials, + options, &args.package_files, azure_opts.force, ) diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 05a1305867..f0fb767639 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -3,10 +3,9 @@ use std::path::{Path, PathBuf}; use futures::{StreamExt, TryStreamExt}; use miette::IntoDiagnostic; use opendal::{Configurator, ErrorKind, Operator}; -use rattler_azure::AzureCredentials; +use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpointOptions}; use tokio::io::AsyncReadExt; use tokio_util::bytes::BytesMut; -use url::Url; use crate::upload::package::ExtractedPackage; @@ -28,21 +27,20 @@ pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "rcw"; /// Uploads packages to a channel in an Azure Blob Storage container. /// -/// The channel URL must be of the form -/// `https://.blob.core.windows.net//`; the account -/// name, endpoint, container, and root prefix are all derived from it (see -/// `azblob_config`). Because the account is derived from the host, upload -/// requires this dotted `.blob...` form and does not support -/// path-style or emulator (Azurite) endpoints. The full blob host lives in the -/// channel URL itself, so no separate account/endpoint configuration is needed. -/// The [`AzureCredentials`] supply only the account key or SAS token. +/// The account name, endpoint, container and root prefix are all derived from the +/// channel URL together with `options` (see `azblob_config`): `options.addressing` +/// decides whether the account is the first host label or the first path segment, +/// and `options.scheme` decides what `az://` is sent over. A path-style entry is +/// therefore what makes an IP, single-label or emulator (Azurite) endpoint +/// uploadable. The [`AzureCredentials`] supply only the account key or SAS token. pub async fn upload_package_to_azure( - channel: Url, + channel: AzureChannelUrl, credentials: AzureCredentials, + options: AzureEndpointOptions, package_files: &[PathBuf], force: bool, ) -> miette::Result<()> { - let config = rattler_azure::azblob_config(&credentials, &channel).into_diagnostic()?; + let config = rattler_azure::azblob_config(&credentials, &channel, options).into_diagnostic()?; let builder = config.into_builder(); let op = Operator::new(builder).into_diagnostic()?.finish(); @@ -65,7 +63,7 @@ pub async fn upload_package_to_azure( /// Uploads a single package file to the Azure Blob container via the given operator. async fn upload_single_package( op: &Operator, - channel: &Url, + channel: &AzureChannelUrl, package_file: &Path, force: bool, ) -> miette::Result<()> { @@ -78,14 +76,10 @@ async fn upload_single_package( .ok_or_else(|| miette::miette!("Failed to get filename"))?; let key = format!("{subdir}/{filename}"); - // The blob's on-the-wire address, used only for diagnostics. `channel.path()` - // already carries `//`, so the full `az://` URL is the host - // followed by that path and the key; do not prepend the container again. - let blob_url = format!( - "az://{}{}/{key}", - channel.host_str().unwrap_or_default(), - channel.path() - ); + // The blob's address as the user wrote the channel, used only for + // diagnostics. The canonical spelling already carries `//`, + // so the key is appended to it; do not prepend the container again. + let blob_url = format!("{}/{key}", channel.canonical()); // Guard against overwriting an existing blob when `--force` was not passed. // opendal 0.57 only honours `if_not_exists` on the single-shot Put Blob path, @@ -182,7 +176,7 @@ async fn upload_single_package( #[cfg(test)] mod test { use opendal::{Operator, services::Memory}; - use url::Url; + use rattler_azure::AzureChannelUrl; use super::upload_single_package; use crate::upload::package::ExtractedPackage; @@ -192,8 +186,8 @@ mod test { Operator::new(Memory::default()).unwrap().finish() } - fn test_channel() -> Url { - Url::parse("https://account.blob.core.windows.net/container/prefix").unwrap() + fn test_channel() -> AzureChannelUrl { + AzureChannelUrl::parse("az://account.blob.core.windows.net/container/prefix").unwrap() } fn package_key() -> String { diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index b1b3426532..28ab7b17e0 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -389,20 +389,6 @@ pub struct AnacondaOpts { pub force: bool, } -/// Parse an `az://` channel URL into the wire URL the upload path still works -/// with. -/// -/// The scheme is fixed to https because a clap `value_parser` runs before any -/// config is loaded, so the host's `azure-options` entry — which decides the -/// scheme — is not known yet. The plumbing step that loads those options carries -/// the [`rattler_azure::AzureChannelUrl`] itself instead, which is what lets a -/// custom-scheme (emulator) endpoint be uploaded to. -#[cfg(feature = "azure")] -fn parse_azure_channel_url(value: &str) -> Result { - rattler_azure::AzureChannelUrl::parse(value) - .map(|channel| channel.wire(rattler_azure::AzureScheme::Https)) -} - #[cfg(feature = "s3")] fn parse_s3_url(value: &str) -> Result { let url: Url = @@ -442,8 +428,12 @@ pub struct S3Opts { pub struct AzureOpts { /// The channel URL in the Azure Blob container to upload the package to, /// e.g., `az://myaccount.blob.core.windows.net/my-container/my-channel` - #[arg(short, long, env = "AZURE_CHANNEL", value_parser = parse_azure_channel_url)] - pub channel: Url, + /// + /// Kept as an [`AzureChannelUrl`](rattler_azure::AzureChannelUrl) rather than + /// a wire `Url`, so the scheme the request goes out over is chosen where the + /// endpoint options are known instead of being fixed at parse time. + #[arg(short, long, env = "AZURE_CHANNEL")] + pub channel: rattler_azure::AzureChannelUrl, #[clap(flatten)] pub credentials: rattler_azure::clap::AzureCredentialsOpts, diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index f379dd0f10..dddf3e2d82 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -4364,6 +4364,7 @@ dependencies = [ "itertools 0.15.0", "keyring-core", "netrc-rs", + "rattler_azure", "regex", "reqsign-azure-storage", "reqsign-command-execute-tokio", From 874d79bcc8ce8ece9d23777fd297722e16512e24 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:58:18 +1000 Subject: [PATCH 69/98] docs(azure): drop tool-branded comment prefixes --- crates/rattler_azure/src/lib.rs | 2 +- .../rattler_networking/src/azure_middleware.rs | 14 +++++++------- crates/rattler_upload/src/lib.rs | 17 ++++++++--------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 253961c0d5..279c50cc3c 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -522,7 +522,7 @@ impl From for String { /// /// Nothing in the type ties the argument to an options entry. `rattler-index` /// takes it from the channel host's entry; `rattler_upload` passes the default, -/// because it reads no config file at all (see the `ponytail:` note in +/// because it reads no config file at all (see the note in /// `rattler_upload::upload_from_args`). #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct AzureChannelUrl { diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 13e420519e..db07943e63 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -79,14 +79,14 @@ pub struct AzureMiddleware { /// behave as a defaulted entry (anonymous, https), so a miss is never a /// separate code path. /// - /// ponytail: a plain `HashMap` rather than `rattler_config::AzureOptionsMap`, - /// mirroring [`crate::S3Middleware`]. No caller has a `rattler_config::Config` - /// in hand today — every one of them passes an empty table — so taking the - /// config type would buy a mandatory `rattler_config` edge on the `azure` - /// feature for zero saved conversions. When a caller does grow one, add a + /// A plain `HashMap` rather than `rattler_config::AzureOptionsMap`, mirroring + /// [`crate::S3Middleware`]. No caller has a `rattler_config::Config` in hand + /// today — every one of them passes an empty table — so taking the config type + /// would buy a mandatory `rattler_config` edge on the `azure` feature for zero + /// saved conversions. When a caller does grow one, add a /// `#[cfg(feature = "rattler_config")]` helper next to - /// [`crate::s3_middleware::compute_s3_config_from_config`] rather than - /// changing this signature. + /// [`crate::s3_middleware::compute_s3_config_from_config`] rather than changing + /// this signature. options: HashMap, } diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index dcae7c800c..2bddb581a0 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -84,15 +84,14 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { #[cfg(feature = "azure")] ServerType::Azure(azure_opts) => { let channel = azure_opts.channel; - // ponytail: the endpoint options are the anonymous https host-style - // defaults, because `upload_from_args` reads no configuration file at - // all — it only opens the auth store — so there is nowhere for an - // `[azure-options.""]` entry to come from. That ceiling is - // exactly real Azure: an emulator or path-style endpoint cannot be - // uploaded to from this entry point. Lifting it means giving - // `rattler_upload` a `--config` of its own (rattler-build, its only - // caller, already parses one) and passing the host's entry through - // here; the plumbing below it already honours whatever it is handed. + // The anonymous https host-style defaults, because `upload_from_args` + // reads no configuration file at all — it only opens the auth store — + // so there is nowhere for an `[azure-options.""]` entry to come + // from. The ceiling is exactly real Azure: an emulator or path-style + // endpoint cannot be uploaded to from this entry point. Lifting it + // means giving `rattler_upload` a `--config` of its own (rattler-build, + // its only caller, already parses one) and passing the host's entry + // through here; everything below already honours what it is handed. let options = rattler_azure::AzureEndpointOptions::default(); let credentials = azure_opts .credentials From d90771131ecdb925ea38a619b9d627124f5da203 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:19:50 +1000 Subject: [PATCH 70/98] fix(bin): forward rattler_upload tls and sigstore features, scope shard cache synthesis to az:// --- crates/rattler-bin/Cargo.toml | 18 +++++- .../src/gateway/sharded_subdir/tokio/index.rs | 61 ++++++++++++++++--- 2 files changed, 69 insertions(+), 10 deletions(-) diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index ef796fad22..3c64057885 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -23,6 +23,7 @@ native-tls = [ "rattler_repodata_gateway/native-tls", "rattler_networking/native-tls", "rattler_cache/native-tls", + "rattler_upload/native-tls", ] rustls = [ "reqwest/rustls", @@ -30,6 +31,7 @@ rustls = [ "rattler_repodata_gateway/rustls", "rattler_networking/rustls", "rattler_cache/rustls", + "rattler_upload/rustls", ] s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] @@ -69,7 +71,21 @@ rattler_menuinst = { workspace = true, default-features = false } rattler_package_streaming = { workspace = true, default-features = false, features = [ "reqwest", ] } -rattler_upload = { workspace = true } +# Default features off, stated explicitly rather than inherited from the +# workspace dependency, so this crate's own `s3` / `azure` flags remain the only +# thing that compiles in the S3 and Azure upload backends. +# +# Defaults being off also meant none of `rattler_upload`'s own default features +# reached it: neither its TLS forwarding nor `sigstore-sign`, so `rattler upload` +# rejected `--attestation` outright. https kept working only because feature +# unification enabled `reqwest`'s TLS from the sibling rattler crates — a +# coincidence of the dependency graph, not something this crate asked for. TLS is +# now forwarded from the `rustls` / `native-tls` features above so it is chosen +# rather than inherited, and `sigstore-sign` is unconditional because +# attestations have no flag of their own to gate them. +rattler_upload = { workspace = true, default-features = false, features = [ + "sigstore-sign", +] } rattler_shell = { workspace = true, default-features = false } reqwest = { workspace = true, features = ["stream", "query"] } reqwest-middleware = { workspace = true } diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs index 8128011a20..fb03303418 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs @@ -41,14 +41,29 @@ use url::Url; /// policy is never weakened. const SHARD_INDEX_SYNTHETIC_CACHE_CONTROL: &str = "max-age=60"; +/// Scheme of a channel served from Azure Blob Storage. +/// +/// The gateway keeps the channel URL in its `az://` form throughout; the +/// networking middleware rewrites the scheme only at send time, on its own copy +/// of the request. So the scheme observed here still identifies an Azure +/// channel. +const AZURE_CHANNEL_SCHEME: &str = "az"; + /// Build a [`CachePolicy`] for a shard-index response. /// -/// Responses that already carry a `Cache-Control` header are used as-is. -/// Responses without one (notably Azure Blob Storage) are given a small -/// synthetic `max-age` so freshness accumulates instead of forcing a -/// revalidation on every fetch. Validators are preserved either way. +/// Responses that already carry a `Cache-Control` header are used as-is. An +/// `az://` channel that answers without one is given a small synthetic `max-age` +/// so freshness accumulates instead of forcing a revalidation on every fetch. +/// Validators are preserved either way. +/// +/// The synthesis is limited to `az://` because a missing `Cache-Control` is +/// only known to be an unconfigurable property of the origin for Azure Blob +/// Storage. For any other origin the absence is a deliberate policy that the +/// client must not override. fn shard_index_cache_policy(request: &SimpleRequest, response: &Response) -> CachePolicy { - if response.headers().contains_key(http::header::CACHE_CONTROL) { + if response.headers().contains_key(http::header::CACHE_CONTROL) + || request.uri().scheme_str() != Some(AZURE_CHANNEL_SCHEME) + { return CachePolicy::new(request, response); } @@ -642,15 +657,43 @@ mod tests { reqwest::Response::from(builder.body(Vec::new()).unwrap()) } + /// Shard index URL for a channel, in the scheme the gateway holds it in. + fn shards_url(scheme: &str) -> Url { + Url::parse(&format!( + "{scheme}://example.blob.core.windows.net/channel/noarch/repodata_shards.msgpack.zst" + )) + .unwrap() + } + + /// The synthesis is scoped to Azure channels: an `az://` response without a + /// `Cache-Control` gains a freshness window, while any other origin keeps the + /// zero-freshness policy that the absent header implies. + #[test] + fn synthesis_is_limited_to_azure_channels() { + let now = SystemTime::now(); + let no_cache_control = || response(StatusCode::OK, "\"v1\"", None); + + let azure = SimpleRequest::get(&shards_url("az")); + assert!( + shard_index_cache_policy(&azure, &no_cache_control()).time_to_live(now) + > Duration::ZERO, + "an az:// shard index must be given a synthetic freshness window" + ); + + let https = SimpleRequest::get(&shards_url("https")); + assert_eq!( + shard_index_cache_policy(&https, &no_cache_control()).time_to_live(now), + Duration::ZERO, + "a non-Azure origin's missing Cache-Control must be honoured as-is" + ); + } + /// A 304 revalidation must persist a policy that keeps a non-zero freshness /// window, so a fetch within that window is a local cache hit instead of yet /// another conditional round-trip. #[test] fn revalidation_policy_retains_freshness() { - let url = Url::parse( - "https://example.blob.core.windows.net/channel/noarch/repodata_shards.msgpack.zst", - ) - .unwrap(); + let url = shards_url("az"); let request = SimpleRequest::get(&url); let now = SystemTime::now(); From c6482b504137a347d9428803c4debc18a32e62f2 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Fri, 31 Jul 2026 17:19:50 +1000 Subject: [PATCH 71/98] test(azure): add azurite integration tests and e2e ci workflow --- .github/workflows/e2e-azure-tests.yml | 91 +++++ Cargo.lock | 2 + crates/rattler_index/Cargo.toml | 11 + crates/rattler_index/tests/azure_azurite.rs | 365 ++++++++++++++++++ .../tests/azure_azurite_fetch.rs | 225 +++++++++++ 5 files changed, 694 insertions(+) create mode 100644 .github/workflows/e2e-azure-tests.yml create mode 100644 crates/rattler_index/tests/azure_azurite.rs create mode 100644 crates/rattler_networking/tests/azure_azurite_fetch.rs diff --git a/.github/workflows/e2e-azure-tests.yml b/.github/workflows/e2e-azure-tests.yml new file mode 100644 index 0000000000..92d757efba --- /dev/null +++ b/.github/workflows/e2e-azure-tests.yml @@ -0,0 +1,91 @@ +on: + push: + branches: [main] + pull_request: + paths: + - crates/rattler_azure/** + - crates/rattler_index/** + - crates/rattler_networking/** + - crates/rattler_upload/** + - pixi.toml + - pixi.lock + - .github/workflows/e2e-azure-tests.yml + +name: E2E Azure Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + RUST_LOG: info + RUST_BACKTRACE: 1 + CARGO_TERM_COLOR: always + +jobs: + e2e-azurite-test: + name: E2E Index/Fetch [Azurite] + runs-on: ubuntu-latest + + env: + # Enable sccache. + # + # This environment variable is picked up by pixi build which will then + # set up the rust build using sccache. + SCCACHE_GHA_ENABLED: "true" + + steps: + - name: Checkout source code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + submodules: recursive + + # Not a `services:` container. A service container's command cannot be + # overridden, and `--skipApiVersionCheck` is required: opendal pins an + # `x-ms-version` at or beyond what Azurite V3 accepts on its own. Launching + # the server from the job also matches how the Minio E2E job does it. + # + # No secrets: the emulator's `devstoreaccount1` credentials are published + # constants, spelled out in the tests, and only ever reach loopback. + - name: Start Azurite + run: | + docker run --detach --name azurite --publish 10000:10000 \ + mcr.microsoft.com/azure-storage/azurite \ + azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck + + # An unsigned request answered with any status means the blob service is + # listening; 403 is the expected answer and is as good a readiness signal as + # 200 would be. + - name: Wait for Azurite + run: | + for _ in $(seq 1 60); do + if curl --silent --output /dev/null "http://127.0.0.1:10000/devstoreaccount1?comp=list"; then + exit 0 + fi + sleep 2 + done + echo "Azurite did not become ready within 120 seconds" + docker logs azurite + exit 1 + + - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 + with: + environments: default + + # The feature is package-qualified because `azure` is a default feature of + # rattler_index but opt-in for rattler_networking, so an unqualified name + # across both packages would be ambiguous. + - name: Run the Azurite integration tests + run: | + pixi run -vv -- cargo nextest run --run-ignored all \ + --features rattler_networking/azure \ + -p rattler_index --test azure_azurite \ + -p rattler_networking --test azure_azurite_fetch + + - name: Azurite logs + if: always() + run: docker logs azurite + + - name: Stop Azurite + if: always() + run: docker rm --force azurite diff --git a/Cargo.lock b/Cargo.lock index f1a16d1fa9..ef35680e47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5228,6 +5228,7 @@ version = "0.30.10" dependencies = [ "ahash", "anyhow", + "astral-reqwest-middleware", "bytes", "clap", "clap-verbosity-flag", @@ -5254,6 +5255,7 @@ dependencies = [ "serde_json", "sha2 0.11.0", "tar", + "temp-env", "tempfile", "thiserror 2.0.19", "tokio", diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index b431304780..fb65e80be8 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -84,3 +84,14 @@ rattler_repodata_gateway = { path = "../rattler_repodata_gateway", default-featu ] } tempfile = { workspace = true } tools = { path = "../tools", default-features = false } +# `AzureMiddleware` is the only shared-key signer reachable from a test here, and +# the Azurite tests need one to create a container: opendal has no +# container-creation operation. +rattler_networking = { workspace = true, default-features = false, features = [ + "system-integration", + "azure", +] } +reqwest-middleware = { workspace = true } +# Puts Azurite's development credentials in the environment for reqsign's env +# provider without an unsafe `set_var`. +temp-env = { workspace = true } diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs new file mode 100644 index 0000000000..5ff92644aa --- /dev/null +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -0,0 +1,365 @@ +//! Live write-path integration tests against a local Azurite emulator. +//! +//! `index_azure` builds its opendal config from a channel URL plus one +//! `azure-options` entry, and under `path-style = true` the account moves from the +//! host into the endpoint path. Unit tests can only assert the strings that +//! construction produces; whether a real Azure Blob implementation accepts them is +//! a different question, and this is where it gets answered: +//! +//! ```toml +//! [azure-options."127.0.0.1:10000"] +//! auth = true +//! scheme = "http" +//! path-style = true +//! ``` +//! +//! The rest of the file covers two opendal behaviours that the production code +//! deliberately works around, and that only a real server can demonstrate: the +//! multi-block write path silently ignores `if_not_exists`, and it does carry +//! `Cache-Control` through its commit. +//! +//! Run with: +//! +//! ```text +//! docker run --rm -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ +//! azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck +//! cargo nextest run -p rattler_index --test azure_azurite --run-ignored all +//! ``` +//! +//! `--skipApiVersionCheck` is required: opendal pins an `x-ms-version` at or +//! beyond what Azurite V3 accepts on its own. +#![cfg(feature = "azure")] + +use std::{collections::HashMap, path::PathBuf}; + +use opendal::{Configurator, ErrorKind, Operator, services::AzblobConfig}; +use rattler_azure::{ + Addressing, Auth, AzureChannelUrl, AzureCredentials, AzureEndpointOptions, AzureHost, + AzureScheme, +}; +use rattler_index::{IndexAzureConfig, PackageRevisionAssignment, index_azure}; + +/// Azurite's development account and its fixed key. Not a secret: both are +/// published constants of the emulator, hardcoded in opendal's own source, and +/// they only ever address a loopback port. +const ACCOUNT: &str = "devstoreaccount1"; +const ACCOUNT_KEY: &str = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + +/// The authority, which is also the exact `azure-options` table key. An IP with a +/// port is precisely the host shape host-style addressing cannot read an account +/// out of, so it only works through a `path-style = true` entry. +const AUTHORITY: &str = "127.0.0.1:10000"; + +const CONTAINER: &str = "test-channel"; + +/// The `Cache-Control` `rattler_index` writes on repodata (`lib.rs`'s +/// `CACHE_CONTROL_REPODATA`), duplicated because the constant is private. +const CACHE_CONTROL_REPODATA: &str = "public, max-age=300"; + +const MIB: usize = 1024 * 1024; + +/// `rattler_upload`'s `DESIRED_CHUNK_SIZE`, which is what decides whether a +/// package upload takes opendal's single-shot or multi-block path. +const UPLOAD_CHUNK_SIZE: usize = 10 * MIB; + +const PACKAGE: &str = "empty-0.1.0-h4616a5c_0.conda"; + +fn package_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../test-data/packages") + .join(PACKAGE) +} + +/// The channel as a user would write it: the account is the first path segment, +/// which is what `path-style = true` means. `prefix` keeps each test in its own +/// subtree so they can run in parallel. +fn channel(prefix: &str) -> AzureChannelUrl { + AzureChannelUrl::parse(&format!("az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}/{prefix}")) + .expect("azurite channel url") +} + +/// The `azure-options` entry for the emulator: the only configuration these tests +/// hand to the indexer. +fn azurite_options() -> AzureEndpointOptions { + AzureEndpointOptions { + auth: Auth::DefaultChain, + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + } +} + +/// An operator built exactly the way `index_azure` builds one, so the opendal-level +/// tests below run against the config the production path derives rather than a +/// hand-written stand-in. +fn production_operator(channel: &AzureChannelUrl) -> Operator { + let config = rattler_azure::azblob_config( + &AzureCredentials::AccountKey(ACCOUNT_KEY.to_string()), + channel, + azurite_options(), + ) + .expect("azblob config for an azurite path-style channel"); + Operator::new(config.into_builder()) + .expect("azblob operator") + .finish() +} + +/// An operator written out by hand, *not* derived from the code under test. +/// +/// This is what makes the round-trip assertion mean something: if +/// `azblob_config`'s path-style derivation put the blobs somewhere else, this +/// operator would not find them. It is the verbatim shape the emulator wants — the +/// account appears both inside `endpoint` and in `account_name`, `container` is +/// separate, and `root` is the channel prefix without repeating the container. +fn verify_operator(prefix: &str) -> Operator { + let config = AzblobConfig { + endpoint: Some(format!("http://{AUTHORITY}/{ACCOUNT}")), + account_name: Some(ACCOUNT.to_string()), + account_key: Some(ACCOUNT_KEY.to_string()), + container: CONTAINER.to_string(), + root: Some(format!("/{prefix}")), + ..Default::default() + }; + Operator::new(config.into_builder()) + .expect("azblob operator") + .finish() +} + +/// Create the channel's container, which Azurite never does implicitly. +/// +/// This goes through `AzureMiddleware` because it is the one signer already +/// reachable from here: opendal exposes no container-creation operation, and +/// hand-rolling shared-key signing in a test fixture would be more code than the +/// tests it supports. +async fn ensure_container() { + let options = HashMap::from([( + AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), + azurite_options(), + )]); + let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) + .with(rattler_networking::AzureMiddleware::new( + reqwest::Client::new(), + options, + )) + .build(); + + let created = client + .put(format!( + "az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}?restype=container" + )) + .send() + .await + .expect("container create request failed"); + assert!( + // 409 is `ContainerAlreadyExists` — a re-run, not a failure. + created.status().is_success() || created.status() == reqwest::StatusCode::CONFLICT, + "could not create container {CONTAINER}: {}", + created.status() + ); +} + +/// Run `body` with the emulator credentials in the environment. +/// +/// reqsign's env provider sits first in its default chain, so a shared key is how +/// an `auth = true` grant resolves against Azurite — it rejects the AAD bearer +/// tokens the rest of the chain produces. The chain itself is left alone. +async fn with_azurite_credentials>(body: F) { + temp_env::async_with_vars( + [ + ("AZURE_STORAGE_ACCOUNT_NAME", Some(ACCOUNT)), + ("AZURE_STORAGE_ACCOUNT_KEY", Some(ACCOUNT_KEY)), + ], + body, + ) + .await; +} + +fn index_config(channel: AzureChannelUrl) -> IndexAzureConfig { + IndexAzureConfig { + channel, + credentials: AzureCredentials::AccountKey(ACCOUNT_KEY.to_string()), + options: azurite_options(), + target_platform: None, + repodata_patch: None, + write_zst: false, + write_shards: false, + repodata_revisions: Vec::new(), + package_revision_assignment: PackageRevisionAssignment::default(), + force: true, + max_parallel: 4, + multi_progress: None, + } +} + +/// The round trip: seed a package into a path-style Azurite channel, index it +/// through an `azure-options` entry, and read the written `repodata.json` back. +/// +/// The read side uses the hand-written operator, so this checks that the indexer +/// wrote to the blob the URL names — not merely that it reported success. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_index_round_trip_through_a_path_style_entry() { + with_azurite_credentials(async { + const PREFIX: &str = "round-trip"; + ensure_container().await; + + let seeded = verify_operator(PREFIX); + seeded + .write( + &format!("noarch/{PACKAGE}"), + fs_err::read(package_path()).expect("test package"), + ) + .await + .expect("seeding the package failed"); + + index_azure(index_config(channel(PREFIX))) + .await + .expect("indexing an azurite channel failed"); + + let repodata = seeded + .read("noarch/repodata.json") + .await + .expect("the indexer wrote no repodata.json where the channel URL points"); + let json: serde_json::Value = + serde_json::from_slice(&repodata.to_vec()).expect("repodata was not valid json"); + assert!( + json["packages.conda"] + .as_object() + .is_some_and(|packages| packages.contains_key(PACKAGE)), + "repodata should list the seeded package: {json}" + ); + assert_eq!( + json["info"]["subdir"], "noarch", + "repodata should describe the subdir it was written for: {json}" + ); + + // The same write path that ros-recipes' cache-header sweep exists to + // override. Live evidence that opendal's azblob backend honours it at all. + let metadata = seeded + .stat("noarch/repodata.json") + .await + .expect("stat repodata.json"); + assert_eq!(metadata.cache_control(), Some(CACHE_CONTROL_REPODATA)); + }) + .await; +} + +/// A write larger than the chunk size commits through Put Block List, and +/// `Cache-Control` survives that commit. +/// +/// Nothing in production reaches this yet. opendal's azblob backend declares no +/// `write_multi_min_size`, so `Operator::write_with` hands the whole buffer over in +/// a single `write` call and always takes the single-shot Put Blob path, however +/// large the repodata gets; only an explicit `chunk` splits it. So this locks the +/// header down for the commit path a chunked repodata write would take, which is +/// the gap that made the live `Cache-Control` evidence incomplete. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_multi_block_write_keeps_cache_control() { + with_azurite_credentials(async { + const PREFIX: &str = "multi-block-cache-control"; + ensure_container().await; + let op = production_operator(&channel(PREFIX)); + + // Two chunks' worth, so `write` is called more than once and the writer + // switches from `write_once` to staging blocks. + let mut writer = op + .writer_with("noarch/repodata.json") + .chunk(2 * MIB) + .cache_control(CACHE_CONTROL_REPODATA) + .await + .expect("opening a chunked writer failed"); + writer + .write(vec![b'{'; 5 * MIB]) + .await + .expect("chunked write failed"); + writer.close().await.expect("Put Block List commit failed"); + + let metadata = op + .stat("noarch/repodata.json") + .await + .expect("stat the multi-block blob"); + assert!( + metadata.content_length() == (5 * MIB) as u64, + "the blob should have committed all blocks, got {} bytes", + metadata.content_length() + ); + assert_eq!( + metadata.cache_control(), + Some(CACHE_CONTROL_REPODATA), + "Put Block List should carry x-ms-blob-cache-control through its commit" + ); + }) + .await; +} + +/// The gap `upload_package_to_azure`'s pre-write `stat` exists to close: opendal +/// honours `if_not_exists` on the single-shot Put Blob path and silently drops it +/// on the multi-block path. +/// +/// A package over `rattler_upload`'s 10 MiB chunk size is the only way to reach the +/// multi-block path, and this is as close as a test can currently get to that +/// upload: `rattler_upload` reads no config file, so it always builds a host-style +/// endpoint and cannot be pointed at an emulator at all. What it can share is the +/// operator, built here through the same `azblob_config`, and the exact writer +/// options the uploader uses — so the behaviour being reproduced is the uploader's, +/// even though the entry point is not. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_if_not_exists_is_dropped_on_the_multi_block_path() { + with_azurite_credentials(async { + const PREFIX: &str = "overwrite-guard"; + ensure_container().await; + let op = production_operator(&channel(PREFIX)); + + // Baseline: below the chunk size, the guard works and opendal reports the + // conflict the uploader turns into "already exists, use --force". + let small = "noarch/small.conda"; + op.write(small, b"first".to_vec()) + .await + .expect("small write failed"); + let refused = op + .write_with(small, b"second".to_vec()) + .if_not_exists(true) + .await + .expect_err("if_not_exists should refuse to overwrite a small blob"); + assert_eq!(refused.kind(), ErrorKind::ConditionNotMatch); + + // Over the chunk size, the same option is accepted and then ignored. + let large = "noarch/large.conda"; + let payload = vec![0u8; UPLOAD_CHUNK_SIZE + 2 * MIB]; + write_chunked(&op, large, &payload, false).await; + write_chunked(&op, large, &payload, true).await; + + // Which is why the uploader stats first. That check does see the blob, so + // the guard holds for large packages despite opendal dropping the option. + assert!( + op.stat(large).await.is_ok(), + "the pre-write stat must see an existing large blob, since if_not_exists does not" + ); + }) + .await; +} + +/// Write `payload` the way `upload_single_package` does, and assert it succeeded. +/// +/// `guard` is that function's `if_not_exists(!force)`. Passing `true` over a blob +/// that already exists still succeeds, which is the point being demonstrated: on +/// the multi-block path the option is a no-op, so a large upload would clobber +/// without the separate `stat`. +async fn write_chunked(op: &Operator, path: &str, payload: &[u8], guard: bool) { + let mut writer = op + .writer_with(path) + .chunk(UPLOAD_CHUNK_SIZE) + .if_not_exists(guard) + .await + .expect("opening a chunked writer failed"); + writer + .write(payload.to_vec()) + .await + .expect("chunked write failed"); + writer + .close() + .await + .expect("a multi-block commit should succeed even with if_not_exists set"); +} diff --git a/crates/rattler_networking/tests/azure_azurite_fetch.rs b/crates/rattler_networking/tests/azure_azurite_fetch.rs new file mode 100644 index 0000000000..d584ba3d43 --- /dev/null +++ b/crates/rattler_networking/tests/azure_azurite_fetch.rs @@ -0,0 +1,225 @@ +//! Live fetch-path integration tests against a local Azurite emulator. +//! +//! These are the read-side half of the answer to "has any of this been tested +//! against a real Azure-compatible backend?". Everything else about the grant +//! model is unit-tested with mocks; only Azurite can show that a granted entry +//! produces a signature a real Azure Blob implementation accepts, and that an +//! ungranted one does not. +//! +//! Everything is driven through a single `azure-options` entry, which is the +//! point of the exercise — there is no out-of-band account or endpoint +//! configuration on the fetch path: +//! +//! ```toml +//! [azure-options."127.0.0.1:10000"] +//! auth = true +//! scheme = "http" +//! path-style = true +//! ``` +//! +//! Run with: +//! +//! ```text +//! docker run --rm -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ +//! azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck +//! cargo nextest run -p rattler_networking --features azure --test azure_azurite_fetch \ +//! --run-ignored all +//! ``` +//! +//! `--skipApiVersionCheck` is required: `AzureMiddleware` pins an `x-ms-version` +//! at or beyond what Azurite V3 accepts on its own. +#![cfg(feature = "azure")] + +use std::collections::HashMap; + +use rattler_azure::{Addressing, Auth, AzureEndpointOptions, AzureHost, AzureScheme}; +use rattler_networking::AzureMiddleware; +use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; + +/// Azurite's development account and its fixed key. Not a secret: both are +/// published constants of the emulator, hardcoded in opendal's own source, and +/// they only ever address a loopback port. +const ACCOUNT: &str = "devstoreaccount1"; +const ACCOUNT_KEY: &str = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + +/// The authority, which is also the exact `azure-options` table key. An IP with a +/// port is precisely the host shape that host-style addressing cannot read an +/// account out of, so it only works through a `path-style = true` entry. +const AUTHORITY: &str = "127.0.0.1:10000"; + +/// Container name from the test this one restores. Azurite creates containers as +/// private, which is what makes the ungranted case below meaningful. +const CONTAINER: &str = "cli-channel"; + +/// Minimal but structurally real repodata, so the assertions can be about +/// content rather than just a status code. +const REPODATA: &str = r#"{ + "info": { "subdir": "noarch" }, + "packages": {}, + "packages.conda": { + "empty-0.1.0-h4616a5c_0.conda": { + "build": "h4616a5c_0", + "build_number": 0, + "depends": [], + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "name": "empty", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size": 1538, + "subdir": "noarch", + "version": "0.1.0" + } + } +}"#; + +/// The channel as a user would write it: the account is the first path segment, +/// which is what `path-style = true` means. +fn channel_url() -> String { + format!("az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}") +} + +/// The one `azure-options` entry these tests run on, with `auth` as the only +/// variable. `scheme` and `path-style` stay set even in the ungranted case: the +/// entry is what makes the emulator reachable at all, and keeping it identical +/// means the two tests differ in the grant and nothing else. +fn azurite_entry(auth: Auth) -> HashMap { + HashMap::from([( + AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), + AzureEndpointOptions { + auth, + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + )]) +} + +fn client(auth: Auth) -> ClientWithMiddleware { + ClientBuilder::new(reqwest::Client::new()) + .with(AzureMiddleware::new( + reqwest::Client::new(), + azurite_entry(auth), + )) + .build() +} + +/// Create the container and put a `noarch/repodata.json` in it. +/// +/// Seeding runs through the granted middleware rather than a separate SDK, which +/// keeps the test dependency-free and doubles as proof that the signature works +/// for writes and for a request carrying a query string (`?restype=container` +/// participates in the canonicalized signing resource, so a wrong signature +/// fails here first). +async fn seed(client: &ClientWithMiddleware) { + let created = client + .put(format!("{}?restype=container", channel_url())) + .send() + .await + .expect("container create request failed"); + assert!( + // 409 is `ContainerAlreadyExists` — a re-run, not a failure. + created.status().is_success() || created.status() == reqwest::StatusCode::CONFLICT, + "could not create container {CONTAINER}: {}", + created.status() + ); + + // `Content-Length` is set by hand because shared-key signing covers it, and + // reqwest only materializes the header inside hyper at send time — after the + // middleware has already signed. That gap is invisible to production, where the + // middleware only ever carries bodyless `az://` reads, but a seeding PUT walks + // straight into it and gets a 403 for a length mismatch. + let put = client + .put(format!("{}/noarch/repodata.json", channel_url())) + .header("x-ms-blob-type", "BlockBlob") + .header(reqwest::header::CONTENT_LENGTH, REPODATA.len()) + .body(REPODATA) + .send() + .await + .expect("blob upload request failed"); + assert!( + put.status().is_success(), + "could not seed repodata.json: {}", + put.status() + ); +} + +/// A granted, `http`, path-style entry fetches a blob out of a private Azurite +/// container — the whole fetch path end to end, with the entry as the only +/// configuration. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_granted_entry_fetches_repodata() { + // reqsign's env provider sits first in the default chain, so the shared key + // is how a grant resolves against the emulator. Azurite rejects the AAD + // bearer tokens the rest of the chain produces, so this is the only credential + // shape that can work here — the chain itself is untouched. + temp_env::async_with_vars( + [ + ("AZURE_STORAGE_ACCOUNT_NAME", Some(ACCOUNT)), + ("AZURE_STORAGE_ACCOUNT_KEY", Some(ACCOUNT_KEY)), + ], + async { + let client = client(Auth::DefaultChain); + seed(&client).await; + + let url = format!("{}/noarch/repodata.json", channel_url()); + let resp = client + .get(&url) + .send() + .await + .expect("request through azure middleware failed"); + let status = resp.status(); + let body = resp.bytes().await.expect("failed to read body"); + assert!(status.is_success(), "unexpected status {status} for {url}"); + + let json: serde_json::Value = + serde_json::from_slice(&body).expect("fetched repodata was not valid json"); + assert!( + json["packages.conda"] + .as_object() + .is_some_and(|packages| packages.contains_key("empty-0.1.0-h4616a5c_0.conda")), + "repodata fetched via az:// should list the seeded package: {json}" + ); + }, + ) + .await; +} + +/// Without `auth = true` the request goes out unsigned, and a private container +/// refuses it. This is the core claim of the anonymous-by-default model, and the +/// only way to check it is against a server that actually enforces authorization. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_ungranted_entry_is_refused_by_a_private_container() { + temp_env::async_with_vars( + [ + ("AZURE_STORAGE_ACCOUNT_NAME", Some(ACCOUNT)), + ("AZURE_STORAGE_ACCOUNT_KEY", Some(ACCOUNT_KEY)), + ], + async { + // Seed with a grant, then read without one. The credential is present in + // the environment throughout, so a success below would mean the grant + // check leaked it — not that the test was misconfigured. + seed(&client(Auth::DefaultChain)).await; + + let url = format!("{}/noarch/repodata.json", channel_url()); + let resp = client(Auth::Anonymous) + .get(&url) + .send() + .await + .expect("request through azure middleware failed"); + + let status = resp.status(); + assert!( + // Both statuses are correct answers to an unsigned read of a private + // container: Azurite says 403, real Azure says 404 so that a missing + // grant is indistinguishable from a missing blob. Accepting either + // keeps the assertion about "refused", which is the actual claim. + status == reqwest::StatusCode::FORBIDDEN + || status == reqwest::StatusCode::NOT_FOUND, + "an ungranted read of a private container should be refused, got {status} for \ + {url}" + ); + }, + ) + .await; +} From cd8be49c0e8e47b7069b8aa1806dba0cadf470b0 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 09:50:37 +1000 Subject: [PATCH 72/98] fix(ci): drop unneeded azurite --skipApiVersionCheck and correct its rationale --- .github/workflows/e2e-azure-tests.yml | 14 ++++++-------- crates/rattler_index/tests/azure_azurite.rs | 9 ++++++--- .../tests/azure_azurite_fetch.rs | 9 ++++++--- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e-azure-tests.yml b/.github/workflows/e2e-azure-tests.yml index 92d757efba..3774664e1c 100644 --- a/.github/workflows/e2e-azure-tests.yml +++ b/.github/workflows/e2e-azure-tests.yml @@ -40,10 +40,11 @@ jobs: with: submodules: recursive - # Not a `services:` container. A service container's command cannot be - # overridden, and `--skipApiVersionCheck` is required: opendal pins an - # `x-ms-version` at or beyond what Azurite V3 accepts on its own. Launching - # the server from the job also matches how the Minio E2E job does it. + # Not a `services:` container, for the same reason the Minio E2E job is not + # one: launching the server from the job keeps `docker logs` available as a + # diagnosis step when a test fails. A service container would work here — + # the image's default command already binds the blob host to 0.0.0.0 — so + # this is precedent, not necessity. # # No secrets: the emulator's `devstoreaccount1` credentials are published # constants, spelled out in the tests, and only ever reach loopback. @@ -51,7 +52,7 @@ jobs: run: | docker run --detach --name azurite --publish 10000:10000 \ mcr.microsoft.com/azure-storage/azurite \ - azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck + azurite-blob --blobHost 0.0.0.0 # An unsigned request answered with any status means the blob service is # listening; 403 is the expected answer and is as good a readiness signal as @@ -72,9 +73,6 @@ jobs: with: environments: default - # The feature is package-qualified because `azure` is a default feature of - # rattler_index but opt-in for rattler_networking, so an unqualified name - # across both packages would be ambiguous. - name: Run the Azurite integration tests run: | pixi run -vv -- cargo nextest run --run-ignored all \ diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs index 5ff92644aa..62bdfc4ae3 100644 --- a/crates/rattler_index/tests/azure_azurite.rs +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -22,12 +22,15 @@ //! //! ```text //! docker run --rm -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ -//! azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck +//! azurite-blob --blobHost 0.0.0.0 //! cargo nextest run -p rattler_index --test azure_azurite --run-ignored all //! ``` //! -//! `--skipApiVersionCheck` is required: opendal pins an `x-ms-version` at or -//! beyond what Azurite V3 accepts on its own. +//! No `--skipApiVersionCheck` needed: the `x-ms-version` opendal pins is older +//! than what current Azurite accepts. Verified on 3.36.0, which answers that +//! version with `AuthorizationFailure` rather than `InvalidHeaderValue`, i.e. it +//! validates the signature instead of rejecting the version. Add the flag only if +//! an older emulator rejects the version outright. #![cfg(feature = "azure")] use std::{collections::HashMap, path::PathBuf}; diff --git a/crates/rattler_networking/tests/azure_azurite_fetch.rs b/crates/rattler_networking/tests/azure_azurite_fetch.rs index d584ba3d43..d66f5e0bfc 100644 --- a/crates/rattler_networking/tests/azure_azurite_fetch.rs +++ b/crates/rattler_networking/tests/azure_azurite_fetch.rs @@ -21,13 +21,16 @@ //! //! ```text //! docker run --rm -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ -//! azurite-blob --blobHost 0.0.0.0 --skipApiVersionCheck +//! azurite-blob --blobHost 0.0.0.0 //! cargo nextest run -p rattler_networking --features azure --test azure_azurite_fetch \ //! --run-ignored all //! ``` //! -//! `--skipApiVersionCheck` is required: `AzureMiddleware` pins an `x-ms-version` -//! at or beyond what Azurite V3 accepts on its own. +//! No `--skipApiVersionCheck` needed: the `x-ms-version` this middleware pins is +//! older than what current Azurite accepts. Verified on 3.36.0, which answers +//! that version with `AuthorizationFailure` rather than `InvalidHeaderValue`, +//! i.e. it validates the signature instead of rejecting the version. Add the flag +//! only if an older emulator rejects the version outright. #![cfg(feature = "azure")] use std::collections::HashMap; From c10acc0513299fcf1c73a279e79f944dfafdd9ac Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 09:57:02 +1000 Subject: [PATCH 73/98] refactor(ci): run azurite as a service container with a health check --- .github/workflows/e2e-azure-tests.yml | 65 ++++++++++++--------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/.github/workflows/e2e-azure-tests.yml b/.github/workflows/e2e-azure-tests.yml index 3774664e1c..8c4ff3dd70 100644 --- a/.github/workflows/e2e-azure-tests.yml +++ b/.github/workflows/e2e-azure-tests.yml @@ -34,41 +34,40 @@ jobs: # set up the rust build using sccache. SCCACHE_GHA_ENABLED: "true" + # A service container, so the runner does the waiting and the log collection: + # the health check gates every step below on the blob service actually + # listening, and the runner prints the container's logs when the job ends. + # The S3 E2E job is not a precedent either way — its Minio is a native binary + # spawned by `scripts/e2e/s3-minio.nu`, not a container. + # + # The image's default command is used as-is, since a service container cannot + # override one. That starts queue and table alongside blob; only the blob port + # is published, and the tests are unaffected. + # + # `nc -z` and not an HTTP request: every unsigned request is answered with a + # 4xx, which most HTTP clients treat as failure, so a listening socket is both + # the simplest and the most honest readiness signal. `curl` is not in the image + # anyway. + # + # No secrets: the emulator's `devstoreaccount1` credentials are published + # constants, spelled out in the tests, and only ever reach loopback. + services: + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.36.0 + ports: + - 10000:10000 + options: >- + --health-cmd "nc -z 127.0.0.1 10000" + --health-interval 2s + --health-timeout 2s + --health-retries 30 + steps: - name: Checkout source code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: submodules: recursive - # Not a `services:` container, for the same reason the Minio E2E job is not - # one: launching the server from the job keeps `docker logs` available as a - # diagnosis step when a test fails. A service container would work here — - # the image's default command already binds the blob host to 0.0.0.0 — so - # this is precedent, not necessity. - # - # No secrets: the emulator's `devstoreaccount1` credentials are published - # constants, spelled out in the tests, and only ever reach loopback. - - name: Start Azurite - run: | - docker run --detach --name azurite --publish 10000:10000 \ - mcr.microsoft.com/azure-storage/azurite \ - azurite-blob --blobHost 0.0.0.0 - - # An unsigned request answered with any status means the blob service is - # listening; 403 is the expected answer and is as good a readiness signal as - # 200 would be. - - name: Wait for Azurite - run: | - for _ in $(seq 1 60); do - if curl --silent --output /dev/null "http://127.0.0.1:10000/devstoreaccount1?comp=list"; then - exit 0 - fi - sleep 2 - done - echo "Azurite did not become ready within 120 seconds" - docker logs azurite - exit 1 - - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 with: environments: default @@ -79,11 +78,3 @@ jobs: --features rattler_networking/azure \ -p rattler_index --test azure_azurite \ -p rattler_networking --test azure_azurite_fetch - - - name: Azurite logs - if: always() - run: docker logs azurite - - - name: Stop Azurite - if: always() - run: docker rm --force azurite From ba70b11576ffd7ec63954bab6bff8ff6da929fda Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:00:21 +1000 Subject: [PATCH 74/98] refactor(bin): gate attestation behind a feature instead of the dependency line --- crates/rattler-bin/Cargo.toml | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index 3c64057885..015b363ff4 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -16,7 +16,7 @@ name = "rattler" path = "src/main.rs" [features] -default = ["rustls", "s3", "gcs", "azure", "oauth"] +default = ["rustls", "s3", "gcs", "azure", "oauth", "attestation"] native-tls = [ "reqwest/native-tls", "rattler/native-tls", @@ -37,6 +37,7 @@ s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] azure = ["rattler_networking/azure", "rattler_upload/azure"] oauth = ["rattler/oauth"] +attestation = ["rattler_upload/sigstore-sign"] [dependencies] anyhow = { workspace = true } @@ -71,21 +72,7 @@ rattler_menuinst = { workspace = true, default-features = false } rattler_package_streaming = { workspace = true, default-features = false, features = [ "reqwest", ] } -# Default features off, stated explicitly rather than inherited from the -# workspace dependency, so this crate's own `s3` / `azure` flags remain the only -# thing that compiles in the S3 and Azure upload backends. -# -# Defaults being off also meant none of `rattler_upload`'s own default features -# reached it: neither its TLS forwarding nor `sigstore-sign`, so `rattler upload` -# rejected `--attestation` outright. https kept working only because feature -# unification enabled `reqwest`'s TLS from the sibling rattler crates — a -# coincidence of the dependency graph, not something this crate asked for. TLS is -# now forwarded from the `rustls` / `native-tls` features above so it is chosen -# rather than inherited, and `sigstore-sign` is unconditional because -# attestations have no flag of their own to gate them. -rattler_upload = { workspace = true, default-features = false, features = [ - "sigstore-sign", -] } +rattler_upload = { workspace = true, default-features = false } rattler_shell = { workspace = true, default-features = false } reqwest = { workspace = true, features = ["stream", "query"] } reqwest-middleware = { workspace = true } From 0cce4a77bd812648d944838dfb6a5e65a0b5b3c4 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:24:06 +1000 Subject: [PATCH 75/98] refactor(bin): drop attestation feature, keep sigstore-sign on the dependency line --- crates/rattler-bin/Cargo.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index 015b363ff4..c9a0fa12e3 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -16,7 +16,7 @@ name = "rattler" path = "src/main.rs" [features] -default = ["rustls", "s3", "gcs", "azure", "oauth", "attestation"] +default = ["rustls", "s3", "gcs", "azure", "oauth"] native-tls = [ "reqwest/native-tls", "rattler/native-tls", @@ -37,7 +37,6 @@ s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] azure = ["rattler_networking/azure", "rattler_upload/azure"] oauth = ["rattler/oauth"] -attestation = ["rattler_upload/sigstore-sign"] [dependencies] anyhow = { workspace = true } @@ -72,7 +71,9 @@ rattler_menuinst = { workspace = true, default-features = false } rattler_package_streaming = { workspace = true, default-features = false, features = [ "reqwest", ] } -rattler_upload = { workspace = true, default-features = false } +rattler_upload = { workspace = true, default-features = false, features = [ + "sigstore-sign", +] } rattler_shell = { workspace = true, default-features = false } reqwest = { workspace = true, features = ["stream", "query"] } reqwest-middleware = { workspace = true } From fcf06fd4dfc01e406122a8c50c637e5ce512bda6 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:02:51 +1000 Subject: [PATCH 76/98] revert(gateway): drop the shard-index Cache-Control synthesis --- .../src/gateway/sharded_subdir/tokio/index.rs | 207 ++---------------- 1 file changed, 14 insertions(+), 193 deletions(-) diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs index fb03303418..fbac0cb7a9 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs @@ -29,54 +29,6 @@ use tokio::{ }; use url::Url; -/// `Cache-Control` synthesized for a shard index served without one. -/// -/// Azure Blob Storage sends no `Cache-Control` on the shard index, so the -/// derived [`CachePolicy`] has a zero freshness lifetime and every fetch issues -/// a conditional revalidation round-trip forever. Granting a small `max-age` -/// (60 seconds) lets repeated fetches inside the window be served straight from -/// the local cache; once it lapses the retained validators (`ETag` / -/// `Last-Modified`) still drive a cheap revalidation. Responses that carry their -/// own `Cache-Control` are used verbatim, so an origin's explicit freshness -/// policy is never weakened. -const SHARD_INDEX_SYNTHETIC_CACHE_CONTROL: &str = "max-age=60"; - -/// Scheme of a channel served from Azure Blob Storage. -/// -/// The gateway keeps the channel URL in its `az://` form throughout; the -/// networking middleware rewrites the scheme only at send time, on its own copy -/// of the request. So the scheme observed here still identifies an Azure -/// channel. -const AZURE_CHANNEL_SCHEME: &str = "az"; - -/// Build a [`CachePolicy`] for a shard-index response. -/// -/// Responses that already carry a `Cache-Control` header are used as-is. An -/// `az://` channel that answers without one is given a small synthetic `max-age` -/// so freshness accumulates instead of forcing a revalidation on every fetch. -/// Validators are preserved either way. -/// -/// The synthesis is limited to `az://` because a missing `Cache-Control` is -/// only known to be an unconfigurable property of the origin for Azure Blob -/// Storage. For any other origin the absence is a deliberate policy that the -/// client must not override. -fn shard_index_cache_policy(request: &SimpleRequest, response: &Response) -> CachePolicy { - if response.headers().contains_key(http::header::CACHE_CONTROL) - || request.uri().scheme_str() != Some(AZURE_CHANNEL_SCHEME) - { - return CachePolicy::new(request, response); - } - - let mut synthetic = http::Response::new(()); - *synthetic.status_mut() = response.status(); - *synthetic.headers_mut() = response.headers().clone(); - synthetic.headers_mut().insert( - http::header::CACHE_CONTROL, - http::HeaderValue::from_static(SHARD_INDEX_SYNTHETIC_CACHE_CONTROL), - ); - CachePolicy::new(request, &synthetic) -} - /// Creates a `SubdirNotFoundError` for when sharded repodata is not available. fn create_subdir_not_found_error(channel_base_url: &Url) -> GatewayError { GatewayError::SubdirNotFoundError(Box::new(SubdirNotFoundError { @@ -320,33 +272,17 @@ pub async fn fetch_index( &response, SystemTime::now(), ) { - AfterResponse::NotModified(refreshed, _) => { - // The cached file is still valid. `after_response` - // returns a refreshed policy derived from the stored - // 200 response with the 304's headers merged, so it - // stays storable and retains the original (or - // synthesized) freshness window. Persist it so the - // next fetch inside the window is a local cache hit - // instead of yet another conditional round-trip. - match read_cached_body(&mut cache_reader).await { - Ok(body) => { + AfterResponse::NotModified(_policy, _) => { + // The cached file is still valid + match read_shard_index_from_reader(&mut cache_reader).await { + Ok(shard_index) => { tracing::debug!("shard index cache was not modified"); - let mut guard = cache_reader.into_inner(); - if let Err(e) = write_shard_index_cache( - guard.inner_mut(), - refreshed, - Bytes::from(body.clone()), - ) - .await - { - tracing::warn!( - "failed to persist refreshed shard index cache policy: {e}" - ); - } if let Some((reporter, index)) = download_reporter { reporter.on_download_complete(response.url(), index); } - return parse_shard_index(body).await; + // If reading the file failed for some reason we'll just + // fetch it again. + return Ok(shard_index); } Err(e) => { tracing::warn!( @@ -358,13 +294,9 @@ pub async fn fetch_index( } } } - AfterResponse::Modified(_, _) => { + AfterResponse::Modified(policy, _) => { // Close the old file so we can create a new one. tracing::debug!("shard index cache has become stale"); - // Synthesize freshness for a Cache-Control-less - // response so the re-cached index does not revalidate - // on every subsequent fetch. - let policy = shard_index_cache_policy(&canonical_request, &response); return from_response( cache_reader.into_inner(), &cache_path, @@ -451,7 +383,7 @@ pub async fn fetch_index( return Err(create_subdir_not_found_error(channel_base_url)); } - let policy = shard_index_cache_policy(&canonical_request, &response); + let policy = CachePolicy::new(&canonical_request, &response); from_response( cache_reader.into_inner(), &cache_path, @@ -531,20 +463,18 @@ async fn write_not_found_cache(cache_file: &mut File, policy: CachePolicy) -> st .await } -/// Read the remaining bytes (the cached shard-index body) from a reader. -async fn read_cached_body( +/// Read the shard index from a reader and deserialize it. +pub async fn read_shard_index_from_reader( reader: &mut BufReader, -) -> Result, GatewayError> { +) -> Result { + // Read the file to memory let mut bytes = Vec::new(); reader .read_to_end(&mut bytes) .await .map_err(|e| GatewayError::IoError("failed to read shard index buffer".to_string(), e))?; - Ok(bytes) -} -/// Deserialize a shard index from raw `msgpack` bytes. -async fn parse_shard_index(bytes: Vec) -> Result { + // Deserialize the bytes run_blocking_task(move || { rmp_serde::from_slice(&bytes) .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) @@ -553,14 +483,6 @@ async fn parse_shard_index(bytes: Vec) -> Result( - reader: &mut BufReader, -) -> Result { - let bytes = read_cached_body(reader).await?; - parse_shard_index(bytes).await -} - /// Cache information stored at the start of the cache file. #[derive(Clone, Debug, Serialize, Deserialize)] struct CacheHeader { @@ -634,104 +556,3 @@ impl RequestLike for SimpleRequest { &self.uri() == other } } - -#[cfg(test)] -mod tests { - use std::time::{Duration, SystemTime}; - - use http::{StatusCode, header}; - use http_cache_semantics::{AfterResponse, BeforeRequest}; - use url::Url; - - use super::{SimpleRequest, shard_index_cache_policy}; - - /// Builds a bodyless `reqwest::Response`, mirroring how an origin (e.g. - /// Azure Blob Storage) answers for a shard index. - fn response(status: StatusCode, etag: &str, cache_control: Option<&str>) -> reqwest::Response { - let mut builder = http::Response::builder() - .status(status) - .header(header::ETAG, etag); - if let Some(cache_control) = cache_control { - builder = builder.header(header::CACHE_CONTROL, cache_control); - } - reqwest::Response::from(builder.body(Vec::new()).unwrap()) - } - - /// Shard index URL for a channel, in the scheme the gateway holds it in. - fn shards_url(scheme: &str) -> Url { - Url::parse(&format!( - "{scheme}://example.blob.core.windows.net/channel/noarch/repodata_shards.msgpack.zst" - )) - .unwrap() - } - - /// The synthesis is scoped to Azure channels: an `az://` response without a - /// `Cache-Control` gains a freshness window, while any other origin keeps the - /// zero-freshness policy that the absent header implies. - #[test] - fn synthesis_is_limited_to_azure_channels() { - let now = SystemTime::now(); - let no_cache_control = || response(StatusCode::OK, "\"v1\"", None); - - let azure = SimpleRequest::get(&shards_url("az")); - assert!( - shard_index_cache_policy(&azure, &no_cache_control()).time_to_live(now) - > Duration::ZERO, - "an az:// shard index must be given a synthetic freshness window" - ); - - let https = SimpleRequest::get(&shards_url("https")); - assert_eq!( - shard_index_cache_policy(&https, &no_cache_control()).time_to_live(now), - Duration::ZERO, - "a non-Azure origin's missing Cache-Control must be honoured as-is" - ); - } - - /// A 304 revalidation must persist a policy that keeps a non-zero freshness - /// window, so a fetch within that window is a local cache hit instead of yet - /// another conditional round-trip. - #[test] - fn revalidation_policy_retains_freshness() { - let url = shards_url("az"); - let request = SimpleRequest::get(&url); - let now = SystemTime::now(); - - // The original 200 carried no `Cache-Control`, so `shard_index_cache_policy` - // synthesizes a small `max-age` and the stored policy starts fresh. - let original = response(StatusCode::OK, "\"v1\"", None); - let stored = shard_index_cache_policy(&request, &original); - assert!(matches!( - stored.before_request(&request, now), - BeforeRequest::Fresh(_) - )); - - // A later revalidation is answered with a 304 that, like Azure, carries no - // `Cache-Control`. Routing it through `after_response` yields the policy - // that is persisted for the next fetch. - let not_modified = response(StatusCode::NOT_MODIFIED, "\"v1\"", None); - let refreshed = match stored.after_response(&request, ¬_modified, now) { - AfterResponse::NotModified(policy, _) => policy, - AfterResponse::Modified(_, _) => { - panic!("a matching ETag must revalidate as NotModified") - } - }; - - // The refreshed policy must retain a non-zero freshness window so the next - // fetch is served straight from the cache. - assert!( - refreshed.time_to_live(now) > Duration::ZERO, - "refreshed policy lost its freshness window" - ); - assert!(matches!( - refreshed.before_request(&request, now), - BeforeRequest::Fresh(_) - )); - - // Regression guard: building a policy directly from the 304 (a status that - // is not storable) yields zero freshness, which would force perpetual - // revalidation. - let from_304 = shard_index_cache_policy(&request, ¬_modified); - assert_eq!(from_304.time_to_live(now), Duration::ZERO); - } -} From 494a6b01ea48ba7f56f3dc57ce152e1a8c359263 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:02:51 +1000 Subject: [PATCH 77/98] fix(networking): gate the auth store to http and https --- .../src/authentication_middleware.rs | 37 +++++++++++++++++++ .../src/azure_middleware.rs | 14 ++++--- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 272db69f0f..88ca981f3a 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -35,6 +35,13 @@ impl Middleware for AuthenticationMiddleware { return next.run(req, extensions).await; } + // Entries here are keyed by host alone, so without this gate a + // `*.blob.core.windows.net` token would attach to `az://` too. Those schemes + // carry their own signing middleware and their own grant model. + if !matches!(req.url().scheme(), "http" | "https") { + return next.run(req, extensions).await; + } + let url = req.url().clone(); match self.auth_storage.get_by_url_with_host(url) { Err(_) => { @@ -431,6 +438,36 @@ mod tests { Ok(()) } + #[cfg(feature = "keyring")] + #[tokio::test] + async fn non_http_schemes_are_left_unauthenticated() -> anyhow::Result<()> { + let tdir = tempdir()?; + let mut storage = AuthenticationStorage::empty(); + storage.add_backend(Arc::from(FileStorage::from_path( + tdir.path().to_path_buf().join("auth.json"), + )?)); + storage.store( + "*.blob.core.windows.net", + &Authentication::BearerToken("xyztokytoken".to_string()), + )?; + + let (client, mut captured_rx) = make_client_harness(&storage); + let request = client + .get("az://acct.blob.core.windows.net/channel/noarch/repodata.json") + .build()?; + let _ = client.execute(request).await; + + let captured_request = captured_rx.recv().await.unwrap(); + assert_eq!( + captured_request + .headers() + .get(reqwest::header::AUTHORIZATION), + None + ); + + Ok(()) + } + #[test] fn test_host_wildcard_expansion() -> anyhow::Result<()> { for (host, should_succeed) in [ diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index db07943e63..d60f400344 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -12,7 +12,9 @@ use reqwest::{Client, Request, Response}; use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; use url::Url; -/// The Azure Storage REST API version sent on every request. +/// The Azure Storage REST API version sent on every signed request. A URL that +/// already carries a SAS returns before this is attached, and the write path pins +/// its own version inside opendal. const X_MS_VERSION: &str = "2021-12-02"; /// Middleware that rewrites `az://` URLs to their wire form and, where a host is @@ -64,11 +66,11 @@ const X_MS_VERSION: &str = "2021-12-02"; /// hide behind it — and userinfo is invalid in a blob URL anyway. /// /// Granted credentials are resolved by reqsign's [`DefaultCredentialProvider`] -/// chain, in its usual order: environment variables, then workload/managed -/// identity, then the Azure CLI (`az login`). rattler's -/// [`crate::AuthenticationStorage`] is not consulted for Azure — there is no -/// `Authentication` Azure variant — so per-host credentials configured there do -/// not apply to `az://` requests. +/// chain, in its order: environment variables, the Azure CLI (`az login`), +/// client certificate, client secret, pipelines, workload identity, IMDS. +/// rattler's [`crate::AuthenticationStorage`] is not consulted — it has no Azure +/// variant, and [`crate::AuthenticationMiddleware`] handles only `http`/`https`, +/// so its host-keyed entries cannot reach an `az://` request either. #[derive(Clone)] pub struct AzureMiddleware { /// reqsign signer; caches the resolved credential internally. From 16c41e18cfdb304fd004be3945260b9a5996f661 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:02:51 +1000 Subject: [PATCH 78/98] fix(config): refuse cleartext grants to routable hosts and colliding host keys --- crates/rattler_config/src/config.rs | 4 + crates/rattler_config/src/config/azure.rs | 126 ++++++++++++++++++++-- crates/rattler_config/tests/compat.rs | 5 + 3 files changed, 127 insertions(+), 8 deletions(-) diff --git a/crates/rattler_config/src/config.rs b/crates/rattler_config/src/config.rs index 6c0421a226..67d3ae556b 100644 --- a/crates/rattler_config/src/config.rs +++ b/crates/rattler_config/src/config.rs @@ -351,6 +351,7 @@ impl Config for CommonConfig { "allow-hard-links".to_string(), "allow-ref-links".to_string(), "s3-options".to_string(), + "azure-options".to_string(), "index-config".to_string(), ]; keys.extend(prefixed_keys("build", self.build.keys())); @@ -446,6 +447,9 @@ where /// should surface these to the user as warnings (they are typos or /// keys of other tools). pub fn from_toml_str(input: &str) -> Result<(Self, BTreeSet), toml::de::Error> { + azure::ensure_no_colliding_hosts(&input.parse()?) + .map_err(serde::de::Error::custom::)?; + // The document is deserialized twice: once into the common // configuration and once into the extension. Each pass records the // keys it did not recognize; only keys unknown to *both* passes are diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs index 7155b2d928..b2f290757f 100644 --- a/crates/rattler_config/src/config/azure.rs +++ b/crates/rattler_config/src/config/azure.rs @@ -1,9 +1,23 @@ use indexmap::IndexMap; -use rattler_azure::{AzureEndpointOptions, AzureHost}; +use rattler_azure::{Auth, AzureEndpointOptions, AzureHost, AzureScheme}; use serde::{Deserialize, Serialize}; use crate::config::Config; +/// Whether a credential may cross this host's network unencrypted. +/// +/// A single-label name (`localhost`, a `docker compose` service) has no public DNS +/// resolution, so it counts as local; anything with a dot does not. +fn is_local(host: &AzureHost) -> bool { + match host.host() { + url::Host::Domain(domain) => !domain.contains('.'), + url::Host::Ipv4(ip) => { + ip.is_loopback() || ip.is_private() || ip.is_link_local() || ip.is_unspecified() + } + url::Host::Ipv6(ip) => ip.is_loopback() || ip.is_unspecified(), + } +} + /// Per-host options for Azure Blob channels, keyed by endpoint authority /// (including a port where one is used, e.g. `127.0.0.1:10000`). /// @@ -49,12 +63,43 @@ impl AzureOptionsMap { self.0.get(host).copied().unwrap_or_default() } - /// The configured hosts, in the order they were written. + /// The configured hosts, in the order the document's table iterated them + /// (`toml::Table` is a `BTreeMap`, so that is byte order, not write order). pub fn hosts(&self) -> impl Iterator { self.0.keys() } } +/// Reject a document that spells one host two ways. +/// +/// Both spellings reach serde, which keeps whichever the table iterated last — +/// silently overriding an `auth = false` or dropping a grant. TOML's own +/// duplicate-key check runs on the raw text, so it cannot see the collision; this +/// has to run while both spellings are still visible. +pub(crate) fn ensure_no_colliding_hosts(document: &toml::Table) -> Result<(), String> { + let Some(table) = document + .get("azure-options") + .and_then(toml::Value::as_table) + else { + return Ok(()); + }; + + let mut seen: IndexMap = IndexMap::new(); + for written in table.keys() { + // An unparseable key is serde's error to report, not ours. + let Ok(host) = AzureHost::parse(written) else { + continue; + }; + if let Some(first) = seen.insert(host.clone(), written) { + return Err(format!( + "`azure-options` names one host twice: \"{first}\" and \"{written}\" are both \ + `{host}`" + )); + } + } + Ok(()) +} + impl Config for AzureOptionsMap { fn is_default(&self) -> bool { self.0.is_empty() @@ -62,8 +107,9 @@ impl Config for AzureOptionsMap { fn merge_config(self, other: &Self) -> Result { // Merge the two maps, with `other`'s entries overwriting existing keys. - // A host is granted or not as a whole; entries are not merged field-wise, - // so a higher-precedence file cannot partially relax a grant. + // A host is granted or not as a whole, so mentioning a host in a + // higher-precedence file replaces the lower file's entry outright rather + // than merging field-wise the way `repodata-config` does. let mut merged = self.0.clone(); for (key, value) in &other.0 { merged.insert(key.clone(), *value); @@ -72,11 +118,28 @@ impl Config for AzureOptionsMap { } fn validate(&self) -> Result<(), super::ValidationError> { + for (host, options) in &self.0 { + if options.auth == Auth::DefaultChain + && options.scheme == AzureScheme::Http + && !is_local(host) + { + return Err(super::ValidationError::Invalid(format!( + "`azure-options.\"{host}\"` grants credentials over cleartext http. A \ + credential may only be sent unencrypted to a local endpoint: use an \ + https scheme, or address the emulator by loopback address." + ))); + } + } Ok(()) } fn keys(&self) -> Vec { - self.0.keys().map(ToString::to_string).collect() + // Quoted, because every Azure authority contains dots and an unquoted key + // is not the TOML path the user must pass to `config set`/`unset`. + self.0 + .keys() + .map(|host| toml::Value::from(host.to_string()).to_string()) + .collect() } } @@ -123,6 +186,52 @@ mod tests { assert_eq!(unlisted, AzureEndpointOptions::default()); } + /// A grant may only ride cleartext to an endpoint that is not routable off + /// the machine or its LAN. + #[test] + fn cleartext_grants_are_confined_to_local_endpoints() { + for authority in ["127.0.0.1:10000", "[::1]:10000", "azurite:10000"] { + let map: AzureOptionsMap = toml::from_str(&format!( + "[\"{authority}\"]\nauth = true\nscheme = \"http\"\n" + )) + .unwrap(); + assert!(map.validate().is_ok(), "{authority} is local"); + } + + for authority in ["mycompany.blob.core.windows.net", "internal.example.com"] { + let map: AzureOptionsMap = toml::from_str(&format!( + "[\"{authority}\"]\nauth = true\nscheme = \"http\"\n" + )) + .unwrap(); + assert!(map.validate().is_err(), "{authority} is routable"); + + // The same host over https, and the same cleartext scheme without a + // grant, are both fine — it is only the pair that is refused. + let https: AzureOptionsMap = + toml::from_str(&format!("[\"{authority}\"]\nauth = true\n")).unwrap(); + assert!(https.validate().is_ok()); + let anonymous: AzureOptionsMap = + toml::from_str(&format!("[\"{authority}\"]\nscheme = \"http\"\n")).unwrap(); + assert!(anonymous.validate().is_ok()); + } + } + + /// Two spellings of one host must be refused rather than one silently + /// winning: the loser here is an explicit `auth = false`. + #[test] + fn a_document_naming_one_host_twice_is_refused() { + let document = r#" +[azure-options."acct.blob.example"] +auth = false + +[azure-options."ACCT.blob.example."] +auth = true +"#; + let error = ensure_no_colliding_hosts(&document.parse().unwrap()) + .expect_err("a collision must be reported"); + assert!(error.contains("acct.blob.example"), "{error}"); + } + /// A later config file overwrites a host wholesale. It must not be able to /// keep an earlier `auth = true` while changing only the scheme. #[test] @@ -170,9 +279,10 @@ mod tests { "the grant written as `{written}` did not apply to `{looked_up}`" ); // The key is stored canonically, so `keys()` reports what a lookup - // would need rather than what happened to be typed, and writing the - // table back out produces a key that parses to the same host. - assert_eq!(map.keys(), vec![looked_up.to_string()], "{written}"); + // would need rather than what happened to be typed — quoted, as the + // TOML path it has to be written as — and writing the table back out + // produces a key that parses to the same host. + assert_eq!(map.keys(), vec![format!("\"{looked_up}\"")], "{written}"); let written_back = toml::to_string(&map).unwrap(); assert!( written_back.contains(&format!("[\"{looked_up}\"]")), diff --git a/crates/rattler_config/tests/compat.rs b/crates/rattler_config/tests/compat.rs index f929084c0d..f3d6a78290 100644 --- a/crates/rattler_config/tests/compat.rs +++ b/crates/rattler_config/tests/compat.rs @@ -185,6 +185,11 @@ const EDIT_MATRIX: &[(&str, &str)] = &[ "s3-options.some-bucket", r#"{"endpoint-url": "https://s3.example.com", "region": "auto", "force-path-style": true}"#, ), + // Quoted, because every Azure authority contains dots. + ( + r#"azure-options."acct.blob.core.windows.net""#, + r#"{"auth": true}"#, + ), ]; /// Every key in the matrix can be set on a fully populated config, the From 19db4df3ad3329a2be6a76c1cf7fcdeb6a478943 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:02:51 +1000 Subject: [PATCH 79/98] feat(azure): accept path_style as an alias for path-style --- crates/rattler_azure/src/options.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_azure/src/options.rs b/crates/rattler_azure/src/options.rs index 5540cbd265..56722c528a 100644 --- a/crates/rattler_azure/src/options.rs +++ b/crates/rattler_azure/src/options.rs @@ -175,7 +175,7 @@ pub struct AzureEndpointOptions { /// The field is named for what it holds, but the config key stays /// `path-style`: that is the spelling users have written, and the bool bridge /// is what the key means. - #[cfg_attr(feature = "serde", serde(rename = "path-style"))] + #[cfg_attr(feature = "serde", serde(rename = "path-style", alias = "path_style"))] pub addressing: Addressing, } From 227aa2094f5a05de2a9507d79cfb5c2d93eb0e0e Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:02:59 +1000 Subject: [PATCH 80/98] fix(index): scope the url import to s3 and drop the dead typos entry --- crates/rattler_index/src/lib.rs | 11 ++++++----- typos.toml | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index af6d22ef12..1365021432 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -56,7 +56,7 @@ use serde::Serialize; use sha2::{Digest, Sha256}; use tokio::sync::Semaphore; use tracing::Instrument; -#[cfg(any(feature = "s3", feature = "azure"))] +#[cfg(feature = "s3")] use url::Url; /// Channel metadata written into generated repodata. @@ -1544,10 +1544,11 @@ pub struct IndexAzureConfig { pub max_parallel: usize, /// The multi-progress bar to use for the index. pub multi_progress: Option, - // NOTE: no `precondition_checks` field. opendal's azblob service does not - // support conditional (`if_match`) writes, so precondition checks can only - // ever be disabled here; the Azure path hardcodes `Disabled` rather than - // exposing a knob whose enabled state always fails. + // NOTE: no `precondition_checks` field. opendal's azblob service supports + // `if_not_exists` and conditional reads but not conditional (`if_match`) + // writes, so any index of an already-populated channel fails under `Enabled`. + // The Azure path hardcodes `Disabled` rather than exposing a knob that only + // works on a first index; the conditional read is lost along with it. } /// Create a new `repodata.json` for all packages in the channel at the given diff --git a/typos.toml b/typos.toml index bffe0904e9..0fd1f25226 100644 --- a/typos.toml +++ b/typos.toml @@ -19,4 +19,3 @@ strat = "strat" haa = "haa" intoto = "intoto" certifi = "certifi" -AKS = "AKS" From 7c86cc5f214f387c6845e65025f10ad6bbe9b048 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:02:59 +1000 Subject: [PATCH 81/98] docs: correct false claims in the azure docs and comments --- crates/rattler_index/README.md | 23 +++++++++++---------- crates/rattler_index/tests/azure_azurite.rs | 4 ++-- crates/rattler_upload/src/lib.rs | 10 ++++----- crates/rattler_upload/src/upload/azure.rs | 17 ++++++++------- py-rattler/rattler/networking/middleware.py | 9 +++++--- 5 files changed, 34 insertions(+), 29 deletions(-) diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index 9b73671b41..6468ffebb9 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -36,8 +36,9 @@ login` AAD bearer token directly. Supply one of: - `--azure-cli`: mint a short-lived user-delegation SAS from the current `az login` session automatically. Requires the Azure CLI (`az`) on `PATH` and a prior `az login`. The minted SAS is scoped to the target container, granted - only the permissions indexing needs, and expires after 30 minutes (a SAS - cannot be individually revoked, so it is kept short-lived on purpose). + only the permissions indexing needs, and expires after 30 minutes by default + (`--azure-cli-sas-ttl-minutes`, plus two minutes of clock-skew headroom). A SAS + cannot be individually revoked, so it is kept short-lived on purpose. - `--account-key` / `AZURE_STORAGE_KEY`: a storage account key. - `--sas-token` / `AZURE_STORAGE_SAS_TOKEN`: a SAS token you supply yourself. @@ -48,15 +49,15 @@ via `--sas-token`: ```shell export AZURE_STORAGE_SAS_TOKEN=$(az storage container generate-sas \ --account-name my-storage-account --name my-container \ - --permissions rwlc --expiry 2026-01-01T00:00Z \ + --permissions rwlc --expiry "$(date -u -d '+30 minutes' +%Y-%m-%dT%H:%MZ)" \ --auth-mode login --as-user --https-only -o tsv) rattler-index --config ./rattler-config.toml az \ az://my-storage-account.blob.core.windows.net/my-container/my-channel ``` The `--config` flag points at the same TOML configuration file used by pixi. It -configures S3 and Azure credentials, concurrency, and per-channel index options -under the `[index-config]` section. +configures S3 credentials, Azure endpoint options, concurrency, and per-channel +index options under the `[index-config]` section. When `--config` is omitted, `rattler-index` falls back to its built-in defaults (`write-zst = true`, `write-shards = true`, no advertised repodata revisions, @@ -82,21 +83,21 @@ is rejected. Credentials are never stored in the config — they are resolved at runtime from `--account-key` / `--sas-token`, an `az login` session (`--azure-cli`), or the `DefaultCredentialProvider` chain. -A host that is not a plain `.blob.` domain, or that is not -reached over https, needs an entry under `[azure-options.""]`. The key is -the host with its port when the URL has one: +A host whose first label is not the storage account, or that is not reached over +https, needs an entry under `[azure-options.""]`. The key is the host with +its port when the URL has one: | Field | Type | Description | | --- | --- | --- | -| `auth` | boolean | Whether credentials may be sent to this host. Defaults to `false`, which fetches anonymously. This is the only way a credential attaches to a host, so keep these entries in your user-level config file, never in a checked-in project file. | +| `auth` | boolean | Whether credentials may be sent to this host **when fetching**. Defaults to `false`, which fetches anonymously, and is the only way a credential attaches to a host on the fetch path — so keep these entries in your user-level config file, never in a checked-in project file. It has no effect on `rattler-index` or `rattler upload`, which take their credentials from the command line. | | `scheme` | string | The scheme `az://` is rewritten to: `"https"` (default) or `"http"`. Use `http` for local emulators only. | | `path-style` | boolean | Where the storage account is found. `false` (default) reads it from the first host label. `true` reads it from the first path segment instead, which is the only form that works for an IP-literal or single-label host. | -Indexing a channel in the Azurite emulator needs all three: +Indexing a channel in the Azurite emulator needs the two wire settings (add +`auth = true` as well if the same host is also fetched from): ```toml [azure-options."127.0.0.1:10000"] -auth = true scheme = "http" path-style = true ``` diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs index 62bdfc4ae3..c50d75af66 100644 --- a/crates/rattler_index/tests/azure_azurite.rs +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -304,8 +304,8 @@ async fn azurite_multi_block_write_keeps_cache_control() { /// multi-block path, and this is as close as a test can currently get to that /// upload: `rattler_upload` reads no config file, so it always builds a host-style /// endpoint and cannot be pointed at an emulator at all. What it can share is the -/// operator, built here through the same `azblob_config`, and the exact writer -/// options the uploader uses — so the behaviour being reproduced is the uploader's, +/// operator, built here through the same `azblob_config`, and the uploader's chunk +/// size and overwrite guard — so the behaviour being reproduced is the uploader's, /// even though the entry point is not. #[tokio::test] #[ignore = "requires a running Azurite emulator; see the module docs"] diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index 2bddb581a0..7c300f406d 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -87,11 +87,11 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { // The anonymous https host-style defaults, because `upload_from_args` // reads no configuration file at all — it only opens the auth store — // so there is nowhere for an `[azure-options.""]` entry to come - // from. The ceiling is exactly real Azure: an emulator or path-style - // endpoint cannot be uploaded to from this entry point. Lifting it - // means giving `rattler_upload` a `--config` of its own (rattler-build, - // its only caller, already parses one) and passing the host's entry - // through here; everything below already honours what it is handed. + // from. The ceiling is https plus host-style addressing — any host + // whose first label is the account, Azure or not; an http or + // path-style endpoint (Azurite as normally run) is unreachable. + // Lifting it means giving `rattler_upload` a `--config` of its own, + // which neither `rattler upload` nor rattler-build passes today. let options = rattler_azure::AzureEndpointOptions::default(); let credentials = azure_opts .credentials diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index f0fb767639..bcbacf11e8 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -85,11 +85,11 @@ async fn upload_single_package( // opendal 0.57 only honours `if_not_exists` on the single-shot Put Blob path, // never the multi-block Put Block List path used for packages larger than a // single block, so the writer-level `if_not_exists(!force)` below silently - // does nothing for large uploads. An explicit `stat` closes that gap at all - // sizes. The residual stat->write TOCTOU (another writer could create the - // blob between this check and `close`) is acceptable: it is strictly better - // than a silent clobber, and the writer-level `if_not_exists` still - // guards the small-file and racing-writer cases. + // does nothing for large uploads. An explicit `stat` closes that gap for a + // blob that already exists, but not for two writers racing to create one: + // above the chunk size both stat as absent and the second commit wins + // silently. Concurrent uploads of the same package are unsafe until opendal + // carries `if_none_match` onto the Put Block List path. if !force { match op.stat(&key).await { Ok(_) => { @@ -201,9 +201,10 @@ mod test { } /// C2: without `--force`, uploading over an existing blob must error rather - /// than silently overwrite it, at all package sizes. This exercises the - /// explicit pre-write `stat` guard that backstops opendal's - /// `if_not_exists`, which is dropped on the multi-block upload path. + /// than silently overwrite it. The memory backend honours `if_not_exists` on + /// every path and the fixture is a single block, so this covers the small-blob + /// path only — for the multi-block behaviour see + /// `rattler_index/tests/azure_azurite.rs::azurite_if_not_exists_is_dropped_on_the_multi_block_path`. #[tokio::test] async fn test_existing_blob_without_force_errors() { let op = memory_operator(); diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index fc1c1a6047..821cbc90e1 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -161,9 +161,12 @@ class AzureMiddleware: """ Middleware to work with az:// URLs. - Fetches are anonymous: the per-host `azure-options` table (credential grants, - `scheme`, `path-style`) is not exposed to Python yet — that mirrors what - `S3Config` does for buckets and is follow-up work. + Fetches are anonymous. The per-host `azure-options` table (credential grants, + `scheme`, `path-style`) is not exposed to Python yet — unlike `S3Config`, which + does expose per-bucket configuration — so a private container cannot be read + from Python, and an emulator or custom endpoint cannot be reached at all. Azure + answers an unauthorized read of a private container with a 404, so that is what + surfaces. Follow-up work. Examples -------- From 68bc60c87edc44bd8bb9d2b9c065b57d6455212f Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:10:25 +1000 Subject: [PATCH 82/98] docs: trim comments that do not carry their weight --- .github/workflows/e2e-azure-tests.yml | 15 ------ crates/rattler-bin/src/commands/client.rs | 5 +- crates/rattler_azure/src/clap.rs | 12 +---- crates/rattler_azure/src/lib.rs | 64 ++--------------------- 4 files changed, 7 insertions(+), 89 deletions(-) diff --git a/.github/workflows/e2e-azure-tests.yml b/.github/workflows/e2e-azure-tests.yml index 8c4ff3dd70..8b2b0950ac 100644 --- a/.github/workflows/e2e-azure-tests.yml +++ b/.github/workflows/e2e-azure-tests.yml @@ -34,21 +34,6 @@ jobs: # set up the rust build using sccache. SCCACHE_GHA_ENABLED: "true" - # A service container, so the runner does the waiting and the log collection: - # the health check gates every step below on the blob service actually - # listening, and the runner prints the container's logs when the job ends. - # The S3 E2E job is not a precedent either way — its Minio is a native binary - # spawned by `scripts/e2e/s3-minio.nu`, not a container. - # - # The image's default command is used as-is, since a service container cannot - # override one. That starts queue and table alongside blob; only the blob port - # is published, and the tests are unaffected. - # - # `nc -z` and not an HTTP request: every unsigned request is answered with a - # 4xx, which most HTTP clients treat as failure, so a listening socket is both - # the simplest and the most honest readiness signal. `curl` is not in the image - # anyway. - # # No secrets: the emulator's `devstoreaccount1` credentials are published # constants, spelled out in the tests, and only ever reach loopback. services: diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index 8379b32c77..01bc98d7db 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -63,10 +63,7 @@ pub fn create_client_with_middleware( let client = client.with(rattler_networking::GCSMiddleware::default()); // `az://` URLs carry the full blob endpoint, so the middleware needs no // configuration to *reach* one — it just swaps the scheme. The empty options - // table means every `az://` fetch here is anonymous; this CLI reads no config - // file, so there is no `azure-options` to hand it (the same reason S3 gets an - // empty map above). It reuses the configured download client for reqsign's - // credential resolution so proxy/CA/TLS settings carry through. + // table means every `az://` fetch here is anonymous. #[cfg(feature = "azure")] let client = client.with(rattler_networking::AzureMiddleware::new( download_client, diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index eea35c5c76..5319373f7e 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -45,9 +45,6 @@ pub enum AzureCredentialsError { /// reason about combinations. Only [`AzureAuthSource::AzureCli`] carries state /// (the minting TTL), which is why account/container derivation is needed for /// that arm alone. -/// -/// Two of the three arms carry a secret, so `Debug` is implemented by hand to -/// redact them rather than derived (see [`AzureCredentials`]). #[derive(Clone, PartialEq, Eq)] pub enum AzureAuthSource { /// Use a shared storage account key verbatim. @@ -113,11 +110,6 @@ impl AzureAuthSource { /// exported *and* `--azure-cli` is passed), so [`AzureCredentialsOpts::source`] /// applies an explicit precedence rather than treating the combination as an /// error — see that method for the exact ordering. -/// -/// `account_key` and `sas_token` are secrets, so `Debug` is implemented by hand -/// to redact them rather than derived (see [`AzureCredentials`]). This matters -/// more here than elsewhere: these options are typically part of a larger clap -/// struct that gets logged wholesale on a parse or startup failure. #[derive(Clone, PartialEq, Parser)] pub struct AzureCredentialsOpts { /// The Azure Storage account key. @@ -343,9 +335,7 @@ mod tests { assert_eq!(err.kind(), clap::error::ErrorKind::ArgumentConflict); } - /// Neither the resolved source nor the raw options may print a secret. Both - /// types hand-write `Debug` precisely so that a wholesale `{:?}` of an - /// enclosing CLI struct cannot leak a key or token into a log. + /// Neither the resolved source nor the raw options may print a secret. #[test] fn debug_never_prints_secrets() { let sources = [ diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 279c50cc3c..3f4a483b1e 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -13,7 +13,7 @@ //! //! Userinfo (`user:pass@host`) is rejected wherever a host is parsed, because //! `az://real.host@evil.example/…` reads as the real host while addressing the -//! attacker's. +//! attacker's and provides no real functionality. #[cfg(feature = "clap")] pub mod clap; @@ -185,9 +185,7 @@ pub struct AzureCoordinates { /// derived account and container are additionally held to Azure's own naming /// rules — under *both* addressing styles, since path-style takes the account /// from user-controlled path text. Those rules reject an empty name, any -/// character outside `[a-z0-9-]`, and a leading `-`, which together are what keep -/// an option-shaped value such as `--as-user` from ever reaching the `az` argv in -/// [`mint_user_delegation_sas`]. +/// character outside `[a-z0-9-]`, and a leading `-`. /// /// [crate-level docs]: crate pub fn account_and_container( @@ -197,10 +195,6 @@ pub fn account_and_container( if !url.username().is_empty() || url.password().is_some() { return Err(AzureUrlError::UserInfoNotAllowed); } - // Re-normalize through `AzureHost` rather than reading `host_str()` directly, - // so the account label is taken from the same spelling the config table is - // keyed by — trailing dot stripped, empty labels rejected, IP literals - // discriminated by type rather than by counting dots. let host = AzureHost::from_url(url)?; // Empty segments are never a valid name, so an empty one is a missing one. let mut segments = url @@ -233,15 +227,6 @@ pub fn account_and_container( } /// Check a derived name against Azure's storage account naming rules. -/// -/// Azure's rules are stricter than "whatever a URL host label may contain": 3-24 -/// characters of lowercase letters and digits, no hyphens at all. Enforcing the -/// real rules rather than a permissive charset is what makes the guarantee in -/// [`account_and_container`] true — a name that cannot contain `-` can never be -/// read as an option by a subprocess — and it rejects names Azure would reject -/// anyway, while the error can still name the URL that produced them. The length -/// bound is also what rejects an empty name, so emptiness is not left to a -/// caller's `filter`. fn validate_account(name: &str) -> Result<(), AzureUrlError> { let valid = (3..=24).contains(&name.len()) && name @@ -255,9 +240,7 @@ fn validate_account(name: &str) -> Result<(), AzureUrlError> { /// Check a derived name against Azure's blob container naming rules. /// /// 3-63 characters of lowercase letters, digits and hyphens, with no leading or -/// trailing hyphen and no consecutive hyphens. The leading-hyphen rule is the one -/// carrying security weight: `--https-only` is a perfectly good `[a-z0-9-]` -/// string, so a charset check alone would hand it to the `az` argv as an option. +/// trailing hyphen and no consecutive hyphens. fn validate_container(name: &str) -> Result<(), AzureUrlError> { let valid = (3..=63).contains(&name.len()) && name @@ -273,42 +256,6 @@ fn validate_container(name: &str) -> Result<(), AzureUrlError> { /// A normalized Azure Blob endpoint authority: a host, and its port when one is /// written. -/// -/// This is the identity of an endpoint everywhere on the Azure path — the -/// `azure-options` table key, the authority of an [`AzureChannelUrl`], and the -/// text the guided errors tell a user to write. Because the *same* type -/// deserializes a config key and is handed out by [`AzureChannelUrl::host`], a -/// written key and a lookup cannot disagree about spelling: both are the output of -/// [`AzureHost::parse`]. -/// -/// # Why not a `Url` -/// -/// A `Url`'s port is scheme-relative: `https://host:443` serializes back as -/// `https://host`, because 443 is the https default. Keeping the authority in a -/// `Url` with a fixed scheme therefore *loses* a user-written `:443`, and makes -/// equality scheme-relative along with it (`host:443` == `host`). Holding the host -/// and the port as separate values keeps a written port a written port, whatever -/// scheme the request eventually uses. -/// -/// # Normalization -/// -/// [`parse`](Self::parse) is the only way in. It applies the URL Standard's host -/// parser (via [`url::Host`]) — lowercasing, IDNA (`ünï.example` → -/// `xn--n-nga1b.example`), and canonical, *typed* IP literals (`0x7f.1` → -/// [`url::Host::Ipv4`], `[0:0:0:0:0:0:0:1]` → [`url::Host::Ipv6`]) — plus two -/// rules of its own: -/// -/// - a single trailing dot, the DNS root label, is stripped, so -/// `acct.blob.example.` and `acct.blob.example` are one host; -/// - no label may be empty, so `acct..blob.example` is rejected rather than -/// silently handing an empty label to account derivation. -/// -/// The port is deliberately *not* normalized: which scheme this host is reached -/// over is decided later, by its options entry, so no port here can be known to -/// be a default. A written port is kept verbatim and an absent one stays absent. -/// -/// [`Display`](std::fmt::Display) writes the canonical `host[:port]` text with an -/// IPv6 literal bracketed, and round-trips through [`parse`](Self::parse). #[derive(Clone, PartialEq, Eq, Hash)] #[cfg_attr( feature = "serde", @@ -823,9 +770,8 @@ pub async fn mint_user_delegation_sas( permissions: &str, valid_for: std::time::Duration, ) -> Result { - /// Extra slack added to the requested lifetime so a slightly fast client - /// clock (the expiry is computed from *this* machine's time) does not shrink - /// the usable window toward zero at the Azure end. + /// Slack for a client clock running up to two minutes slow, since the expiry + /// is computed here and evaluated by Azure. const CLOCK_SKEW_HEADROOM: std::time::Duration = std::time::Duration::from_secs(120); let signed = jiff::SignedDuration::try_from(valid_for.saturating_add(CLOCK_SKEW_HEADROOM)) From 144f527b53533980cb57d85a58e5e1f738291b96 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:22:52 +1000 Subject: [PATCH 83/98] refactor: hold azure credentials in secrecy::SecretString --- Cargo.lock | 1 + Cargo.toml | 1 + crates/rattler_azure/Cargo.toml | 1 + crates/rattler_azure/src/clap.rs | 96 +++++++-------- crates/rattler_azure/src/lib.rs | 128 ++++++++++++++------ crates/rattler_index/src/lib.rs | 6 +- crates/rattler_index/src/main.rs | 4 +- crates/rattler_index/tests/azure_azurite.rs | 4 +- crates/rattler_upload/src/upload/azure.rs | 10 +- 9 files changed, 159 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef35680e47..8d014874dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5057,6 +5057,7 @@ dependencies = [ "jiff", "opendal", "percent-encoding", + "secrecy", "serde", "thiserror 2.0.19", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 5e81981696..9add3b1373 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -153,6 +153,7 @@ rstest = { version = "0.26" } rstest_reuse = "0.7" rustix = { version = "1.1", default-features = false } simd-json = { version = "0.17", features = ["serde_impl"] } +secrecy = "0.10" self_cell = "1" serde = { version = "1" } serde_bytes = { version = "0.11" } diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index 690a1f68b1..631c45083f 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -30,6 +30,7 @@ opendal = { workspace = true, default-features = false, features = [ "services-azblob", ], optional = true } percent-encoding = { workspace = true, optional = true } +secrecy = { workspace = true } serde = { workspace = true, features = ["derive"], optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["process"], optional = true } diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index 5319373f7e..1bf1d0a75d 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -2,6 +2,8 @@ use std::time::Duration; use clap::Parser; +use secrecy::{ExposeSecret, SecretString}; + use crate::{ AzureCliSasError, AzureCoordinates, AzureCredentials, AzureUrlError, mint_user_delegation_sas, }; @@ -45,13 +47,13 @@ pub enum AzureCredentialsError { /// reason about combinations. Only [`AzureAuthSource::AzureCli`] carries state /// (the minting TTL), which is why account/container derivation is needed for /// that arm alone. -#[derive(Clone, PartialEq, Eq)] +#[derive(Clone, Debug)] pub enum AzureAuthSource { /// Use a shared storage account key verbatim. - AccountKey(String), + AccountKey(SecretString), /// Use a supplied SAS token verbatim. - SasToken(String), + SasToken(SecretString), /// Mint a short-lived user-delegation SAS from the current `az login` /// session, valid for `ttl`. @@ -61,22 +63,6 @@ pub enum AzureAuthSource { }, } -impl std::fmt::Debug for AzureAuthSource { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - // Print only the variant, never the secret it carries. - AzureAuthSource::AccountKey(_) => { - f.debug_tuple("AccountKey").field(&"").finish() - } - AzureAuthSource::SasToken(_) => f.debug_tuple("SasToken").field(&"").finish(), - // The TTL is not a secret. - AzureAuthSource::AzureCli { ttl } => { - f.debug_struct("AzureCli").field("ttl", ttl).finish() - } - } - } -} - impl AzureAuthSource { /// Resolve this source into concrete [`AzureCredentials`]. /// @@ -110,7 +96,7 @@ impl AzureAuthSource { /// exported *and* `--azure-cli` is passed), so [`AzureCredentialsOpts::source`] /// applies an explicit precedence rather than treating the combination as an /// error — see that method for the exact ordering. -#[derive(Clone, PartialEq, Parser)] +#[derive(Clone, Debug, Parser)] pub struct AzureCredentialsOpts { /// The Azure Storage account key. /// @@ -121,17 +107,19 @@ pub struct AzureCredentialsOpts { long, env = "AZURE_STORAGE_KEY", conflicts_with = "sas_token", - help_heading = "Azure Credentials" + help_heading = "Azure Credentials", + value_parser = secret )] - pub account_key: Option, + pub account_key: Option, /// A shared access signature (SAS) token, with or without a leading `?`. #[arg( long, env = "AZURE_STORAGE_SAS_TOKEN", - help_heading = "Azure Credentials" + help_heading = "Azure Credentials", + value_parser = secret )] - pub sas_token: Option, + pub sas_token: Option, /// Mint a short-lived user-delegation SAS from the current `az login` /// session (requires the Azure CLI). @@ -157,17 +145,29 @@ pub struct AzureCredentialsOpts { pub azure_cli_sas_ttl_minutes: u64, } -impl std::fmt::Debug for AzureCredentialsOpts { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Whether a secret was supplied is useful for diagnosing precedence; the - // secret itself never is. - let redact = |value: &Option| value.as_ref().map(|_| ""); - f.debug_struct("AzureCredentialsOpts") - .field("account_key", &redact(&self.account_key)) - .field("sas_token", &redact(&self.sas_token)) - .field("azure_cli", &self.azure_cli) - .field("azure_cli_sas_ttl_minutes", &self.azure_cli_sas_ttl_minutes) - .finish() +/// Take a command-line or environment value straight into a [`SecretString`], so +/// it is never held as a plain `String` that a `{:?}` could reach. +fn secret(value: &str) -> Result { + Ok(value.into()) +} + +impl PartialEq for AzureCredentialsOpts { + /// Hand-written because [`SecretString`] withholds `PartialEq` — comparing + /// secrets is not constant-time. The containing `UploadOpts` tree derives + /// `PartialEq`, and comparing parsed command lines is not a secrets check. + fn eq(&self, other: &Self) -> bool { + fn same(left: Option<&SecretString>, right: Option<&SecretString>) -> bool { + match (left, right) { + (Some(left), Some(right)) => left.expose_secret() == right.expose_secret(), + (None, None) => true, + _ => false, + } + } + + same(self.account_key.as_ref(), other.account_key.as_ref()) + && same(self.sas_token.as_ref(), other.sas_token.as_ref()) + && self.azure_cli == other.azure_cli + && self.azure_cli_sas_ttl_minutes == other.azure_cli_sas_ttl_minutes } } @@ -222,8 +222,8 @@ mod tests { azure_cli: bool, ) -> AzureCredentialsOpts { AzureCredentialsOpts { - account_key: account_key.map(str::to_string), - sas_token: sas_token.map(str::to_string), + account_key: account_key.map(Into::into), + sas_token: sas_token.map(Into::into), azure_cli, azure_cli_sas_ttl_minutes: DEFAULT_AZURE_CLI_SAS_TTL_MINUTES, } @@ -238,7 +238,7 @@ mod tests { async fn account_key_resolves() { assert!(matches!( opts(Some("key"), None, false).resolve("cw", unreachable_context).await, - Ok(AzureCredentials::AccountKey(k)) if k == "key" + Ok(AzureCredentials::AccountKey(k)) if k.expose_secret() == "key" )); } @@ -246,7 +246,7 @@ mod tests { async fn sas_token_resolves() { assert!(matches!( opts(None, Some("sv=..."), false).resolve("cw", unreachable_context).await, - Ok(AzureCredentials::SasToken(t)) if t == "sv=..." + Ok(AzureCredentials::SasToken(t)) if t.expose_secret() == "sv=..." )); } @@ -270,12 +270,12 @@ mod tests { // SAS token beats an account key when `--azure-cli` is absent. assert!(matches!( opts(Some("key"), Some("sv=..."), false).source(), - Ok(AzureAuthSource::SasToken(t)) if t == "sv=..." + Ok(AzureAuthSource::SasToken(t)) if t.expose_secret() == "sv=..." )); // Account key is the last resort. assert!(matches!( opts(Some("key"), None, false).source(), - Ok(AzureAuthSource::AccountKey(k)) if k == "key" + Ok(AzureAuthSource::AccountKey(k)) if k.expose_secret() == "key" )); } @@ -283,12 +283,10 @@ mod tests { fn azure_cli_ttl_is_carried_through() { let mut opts = opts(None, None, true); opts.azure_cli_sas_ttl_minutes = 45; - assert_eq!( + assert!(matches!( opts.source().unwrap(), - AzureAuthSource::AzureCli { - ttl: Duration::from_secs(45 * 60), - } - ); + AzureAuthSource::AzureCli { ttl } if ttl == Duration::from_secs(45 * 60) + )); } // The `--azure-cli` resolve path shells out to `az`, which isn't available in @@ -344,7 +342,7 @@ mod tests { ]; for source in &sources { let out = format!("{source:?}"); - assert!(out.contains(""), "not redacted: {out}"); + assert!(out.contains("REDACTED"), "not redacted: {out}"); assert!(!out.contains("supersecret"), "leaked key: {out}"); assert!(!out.contains("deadbeef"), "leaked token: {out}"); } @@ -356,11 +354,11 @@ mod tests { assert!(format!("{cli:?}").contains("60")); let out = format!("{:?}", opts(Some("supersecretkey"), None, false)); - assert!(out.contains(""), "not redacted: {out}"); + assert!(out.contains("REDACTED"), "not redacted: {out}"); assert!(!out.contains("supersecret"), "leaked key: {out}"); let out = format!("{:?}", opts(None, Some("sig=deadbeef"), false)); - assert!(out.contains(""), "not redacted: {out}"); + assert!(out.contains("REDACTED"), "not redacted: {out}"); assert!(!out.contains("deadbeef"), "leaked token: {out}"); // Absent secrets print as `None`, so the redaction cannot be mistaken for diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 3f4a483b1e..e0737b012e 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -22,6 +22,7 @@ pub mod options; pub use options::{Addressing, Auth, AzureEndpointOptions, AzureScheme}; +pub use secrecy::{ExposeSecret, SecretString}; use url::Url; /// Credentials for authenticating to Azure Blob storage. @@ -32,28 +33,16 @@ use url::Url; /// by the consumer from the channel URL together with the host's addressing /// style (see [`account_and_container`]). /// -/// The type deliberately has no `Serialize`/`Deserialize`: it holds raw account -/// keys and SAS tokens, so serialization would risk leaking secrets to disk. For -/// the same reason `Debug` is implemented by hand to redact the secret values -/// rather than derived. -#[derive(Clone)] +/// Both variants hold a [`SecretString`], so `Debug` redacts them, the bytes are +/// zeroized on drop, and every read is a visible `expose_secret()`. The type has +/// no `Serialize`/`Deserialize` either, so it cannot reach disk. +#[derive(Clone, Debug)] pub enum AzureCredentials { /// A shared storage account key. - AccountKey(String), + AccountKey(SecretString), /// A shared access signature (SAS) token. - SasToken(String), -} - -impl std::fmt::Debug for AzureCredentials { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Print only the variant, never the secret it carries. - let variant = match self { - AzureCredentials::AccountKey(_) => "AccountKey", - AzureCredentials::SasToken(_) => "SasToken", - }; - f.debug_tuple(variant).field(&"").finish() - } + SasToken(SecretString), } /// Strip a single leading `?` from a SAS token. @@ -471,7 +460,7 @@ impl From for String { /// takes it from the channel host's entry; `rattler_upload` passes the default, /// because it reads no config file at all (see the note in /// `rattler_upload::upload_from_args`). -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Clone, PartialEq, Eq, Hash)] pub struct AzureChannelUrl { /// The authority, normalized independently of any scheme. host: AzureHost, @@ -543,26 +532,31 @@ impl AzureChannelUrl { /// through this spelling, and `[azure-options."…"]` through [`Self::host`]. /// Matching the wire string instead was reviewer issue 5 — the two spellings /// exist so a config key never has to guess which one a channel was stored as. + /// A SAS written inline is masked: this spelling is the one that reaches logs + /// and error messages, and [`Self::wire`] is the only way to the signature. pub fn canonical(&self) -> Url { - self.spelled("az") + self.spelled("az", Sas::Masked) } /// The `http(s)://host/path` spelling used for actual requests, over the /// scheme the host's options entry asks for. pub fn wire(&self, scheme: AzureScheme) -> Url { - self.spelled(scheme.as_str()) + self.spelled(scheme.as_str(), Sas::Exposed) } /// Build one spelling of this URL. /// /// Both public spellings go through here, so they cannot differ in anything - /// but the scheme: the host, port, path, query and fragment they are built - /// from are literally the same values. - fn spelled(&self, scheme: &str) -> Url { + /// but the scheme and whether the signature is masked: the host, port, path, + /// query and fragment they are built from are literally the same values. + fn spelled(&self, scheme: &str, sas: Sas) -> Url { let mut text = format!("{scheme}://{}{}", self.host, self.path); if let Some(query) = &self.query { text.push('?'); - text.push_str(query); + match sas { + Sas::Exposed => text.push_str(query), + Sas::Masked => text.push_str(&mask_sas_signature(query)), + } } if let Some(fragment) = &self.fragment { text.push('#'); @@ -594,6 +588,40 @@ impl std::fmt::Display for AzureChannelUrl { } } +impl std::fmt::Debug for AzureChannelUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Derived, this would print the raw query and hand a `{:?}` on any struct + // holding a channel the signature that `canonical()` exists to withhold. + f.debug_tuple("AzureChannelUrl") + .field(&self.canonical().as_str()) + .finish() + } +} + +/// Whether a spelling of a channel URL may carry the SAS signature. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Sas { + /// For the wire: the signature is what makes the request authentic. + Exposed, + /// For anything a human or a log sees. + Masked, +} + +/// Replace the value of a query's `sig` parameter, leaving the rest intact. +/// +/// The other SAS parameters (`sv`, `se`, `sp`, …) describe the grant and are worth +/// showing; `sig` is the secret that makes it usable. +fn mask_sas_signature(query: &str) -> String { + query + .split('&') + .map(|parameter| match parameter.split_once('=') { + Some((name, _)) if name.eq_ignore_ascii_case("sig") => format!("{name}=REDACTED"), + _ => parameter.to_string(), + }) + .collect::>() + .join("&") +} + impl std::str::FromStr for AzureChannelUrl { type Err = AzureUrlError; @@ -691,8 +719,11 @@ pub fn azblob_config( ); let (account_key, sas_token) = match credentials { - AzureCredentials::AccountKey(key) => (Some(key.clone()), None), - AzureCredentials::SasToken(token) => (None, Some(normalize_sas_token(token).to_string())), + AzureCredentials::AccountKey(key) => (Some(key.expose_secret().to_string()), None), + AzureCredentials::SasToken(token) => ( + None, + Some(normalize_sas_token(token.expose_secret()).to_string()), + ), }; Ok(opendal::services::AzblobConfig { @@ -769,7 +800,7 @@ pub async fn mint_user_delegation_sas( container: &str, permissions: &str, valid_for: std::time::Duration, -) -> Result { +) -> Result { /// Slack for a client clock running up to two minutes slow, since the expiry /// is computed here and evaluated by Azure. const CLOCK_SKEW_HEADROOM: std::time::Duration = std::time::Duration::from_secs(120); @@ -824,7 +855,7 @@ pub async fn mint_user_delegation_sas( if token.is_empty() { return Err(AzureCliSasError::EmptyOutput); } - Ok(token) + Ok(token.into()) } /// Build the [`tokio::process::Command`] used to invoke the Azure CLI. @@ -1297,7 +1328,7 @@ mod tests { }; let config = azblob_config( - &AzureCredentials::AccountKey("key".to_string()), + &AzureCredentials::AccountKey("key".into()), &channel, options, ) @@ -1334,7 +1365,7 @@ mod tests { let channel = AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); let config = azblob_config( - &AzureCredentials::SasToken("?sv=token".to_string()), + &AzureCredentials::SasToken("?sv=token".into()), &channel, AzureEndpointOptions { auth: Auth::Anonymous, @@ -1359,7 +1390,7 @@ mod tests { AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general/sub/dir") .unwrap(); let config = azblob_config( - &AzureCredentials::SasToken("sv=token".to_string()), + &AzureCredentials::SasToken("sv=token".into()), &channel, AzureEndpointOptions::default(), ) @@ -1384,7 +1415,7 @@ mod tests { let channel = AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/with%20space").unwrap(); let config = azblob_config( - &AzureCredentials::AccountKey("key".to_string()), + &AzureCredentials::AccountKey("key".into()), &channel, AzureEndpointOptions::default(), ) @@ -1419,9 +1450,38 @@ mod debug_redaction_tests { AzureCredentials::SasToken("sig=deadbeef".into()), ] { let out = format!("{creds:?}"); - assert!(out.contains(""), "not redacted: {out}"); + assert!(out.contains("REDACTED"), "not redacted: {out}"); assert!(!out.contains("supersecret")); assert!(!out.contains("deadbeef")); } } + + /// An inline SAS reaches the wire and nothing else. Every other spelling of the + /// channel is a log line or an error message waiting to happen. + #[test] + fn only_the_wire_spelling_carries_the_signature() { + let channel = AzureChannelUrl::parse( + "az://acct.blob.core.windows.net/general/p?sv=2024-11-04&sig=SECRETSIG&se=z", + ) + .unwrap(); + + for shown in [ + channel.canonical().to_string(), + channel.to_string(), + format!("{channel:?}"), + ] { + assert!(!shown.contains("SECRETSIG"), "signature leaked: {shown}"); + // The rest of the grant is not secret and is worth showing. + assert!(shown.contains("sv=2024-11-04"), "over-redacted: {shown}"); + assert!(shown.contains("se=z"), "over-redacted: {shown}"); + } + + assert!( + channel + .wire(AzureScheme::Https) + .to_string() + .contains("sig=SECRETSIG"), + "the wire spelling must keep the signature that authenticates the request" + ); + } } diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 1365021432..4f0ccb145c 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -1909,7 +1909,7 @@ mod tests { let channel = AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general/sub/dir") .unwrap(); - let credentials = AzureCredentials::SasToken("sv=token".to_string()); + let credentials = AzureCredentials::SasToken("sv=token".into()); let config = rattler_azure::azblob_config(&credentials, &channel, AzureEndpointOptions::default()) @@ -1931,7 +1931,7 @@ mod tests { fn azblob_config_container_only_url() { let channel = AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general").unwrap(); - let credentials = AzureCredentials::AccountKey("key".to_string()); + let credentials = AzureCredentials::AccountKey("key".into()); let config = rattler_azure::azblob_config(&credentials, &channel, AzureEndpointOptions::default()) @@ -1949,7 +1949,7 @@ mod tests { let channel = AzureChannelUrl::parse("az://devstoreaccount1.blob.localhost:10000/testcontainer/ch") .unwrap(); - let credentials = AzureCredentials::AccountKey("key".to_string()); + let credentials = AzureCredentials::AccountKey("key".into()); let config = rattler_azure::azblob_config( &credentials, diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 719573abf5..cbb0a0aac6 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -396,7 +396,7 @@ mod tests { assert_eq!(options.addressing, Addressing::PathStyle); let azblob = rattler_azure::azblob_config( - &AzureCredentials::AccountKey("key".to_string()), + &AzureCredentials::AccountKey("key".into()), &channel, options, ) @@ -421,7 +421,7 @@ mod tests { let options = azure_endpoint_options(&None, channel.host()); let err = rattler_azure::azblob_config( - &AzureCredentials::AccountKey("key".to_string()), + &AzureCredentials::AccountKey("key".into()), &channel, options, ) diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs index c50d75af66..41300dfa2c 100644 --- a/crates/rattler_index/tests/azure_azurite.rs +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -97,7 +97,7 @@ fn azurite_options() -> AzureEndpointOptions { /// hand-written stand-in. fn production_operator(channel: &AzureChannelUrl) -> Operator { let config = rattler_azure::azblob_config( - &AzureCredentials::AccountKey(ACCOUNT_KEY.to_string()), + &AzureCredentials::AccountKey(ACCOUNT_KEY.into()), channel, azurite_options(), ) @@ -180,7 +180,7 @@ async fn with_azurite_credentials>(body: F) { fn index_config(channel: AzureChannelUrl) -> IndexAzureConfig { IndexAzureConfig { channel, - credentials: AzureCredentials::AccountKey(ACCOUNT_KEY.to_string()), + credentials: AzureCredentials::AccountKey(ACCOUNT_KEY.into()), options: azurite_options(), target_platform: None, repodata_patch: None, diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index bcbacf11e8..b6d84b3000 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -78,8 +78,14 @@ async fn upload_single_package( // The blob's address as the user wrote the channel, used only for // diagnostics. The canonical spelling already carries `//`, - // so the key is appended to it; do not prepend the container again. - let blob_url = format!("{}/{key}", channel.canonical()); + // so the key is appended to it; do not prepend the container again. Query and + // fragment go first, or an inline SAS would land in the middle of the path. + let blob_url = { + let mut channel_url = channel.canonical(); + channel_url.set_query(None); + channel_url.set_fragment(None); + format!("{channel_url}/{key}") + }; // Guard against overwriting an existing blob when `--force` was not passed. // opendal 0.57 only honours `if_not_exists` on the single-shot Put Blob path, From d3ab47207c59e64db2eca356d13714dc116a787d Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:53:43 +1000 Subject: [PATCH 84/98] fix(azure): reject rewritten paths, mismatched addressing, and port-lossy coordinates --- crates/rattler_azure/src/clap.rs | 58 ++-- crates/rattler_azure/src/lib.rs | 436 +++++++++++++++++++++++-------- crates/rattler_index/src/main.rs | 7 +- crates/rattler_upload/src/lib.rs | 7 +- 4 files changed, 362 insertions(+), 146 deletions(-) diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index 1bf1d0a75d..372a8ec6dc 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -5,7 +5,8 @@ use clap::Parser; use secrecy::{ExposeSecret, SecretString}; use crate::{ - AzureCliSasError, AzureCoordinates, AzureCredentials, AzureUrlError, mint_user_delegation_sas, + AzureChannelUrl, AzureCliSasError, AzureCoordinates, AzureCredentials, AzureEndpointOptions, + AzureUrlError, account_and_container, mint_user_delegation_sas, }; /// Default lifetime, in minutes, of a SAS minted from an `az login` session. @@ -66,23 +67,31 @@ pub enum AzureAuthSource { impl AzureAuthSource { /// Resolve this source into concrete [`AzureCredentials`]. /// - /// `permissions` and `cli_context` are consulted **only** for the - /// [`AzureAuthSource::AzureCli`] arm, which mints a SAS scoped to the - /// container returned by `cli_context` with those permissions. The account - /// key and SAS token arms never invoke `cli_context`, so callers pay for - /// account/container derivation only on the minting path. + /// `permissions`, `channel` and `options` are consulted **only** for the + /// [`AzureAuthSource::AzureCli`] arm, which mints a SAS scoped to the channel's + /// container with those permissions. Taking the channel and its options rather + /// than pre-derived coordinates is what keeps the account the SAS is minted for + /// and the scheme it is restricted to from coming from two different places. pub async fn resolve( self, permissions: &str, - cli_context: impl FnOnce() -> Result, + channel: &AzureChannelUrl, + options: AzureEndpointOptions, ) -> Result { match self { AzureAuthSource::AccountKey(key) => Ok(AzureCredentials::AccountKey(key)), AzureAuthSource::SasToken(token) => Ok(AzureCredentials::SasToken(token)), AzureAuthSource::AzureCli { ttl } => { - let AzureCoordinates { account, container } = cli_context()?; - let token = - mint_user_delegation_sas(&account, &container, permissions, ttl).await?; + let AzureCoordinates { account, container } = + account_and_container(channel, options.addressing)?; + let token = mint_user_delegation_sas( + &account, + &container, + permissions, + ttl, + options.scheme, + ) + .await?; Ok(AzureCredentials::SasToken(token)) } } @@ -200,15 +209,16 @@ impl AzureCredentialsOpts { /// Resolve the supplied options into concrete [`AzureCredentials`]. /// - /// Precedence is applied by [`AzureCredentialsOpts::source`]. `permissions` - /// and `cli_context` are consulted only when the winning source is + /// Precedence is applied by [`AzureCredentialsOpts::source`]. `permissions`, + /// `channel` and `options` are consulted only when the winning source is /// `--azure-cli`; see [`AzureAuthSource::resolve`]. pub async fn resolve( self, permissions: &str, - cli_context: impl FnOnce() -> Result, + channel: &AzureChannelUrl, + options: AzureEndpointOptions, ) -> Result { - self.source()?.resolve(permissions, cli_context).await + self.source()?.resolve(permissions, channel, options).await } } @@ -229,32 +239,40 @@ mod tests { } } - /// `cli_context` must not be invoked for the account-key/SAS-token paths. - fn unreachable_context() -> Result { - panic!("cli_context should not be called for non-azure-cli sources"); + /// A channel whose coordinates cannot be derived under `options`: resolving a + /// verbatim credential must not need them, and this is what proves it. + fn underivable() -> (AzureChannelUrl, AzureEndpointOptions) { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + let options = AzureEndpointOptions::default(); + assert!(account_and_container(&channel, options.addressing).is_err()); + (channel, options) } #[tokio::test] async fn account_key_resolves() { + let (channel, options) = underivable(); assert!(matches!( - opts(Some("key"), None, false).resolve("cw", unreachable_context).await, + opts(Some("key"), None, false).resolve("cw", &channel, options).await, Ok(AzureCredentials::AccountKey(k)) if k.expose_secret() == "key" )); } #[tokio::test] async fn sas_token_resolves() { + let (channel, options) = underivable(); assert!(matches!( - opts(None, Some("sv=..."), false).resolve("cw", unreachable_context).await, + opts(None, Some("sv=..."), false).resolve("cw", &channel, options).await, Ok(AzureCredentials::SasToken(t)) if t.expose_secret() == "sv=..." )); } #[tokio::test] async fn none_is_rejected() { + let (channel, options) = underivable(); assert!(matches!( opts(None, None, false) - .resolve("cw", unreachable_context) + .resolve("cw", &channel, options) .await, Err(AzureCredentialsError::Missing) )); diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index e0737b012e..e8e8d4866f 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -99,6 +99,28 @@ pub enum AzureUrlError { )] InvalidHost(String), + /// Path-style addressing was configured for a host that already carries the + /// account in its first label. + /// + /// Both styles produce identical request URLs for such a host, so the mistake + /// is invisible on the wire and only shows up when an account name is needed + /// on its own — where it would be taken from path text the user meant as a + /// container. + #[error( + "Azure blob URL host `{host}` is a dotted domain whose first label `{label}` is already a \ + valid storage account name, so it is a host-style endpoint. Reading it path-style would \ + take the storage account from the URL path instead and mint credentials for whatever \ + account that text names; remove `path-style = true` from `[azure-options.\"{host}\"]`, or \ + remove the entry" + )] + PathStyleOnAccountHost { + /// The host as [`AzureHost`] spells it, which is how the config table is + /// keyed. + host: String, + /// Its first label. + label: String, + }, + /// The URL has no path segment to read the account from (path-style only). #[error("could not derive account name from Azure blob URL")] NoAccount, @@ -132,6 +154,24 @@ pub enum AzureUrlError { source: url::ParseError, }, + /// The written path is not the path the URL Standard resolves it to. + /// + /// `..` segments — including percent-encoded ones — are resolved before any + /// segment is validated, so a path that reads as one container (or, path-style, + /// one account) can address another. The resolved form is quoted so the user + /// can see where the URL would actually have gone. + #[error( + "Azure blob channel URL path `{written}` is not the path it resolves to, `{resolved}`; a \ + channel URL must name the location it addresses, so write `{resolved}` if that is the \ + location you mean" + )] + NonCanonicalPath { + /// The path as written. + written: String, + /// The path it resolves to. + resolved: String, + }, + /// The channel URL does not use the `az://` scheme. #[error( "Azure blob channel URL must use the `az://` scheme, e.g. \ @@ -165,12 +205,13 @@ pub struct AzureCoordinates { /// [`AzureUrlError::InvalidHost`], whose message names the config line that /// switches to path-style. /// - [`Addressing::PathStyle`] (Azurite and other emulators): account = first -/// path segment, container = second. Any host shape is accepted, since the host -/// carries no account information at all. +/// path segment, container = second. The host must *not* be one that could +/// carry an account itself, or the two styles disagree about which name is the +/// account while producing identical request URLs — see +/// [`AzureUrlError::PathStyleOnAccountHost`]. /// /// The host is otherwise trusted verbatim (see the [crate-level docs] for the -/// host model): userinfo (`user:pass@host`) is rejected as a host-spoofing -/// vector, but an honest, arbitrary host is the caller's responsibility. The +/// host model): an honest, arbitrary host is the caller's responsibility. The /// derived account and container are additionally held to Azure's own naming /// rules — under *both* addressing styles, since path-style takes the account /// from user-controlled path text. Those rules reject an empty name, any @@ -178,18 +219,13 @@ pub struct AzureCoordinates { /// /// [crate-level docs]: crate pub fn account_and_container( - url: &Url, + channel: &AzureChannelUrl, addressing: Addressing, ) -> Result { - if !url.username().is_empty() || url.password().is_some() { - return Err(AzureUrlError::UserInfoNotAllowed); - } - let host = AzureHost::from_url(url)?; + let host = channel.host(); // Empty segments are never a valid name, so an empty one is a missing one. - let mut segments = url + let mut segments = channel .path_segments() - .into_iter() - .flatten() .map(|segment| (!segment.is_empty()).then_some(segment)); let mut next_segment = || segments.next().flatten(); @@ -200,10 +236,21 @@ pub fn account_and_container( .ok_or_else(|| AzureUrlError::InvalidHost(host.to_string()))?; (account, next_segment().ok_or(AzureUrlError::NoContainer)?) } - Addressing::PathStyle => ( - next_segment().ok_or(AzureUrlError::NoAccount)?, - next_segment().ok_or(AzureUrlError::NoContainer)?, - ), + Addressing::PathStyle => { + if let Some(label) = host + .account_label() + .filter(|label| validate_account(label).is_ok()) + { + return Err(AzureUrlError::PathStyleOnAccountHost { + host: host.to_string(), + label: label.to_string(), + }); + } + ( + next_segment().ok_or(AzureUrlError::NoAccount)?, + next_segment().ok_or(AzureUrlError::NoContainer)?, + ) + } }; validate_account(account)?; @@ -332,21 +379,6 @@ impl AzureHost { Ok(Self { host, port }) } - /// The authority of an already-parsed URL, re-normalized. - /// - /// Note the asymmetry with [`AzureChannelUrl`], which stores an `AzureHost` - /// rather than a `Url` for exactly this reason: a `Url` has *already* dropped - /// a port equal to its scheme's default, so `https://host:443/…` arrives here - /// as `host` and cannot be told apart from `https://host/…`. Prefer - /// [`AzureChannelUrl::host`] wherever the channel URL itself is in reach. - pub fn from_url(url: &Url) -> Result { - let host = url.host_str().ok_or(AzureUrlError::NoHost)?; - match url.port() { - Some(port) => Self::parse(&format!("{host}:{port}")), - None => Self::parse(host), - } - } - /// The parsed host, without the port. pub fn host(&self) -> &url::Host { &self.host @@ -517,6 +549,22 @@ impl AzureChannelUrl { } })?; + // Dot segments — `%2e%2e` as much as `..` — are resolved by that parser + // before anything here has looked at a segment, so a path reading as one + // container (path-style: one *account*) can address another. Nothing needs + // to guess which rewrites are benign: a path that is not already the path + // it resolves to is not the path the user can be assumed to have meant. + let written = match tail.split(['?', '#']).next().unwrap_or_default() { + "" => "/", + path => path, + }; + if written != url.path() { + return Err(AzureUrlError::NonCanonicalPath { + written: written.to_string(), + resolved: url.path().to_string(), + }); + } + Ok(Self { host, path: url.path().to_string(), @@ -578,6 +626,12 @@ impl AzureChannelUrl { pub fn host(&self) -> &AzureHost { &self.host } + + /// The still-encoded path segments, exactly as [`Url::path_segments`] would + /// yield them for the wire spelling. + fn path_segments(&self) -> std::str::Split<'_, char> { + self.path.strip_prefix('/').unwrap_or(&self.path).split('/') + } } impl std::fmt::Display for AzureChannelUrl { @@ -681,10 +735,10 @@ pub fn azblob_config( channel: &AzureChannelUrl, options: AzureEndpointOptions, ) -> Result { - let wire = channel.wire(options.scheme); - let AzureCoordinates { account, container } = account_and_container(&wire, options.addressing)?; + let AzureCoordinates { account, container } = + account_and_container(channel, options.addressing)?; - // The authority comes from `AzureHost`, not from the wire URL: a `Url` has + // The authority comes from `AzureHost`, not from a wire URL: a `Url` has // already dropped a port equal to its scheme's default, so reading it back // would turn a written `:443` into no port at all. let authority = channel.host(); @@ -709,9 +763,8 @@ pub fn azblob_config( // `account_and_container` has already confirmed the consumed segments exist. let root = format!( "/{}", - wire.path_segments() - .into_iter() - .flatten() + channel + .path_segments() .skip(consumed) .map(|segment| percent_encoding::percent_decode_str(segment).decode_utf8_lossy()) .collect::>() @@ -777,12 +830,16 @@ pub enum AzureCliSasError { /// ```text /// az storage container generate-sas --account-name --name /// --permissions --expiry --auth-mode login --as-user -/// --https-only -o tsv +/// [--https-only] -o tsv /// ``` /// /// `permissions` is the Azure SAS permission string (e.g. `"cw"`). The returned /// token has no leading `?`. Requires `az` on `PATH` and a prior `az login`. /// +/// `scheme` is the wire scheme the host's options entry asks for: `--https-only` +/// restricts the SAS to TLS, which would make it unusable against a host reached +/// over http. +/// /// Runs the `az` process on the tokio runtime; it is meant to be called once at /// setup time. /// @@ -800,6 +857,7 @@ pub async fn mint_user_delegation_sas( container: &str, permissions: &str, valid_for: std::time::Duration, + scheme: AzureScheme, ) -> Result { /// Slack for a client clock running up to two minutes slow, since the expiry /// is computed here and evaluated by Azure. @@ -817,25 +875,13 @@ pub async fn mint_user_delegation_sas( let mut command = az_command()?; let output = command - .args([ - "storage", - "container", - "generate-sas", - "--account-name", + .args(generate_sas_args( account, - "--name", container, - "--permissions", permissions, - "--expiry", &expiry, - "--auth-mode", - "login", - "--as-user", - "--https-only", - "-o", - "tsv", - ]) + scheme, + )) .output() .await .map_err(|err| { @@ -858,6 +904,42 @@ pub async fn mint_user_delegation_sas( Ok(token.into()) } +/// The argv for the `az storage container generate-sas` call. +/// +/// Split out from the spawn so the argument list is testable without an `az` on +/// `PATH`. It stays a list of separate arguments — never a command line — so no +/// value can be read as anything but one argument. +#[cfg(feature = "clap")] +fn generate_sas_args<'a>( + account: &'a str, + container: &'a str, + permissions: &'a str, + expiry: &'a str, + scheme: AzureScheme, +) -> Vec<&'a str> { + let mut args = vec![ + "storage", + "container", + "generate-sas", + "--account-name", + account, + "--name", + container, + "--permissions", + permissions, + "--expiry", + expiry, + "--auth-mode", + "login", + "--as-user", + ]; + if let AzureScheme::Https = scheme { + args.push("--https-only"); + } + args.extend(["-o", "tsv"]); + args +} + /// Build the [`tokio::process::Command`] used to invoke the Azure CLI. /// /// `which` resolves `az` up front so a missing CLI surfaces as [`AzureCliSasError::AzResolve`] @@ -876,11 +958,19 @@ fn az_command() -> Result { mod tests { use super::*; + /// Every derivation runs off an [`AzureChannelUrl`], so the tests build one. + fn channel(url: &str) -> AzureChannelUrl { + AzureChannelUrl::parse(url).unwrap_or_else(|err| panic!("{url} should parse: {err}")) + } + #[test] fn normal_url_resolves() { - let url = Url::parse("https://acct.blob.core.windows.net/general/noarch").unwrap(); assert_eq!( - account_and_container(&url, Addressing::HostStyle).unwrap(), + account_and_container( + &channel("az://acct.blob.core.windows.net/general/noarch"), + Addressing::HostStyle + ) + .unwrap(), AzureCoordinates { account: "acct".to_string(), container: "general".to_string(), @@ -890,13 +980,6 @@ mod tests { #[test] fn userinfo_is_rejected() { - let url = Url::parse("https://acct.blob.core.windows.net@evil.example/general").unwrap(); - for addressing in [Addressing::HostStyle, Addressing::PathStyle] { - assert!(matches!( - account_and_container(&url, addressing), - Err(AzureUrlError::UserInfoNotAllowed) - )); - } assert!(matches!( AzureChannelUrl::parse("az://acct.blob.core.windows.net@evil.example/general"), Err(AzureUrlError::UserInfoNotAllowed) @@ -912,31 +995,28 @@ mod tests { /// comes from user-controlled path text rather than a host label. #[test] fn invalid_component_names_are_rejected_under_both_styles() { - let host_style = - Url::parse("https://acct.blob.core.windows.net/general;evil/noarch").unwrap(); assert!(matches!( - account_and_container(&host_style, Addressing::HostStyle), + account_and_container( + &channel("az://acct.blob.core.windows.net/general;evil/noarch"), + Addressing::HostStyle + ), Err(AzureUrlError::InvalidContainerName(_)) )); for (path, account_at_fault) in [ - ("http://127.0.0.1:10000/devstore;evil/general", true), - ("http://127.0.0.1:10000/DevStoreAccount1/general", true), + ("az://127.0.0.1:10000/devstore;evil/general", true), + ("az://127.0.0.1:10000/DevStoreAccount1/general", true), // Azure allows no hyphen at all in an account name. - ("http://127.0.0.1:10000/dev-store/general", true), + ("az://127.0.0.1:10000/dev-store/general", true), // Too short for Azure, whatever the charset says. - ("http://127.0.0.1:10000/ab/general", true), - ( - "http://127.0.0.1:10000/devstoreaccount1/general;evil", - false, - ), - ("http://127.0.0.1:10000/devstoreaccount1/ab", false), - ("http://127.0.0.1:10000/devstoreaccount1/a--b", false), - ("http://127.0.0.1:10000/devstoreaccount1/-general", false), - ("http://127.0.0.1:10000/devstoreaccount1/general-", false), + ("az://127.0.0.1:10000/ab/general", true), + ("az://127.0.0.1:10000/devstoreaccount1/general;evil", false), + ("az://127.0.0.1:10000/devstoreaccount1/ab", false), + ("az://127.0.0.1:10000/devstoreaccount1/a--b", false), + ("az://127.0.0.1:10000/devstoreaccount1/-general", false), + ("az://127.0.0.1:10000/devstoreaccount1/general-", false), ] { - let url = Url::parse(path).unwrap(); - let Err(err) = account_and_container(&url, Addressing::PathStyle) else { + let Err(err) = account_and_container(&channel(path), Addressing::PathStyle) else { panic!("expected a rejection for {path}"); }; let matched = if account_at_fault { @@ -954,16 +1034,12 @@ mod tests { #[test] fn option_shaped_components_are_rejected() { for (url, account_at_fault) in [ - ("https://--as-user.blob.core.windows.net/general", true), - ("https://-o.blob.core.windows.net/general", true), - ( - "https://acct.blob.core.windows.net/--https-only/noarch", - false, - ), - ("https://acct.blob.core.windows.net/-o/noarch", false), + ("az://--as-user.blob.core.windows.net/general", true), + ("az://-o.blob.core.windows.net/general", true), + ("az://acct.blob.core.windows.net/--https-only/noarch", false), + ("az://acct.blob.core.windows.net/-o/noarch", false), ] { - let parsed = Url::parse(url).unwrap(); - let Err(err) = account_and_container(&parsed, Addressing::HostStyle) else { + let Err(err) = account_and_container(&channel(url), Addressing::HostStyle) else { panic!("expected a rejection for {url}"); }; let matched = if account_at_fault { @@ -985,11 +1061,21 @@ mod tests { #[test] fn path_style_derives_account_from_first_segment() { - for host in ["127.0.0.1:10000", "azurite:10000", "localhost"] { - let url = - Url::parse(&format!("http://{host}/devstoreaccount1/general/noarch")).unwrap(); + for host in [ + "127.0.0.1:10000", + "[::1]:10000", + "azurite:10000", + "localhost:10000", + // A bare docker service name, which is the shape a compose file gives. + "azurite", + "localhost", + ] { assert_eq!( - account_and_container(&url, Addressing::PathStyle).unwrap(), + account_and_container( + &channel(&format!("az://{host}/devstoreaccount1/general/noarch")), + Addressing::PathStyle + ) + .unwrap(), AzureCoordinates { account: "devstoreaccount1".to_string(), container: "general".to_string(), @@ -1001,17 +1087,51 @@ mod tests { #[test] fn path_style_needs_two_segments() { - let url = Url::parse("http://127.0.0.1:10000/devstoreaccount1").unwrap(); assert!(matches!( - account_and_container(&url, Addressing::PathStyle), + account_and_container( + &channel("az://127.0.0.1:10000/devstoreaccount1"), + Addressing::PathStyle + ), Err(AzureUrlError::NoContainer) )); - let url = Url::parse("http://127.0.0.1:10000/").unwrap(); - assert!(matches!( - account_and_container(&url, Addressing::PathStyle), - Err(AzureUrlError::NoAccount) - )); + for empty in ["az://127.0.0.1:10000/", "az://127.0.0.1:10000"] { + assert!(matches!( + account_and_container(&channel(empty), Addressing::PathStyle), + Err(AzureUrlError::NoAccount) + )); + } + } + + /// A `path-style = true` entry on a host that already carries the account is a + /// config mistake with no visible symptom — request URLs come out identical + /// under both styles — right up to a mint asking for a delegation SAS on the + /// account name the *path* happened to spell. + #[test] + fn path_style_rejects_a_host_that_carries_the_account() { + let err = account_and_container( + &channel("az://acct.blob.core.windows.net/general/mychannel"), + Addressing::PathStyle, + ) + .expect_err("path-style must not read an account out of a host-style URL"); + + assert!( + matches!(err, AzureUrlError::PathStyleOnAccountHost { .. }), + "{err}" + ); + let message = err.to_string(); + assert!(message.contains("acct.blob.core.windows.net"), "{message}"); + assert!(message.contains("path-style = true"), "{message}"); + + // A first label that is not a legal account name carries no account, so + // path-style is the only reading left. + assert!( + account_and_container( + &channel("az://my-emulator.internal/devstoreaccount1/general"), + Addressing::PathStyle + ) + .is_ok() + ); } /// Host-style must keep rejecting hosts it cannot derive an account from — and @@ -1029,9 +1149,13 @@ mod tests { // must not sneak past the dotted-domain gate. ("localhost.", "localhost"), ("LocalHost", "localhost"), + // A port equal to a wire scheme's default: it is part of the key, and + // only survives because the channel — not a `Url` — is what is read. + ("azurite:443", "azurite:443"), + ("azurite:80", "azurite:80"), ] { - let url = Url::parse(&format!("http://{host}/devstoreaccount1/general")).unwrap(); - let err = account_and_container(&url, Addressing::HostStyle) + let channel = channel(&format!("az://{host}/devstoreaccount1/general")); + let err = account_and_container(&channel, Addressing::HostStyle) .expect_err("host-style must not accept an undottable host"); assert!(matches!(err, AzureUrlError::InvalidHost(_)), "{err}"); @@ -1039,6 +1163,9 @@ mod tests { assert!(message.contains("path-style = true"), "{message}"); let key = format!("[azure-options.\"{expected_key}\"]"); assert!(message.contains(&key), "{message}"); + // The key named must be the one an `azure-options` lookup is made + // with, or the entry the user writes cannot ever apply. + assert_eq!(expected_key, channel.host().to_string(), "{host}"); } } @@ -1065,14 +1192,6 @@ mod tests { ), "expected a rejection for {host}" ); - let url = Url::parse(&format!("https://{host}/general/noarch")).unwrap(); - assert!( - matches!( - account_and_container(&url, Addressing::HostStyle), - Err(AzureUrlError::InvalidHostAuthority { .. }) - ), - "expected a rejection for {host}" - ); } } @@ -1430,12 +1549,101 @@ mod tests { /// later, once its options entry is known. #[test] fn parse_defers_account_derivation() { - let channel = - AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); - let wire = channel.wire(AzureScheme::Http); + let channel = channel("az://127.0.0.1:10000/devstoreaccount1/general"); + + assert!(account_and_container(&channel, Addressing::HostStyle).is_err()); + assert!(account_and_container(&channel, Addressing::PathStyle).is_ok()); + } + + /// The rewrite is invisible in the URL a user reads back: under path-style it + /// moves the *account* too, so a channel URL that says `devstoreaccount1` mints + /// a SAS for whatever account the escaped `..` climbs out to. + #[test] + fn a_rewritten_path_is_rejected() { + for (input, resolved) in [ + ( + "az://acct.blob.core.windows.net/general/%2e%2e/%2e%2e/othercontainer/x", + "/othercontainer/x", + ), + ( + "az://127.0.0.1:10000/devstoreaccount1/general/%2e%2e/%2e%2e/otheraccount/othercontainer", + "/otheraccount/othercontainer", + ), + ( + "az://acct.blob.core.windows.net/general/../../othercontainer", + "/othercontainer", + ), + ( + "az://acct.blob.core.windows.net/general/./noarch", + "/general/noarch", + ), + ] { + let Err(err) = AzureChannelUrl::parse(input) else { + panic!("expected a rejection for {input}"); + }; + assert!( + matches!(&err, AzureUrlError::NonCanonicalPath { resolved: got, .. } if got == resolved), + "unexpected error for {input}: {err}" + ); + // Both spellings are in the message, so the user can see where the URL + // they wrote would have gone. + let message = err.to_string(); + assert!(message.contains(resolved), "{message}"); + assert!(message.contains("/general/"), "{message}"); + } + } - assert!(account_and_container(&wire, Addressing::HostStyle).is_err()); - assert!(account_and_container(&wire, Addressing::PathStyle).is_ok()); + /// The paths that must keep parsing: nothing about rejecting rewrites may + /// narrow what an ordinary channel URL can say. + #[test] + fn unrewritten_paths_still_parse() { + for (input, path) in [ + ( + "az://acct.blob.core.windows.net/general/prefix", + "/general/prefix", + ), + ("az://acct.blob.core.windows.net/general/", "/general/"), + ("az://acct.blob.core.windows.net/", "/"), + ("az://acct.blob.core.windows.net", "/"), + ( + "az://acct.blob.core.windows.net/general/with%20space", + "/general/with%20space", + ), + ( + "az://acct.blob.core.windows.net/general/p?sv=token#frag", + "/general/p", + ), + // A dot inside a segment is not a dot segment. + ( + "az://acct.blob.core.windows.net/general/..hidden/...", + "/general/..hidden/...", + ), + ] { + assert_eq!(channel(input).canonical().path(), path, "{input}"); + } + } + + /// `--https-only` restricts the SAS to TLS, so a host configured for http would + /// get a token it cannot use. Nothing else about the grant may move with it. + #[cfg(feature = "clap")] + #[test] + fn https_only_follows_the_configured_scheme() { + let args = + |scheme| generate_sas_args("acct", "general", "cw", "2030-01-01T00:00:00Z", scheme); + + assert!(args(AzureScheme::Https).contains(&"--https-only")); + assert!(!args(AzureScheme::Http).contains(&"--https-only")); + + for scheme in [AzureScheme::Https, AzureScheme::Http] { + let args = args(scheme); + assert!(args.windows(2).any(|pair| pair == ["--permissions", "cw"])); + assert!( + args.windows(2) + .any(|pair| pair == ["--expiry", "2030-01-01T00:00:00Z"]) + ); + assert!(args.contains(&"--as-user")); + assert!(args.windows(2).any(|pair| pair == ["--auth-mode", "login"])); + } } } diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index cbb0a0aac6..06f77ec2b3 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -260,12 +260,7 @@ async fn main() -> anyhow::Result<()> { let options = azure_endpoint_options(&config, channel.host()); let credentials = credentials - .resolve(AZURE_INDEX_SAS_PERMISSIONS, || { - Ok(rattler_azure::account_and_container( - &channel.wire(options.scheme), - options.addressing, - )?) - }) + .resolve(AZURE_INDEX_SAS_PERMISSIONS, &channel, options) .await?; index_azure_with_channel_metadata( diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index 7c300f406d..f1b19eaca5 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -95,12 +95,7 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { let options = rattler_azure::AzureEndpointOptions::default(); let credentials = azure_opts .credentials - .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, || { - Ok(rattler_azure::account_and_container( - &channel.wire(options.scheme), - options.addressing, - )?) - }) + .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, &channel, options) .await .into_diagnostic()?; upload::upload_package_to_azure( From d2c492a9bcaa7d18a2c1368595bcf57db162bd94 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:53:43 +1000 Subject: [PATCH 85/98] refactor(upload): share one streaming blob writer between azure and s3 --- crates/rattler_upload/src/upload/azure.rs | 268 +++++++++++++--------- crates/rattler_upload/src/upload/mod.rs | 210 +++++++++++++++++ crates/rattler_upload/src/upload/opt.rs | 27 ++- crates/rattler_upload/src/upload/s3.rs | 125 ++-------- 4 files changed, 401 insertions(+), 229 deletions(-) diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index b6d84b3000..facccd520f 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -1,23 +1,19 @@ -use std::path::{Path, PathBuf}; +use std::{ + fmt::Write as _, + path::{Path, PathBuf}, + sync::Mutex, +}; use futures::{StreamExt, TryStreamExt}; use miette::IntoDiagnostic; use opendal::{Configurator, ErrorKind, Operator}; use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpointOptions}; -use tokio::io::AsyncReadExt; -use tokio_util::bytes::BytesMut; -use crate::upload::package::ExtractedPackage; - -/// Size of a single block handed to the writer. Azure Blob block uploads keep -/// the number of blocks low with larger blocks; we use 10 MiB. -const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; - -/// Number of blocks of a single package that are uploaded concurrently. -const PART_CONCURRENCY: usize = 4; - -/// Number of packages that are uploaded concurrently. -const PACKAGE_CONCURRENCY: usize = 4; +use crate::upload::{ + object_store::{BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store}, + opt::ForceOverwrite, + package::ExtractedPackage, +}; /// SAS permissions requested when minting a user-delegation SAS for uploads. /// Creating and writing blobs needs `c` + `w`; `r` is required on top of those @@ -25,6 +21,12 @@ const PACKAGE_CONCURRENCY: usize = 4; /// `stat` (HEAD Blob) is a read. The SAS stays container-scoped and short-lived. pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "rcw"; +/// What became of one package in a concurrent upload run. +enum PackageOutcome { + Uploaded, + Failed(String), +} + /// Uploads packages to a channel in an Azure Blob Storage container. /// /// The account name, endpoint, container and root prefix are all derived from the @@ -38,7 +40,7 @@ pub async fn upload_package_to_azure( credentials: AzureCredentials, options: AzureEndpointOptions, package_files: &[PathBuf], - force: bool, + force: ForceOverwrite, ) -> miette::Result<()> { let config = rattler_azure::azblob_config(&credentials, &channel, options).into_diagnostic()?; @@ -47,17 +49,74 @@ pub async fn upload_package_to_azure( // Upload multiple packages concurrently. Each package is written to its own // key, so the individual uploads are independent. The first failure aborts - // the remaining uploads rather than letting them run to completion. - futures::stream::iter(package_files.iter()) + // the remaining uploads rather than letting them run to completion, so the + // outcomes are recorded as they land and summarised below — otherwise a run + // that failed halfway would report one package and stay silent about the + // rest. + let outcomes = Mutex::new(Vec::new()); + let result = futures::stream::iter(package_files.iter()) .map(Ok) .try_for_each_concurrent(PACKAGE_CONCURRENCY, |package_file| { let op = op.clone(); let channel = &channel; - async move { upload_single_package(&op, channel, package_file, force).await } + let outcomes = &outcomes; + async move { + let result = upload_single_package(&op, channel, package_file, force).await; + let outcome = match &result { + Ok(()) => PackageOutcome::Uploaded, + Err(e) => PackageOutcome::Failed(e.to_string()), + }; + outcomes + .lock() + .expect("upload outcome mutex poisoned") + .push((package_file.clone(), outcome)); + result + } + }) + .await; + + let outcomes = outcomes + .into_inner() + .expect("upload outcome mutex poisoned"); + let summary = summarize(&outcomes, package_files.len()); + match result { + Ok(()) => { + tracing::info!("{summary}"); + Ok(()) + } + // Logged rather than attached to the error, so the counts are visible + // next to the failure that stopped the run without repeating it. + Err(e) => { + tracing::error!("{summary}"); + Err(e) + } + } +} + +/// Renders the per-package outcomes of a run. Packages without an outcome were +/// dropped mid-upload by the fail-fast stream or never started. +fn summarize(outcomes: &[(PathBuf, PackageOutcome)], total: usize) -> String { + let failed: Vec<_> = outcomes + .iter() + .filter_map(|(path, outcome)| match outcome { + PackageOutcome::Failed(message) => Some((path, message)), + PackageOutcome::Uploaded => None, }) - .await?; + .collect(); + let uploaded = outcomes.len() - failed.len(); + let not_attempted = total - outcomes.len(); - Ok(()) + let mut summary = format!( + "Azure upload summary: uploaded {uploaded} / failed {} / not attempted {not_attempted}", + failed.len() + ); + for (path, message) in failed { + let _ = write!(summary, "\n failed: {}: {message}", path.display()); + } + if not_attempted > 0 { + summary.push_str("\n not attempted: cancelled mid-upload or never started"); + } + summary } /// Uploads a single package file to the Azure Blob container via the given operator. @@ -65,16 +124,10 @@ async fn upload_single_package( op: &Operator, channel: &AzureChannelUrl, package_file: &Path, - force: bool, + force: ForceOverwrite, ) -> miette::Result<()> { let package = ExtractedPackage::from_package_file(package_file)?; - let subdir = package - .subdir() - .ok_or_else(|| miette::miette!("Failed to get subdir"))?; - let filename = package - .filename() - .ok_or_else(|| miette::miette!("Failed to get filename"))?; - let key = format!("{subdir}/{filename}"); + let target = BlobUploadTarget::from_package(&package)?; // The blob's address as the user wrote the channel, used only for // diagnostics. The canonical spelling already carries `//`, @@ -84,20 +137,24 @@ async fn upload_single_package( let mut channel_url = channel.canonical(); channel_url.set_query(None); channel_url.set_fragment(None); - format!("{channel_url}/{key}") + format!("{channel_url}/{}", target.key()) }; // Guard against overwriting an existing blob when `--force` was not passed. // opendal 0.57 only honours `if_not_exists` on the single-shot Put Blob path, // never the multi-block Put Block List path used for packages larger than a - // single block, so the writer-level `if_not_exists(!force)` below silently - // does nothing for large uploads. An explicit `stat` closes that gap for a - // blob that already exists, but not for two writers racing to create one: - // above the chunk size both stat as absent and the second commit wins - // silently. Concurrent uploads of the same package are unsafe until opendal - // carries `if_none_match` onto the Put Block List path. - if !force { - match op.stat(&key).await { + // single block, so the writer-level `if_not_exists` silently does nothing for + // large uploads. An explicit `stat` closes that gap for a blob that already + // exists, but not for two writers racing to create one: above the chunk size + // both stat as absent and the second commit wins silently. Concurrent uploads + // of the same package are unsafe until opendal carries `if_none_match` onto + // the Put Block List path. + // + // TODO: delete this guard and its TOCTOU window once + // merges — it carries changes + // in this area. + if !force.is_enabled() { + match op.stat(target.key()).await { Ok(_) => { miette::bail!("Package {blob_url} already exists. Use --force to overwrite."); } @@ -106,85 +163,18 @@ async fn upload_single_package( } } - // The streaming loop below needs the package size to chunk the upload. - let size = fs_err::tokio::metadata(package_file) - .await - .into_diagnostic()? - .len(); - let mut file = tokio::io::BufReader::new( - fs_err::tokio::File::open(package_file) - .await - .into_diagnostic()?, - ); - - // Construct a writer for the package. Setting `chunk` and `concurrent` - // enables opendal's concurrent block upload: data is buffered into - // `DESIRED_CHUNK_SIZE` blocks and up to `PART_CONCURRENCY` blocks are - // uploaded in parallel. `if_not_exists(!force)` maps to an `If-None-Match: *` - // precondition so an existing blob is not silently overwritten unless - // `--force` was passed; note opendal 0.57 only honours it on the single-shot - // Put Blob path, so the pre-write `stat` above is what guards large uploads. - let mut writer = match op - .writer_with(&key) - .chunk(DESIRED_CHUNK_SIZE) - .concurrent(PART_CONCURRENCY) - .if_not_exists(!force) - .await - { - Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { - miette::bail!("Package {blob_url} already exists. Use --force to overwrite."); - } - Ok(writer) => writer, - Err(e) => { - return Err(e).into_diagnostic(); - } - }; - - // Stream the file to the writer in `DESIRED_CHUNK_SIZE` chunks. opendal takes - // care of buffering these into correctly sized blocks and uploading them - // concurrently. - let mut remaining_size = size as usize; - while remaining_size > 0 { - // Allocate memory for this chunk. - let chunk_size = remaining_size.min(DESIRED_CHUNK_SIZE); - let mut chunk = BytesMut::with_capacity(chunk_size); - // Zero-fill up to `chunk_size`; `read_exact` below overwrites every byte. - chunk.resize(chunk_size, 0); - - // Fill the chunk with data. This reads exactly the number of bytes we want. No - // more, no less. - let bytes_read = file.read_exact(&mut chunk[..]).await.into_diagnostic()?; - debug_assert_eq!(bytes_read, chunk.len()); - - // Hand the chunk to the writer. With concurrent writes enabled this returns - // as soon as the chunk is queued rather than fully uploaded. - writer.write(chunk.freeze()).await.into_diagnostic()?; - - // Update the number of remaining bytes. - remaining_size = remaining_size.saturating_sub(bytes_read); - } - - match writer.close().await { - Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { - miette::bail!("Package {blob_url} already exists. Use --force to overwrite."); - } - Ok(_) => { - tracing::info!("Uploaded package to {blob_url}"); - } - Err(e) => { - return Err(e).into_diagnostic(); - } - } - - Ok(()) + stream_package_to_object_store(op, &target, package_file, &blob_url, force).await } #[cfg(test)] mod test { + use std::path::PathBuf; + use opendal::{Operator, services::Memory}; use rattler_azure::AzureChannelUrl; - use super::upload_single_package; + use super::{PackageOutcome, summarize, upload_single_package}; + use crate::upload::opt::ForceOverwrite; use crate::upload::package::ExtractedPackage; use crate::upload::test_utils::test_package_path; @@ -218,11 +208,11 @@ mod test { let package = test_package_path(); // Seed the target blob so the next upload finds it already present. - upload_single_package(&op, &channel, &package, true) + upload_single_package(&op, &channel, &package, ForceOverwrite(true)) .await .expect("initial force upload should succeed"); - let err = upload_single_package(&op, &channel, &package, false) + let err = upload_single_package(&op, &channel, &package, ForceOverwrite(false)) .await .expect_err("upload over an existing blob without --force must fail"); assert!( @@ -235,12 +225,62 @@ mod test { #[tokio::test] async fn test_upload_into_empty_container_succeeds() { let op = memory_operator(); - upload_single_package(&op, &test_channel(), &test_package_path(), false) - .await - .expect("upload into an empty container should succeed"); + upload_single_package( + &op, + &test_channel(), + &test_package_path(), + ForceOverwrite(false), + ) + .await + .expect("upload into an empty container should succeed"); let meta = op.stat(&package_key()).await.unwrap(); let expected_size = std::fs::metadata(test_package_path()).unwrap().len(); assert_eq!(meta.content_length(), expected_size); } + + /// The upload carries the same download metadata as its S3 twin. Only + /// the content disposition can be asserted here — the memory backend ignores + /// user metadata, as azblob does above a single block. + #[tokio::test] + async fn test_upload_sets_content_disposition() { + let op = memory_operator(); + upload_single_package( + &op, + &test_channel(), + &test_package_path(), + ForceOverwrite(false), + ) + .await + .expect("upload should succeed"); + + let path = test_package_path(); + let package = ExtractedPackage::from_package_file(&path).unwrap(); + let meta = op.stat(&package_key()).await.unwrap(); + assert_eq!( + meta.content_disposition(), + Some(format!("attachment; filename={}", package.filename().unwrap()).as_str()) + ); + } + + /// A run that stops early must still say what landed, what failed and + /// how many uploads never got a verdict. + #[test] + fn test_summary_counts_and_names_outcomes() { + let outcomes = vec![ + (PathBuf::from("a.conda"), PackageOutcome::Uploaded), + ( + PathBuf::from("b.conda"), + PackageOutcome::Failed("Package b already exists".to_string()), + ), + ]; + + let summary = summarize(&outcomes, 4); + assert!( + summary.contains("uploaded 1 / failed 1 / not attempted 2"), + "unexpected summary: {summary}" + ); + assert!(summary.contains("failed: b.conda: Package b already exists")); + assert!(summary.contains("not attempted: cancelled mid-upload or never started")); + } } diff --git a/crates/rattler_upload/src/upload/mod.rs b/crates/rattler_upload/src/upload/mod.rs index 309417c174..fe051f1ea7 100644 --- a/crates/rattler_upload/src/upload/mod.rs +++ b/crates/rattler_upload/src/upload/mod.rs @@ -50,6 +50,216 @@ pub use anaconda::AnacondaError; pub use cloudsmith::CloudsmithError; pub use prefix::{PrefixUploadError, upload_package_to_prefix}; +/// The streaming upload shared by the object-store backends (S3 and Azure Blob), +/// which both drive an opendal writer. +#[cfg(any(feature = "s3", feature = "azure"))] +pub(crate) mod object_store { + use std::{collections::HashMap, path::Path}; + + use miette::IntoDiagnostic; + use opendal::{ErrorKind, Operator, options::WriteOptions}; + use rattler_digest::{HashingReader, Md5, Md5Hash, Sha256, Sha256Hash}; + use tokio::io::{AsyncReadExt, AsyncSeekExt}; + use tokio_util::bytes::BytesMut; + + use crate::upload::{opt::ForceOverwrite, package::ExtractedPackage}; + + /// Size of a single chunk handed to the writer. S3 rejects every multipart + /// part but the last below 5 MiB, and Azure Blob bills per block, so both + /// backends prefer few large chunks. + /// + /// Peak buffered bytes across a run are `PACKAGE_CONCURRENCY * + /// PART_CONCURRENCY * DESIRED_CHUNK_SIZE` = 160 MiB. + const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; + + /// Number of chunks of a single package that are uploaded concurrently. + const PART_CONCURRENCY: usize = 4; + + /// Number of packages that are uploaded concurrently. + pub(crate) const PACKAGE_CONCURRENCY: usize = 4; + + /// A package resolved to the channel-relative key it is stored under. Holding + /// the key and the filename together keeps the two from disagreeing. + pub(crate) struct BlobUploadTarget { + key: String, + filename: String, + } + + impl BlobUploadTarget { + /// Resolves `/` from the package's own `index.json`. + pub(crate) fn from_package(package: &ExtractedPackage<'_>) -> miette::Result { + let subdir = package + .subdir() + .ok_or_else(|| miette::miette!("Failed to get subdir"))?; + let filename = package + .filename() + .ok_or_else(|| miette::miette!("Failed to get filename"))?; + Ok(Self { + key: format!("{subdir}/{filename}"), + filename: filename.to_string(), + }) + } + + /// The channel-relative key the package is written to. + pub(crate) fn key(&self) -> &str { + &self.key + } + } + + /// A file measured and hashed by one pass over a single handle, rewound and + /// ready to be read again. Size and hashes describe the same bytes, so the + /// upload cannot publish a length a concurrent writer changed after a `stat`. + struct HashedFile { + reader: R, + size: u64, + sha256: Sha256Hash, + md5: Md5Hash, + } + + async fn hash_file( + path: &Path, + ) -> miette::Result> { + let file = + tokio::io::BufReader::new(fs_err::tokio::File::open(path).await.into_diagnostic()?); + let sha256_reader = HashingReader::<_, Sha256>::new(file); + let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); + let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) + .await + .into_diagnostic()?; + let (sha256_reader, md5) = md5_reader.finalize(); + let (mut reader, sha256) = sha256_reader.finalize(); + reader.rewind().await.into_diagnostic()?; + Ok(HashedFile { + reader, + size, + sha256, + md5, + }) + } + + /// Streams `package_file` to `target`'s key through `op`. + /// + /// `destination` is the blob as the user addressed it and appears in the + /// success log and in the "already exists" error. `if_not_exists` is asked of + /// the backend, which is free to drop it — the caller is responsible for any + /// guard it needs on top (see `azure::upload_single_package`). + pub(crate) async fn stream_package_to_object_store( + op: &Operator, + target: &BlobUploadTarget, + package_file: &Path, + destination: &str, + force: ForceOverwrite, + ) -> miette::Result<()> { + let HashedFile { + mut reader, + size, + sha256, + md5, + } = hash_file(package_file).await?; + + // S3 honours both. azblob never sends content-disposition, and drops user + // metadata on its Put Block List commit, so a package above + // `DESIRED_CHUNK_SIZE` lands there with neither. + let options = WriteOptions { + chunk: Some(DESIRED_CHUNK_SIZE), + concurrent: PART_CONCURRENCY, + content_disposition: Some(format!("attachment; filename={}", target.filename)), + user_metadata: Some(HashMap::from([ + (String::from("package-sha256"), hex::encode(sha256)), + (String::from("package-md5"), hex::encode(md5)), + ])), + if_not_exists: !force.is_enabled(), + ..WriteOptions::default() + }; + + let already_exists = + || miette::miette!("Package {destination} already exists. Use --force to overwrite."); + + let mut writer = match op.writer_options(target.key(), options).await { + Ok(writer) => writer, + Err(e) if e.kind() == ErrorKind::ConditionNotMatch => return Err(already_exists()), + Err(e) => return Err(e).into_diagnostic(), + }; + + if let Err(e) = stream_chunks(&mut writer, &mut reader, size).await { + discard_partial_upload(&mut writer, destination).await; + return Err(e); + } + + match writer.close().await { + Ok(_) => { + tracing::info!("Uploaded package to {destination}"); + Ok(()) + } + Err(e) => { + discard_partial_upload(&mut writer, destination).await; + if e.kind() == ErrorKind::ConditionNotMatch { + return Err(already_exists()); + } + Err(e).into_diagnostic() + } + } + } + + /// Feeds exactly `size` bytes of `reader` to `writer`. opendal buffers them + /// into correctly sized parts/blocks and uploads `PART_CONCURRENCY` at a time. + async fn stream_chunks( + writer: &mut opendal::Writer, + reader: &mut (impl AsyncReadExt + Unpin), + size: u64, + ) -> miette::Result<()> { + let mut remaining_size = size as usize; + while remaining_size > 0 { + let chunk_size = remaining_size.min(DESIRED_CHUNK_SIZE); + let mut chunk = BytesMut::zeroed(chunk_size); + + let bytes_read = reader.read_exact(&mut chunk[..]).await.into_diagnostic()?; + debug_assert_eq!(bytes_read, chunk.len()); + + writer.write(chunk.freeze()).await.into_diagnostic()?; + + remaining_size = remaining_size.saturating_sub(bytes_read); + } + Ok(()) + } + + /// Uncommitted parts are billed until they are discarded. S3 discards them + /// here; azblob's abort is a no-op, so Azure only collects its uncommitted + /// blocks after a week without further writes to the blob. + async fn discard_partial_upload(writer: &mut opendal::Writer, destination: &str) { + if let Err(e) = writer.abort().await { + tracing::warn!("Failed to discard the partial upload of {destination}: {e}"); + } + } + + #[cfg(test)] + mod test { + use super::hash_file; + use crate::upload::test_utils::test_package_path; + use rattler_digest::{Md5, Sha256, compute_file_digest}; + + /// The size the upload streams and the hashes it records must come + /// from the same pass, so they always describe the same bytes. + #[tokio::test] + async fn test_hash_file_size_and_hashes_agree_with_the_file() { + let path = test_package_path(); + let hashed = hash_file(&path).await.expect("hashing the package failed"); + + assert_eq!(hashed.size, std::fs::metadata(&path).unwrap().len()); + assert_eq!( + hashed.sha256, + compute_file_digest::(&path).unwrap(), + "recorded sha256 must match the file's" + ); + assert_eq!( + hashed.md5, + compute_file_digest::(&path).unwrap(), + "recorded md5 must match the file's" + ); + } + } +} + /// Returns the style to use for a progress bar that is currently in progress. fn default_bytes_style() -> Result { Ok(indicatif::ProgressStyle::default_bar() diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 28ab7b17e0..e962e8051f 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -1,7 +1,7 @@ //! Command-line options. use std::path::PathBuf; -use clap::Parser; +use clap::{Parser, builder::TypedValueParser as _}; use rattler_conda_types::utils::url_with_trailing_slash::UrlWithTrailingSlash; use rattler_networking::AuthenticationStorage; use url::Url; @@ -439,8 +439,12 @@ pub struct AzureOpts { pub credentials: rattler_azure::clap::AzureCredentialsOpts, /// Replace files if it already exists. - #[arg(long)] - pub force: bool, + #[arg( + long, + action = clap::ArgAction::SetTrue, + value_parser = clap::builder::BoolishValueParser::new().map(ForceOverwrite) + )] + pub force: ForceOverwrite, } #[derive(Debug)] @@ -630,3 +634,20 @@ impl CondaForgeData { } } } + +#[cfg(all(test, feature = "azure"))] +mod test { + use super::{AzureOpts, ForceOverwrite}; + use clap::Parser; + + /// `--force` stays a flag even though it parses into a newtype. + #[test] + fn test_azure_force_parses_as_a_flag() { + let args = ["az", "--channel", "az://account.blob.core.windows.net/c"]; + let opts = AzureOpts::try_parse_from(args).unwrap(); + assert_eq!(opts.force, ForceOverwrite(false)); + + let opts = AzureOpts::try_parse_from(args.iter().chain(["--force"].iter())).unwrap(); + assert_eq!(opts.force, ForceOverwrite(true)); + } +} diff --git a/crates/rattler_upload/src/upload/s3.rs b/crates/rattler_upload/src/upload/s3.rs index 98202f8bcd..7c0532004f 100644 --- a/crates/rattler_upload/src/upload/s3.rs +++ b/crates/rattler_upload/src/upload/s3.rs @@ -2,24 +2,15 @@ use std::path::{Path, PathBuf}; use futures::StreamExt; use miette::IntoDiagnostic; -use opendal::{Configurator, ErrorKind, Operator, services::S3Config}; -use rattler_digest::{HashingReader, Md5, Sha256}; +use opendal::{Configurator, Operator, services::S3Config}; use rattler_s3::ResolvedS3Credentials; -use tokio::io::{AsyncReadExt, AsyncSeekExt}; -use tokio_util::bytes::BytesMut; use url::Url; -use crate::upload::package::ExtractedPackage; - -/// Size of a single multipart chunk. S3 requires every part except the last to -/// be at least 5 MiB; we use a larger value to keep the number of parts low. -const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; - -/// Number of chunks of a single package that are uploaded concurrently. -const PART_CONCURRENCY: usize = 4; - -/// Number of packages that are uploaded concurrently. -const PACKAGE_CONCURRENCY: usize = 4; +use crate::upload::{ + object_store::{BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store}, + opt::ForceOverwrite, + package::ExtractedPackage, +}; /// Uploads a package to a channel in an S3 bucket. /// @@ -58,7 +49,9 @@ pub async fn upload_package_to_s3( .map(|package_file| { let op = op.clone(); let channel = &channel; - async move { upload_single_package(&op, channel, bucket, package_file, force).await } + async move { + upload_single_package(&op, channel, bucket, package_file, force.into()).await + } }) .buffer_unordered(PACKAGE_CONCURRENCY) .collect::>() @@ -75,103 +68,11 @@ async fn upload_single_package( channel: &Url, bucket: &str, package_file: &Path, - force: bool, + force: ForceOverwrite, ) -> miette::Result<()> { let package = ExtractedPackage::from_package_file(package_file)?; - let subdir = package - .subdir() - .ok_or_else(|| miette::miette!("Failed to get subdir"))?; - let filename = package - .filename() - .ok_or_else(|| miette::miette!("Failed to get filename"))?; - let key = format!("{subdir}/{filename}"); + let target = BlobUploadTarget::from_package(&package)?; + let destination = format!("s3://{bucket}{}/{}", channel.path(), target.key()); - // Compute the hash of the package by streaming its content. - let file = tokio::io::BufReader::new( - fs_err::tokio::File::open(package_file) - .await - .into_diagnostic()?, - ); - let sha256_reader = HashingReader::<_, Sha256>::new(file); - let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); - let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) - .await - .into_diagnostic()?; - let (sha256_reader, md5hash) = md5_reader.finalize(); - let (mut file, sha256hash) = sha256_reader.finalize(); - - // Rewind the file to the beginning. - file.rewind().await.into_diagnostic()?; - - // Construct a writer for the package. Setting `chunk` and `concurrent` - // enables opendal's concurrent multipart upload: data is buffered into - // `DESIRED_CHUNK_SIZE` parts and up to `PART_CONCURRENCY` parts are uploaded - // in parallel. - let mut writer = match op - .writer_with(&key) - .chunk(DESIRED_CHUNK_SIZE) - .concurrent(PART_CONCURRENCY) - .content_disposition(&format!("attachment; filename={filename}")) - .if_not_exists(!force) - .user_metadata([ - (String::from("package-sha256"), hex::encode(sha256hash)), - (String::from("package-md5"), hex::encode(md5hash)), - ]) - .await - { - Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { - miette::bail!( - "Package s3://{bucket}{}/{key} already exists. Use --force to overwrite.", - channel.path().to_string() - ); - } - Ok(writer) => writer, - Err(e) => { - return Err(e).into_diagnostic(); - } - }; - - // Stream the file to the writer in `DESIRED_CHUNK_SIZE` chunks. opendal takes - // care of buffering these into correctly sized parts and uploading them - // concurrently. - let mut remaining_size = size as usize; - while remaining_size > 0 { - // Allocate memory for this chunk. - let chunk_size = remaining_size.min(DESIRED_CHUNK_SIZE); - let mut chunk = BytesMut::with_capacity(chunk_size); - // SAFE: because we do not care about the bytes that are currently in the buffer - unsafe { chunk.set_len(chunk_size) }; - - // Fill the chunk with data. This reads exactly the number of bytes we want. No - // more, no less. - let bytes_read = file.read_exact(&mut chunk[..]).await.into_diagnostic()?; - debug_assert_eq!(bytes_read, chunk.len()); - - // Hand the chunk to the writer. With concurrent writes enabled this returns - // as soon as the chunk is queued rather than fully uploaded. - writer.write(chunk.freeze()).await.into_diagnostic()?; - - // Update the number of remaining bytes. - remaining_size = remaining_size.saturating_sub(bytes_read); - } - - match writer.close().await { - Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { - miette::bail!( - "Package s3://{bucket}{}/{key} already exists. Use --force to overwrite.", - channel.path().to_string() - ); - } - Ok(_) => { - tracing::info!( - "Uploaded package to s3://{bucket}{}/{key}", - channel.path().to_string() - ); - } - Err(e) => { - return Err(e).into_diagnostic(); - } - } - - Ok(()) + stream_package_to_object_store(op, &target, package_file, &destination, force).await } From 30330fa62f0619eb847c4d218addc0f9fae3e8c8 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:53:43 +1000 Subject: [PATCH 86/98] test(azure): assert the outgoing request carries no credential --- crates/rattler_index/tests/azure_azurite.rs | 64 ++++++++++---- .../tests/azure_azurite_fetch.rs | 85 +++++++++++++++++-- 2 files changed, 124 insertions(+), 25 deletions(-) diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs index 41300dfa2c..c6af1cbe1f 100644 --- a/crates/rattler_index/tests/azure_azurite.rs +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -296,9 +296,16 @@ async fn azurite_multi_block_write_keeps_cache_control() { .await; } -/// The gap `upload_package_to_azure`'s pre-write `stat` exists to close: opendal -/// honours `if_not_exists` on the single-shot Put Blob path and silently drops it -/// on the multi-block path. +/// A canary for an opendal bug, so it asserts the *broken* behaviour and fails when +/// upstream fixes it. Read the failure message before touching the assertions. +/// +/// The bug: `azblob_complete_put_block_list_request` never sets `IF_NONE_MATCH`, so +/// `if_not_exists` is silently dropped above the 10 MiB chunk size, while the +/// sibling copy-block-list function does set it. That is the gap +/// `upload_package_to_azure`'s pre-write `stat` exists to close. +/// +/// TODO: revisit once merges — it +/// carries changes in this area. /// /// A package over `rattler_upload`'s 10 MiB chunk size is the only way to reach the /// multi-block path, and this is as close as a test can currently get to that @@ -328,11 +335,33 @@ async fn azurite_if_not_exists_is_dropped_on_the_multi_block_path() { .expect_err("if_not_exists should refuse to overwrite a small blob"); assert_eq!(refused.kind(), ErrorKind::ConditionNotMatch); - // Over the chunk size, the same option is accepted and then ignored. + // Over the chunk size, the same option is accepted and then ignored: the + // guarded second write goes through and replaces the first payload. let large = "noarch/large.conda"; - let payload = vec![0u8; UPLOAD_CHUNK_SIZE + 2 * MIB]; - write_chunked(&op, large, &payload, false).await; - write_chunked(&op, large, &payload, true).await; + let first = vec![0u8; UPLOAD_CHUNK_SIZE + 2 * MIB]; + let second = vec![1u8; first.len()]; + write_chunked(&op, large, &first, false) + .await + .expect("unguarded multi-block write failed"); + write_chunked(&op, large, &second, true) + .await + .unwrap_or_else(|e| { + panic!( + "opendal now honours `if_not_exists` on the multi-block path ({e}). This is \ + the upstream fix this test was waiting for — do not adjust the assertion. \ + Delete `upload_package_to_azure`'s pre-write `stat` guard (and its TOCTOU \ + window) in crates/rattler_upload/src/upload/azure.rs, then delete this test." + ) + }); + let stored = op + .read(large) + .await + .expect("read back the large blob") + .to_vec(); + assert!( + stored == second, + "the guarded multi-block write should have clobbered the first payload" + ); // Which is why the uploader stats first. That check does see the blob, so // the guard holds for large packages despite opendal dropping the option. @@ -344,13 +373,17 @@ async fn azurite_if_not_exists_is_dropped_on_the_multi_block_path() { .await; } -/// Write `payload` the way `upload_single_package` does, and assert it succeeded. +/// Write `payload` the way `upload_single_package` does. /// -/// `guard` is that function's `if_not_exists(!force)`. Passing `true` over a blob -/// that already exists still succeeds, which is the point being demonstrated: on -/// the multi-block path the option is a no-op, so a large upload would clobber -/// without the separate `stat`. -async fn write_chunked(op: &Operator, path: &str, payload: &[u8], guard: bool) { +/// `guard` is that function's `if_not_exists(!force)`. The commit result is handed +/// back rather than unwrapped: whether a guarded commit over an existing blob +/// succeeds is the thing under test, so the caller owns that message. +async fn write_chunked( + op: &Operator, + path: &str, + payload: &[u8], + guard: bool, +) -> opendal::Result<()> { let mut writer = op .writer_with(path) .chunk(UPLOAD_CHUNK_SIZE) @@ -361,8 +394,5 @@ async fn write_chunked(op: &Operator, path: &str, payload: &[u8], guard: bool) { .write(payload.to_vec()) .await .expect("chunked write failed"); - writer - .close() - .await - .expect("a multi-block commit should succeed even with if_not_exists set"); + writer.close().await.map(|_| ()) } diff --git a/crates/rattler_networking/tests/azure_azurite_fetch.rs b/crates/rattler_networking/tests/azure_azurite_fetch.rs index d66f5e0bfc..88608a35cd 100644 --- a/crates/rattler_networking/tests/azure_azurite_fetch.rs +++ b/crates/rattler_networking/tests/azure_azurite_fetch.rs @@ -33,11 +33,22 @@ //! only if an older emulator rejects the version outright. #![cfg(feature = "azure")] -use std::collections::HashMap; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; +use async_trait::async_trait; use rattler_azure::{Addressing, Auth, AzureEndpointOptions, AzureHost, AzureScheme}; use rattler_networking::AzureMiddleware; -use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; +use reqwest::{ + Request, Response, + header::{AUTHORIZATION, HeaderMap}, +}; +use reqwest_middleware::{ + ClientBuilder, ClientWithMiddleware, Middleware, Next, Result as MiddlewareResult, +}; +use url::Url; /// Azurite's development account and its fixed key. Not a secret: both are /// published constants of the emulator, hardcoded in opendal's own source, and @@ -96,13 +107,52 @@ fn azurite_entry(auth: Auth) -> HashMap { )]) } -fn client(auth: Auth) -> ClientWithMiddleware { - ClientBuilder::new(reqwest::Client::new()) +/// The last request to leave the stack, captured *after* `AzureMiddleware` ran. +/// +/// Azurite answers 403 to any request it cannot authenticate, so a response status +/// cannot tell an unsigned request from a signed one whose signature was rejected — +/// a leak with a bad signature reads exactly like a refusal. The claim being tested +/// is about what goes on the wire, so that is what gets asserted. +#[derive(Clone, Default)] +struct SentRequest(Arc>>); + +impl SentRequest { + fn recorded(&self) -> (Url, HeaderMap) { + self.0 + .lock() + .expect("recorder mutex") + .clone() + .expect("no request reached the recorder") + } +} + +#[async_trait] +impl Middleware for SentRequest { + async fn handle( + &self, + req: Request, + extensions: &mut http::Extensions, + next: Next<'_>, + ) -> MiddlewareResult { + *self.0.lock().expect("recorder mutex") = Some((req.url().clone(), req.headers().clone())); + next.run(req, extensions).await + } +} + +fn recording_client(auth: Auth) -> (ClientWithMiddleware, SentRequest) { + let sent = SentRequest::default(); + let client = ClientBuilder::new(reqwest::Client::new()) .with(AzureMiddleware::new( reqwest::Client::new(), azurite_entry(auth), )) - .build() + .with(sent.clone()) + .build(); + (client, sent) +} + +fn client(auth: Auth) -> ClientWithMiddleware { + recording_client(auth).0 } /// Create the container and put a `noarch/repodata.json` in it. @@ -188,8 +238,13 @@ async fn azurite_granted_entry_fetches_repodata() { } /// Without `auth = true` the request goes out unsigned, and a private container -/// refuses it. This is the core claim of the anonymous-by-default model, and the -/// only way to check it is against a server that actually enforces authorization. +/// refuses it. This is the core claim of the anonymous-by-default model. +/// +/// The primary assertion is on the outgoing request, not the status: no +/// `Authorization` header and no SAS `sig` reaches the wire. The 403 stays as a +/// secondary check that the container really is private, but on its own it would +/// also pass while the account key was being sent with a signature Azurite +/// rejected. #[tokio::test] #[ignore = "requires a running Azurite emulator; see the module docs"] async fn azurite_ungranted_entry_is_refused_by_a_private_container() { @@ -205,12 +260,26 @@ async fn azurite_ungranted_entry_is_refused_by_a_private_container() { seed(&client(Auth::DefaultChain)).await; let url = format!("{}/noarch/repodata.json", channel_url()); - let resp = client(Auth::Anonymous) + let (client, sent) = recording_client(Auth::Anonymous); + let resp = client .get(&url) .send() .await .expect("request through azure middleware failed"); + let (sent_url, sent_headers) = sent.recorded(); + assert!( + !sent_headers.contains_key(AUTHORIZATION), + "an ungranted request must carry no credential, but it went out with an \ + Authorization header: {:?}", + sent_headers.get(AUTHORIZATION) + ); + assert!( + !sent_url.query_pairs().any(|(key, _)| key == "sig"), + "an ungranted request must carry no credential, but its URL went out with a SAS \ + signature: {sent_url}" + ); + let status = resp.status(); assert!( // Both statuses are correct answers to an unsigned read of a private From 2434522acf217ce47ee3a587348929fa261b30a0 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:53:43 +1000 Subject: [PATCH 87/98] test(git): isolate the lfs fixture from a developer global gitignore --- crates/rattler_git/tests/lfs.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/rattler_git/tests/lfs.rs b/crates/rattler_git/tests/lfs.rs index c4ba0bd97b..84efb5ebd4 100644 --- a/crates/rattler_git/tests/lfs.rs +++ b/crates/rattler_git/tests/lfs.rs @@ -71,6 +71,8 @@ impl LfsFixture { git(&["config", "user.name", "Test"]); // Disable signing so a global `commit.gpgSign = true` can't interfere. git(&["config", "commit.gpgsign", "false"]); + // Ignore the developer's global gitignore; e.g. `*.bin` would drop `data.bin`. + git(&["config", "core.excludesFile", ""]); git(&["lfs", "install", "--local"]); fs_err::write( From 8d640cfdbf35afa302025baa14c1e28bb1b5d7af Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:01:57 +1000 Subject: [PATCH 88/98] refactor(azure): give the fetch and write paths types that carry only what they use --- crates/rattler_azure/src/clap.rs | 40 +++--- crates/rattler_azure/src/lib.rs | 34 +++--- crates/rattler_azure/src/options.rs | 115 ++++++++++++++---- crates/rattler_config/src/config/azure.rs | 27 ++-- crates/rattler_index/src/lib.rs | 22 ++-- crates/rattler_index/src/main.rs | 35 +++--- crates/rattler_index/tests/azure_azurite.rs | 22 ++-- .../src/azure_middleware.rs | 34 +++--- .../tests/azure_azurite_fetch.rs | 7 +- .../tests/azure_real_fetch.rs | 4 +- crates/rattler_upload/src/lib.rs | 12 +- crates/rattler_upload/src/upload/azure.rs | 15 ++- 12 files changed, 214 insertions(+), 153 deletions(-) diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs index 372a8ec6dc..72e7abb45a 100644 --- a/crates/rattler_azure/src/clap.rs +++ b/crates/rattler_azure/src/clap.rs @@ -5,7 +5,7 @@ use clap::Parser; use secrecy::{ExposeSecret, SecretString}; use crate::{ - AzureChannelUrl, AzureCliSasError, AzureCoordinates, AzureCredentials, AzureEndpointOptions, + AzureChannelUrl, AzureCliSasError, AzureCoordinates, AzureCredentials, AzureEndpoint, AzureUrlError, account_and_container, mint_user_delegation_sas, }; @@ -67,29 +67,29 @@ pub enum AzureAuthSource { impl AzureAuthSource { /// Resolve this source into concrete [`AzureCredentials`]. /// - /// `permissions`, `channel` and `options` are consulted **only** for the + /// `permissions`, `channel` and `endpoint` are consulted **only** for the /// [`AzureAuthSource::AzureCli`] arm, which mints a SAS scoped to the channel's - /// container with those permissions. Taking the channel and its options rather + /// container with those permissions. Taking the channel and its endpoint rather /// than pre-derived coordinates is what keeps the account the SAS is minted for /// and the scheme it is restricted to from coming from two different places. pub async fn resolve( self, permissions: &str, channel: &AzureChannelUrl, - options: AzureEndpointOptions, + endpoint: AzureEndpoint, ) -> Result { match self { AzureAuthSource::AccountKey(key) => Ok(AzureCredentials::AccountKey(key)), AzureAuthSource::SasToken(token) => Ok(AzureCredentials::SasToken(token)), AzureAuthSource::AzureCli { ttl } => { let AzureCoordinates { account, container } = - account_and_container(channel, options.addressing)?; + account_and_container(channel, endpoint.addressing)?; let token = mint_user_delegation_sas( &account, &container, permissions, ttl, - options.scheme, + endpoint.scheme, ) .await?; Ok(AzureCredentials::SasToken(token)) @@ -210,15 +210,15 @@ impl AzureCredentialsOpts { /// Resolve the supplied options into concrete [`AzureCredentials`]. /// /// Precedence is applied by [`AzureCredentialsOpts::source`]. `permissions`, - /// `channel` and `options` are consulted only when the winning source is + /// `channel` and `endpoint` are consulted only when the winning source is /// `--azure-cli`; see [`AzureAuthSource::resolve`]. pub async fn resolve( self, permissions: &str, channel: &AzureChannelUrl, - options: AzureEndpointOptions, + endpoint: AzureEndpoint, ) -> Result { - self.source()?.resolve(permissions, channel, options).await + self.source()?.resolve(permissions, channel, endpoint).await } } @@ -239,40 +239,40 @@ mod tests { } } - /// A channel whose coordinates cannot be derived under `options`: resolving a + /// A channel whose coordinates cannot be derived under `endpoint`: resolving a /// verbatim credential must not need them, and this is what proves it. - fn underivable() -> (AzureChannelUrl, AzureEndpointOptions) { + fn underivable() -> (AzureChannelUrl, AzureEndpoint) { let channel = AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); - let options = AzureEndpointOptions::default(); - assert!(account_and_container(&channel, options.addressing).is_err()); - (channel, options) + let endpoint = AzureEndpoint::default(); + assert!(account_and_container(&channel, endpoint.addressing).is_err()); + (channel, endpoint) } #[tokio::test] async fn account_key_resolves() { - let (channel, options) = underivable(); + let (channel, endpoint) = underivable(); assert!(matches!( - opts(Some("key"), None, false).resolve("cw", &channel, options).await, + opts(Some("key"), None, false).resolve("cw", &channel, endpoint).await, Ok(AzureCredentials::AccountKey(k)) if k.expose_secret() == "key" )); } #[tokio::test] async fn sas_token_resolves() { - let (channel, options) = underivable(); + let (channel, endpoint) = underivable(); assert!(matches!( - opts(None, Some("sv=..."), false).resolve("cw", &channel, options).await, + opts(None, Some("sv=..."), false).resolve("cw", &channel, endpoint).await, Ok(AzureCredentials::SasToken(t)) if t.expose_secret() == "sv=..." )); } #[tokio::test] async fn none_is_rejected() { - let (channel, options) = underivable(); + let (channel, endpoint) = underivable(); assert!(matches!( opts(None, None, false) - .resolve("cw", &channel, options) + .resolve("cw", &channel, endpoint) .await, Err(AzureCredentialsError::Missing) )); diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index e8e8d4866f..dc9bb31226 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -20,7 +20,9 @@ pub mod clap; pub mod options; -pub use options::{Addressing, Auth, AzureEndpointOptions, AzureScheme}; +pub use options::{ + Addressing, Auth, AzureEndpoint, AzureEndpointOptions, AzureFetchOptions, AzureScheme, +}; pub use secrecy::{ExposeSecret, SecretString}; use url::Url; @@ -733,18 +735,18 @@ fn strip_az_scheme(value: &str) -> Option<&str> { pub fn azblob_config( credentials: &AzureCredentials, channel: &AzureChannelUrl, - options: AzureEndpointOptions, + endpoint_options: AzureEndpoint, ) -> Result { let AzureCoordinates { account, container } = - account_and_container(channel, options.addressing)?; + account_and_container(channel, endpoint_options.addressing)?; // The authority comes from `AzureHost`, not from a wire URL: a `Url` has // already dropped a port equal to its scheme's default, so reading it back // would turn a written `:443` into no port at all. let authority = channel.host(); - let endpoint = match options.addressing { - Addressing::HostStyle => format!("{}://{authority}", options.scheme), - Addressing::PathStyle => format!("{}://{authority}/{account}", options.scheme), + let endpoint = match endpoint_options.addressing { + Addressing::HostStyle => format!("{}://{authority}", endpoint_options.scheme), + Addressing::PathStyle => format!("{}://{authority}/{account}", endpoint_options.scheme), }; // Root prefix = the path after the segments the coordinates already consumed: @@ -752,7 +754,7 @@ pub fn azblob_config( // one too few there leaves the account segment inside `root`, so every blob is // written one directory deeper than the channel actually lives — silently, and // in the right container, which is what makes it hard to spot. - let consumed = match options.addressing { + let consumed = match endpoint_options.addressing { Addressing::HostStyle => 1, Addressing::PathStyle => 2, }; @@ -1440,16 +1442,13 @@ mod tests { let channel = AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general/mychannel") .unwrap(); - let options = AzureEndpointOptions { - auth: Auth::DefaultChain, - scheme: AzureScheme::Http, - addressing: Addressing::PathStyle, - }; - let config = azblob_config( &AzureCredentials::AccountKey("key".into()), &channel, - options, + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, ) .unwrap(); @@ -1486,8 +1485,7 @@ mod tests { let config = azblob_config( &AzureCredentials::SasToken("?sv=token".into()), &channel, - AzureEndpointOptions { - auth: Auth::Anonymous, + AzureEndpoint { scheme: AzureScheme::Http, addressing: Addressing::PathStyle, }, @@ -1511,7 +1509,7 @@ mod tests { let config = azblob_config( &AzureCredentials::SasToken("sv=token".into()), &channel, - AzureEndpointOptions::default(), + AzureEndpoint::default(), ) .unwrap(); @@ -1536,7 +1534,7 @@ mod tests { let config = azblob_config( &AzureCredentials::AccountKey("key".into()), &channel, - AzureEndpointOptions::default(), + AzureEndpoint::default(), ) .unwrap(); diff --git a/crates/rattler_azure/src/options.rs b/crates/rattler_azure/src/options.rs index 56722c528a..cf14734fd3 100644 --- a/crates/rattler_azure/src/options.rs +++ b/crates/rattler_azure/src/options.rs @@ -7,6 +7,15 @@ //! suffix classification carries no security weight, and the absence of the list //! is what lets custom endpoints and the Azurite emulator work at all. //! +//! # Three types, one table +//! +//! [`AzureEndpointOptions`] is the file format. Nothing consumes it directly: +//! the fetch path takes [`AzureFetchOptions`] and the write path takes +//! [`AzureEndpoint`], so `auth` cannot reach a caller that supplies its own +//! credential and [`Addressing`] cannot reach one that never derives an account. +//! A field a consumer would have to ignore reads as a guarantee, and the write +//! path ignoring `auth` looked exactly like a credential gate that was never there. +//! //! # Why enums for what the config spells as bools //! //! The TOML surface stays `auth = true` / `path-style = true`, because that is @@ -151,7 +160,41 @@ impl From for bool { } } -/// How to reach, and whether to authenticate to, one Azure Blob host. +/// How to address one Azure Blob host. Carries no grant. +/// +/// This is what the write path takes. Splitting it out is what stops +/// [`Auth`] from reaching a consumer that cannot act on it: `azblob_config` and +/// the SAS mint are handed a material credential by their caller, so there is no +/// ambient chain for a grant to gate, and a grant they could read would be a +/// promise nothing keeps. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +pub struct AzureEndpoint { + /// The scheme `az://` is rewritten to for this host. + pub scheme: AzureScheme, + + /// Where the account name is found in the URL for this host. + pub addressing: Addressing, +} + +/// What the fetch middleware needs to reach one Azure Blob host. +/// +/// [`Addressing`] is absent rather than ignored: the fetch path forwards a path +/// and never derives an account name from it. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +pub struct AzureFetchOptions { + /// Whether credentials may be sent to this host. + pub auth: Auth, + + /// The scheme `az://` is rewritten to for this host. + pub scheme: AzureScheme, +} + +/// One `azure-options` entry, as the config file spells it. +/// +/// This is the serde surface and nothing else: the three TOML keys live here, and +/// each consumer takes the narrower view it can actually act on, via +/// [`Self::endpoint`] or [`Self::fetch`]. The fields are private so that view is +/// the only way in. /// /// The default value is the no-entry behaviour: anonymous, https, host-style. A /// host with no config entry behaves exactly as if it had a defaulted entry, so @@ -164,19 +207,42 @@ impl From for bool { serde(rename_all = "kebab-case", default) )] pub struct AzureEndpointOptions { - /// Whether credentials may be sent to this host. - pub auth: Auth, + auth: Auth, - /// The scheme `az://` is rewritten to for this host. - pub scheme: AzureScheme, + scheme: AzureScheme, - /// Where the account name is found in the URL for this host. - /// /// The field is named for what it holds, but the config key stays /// `path-style`: that is the spelling users have written, and the bool bridge /// is what the key means. #[cfg_attr(feature = "serde", serde(rename = "path-style", alias = "path_style"))] - pub addressing: Addressing, + addressing: Addressing, +} + +impl AzureEndpointOptions { + /// Build an entry from a grant and the endpoint it applies to. + pub fn new(auth: Auth, endpoint: AzureEndpoint) -> Self { + Self { + auth, + scheme: endpoint.scheme, + addressing: endpoint.addressing, + } + } + + /// How to address this host, for the write path. + pub fn endpoint(self) -> AzureEndpoint { + AzureEndpoint { + scheme: self.scheme, + addressing: self.addressing, + } + } + + /// The grant and wire scheme, for the fetch path. + pub fn fetch(self) -> AzureFetchOptions { + AzureFetchOptions { + auth: self.auth, + scheme: self.scheme, + } + } } #[cfg(all(test, feature = "serde"))] @@ -197,36 +263,39 @@ mod tests { .unwrap(); assert_eq!( opts, - AzureEndpointOptions { - auth: Auth::DefaultChain, - scheme: AzureScheme::Http, - addressing: Addressing::PathStyle, - } + AzureEndpointOptions::new( + Auth::DefaultChain, + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) ); // An empty entry is the same as no entry: anonymous, https, host-style. let empty: AzureEndpointOptions = toml::from_str("").unwrap(); assert_eq!(empty, AzureEndpointOptions::default()); - assert_eq!(empty.auth, Auth::Anonymous); - assert!(!empty.auth.is_granted()); - assert_eq!(empty.scheme, AzureScheme::Https); - assert_eq!(empty.addressing, Addressing::HostStyle); + assert_eq!(empty.fetch(), AzureFetchOptions::default()); + assert!(!empty.fetch().auth.is_granted()); + assert_eq!(empty.endpoint(), AzureEndpoint::default()); // `auth = false` is spelled out explicitly by some users; it must not be // mistaken for a grant. let denied: AzureEndpointOptions = toml::from_str("auth = false").unwrap(); - assert!(!denied.auth.is_granted()); + assert!(!denied.fetch().auth.is_granted()); } /// Round-tripping must preserve the boolean spelling, not leak the enum /// variant names into a written config file. #[test] fn enums_serialize_back_to_bools() { - let toml = toml::to_string(&AzureEndpointOptions { - auth: Auth::DefaultChain, - scheme: AzureScheme::Http, - addressing: Addressing::PathStyle, - }) + let toml = toml::to_string(&AzureEndpointOptions::new( + Auth::DefaultChain, + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + )) .unwrap(); assert!(toml.contains("auth = true"), "{toml}"); assert!(toml.contains("path-style = true"), "{toml}"); diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs index b2f290757f..9e6c38601a 100644 --- a/crates/rattler_config/src/config/azure.rs +++ b/crates/rattler_config/src/config/azure.rs @@ -119,8 +119,9 @@ impl Config for AzureOptionsMap { fn validate(&self) -> Result<(), super::ValidationError> { for (host, options) in &self.0 { - if options.auth == Auth::DefaultChain - && options.scheme == AzureScheme::Http + let fetch = options.fetch(); + if fetch.auth == Auth::DefaultChain + && fetch.scheme == AzureScheme::Http && !is_local(host) { return Err(super::ValidationError::Invalid(format!( @@ -171,18 +172,18 @@ mod tests { .unwrap(); let real = map.get(&host("mycompany.blob.core.windows.net")); - assert_eq!(real.auth, Auth::DefaultChain); - assert_eq!(real.scheme, AzureScheme::Https); - assert_eq!(real.addressing, Addressing::HostStyle); + assert_eq!(real.fetch().auth, Auth::DefaultChain); + assert_eq!(real.endpoint().scheme, AzureScheme::Https); + assert_eq!(real.endpoint().addressing, Addressing::HostStyle); let azurite = map.get(&host("127.0.0.1:10000")); - assert_eq!(azurite.auth, Auth::DefaultChain); - assert_eq!(azurite.scheme, AzureScheme::Http); - assert_eq!(azurite.addressing, Addressing::PathStyle); + assert_eq!(azurite.fetch().auth, Auth::DefaultChain); + assert_eq!(azurite.endpoint().scheme, AzureScheme::Http); + assert_eq!(azurite.endpoint().addressing, Addressing::PathStyle); // An unlisted host gets no grant. let unlisted = map.get(&host("someoneelse.blob.core.windows.net")); - assert!(!unlisted.auth.is_granted()); + assert!(!unlisted.fetch().auth.is_granted()); assert_eq!(unlisted, AzureEndpointOptions::default()); } @@ -243,12 +244,12 @@ auth = true let merged = base.merge_config(&over).unwrap(); let entry = merged.get(&host("host.example")); - assert_eq!(entry.scheme, AzureScheme::Http); + assert_eq!(entry.endpoint().scheme, AzureScheme::Http); assert!( - !entry.auth.is_granted(), + !entry.fetch().auth.is_granted(), "overwriting an entry must not inherit the previous grant" ); - assert_eq!(entry.addressing, Addressing::HostStyle); + assert_eq!(entry.endpoint().addressing, Addressing::HostStyle); } /// The defect this key type exists to kill: every one of these keys is a @@ -275,7 +276,7 @@ auth = true toml::from_str(&format!("[\"{written}\"]\nauth = true\n")).unwrap(); assert!( - map.get(&host(looked_up)).auth.is_granted(), + map.get(&host(looked_up)).fetch().auth.is_granted(), "the grant written as `{written}` did not apply to `{looked_up}`" ); // The key is stored canonically, so `keys()` reports what a lookup diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index 4f0ccb145c..a01d97d641 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -29,7 +29,7 @@ use opendal::layers::RetryLayer; use opendal::services::S3Config; use opendal::{Configurator, Operator, services::FsConfig}; #[cfg(feature = "azure")] -use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpointOptions}; +use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpoint}; use rattler_conda_types::{ ChannelInfo, ChannelRelations, PackageRecord, PatchInstructions, Platform, RepoData, Shard, ShardedRepodata, ShardedSubdirInfo, UrlOrPath, V3Packages, WhlPackageRecord, @@ -1522,10 +1522,10 @@ pub struct IndexAzureConfig { pub channel: AzureChannelUrl, /// The credentials to use for Azure Blob access. pub credentials: AzureCredentials, - /// The `azure-options` entry for the channel's host, which decides the wire - /// scheme and whether the account is read from the host or the path. The - /// defaults (https, host-style) describe real Azure. - pub options: AzureEndpointOptions, + /// How to address the channel's host: the wire scheme, and whether the account + /// is read from the host or the path. The defaults (https, host-style) describe + /// real Azure. + pub endpoint: AzureEndpoint, /// The target platform to index. pub target_platform: Option, /// The path to a repodata patch to apply to the index. @@ -1565,7 +1565,7 @@ pub async fn index_azure_with_channel_metadata( IndexAzureConfig { channel, credentials, - options, + endpoint, target_platform, repodata_patch, write_zst, @@ -1578,7 +1578,7 @@ pub async fn index_azure_with_channel_metadata( }: IndexAzureConfig, channel_metadata: ChannelMetadata, ) -> anyhow::Result<()> { - let azblob_config = rattler_azure::azblob_config(&credentials, &channel, options)?; + let azblob_config = rattler_azure::azblob_config(&credentials, &channel, endpoint)?; let builder = azblob_config.into_builder(); let op = Operator::new(builder)?.layer(RetryLayer::new()).finish(); @@ -1912,8 +1912,7 @@ mod tests { let credentials = AzureCredentials::SasToken("sv=token".into()); let config = - rattler_azure::azblob_config(&credentials, &channel, AzureEndpointOptions::default()) - .unwrap(); + rattler_azure::azblob_config(&credentials, &channel, AzureEndpoint::default()).unwrap(); assert_eq!( config.endpoint.as_deref(), @@ -1934,8 +1933,7 @@ mod tests { let credentials = AzureCredentials::AccountKey("key".into()); let config = - rattler_azure::azblob_config(&credentials, &channel, AzureEndpointOptions::default()) - .unwrap(); + rattler_azure::azblob_config(&credentials, &channel, AzureEndpoint::default()).unwrap(); assert_eq!(config.container, "general"); assert_eq!(config.root.as_deref(), Some("/")); @@ -1954,7 +1952,7 @@ mod tests { let config = rattler_azure::azblob_config( &credentials, &channel, - AzureEndpointOptions { + AzureEndpoint { scheme: rattler_azure::AzureScheme::Http, ..Default::default() }, diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index 06f77ec2b3..fbf5858b22 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -5,7 +5,7 @@ use anyhow::Context; use clap::{Parser, Subcommand}; use clap_verbosity_flag::Verbosity; #[cfg(feature = "azure")] -use rattler_azure::{AzureChannelUrl, AzureEndpointOptions, AzureHost}; +use rattler_azure::{AzureChannelUrl, AzureEndpoint, AzureHost}; use rattler_conda_types::Platform; use rattler_config::config::{ concurrency::default_max_concurrent_solves, index::IndexChannelConfig, @@ -257,17 +257,17 @@ async fn main() -> anyhow::Result<()> { effective_index_options(&resolved); let channel_metadata = ChannelMetadata::from_index_config(&resolved); - let options = azure_endpoint_options(&config, channel.host()); + let endpoint = azure_endpoint(&config, channel.host()); let credentials = credentials - .resolve(AZURE_INDEX_SAS_PERMISSIONS, &channel, options) + .resolve(AZURE_INDEX_SAS_PERMISSIONS, &channel, endpoint) .await?; index_azure_with_channel_metadata( IndexAzureConfig { channel, credentials, - options, + endpoint, target_platform: cli.target_platform, repodata_patch: cli.repodata_patch, write_zst, @@ -287,16 +287,18 @@ async fn main() -> anyhow::Result<()> { Ok(()) } -/// The `[azure-options.""]` entry for a channel's host, or the anonymous -/// https host-style defaults when there is no config file or no entry. +/// How to address a channel's host, from its `[azure-options.""]` entry, or +/// the https host-style defaults when there is no config file or no entry. /// /// A host without an entry and a host with an empty entry are defined to behave -/// identically, so this never has to report which of the two it found. +/// identically, so this never has to report which of the two it found. The entry's +/// `auth` grant is not part of the result: indexing signs with the credential its +/// caller supplied, so there is no ambient chain for a grant to gate. #[cfg(feature = "azure")] -fn azure_endpoint_options(config: &Option, host: &AzureHost) -> AzureEndpointOptions { +fn azure_endpoint(config: &Option, host: &AzureHost) -> AzureEndpoint { config .as_ref() - .map(|config| config.azure_options.get(host)) + .map(|config| config.azure_options.get(host).endpoint()) .unwrap_or_default() } @@ -329,7 +331,7 @@ fn effective_index_options( #[cfg(all(test, feature = "azure"))] mod tests { - use rattler_azure::{Addressing, Auth, AzureCredentials, AzureScheme}; + use rattler_azure::{Addressing, AzureCredentials, AzureScheme}; use super::*; @@ -385,15 +387,14 @@ mod tests { AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general/mychannel") .unwrap(); - let options = azure_endpoint_options(&config, channel.host()); - assert_eq!(options.auth, Auth::DefaultChain); - assert_eq!(options.scheme, AzureScheme::Http); - assert_eq!(options.addressing, Addressing::PathStyle); + let endpoint = azure_endpoint(&config, channel.host()); + assert_eq!(endpoint.scheme, AzureScheme::Http); + assert_eq!(endpoint.addressing, Addressing::PathStyle); let azblob = rattler_azure::azblob_config( &AzureCredentials::AccountKey("key".into()), &channel, - options, + endpoint, ) .expect("an Azurite channel must build an opendal config"); @@ -413,12 +414,12 @@ mod tests { fn an_emulator_host_without_an_entry_is_a_guided_error() { let channel = AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); - let options = azure_endpoint_options(&None, channel.host()); + let endpoint = azure_endpoint(&None, channel.host()); let err = rattler_azure::azblob_config( &AzureCredentials::AccountKey("key".into()), &channel, - options, + endpoint, ) .expect_err("host-style cannot address an IP literal"); let message = err.to_string(); diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs index c6af1cbe1f..1538d7b273 100644 --- a/crates/rattler_index/tests/azure_azurite.rs +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -37,8 +37,8 @@ use std::{collections::HashMap, path::PathBuf}; use opendal::{Configurator, ErrorKind, Operator, services::AzblobConfig}; use rattler_azure::{ - Addressing, Auth, AzureChannelUrl, AzureCredentials, AzureEndpointOptions, AzureHost, - AzureScheme, + Addressing, Auth, AzureChannelUrl, AzureCredentials, AzureEndpoint, AzureEndpointOptions, + AzureHost, AzureScheme, }; use rattler_index::{IndexAzureConfig, PackageRevisionAssignment, index_azure}; @@ -85,11 +85,13 @@ fn channel(prefix: &str) -> AzureChannelUrl { /// The `azure-options` entry for the emulator: the only configuration these tests /// hand to the indexer. fn azurite_options() -> AzureEndpointOptions { - AzureEndpointOptions { - auth: Auth::DefaultChain, - scheme: AzureScheme::Http, - addressing: Addressing::PathStyle, - } + AzureEndpointOptions::new( + Auth::DefaultChain, + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) } /// An operator built exactly the way `index_azure` builds one, so the opendal-level @@ -99,7 +101,7 @@ fn production_operator(channel: &AzureChannelUrl) -> Operator { let config = rattler_azure::azblob_config( &AzureCredentials::AccountKey(ACCOUNT_KEY.into()), channel, - azurite_options(), + azurite_options().endpoint(), ) .expect("azblob config for an azurite path-style channel"); Operator::new(config.into_builder()) @@ -137,7 +139,7 @@ fn verify_operator(prefix: &str) -> Operator { async fn ensure_container() { let options = HashMap::from([( AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), - azurite_options(), + azurite_options().fetch(), )]); let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) .with(rattler_networking::AzureMiddleware::new( @@ -181,7 +183,7 @@ fn index_config(channel: AzureChannelUrl) -> IndexAzureConfig { IndexAzureConfig { channel, credentials: AzureCredentials::AccountKey(ACCOUNT_KEY.into()), - options: azurite_options(), + endpoint: azurite_options().endpoint(), target_platform: None, repodata_patch: None, write_zst: false, diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index d60f400344..109d5d0adb 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use async_trait::async_trait; -use rattler_azure::{Auth, AzureChannelUrl, AzureEndpointOptions, AzureHost}; +use rattler_azure::{Auth, AzureChannelUrl, AzureFetchOptions, AzureHost}; use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; use reqsign_command_execute_tokio::TokioCommandExecute; use reqsign_core::{Context, OsEnv, ProvideCredential, Signer}; @@ -34,7 +34,7 @@ const X_MS_VERSION: &str = "2021-12-02"; /// credential can leak to a host the user never named, and an anonymous read of a /// public container does not block on the managed-identity / IMDS probe. /// -/// A credential attaches to a host only because an [`AzureEndpointOptions`] entry +/// A credential attaches to a host only because an `azure-options` entry /// for it says [`Auth::DefaultChain`], which comes from the user's `azure-options` /// config table: /// @@ -89,7 +89,7 @@ pub struct AzureMiddleware { /// `#[cfg(feature = "rattler_config")]` helper next to /// [`crate::s3_middleware::compute_s3_config_from_config`] rather than changing /// this signature. - options: HashMap, + options: HashMap, } impl AzureMiddleware { @@ -102,7 +102,7 @@ impl AzureMiddleware { /// /// `options` is the `azure-options` table: the per-host grants. An empty map /// means every `az://` request is anonymous. - pub fn new(client: Client, options: HashMap) -> Self { + pub fn new(client: Client, options: HashMap) -> Self { Self::with_credential_provider(client, DefaultCredentialProvider::new(), options) } @@ -115,7 +115,7 @@ impl AzureMiddleware { fn with_credential_provider( client: Client, provider: impl ProvideCredential + 'static, - options: HashMap, + options: HashMap, ) -> Self { let ctx = Context::new() .with_file_read(TokioFileRead) @@ -135,11 +135,7 @@ impl AzureMiddleware { /// the options table is keyed by, so a grant cannot miss over case, a trailing /// dot, an IDNA name or an IP literal written oddly. /// - /// [`AzureEndpointOptions::addressing`] is deliberately unused here: the fetch - /// path never needs an account name, it only forwards a path. Addressing - /// matters to the write path, which derives coordinates via - /// `rattler_azure::account_and_container`. - fn resolve(&self, url: &Url) -> MiddlewareResult<(AzureChannelUrl, AzureEndpointOptions)> { + fn resolve(&self, url: &Url) -> MiddlewareResult<(AzureChannelUrl, AzureFetchOptions)> { let channel = AzureChannelUrl::parse(url.as_str()).map_err(|e| { // The URL is not echoed back: the one rejection a user hits here is // userinfo, and quoting it would print their password. @@ -280,21 +276,21 @@ mod tests { /// The `azure-options` table for one host, as a caller would build it. fn options( authority: &str, - options: AzureEndpointOptions, - ) -> HashMap { + options: AzureFetchOptions, + ) -> HashMap { HashMap::from([(AzureHost::parse(authority).expect("test host"), options)]) } /// A grant with everything else defaulted: anonymous is the only interesting /// axis in most of these tests. - fn granted() -> AzureEndpointOptions { - AzureEndpointOptions { + fn granted() -> AzureFetchOptions { + AzureFetchOptions { auth: Auth::DefaultChain, ..Default::default() } } - fn middleware(options: HashMap) -> AzureMiddleware { + fn middleware(options: HashMap) -> AzureMiddleware { AzureMiddleware::new(Client::new(), options) } @@ -334,10 +330,9 @@ mod tests { fn rewrites_to_http_for_an_emulator_entry() { let emulator = middleware(options( "127.0.0.1:10000", - AzureEndpointOptions { + AzureFetchOptions { auth: Auth::DefaultChain, scheme: AzureScheme::Http, - addressing: rattler_azure::Addressing::PathStyle, }, )); assert_eq!( @@ -568,11 +563,10 @@ mod tests { /// An emulator-shaped entry (http, path-style) with the grant taken from the /// caller, so one server can exercise both sides of the hint. - fn emulator_entry(auth: Auth) -> AzureEndpointOptions { - AzureEndpointOptions { + fn emulator_entry(auth: Auth) -> AzureFetchOptions { + AzureFetchOptions { auth, scheme: AzureScheme::Http, - addressing: rattler_azure::Addressing::PathStyle, } } diff --git a/crates/rattler_networking/tests/azure_azurite_fetch.rs b/crates/rattler_networking/tests/azure_azurite_fetch.rs index 88608a35cd..3405bde2d5 100644 --- a/crates/rattler_networking/tests/azure_azurite_fetch.rs +++ b/crates/rattler_networking/tests/azure_azurite_fetch.rs @@ -39,7 +39,7 @@ use std::{ }; use async_trait::async_trait; -use rattler_azure::{Addressing, Auth, AzureEndpointOptions, AzureHost, AzureScheme}; +use rattler_azure::{Auth, AzureFetchOptions, AzureHost, AzureScheme}; use rattler_networking::AzureMiddleware; use reqwest::{ Request, Response, @@ -96,13 +96,12 @@ fn channel_url() -> String { /// variable. `scheme` and `path-style` stay set even in the ungranted case: the /// entry is what makes the emulator reachable at all, and keeping it identical /// means the two tests differ in the grant and nothing else. -fn azurite_entry(auth: Auth) -> HashMap { +fn azurite_entry(auth: Auth) -> HashMap { HashMap::from([( AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), - AzureEndpointOptions { + AzureFetchOptions { auth, scheme: AzureScheme::Http, - addressing: Addressing::PathStyle, }, )]) } diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs index 7b3cafff51..6e9562708a 100644 --- a/crates/rattler_networking/tests/azure_real_fetch.rs +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; -use rattler_azure::{Auth, AzureEndpointOptions, AzureHost}; +use rattler_azure::{Auth, AzureFetchOptions, AzureHost}; use rattler_networking::AzureMiddleware; use reqwest_middleware::ClientBuilder; @@ -40,7 +40,7 @@ async fn azure_middleware_fetches_real_repodata() { // loudly instead of falling through to an unsigned 404. let options = HashMap::from([( AzureHost::parse(&host).expect("AZURE_TEST_HOST is not a valid host[:port]"), - AzureEndpointOptions { + AzureFetchOptions { auth: Auth::DefaultChain, ..Default::default() }, diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index f1b19eaca5..41850bbe75 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -84,24 +84,24 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { #[cfg(feature = "azure")] ServerType::Azure(azure_opts) => { let channel = azure_opts.channel; - // The anonymous https host-style defaults, because `upload_from_args` - // reads no configuration file at all — it only opens the auth store — - // so there is nowhere for an `[azure-options.""]` entry to come + // The https host-style defaults, because `upload_from_args` reads no + // configuration file at all — it only opens the auth store — so there is + // nowhere for an `[azure-options.""]` entry to come // from. The ceiling is https plus host-style addressing — any host // whose first label is the account, Azure or not; an http or // path-style endpoint (Azurite as normally run) is unreachable. // Lifting it means giving `rattler_upload` a `--config` of its own, // which neither `rattler upload` nor rattler-build passes today. - let options = rattler_azure::AzureEndpointOptions::default(); + let endpoint = rattler_azure::AzureEndpoint::default(); let credentials = azure_opts .credentials - .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, &channel, options) + .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, &channel, endpoint) .await .into_diagnostic()?; upload::upload_package_to_azure( channel, credentials, - options, + endpoint, &args.package_files, azure_opts.force, ) diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index facccd520f..8893b836f4 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -7,7 +7,7 @@ use std::{ use futures::{StreamExt, TryStreamExt}; use miette::IntoDiagnostic; use opendal::{Configurator, ErrorKind, Operator}; -use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpointOptions}; +use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpoint}; use crate::upload::{ object_store::{BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store}, @@ -30,19 +30,18 @@ enum PackageOutcome { /// Uploads packages to a channel in an Azure Blob Storage container. /// /// The account name, endpoint, container and root prefix are all derived from the -/// channel URL together with `options` (see `azblob_config`): `options.addressing` -/// decides whether the account is the first host label or the first path segment, -/// and `options.scheme` decides what `az://` is sent over. A path-style entry is -/// therefore what makes an IP, single-label or emulator (Azurite) endpoint -/// uploadable. The [`AzureCredentials`] supply only the account key or SAS token. +/// channel URL together with `endpoint` (see `azblob_config`), so a path-style entry +/// is what makes an IP, single-label or emulator (Azurite) endpoint uploadable. The +/// [`AzureCredentials`] supply only the account key or SAS token. pub async fn upload_package_to_azure( channel: AzureChannelUrl, credentials: AzureCredentials, - options: AzureEndpointOptions, + endpoint: AzureEndpoint, package_files: &[PathBuf], force: ForceOverwrite, ) -> miette::Result<()> { - let config = rattler_azure::azblob_config(&credentials, &channel, options).into_diagnostic()?; + let config = + rattler_azure::azblob_config(&credentials, &channel, endpoint).into_diagnostic()?; let builder = config.into_builder(); let op = Operator::new(builder).into_diagnostic()?.finish(); From f9cab6a915c8425e04c97305d5cd37fbda14498e Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:35:52 +1000 Subject: [PATCH 89/98] fix(azure): refuse path segments that cannot name a blob --- crates/rattler_azure/src/lib.rs | 90 ++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index dc9bb31226..fa82687e7f 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -174,6 +174,35 @@ pub enum AzureUrlError { resolved: String, }, + /// A path segment percent-decodes to bytes that are not UTF-8. + /// + /// Blob names are UTF-8, so there is nothing to send such a segment as. Decoding + /// lossily would substitute U+FFFD and address a different blob than the URL + /// names, silently and without an error at any layer. + #[error( + "Azure blob channel URL segment `{segment}` percent-decodes to bytes that are not UTF-8, \ + so it cannot name a blob" + )] + NonUtf8Path { + /// The segment as written. + segment: String, + /// Where the decoded bytes stop being UTF-8. + #[source] + source: std::str::Utf8Error, + }, + + /// A path segment contains `%2F`, an encoded slash. + /// + /// One segment holding a slash and two segments are different blob paths, and + /// the URL Standard does not resolve `%2F`, so whichever reading we picked would + /// be a place the URL text does not say. Refusing keeps the written path and the + /// blob path the same shape. + #[error( + "Azure blob channel URL segment `{0}` contains an encoded slash (`%2F`); write the path \ + separator as `/` if you mean a new segment" + )] + EncodedSlashInPath(String), + /// The channel URL does not use the `az://` scheme. #[error( "Azure blob channel URL must use the `az://` scheme, e.g. \ @@ -567,6 +596,21 @@ impl AzureChannelUrl { }); } + // Every segment must survive the round trip to a blob name. `%2F` is checked + // before decoding, because after it there is no telling it from a `/` the + // user wrote. + for segment in url.path_segments().into_iter().flatten() { + if segment.to_ascii_uppercase().contains("%2F") { + return Err(AzureUrlError::EncodedSlashInPath(segment.to_string())); + } + percent_encoding::percent_decode_str(segment) + .decode_utf8() + .map_err(|source| AzureUrlError::NonUtf8Path { + segment: segment.to_string(), + source, + })?; + } + Ok(Self { host, path: url.path().to_string(), @@ -768,8 +812,18 @@ pub fn azblob_config( channel .path_segments() .skip(consumed) - .map(|segment| percent_encoding::percent_decode_str(segment).decode_utf8_lossy()) - .collect::>() + // Infallible in practice: `AzureChannelUrl::parse` rejects a segment that + // does not decode to UTF-8. Erroring rather than substituting U+FFFD is + // what keeps that a guarantee instead of an assumption. + .map(|segment| { + percent_encoding::percent_decode_str(segment) + .decode_utf8() + .map_err(|source| AzureUrlError::NonUtf8Path { + segment: segment.to_string(), + source, + }) + }) + .collect::, _>>()? .join("/") ); @@ -1621,6 +1675,38 @@ mod tests { } } + /// A segment that cannot become a blob name is refused, rather than becoming a + /// different blob name than the URL says. + #[test] + fn segments_that_cannot_name_a_blob_are_rejected() { + assert!(matches!( + AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/%ff"), + Err(AzureUrlError::NonUtf8Path { .. }) + )); + + // Both spellings: `url` normalizes the hex digits' case but not the escape. + for input in [ + "az://acct.blob.core.windows.net/general/a%2Fb", + "az://acct.blob.core.windows.net/general/a%2fb", + ] { + assert!( + matches!( + AzureChannelUrl::parse(input), + Err(AzureUrlError::EncodedSlashInPath(_)) + ), + "{input}" + ); + } + + // A percent escape that is valid UTF-8 is still a legitimate segment. + assert_eq!( + channel("az://acct.blob.core.windows.net/general/caf%C3%A9") + .canonical() + .path(), + "/general/caf%C3%A9" + ); + } + /// `--https-only` restricts the SAS to TLS, so a host configured for http would /// get a token it cannot use. Nothing else about the grant may move with it. #[cfg(feature = "clap")] From b70dd8764e9a0ad1888c20b2f11434f51c67ac48 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:47:48 +1000 Subject: [PATCH 90/98] fix(azure): warn instead of erroring on path-style for a real Azure host --- crates/rattler_azure/Cargo.toml | 2 + crates/rattler_azure/src/lib.rs | 155 ++++++++++++++++++++------------ 2 files changed, 100 insertions(+), 57 deletions(-) diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index 631c45083f..b76b9c8493 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -34,9 +34,11 @@ secrecy = { workspace = true } serde = { workspace = true, features = ["derive"], optional = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["process"], optional = true } +tracing = { workspace = true } url = { workspace = true } which = { workspace = true, optional = true } [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt"] } toml = { workspace = true } +tracing-test = { workspace = true } diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index fa82687e7f..c3fa84e6fb 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -101,28 +101,6 @@ pub enum AzureUrlError { )] InvalidHost(String), - /// Path-style addressing was configured for a host that already carries the - /// account in its first label. - /// - /// Both styles produce identical request URLs for such a host, so the mistake - /// is invisible on the wire and only shows up when an account name is needed - /// on its own — where it would be taken from path text the user meant as a - /// container. - #[error( - "Azure blob URL host `{host}` is a dotted domain whose first label `{label}` is already a \ - valid storage account name, so it is a host-style endpoint. Reading it path-style would \ - take the storage account from the URL path instead and mint credentials for whatever \ - account that text names; remove `path-style = true` from `[azure-options.\"{host}\"]`, or \ - remove the entry" - )] - PathStyleOnAccountHost { - /// The host as [`AzureHost`] spells it, which is how the config table is - /// keyed. - host: String, - /// Its first label. - label: String, - }, - /// The URL has no path segment to read the account from (path-style only). #[error("could not derive account name from Azure blob URL")] NoAccount, @@ -236,10 +214,13 @@ pub struct AzureCoordinates { /// [`AzureUrlError::InvalidHost`], whose message names the config line that /// switches to path-style. /// - [`Addressing::PathStyle`] (Azurite and other emulators): account = first -/// path segment, container = second. The host must *not* be one that could -/// carry an account itself, or the two styles disagree about which name is the -/// account while producing identical request URLs — see -/// [`AzureUrlError::PathStyleOnAccountHost`]. +/// path segment, container = second. On a host under a known Azure Blob suffix +/// this is almost certainly a config mistake — the two styles then disagree +/// about which name is the account while producing identical request URLs, so +/// nothing fails until a mint asks for a delegation SAS on whatever the path +/// spelled. It is only a warning: the list is advisory, cannot cover a +/// proxy or a private endpoint, and choosing the addressing for a host remains +/// the user's call. /// /// The host is otherwise trusted verbatim (see the [crate-level docs] for the /// host model): an honest, arbitrary host is the caller's responsibility. The @@ -268,14 +249,16 @@ pub fn account_and_container( (account, next_segment().ok_or(AzureUrlError::NoContainer)?) } Addressing::PathStyle => { - if let Some(label) = host - .account_label() - .filter(|label| validate_account(label).is_ok()) - { - return Err(AzureUrlError::PathStyleOnAccountHost { - host: host.to_string(), - label: label.to_string(), - }); + if host.is_known_azure_blob_endpoint() { + tracing::warn!( + "`path-style = true` is set for `{host}`, which is a real Azure Blob endpoint \ + addressed host-style: its storage account is `{}`, not the first path \ + segment. Requests still come out identical, but anything that needs the \ + account on its own — minting a user-delegation SAS, for one — will use the \ + path segment instead. Remove `path-style = true` from \ + `[azure-options.\"{host}\"]` unless you meant it", + host.account_label().unwrap_or(""), + ); } ( next_segment().ok_or(AzureUrlError::NoAccount)?, @@ -443,6 +426,33 @@ impl AzureHost { url::Host::Ipv4(_) | url::Host::Ipv6(_) => None, } } + + /// Whether this host sits under a suffix Microsoft operates, where the account + /// is by definition the first label. + /// + /// Advisory only, and deliberately not a security boundary: a grant is written + /// per host, so no behaviour hangs off this answer. It exists to warn about a + /// `path-style = true` that cannot be what the user meant. A proxy or private + /// endpoint in front of real Azure answers `false`, which is why a `false` here + /// is never treated as evidence of anything. + pub fn is_known_azure_blob_endpoint(&self) -> bool { + const SUFFIXES: &[&str] = &[ + "blob.core.windows.net", // global + "blob.core.usgovcloudapi.net", // US Government + "blob.core.chinacloudapi.cn", // China, operated by 21Vianet + ]; + + let url::Host::Domain(domain) = &self.host else { + return false; + }; + SUFFIXES.iter().any(|suffix| { + // The dot has to be part of the match, or `notblob.core.windows.net` + // would pass as `blob.core.windows.net`. + domain + .strip_suffix(suffix) + .is_some_and(|prefix| prefix.ends_with('.')) + }) + } } impl std::fmt::Display for AzureHost { @@ -1159,35 +1169,66 @@ mod tests { } } - /// A `path-style = true` entry on a host that already carries the account is a - /// config mistake with no visible symptom — request URLs come out identical - /// under both styles — right up to a mint asking for a delegation SAS on the - /// account name the *path* happened to spell. + /// A `path-style = true` entry on a real Azure host is a config mistake with no + /// visible symptom — request URLs come out identical under both styles — right + /// up to a mint asking for a delegation SAS on the account name the *path* + /// happened to spell. Which addressing a host uses is still the user's call, so + /// this warns and proceeds. #[test] - fn path_style_rejects_a_host_that_carries_the_account() { - let err = account_and_container( + #[tracing_test::traced_test] + fn path_style_on_a_real_azure_host_warns_and_proceeds() { + let coords = account_and_container( &channel("az://acct.blob.core.windows.net/general/mychannel"), Addressing::PathStyle, ) - .expect_err("path-style must not read an account out of a host-style URL"); + .expect("addressing is the user's call, so this is a warning and not an error"); + assert_eq!(coords.account, "general"); - assert!( - matches!(err, AzureUrlError::PathStyleOnAccountHost { .. }), - "{err}" - ); - let message = err.to_string(); - assert!(message.contains("acct.blob.core.windows.net"), "{message}"); - assert!(message.contains("path-style = true"), "{message}"); + assert!(logs_contain("path-style = true")); + assert!(logs_contain("acct.blob.core.windows.net")); - // A first label that is not a legal account name carries no account, so - // path-style is the only reading left. - assert!( - account_and_container( - &channel("az://my-emulator.internal/devstoreaccount1/general"), - Addressing::PathStyle - ) - .is_ok() - ); + // A host that is not a known Azure endpoint gets no warning, however much + // its first label looks like an account name. + let coords = account_and_container( + &channel("az://acct.blob.example.com/devstoreaccount1/general"), + Addressing::PathStyle, + ) + .unwrap(); + assert_eq!(coords.account, "devstoreaccount1"); + assert!(!logs_contain("acct.blob.example.com")); + } + + /// The suffix list is advisory, but a sloppy match on it would warn about + /// hosts Microsoft does not operate — and stay silent on ones it does. + #[test] + fn known_azure_endpoints_are_matched_on_a_label_boundary() { + for host in [ + "acct.blob.core.windows.net", + "acct.blob.core.usgovcloudapi.net", + "acct.blob.core.chinacloudapi.cn", + ] { + assert!( + AzureHost::parse(host) + .unwrap() + .is_known_azure_blob_endpoint(), + "{host}" + ); + } + + for host in [ + "notblob.core.windows.net", // no label boundary + "blob.core.windows.net", // the suffix alone carries no account + "acct.blob.core.windows.net.evil.test", // suffix in the middle + "127.0.0.1:10000", + "azurite", + ] { + assert!( + !AzureHost::parse(host) + .unwrap() + .is_known_azure_blob_endpoint(), + "{host}" + ); + } } /// Host-style must keep rejecting hosts it cannot derive an account from — and From 1efe9c514280e8c50f5e1838b38b9c978d7c5f1d Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:57:08 +1000 Subject: [PATCH 91/98] fix(azure): keep the SAS signature out of storage error messages --- Cargo.lock | 3 + crates/rattler_azure/Cargo.toml | 4 +- crates/rattler_index/Cargo.toml | 1 + crates/rattler_index/src/lib.rs | 21 +++- crates/rattler_redaction/src/lib.rs | 107 ++++++++++++++++++ crates/rattler_upload/src/upload/azure.rs | 18 +-- crates/rattler_upload/src/upload/mod.rs | 127 +++++++++++++++++++++- crates/rattler_upload/src/upload/s3.rs | 10 +- 8 files changed, 271 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d014874dc..90df8fad74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5062,6 +5062,8 @@ dependencies = [ "thiserror 2.0.19", "tokio", "toml", + "tracing", + "tracing-test", "url", "which", ] @@ -5247,6 +5249,7 @@ dependencies = [ "rattler_digest", "rattler_networking", "rattler_package_streaming", + "rattler_redaction", "rattler_repodata_gateway", "rattler_s3", "reqwest", diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index b76b9c8493..6d3301318d 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -16,7 +16,7 @@ default = [] # `tokio` to run `az` without blocking the async runtime). clap = ["dep:clap", "dep:jiff", "dep:which", "dep:tokio"] # `azblob_config`: build an opendal `AzblobConfig` from a channel URL. -opendal = ["dep:opendal", "dep:percent-encoding"] +opendal = ["dep:opendal"] # Serde derives for the `azure-options` config table: the per-host endpoint # options and the `AzureHost` used as their map key. The types themselves are # always compiled; only the derives are gated. Pulls in `serde` alone, so config @@ -29,7 +29,7 @@ jiff = { workspace = true, optional = true } opendal = { workspace = true, default-features = false, features = [ "services-azblob", ], optional = true } -percent-encoding = { workspace = true, optional = true } +percent-encoding = { workspace = true } secrecy = { workspace = true } serde = { workspace = true, features = ["derive"], optional = true } thiserror = { workspace = true } diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index fb65e80be8..f7014c62b7 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -57,6 +57,7 @@ rattler_networking = { workspace = true, default-features = false, features = [ ] } rattler_conda_types = { workspace = true, default-features = false } rattler_digest = { workspace = true, default-features = false } +rattler_redaction = { workspace = true, default-features = false } rattler_package_streaming = { workspace = true, default-features = false } hex = { workspace = true } rattler_azure = { workspace = true, optional = true, features = ["clap"] } diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index a01d97d641..be590c08da 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -1580,7 +1580,26 @@ pub async fn index_azure_with_channel_metadata( ) -> anyhow::Result<()> { let azblob_config = rattler_azure::azblob_config(&credentials, &channel, endpoint)?; let builder = azblob_config.into_builder(); - let op = Operator::new(builder)?.layer(RetryLayer::new()).finish(); + // opendal's default retry interceptor logs the error with its `url` context, + // and for a SAS the credential is *in* that URL — once per retry, at warn + // level. Same message, signature masked. + let op = Operator::new(builder)? + .layer( + RetryLayer::new().with_notify(|event: opendal::layers::RetryEvent<'_>| { + tracing::warn!( + target: "opendal::layers::retry", + "will retry {:?} (attempt {}) after {}s because: {}", + event.op, + event.attempt, + event.retry_after.as_secs_f64(), + rattler_redaction::redact_signatures_in_text( + &format!("{:?}", event.err), + rattler_redaction::DEFAULT_REDACTION_STR, + ), + ); + }), + ) + .finish(); index_with_channel_metadata( target_platform, diff --git a/crates/rattler_redaction/src/lib.rs b/crates/rattler_redaction/src/lib.rs index bef24b14fd..b9bf34691e 100644 --- a/crates/rattler_redaction/src/lib.rs +++ b/crates/rattler_redaction/src/lib.rs @@ -1,8 +1,61 @@ +use std::borrow::Cow; + use url::Url; /// A default string to use for redaction. pub const DEFAULT_REDACTION_STR: &str = "********"; +/// Query parameters whose value is the signature of a pre-signed URL, and is +/// therefore the credential itself: `sig` is Azure's SAS signature and +/// `x-amz-signature` is the S3/SigV4 equivalent. Everything else in such a URL +/// (the validity window, the permissions) is inert without them. +const SIGNATURE_PARAMS: &[&str] = &["sig", "x-amz-signature"]; + +/// Mask the signature of a pre-signed URL wherever one appears in `text`. +/// +/// Takes text rather than a [`Url`] because that is the shape the leak has: a +/// storage backend quotes the request URL inside an error message, and for a SAS +/// the credential is *in* that URL, so the message must be scrubbed before it is +/// logged or shown. A `?`/`&` and a `=` are all that is needed to find the value; +/// anything that cannot appear in a query value ends it. +pub fn redact_signatures_in_text<'a>(text: &'a str, redaction: &str) -> Cow<'a, str> { + let mut out = String::new(); + // Also the "nothing was masked" flag: a masked value always starts past 0. + let mut written = 0; + + for (separator, _) in text.char_indices().filter(|(_, c)| *c == '?' || *c == '&') { + let pair = &text[separator + 1..]; + let Some(equals) = pair.find('=') else { + continue; + }; + if !SIGNATURE_PARAMS + .iter() + .any(|param| pair[..equals].eq_ignore_ascii_case(param)) + { + continue; + } + + let value = separator + 1 + equals + 1; + let end = value + + text[value..] + .find(|c: char| c == '&' || c.is_whitespace() || "\"',)]}".contains(c)) + .unwrap_or(text.len() - value); + if end == value || value < written { + continue; + } + + out.push_str(&text[written..value]); + out.push_str(redaction); + written = end; + } + + if written == 0 { + return Cow::Borrowed(text); + } + out.push_str(&text[written..]); + Cow::Owned(out) +} + /// Anaconda channels are not always publicly available. This function checks if a URL contains a /// secret by identifying whether it contains certain patterns. If it does, the function returns a /// modified URL where any secret has been masked. @@ -28,6 +81,14 @@ pub fn redact_known_secrets_from_url(url: &Url, redaction: &str) -> Option url.set_password(Some(redaction)).ok()?; } + // A pre-signed URL carries its credential in the query, so a URL that reached + // here from an error or a log line has to lose it. + if let Some(query) = url.query() + && let Cow::Owned(masked) = redact_signatures_in_text(&format!("?{query}"), redaction) + { + url.set_query(Some(&masked[1..])); + } + let mut segments = url.path_segments()?; match (segments.next(), segments.next()) { (Some("t"), Some(_)) => { @@ -95,6 +156,52 @@ mod test { use super::*; use std::str::FromStr; + /// The signature is the credential; the rest of a SAS is inert without it and + /// is worth keeping, because the account, container and expiry are what make + /// the error message useful. + #[test] + fn test_redact_signatures_in_text() { + let message = "unexpected status code 403, url=https://acct.blob.core.windows.net/c/p?sv=2025-01-05&se=2026-08-05T00%3A00Z&sig=aBcD%2Fefg%3D, op=stat"; + assert_eq!( + redact_signatures_in_text(message, DEFAULT_REDACTION_STR), + format!( + "unexpected status code 403, url=https://acct.blob.core.windows.net/c/p?sv=2025-01-05&se=2026-08-05T00%3A00Z&sig={DEFAULT_REDACTION_STR}, op=stat" + ) + ); + + // Presigned S3, and a signature that runs to the end of the text. + assert_eq!( + redact_signatures_in_text( + "https://b.s3.amazonaws.com/k?X-Amz-Credential=AK&X-Amz-Signature=deadbeef", + "X" + ), + "https://b.s3.amazonaws.com/k?X-Amz-Credential=AK&X-Amz-Signature=X" + ); + + // Text with nothing to mask is borrowed, not rebuilt. + assert!(matches!( + redact_signatures_in_text("https://prefix.dev/conda-forge?a=b", "X"), + Cow::Borrowed(_) + )); + + // A query param that merely ends in `sig` is not the signature. + assert_eq!( + redact_signatures_in_text("https://h/p?design=keep&sig=drop", "X"), + "https://h/p?design=keep&sig=X" + ); + + // And the same through the `Url` entry point every existing caller uses. + assert_eq!( + Url::from_str("https://acct.blob.core.windows.net/c/p?sv=2025-01-05&sig=secret") + .unwrap() + .redact() + .to_string(), + format!( + "https://acct.blob.core.windows.net/c/p?sv=2025-01-05&sig={DEFAULT_REDACTION_STR}" + ) + ); + } + #[test] fn test_remove_known_secrets_from_url() { assert_eq!( diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 8893b836f4..32fbe83073 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -6,11 +6,13 @@ use std::{ use futures::{StreamExt, TryStreamExt}; use miette::IntoDiagnostic; -use opendal::{Configurator, ErrorKind, Operator}; +use opendal::{Configurator, ErrorKind}; use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpoint}; use crate::upload::{ - object_store::{BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store}, + object_store::{ + BlobStore, BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store, + }, opt::ForceOverwrite, package::ExtractedPackage, }; @@ -44,7 +46,7 @@ pub async fn upload_package_to_azure( rattler_azure::azblob_config(&credentials, &channel, endpoint).into_diagnostic()?; let builder = config.into_builder(); - let op = Operator::new(builder).into_diagnostic()?.finish(); + let op = BlobStore::new(builder).into_diagnostic()?; // Upload multiple packages concurrently. Each package is written to its own // key, so the individual uploads are independent. The first failure aborts @@ -120,7 +122,7 @@ fn summarize(outcomes: &[(PathBuf, PackageOutcome)], total: usize) -> String { /// Uploads a single package file to the Azure Blob container via the given operator. async fn upload_single_package( - op: &Operator, + op: &BlobStore, channel: &AzureChannelUrl, package_file: &Path, force: ForceOverwrite, @@ -169,16 +171,16 @@ async fn upload_single_package( mod test { use std::path::PathBuf; - use opendal::{Operator, services::Memory}; + use opendal::services::Memory; use rattler_azure::AzureChannelUrl; - use super::{PackageOutcome, summarize, upload_single_package}; + use super::{BlobStore, PackageOutcome, summarize, upload_single_package}; use crate::upload::opt::ForceOverwrite; use crate::upload::package::ExtractedPackage; use crate::upload::test_utils::test_package_path; - fn memory_operator() -> Operator { - Operator::new(Memory::default()).unwrap().finish() + fn memory_operator() -> BlobStore { + BlobStore::new(Memory::default()).unwrap() } fn test_channel() -> AzureChannelUrl { diff --git a/crates/rattler_upload/src/upload/mod.rs b/crates/rattler_upload/src/upload/mod.rs index fe051f1ea7..29b1b9ab76 100644 --- a/crates/rattler_upload/src/upload/mod.rs +++ b/crates/rattler_upload/src/upload/mod.rs @@ -64,6 +64,101 @@ pub(crate) mod object_store { use crate::upload::{opt::ForceOverwrite, package::ExtractedPackage}; + /// An object store whose errors cannot carry a credential. + /// + /// opendal stamps the request URL into the context of every HTTP error it + /// builds, and prints that context from both `Display` and `Debug`. For Azure + /// the SAS *is* in the URL, so any opendal error that escapes unmasked is a + /// leaked credential — into a log, a `miette` report, or a CI transcript. + /// + /// The inner [`Operator`] is private and the only error type out is + /// [`BlobStoreError`], which is built by masking. Reaching for an opendal + /// method this does not have means adding it here, where leaving the masking + /// out is a visible omission rather than a silent leak. + #[derive(Clone)] + pub(crate) struct BlobStore(Operator); + + /// An opendal error with any pre-signed signature masked out of its text. + /// + /// Carries the [`ErrorKind`] separately because callers branch on it — a + /// `NotFound` from the overwrite guard, a `ConditionNotMatch` from a write that + /// lost a race — and must not have to read the message to do so. + #[derive(Debug, thiserror::Error)] + #[error("{message}")] + pub(crate) struct BlobStoreError { + kind: ErrorKind, + message: String, + } + + impl BlobStoreError { + fn new(err: opendal::Error) -> Self { + Self { + kind: err.kind(), + // `Debug` rather than `Display`: it is the spelling that keeps the + // source chain and the operation, and it is also the one the leak + // was found in. + message: rattler_redaction::redact_signatures_in_text( + &format!("{err:?}"), + rattler_redaction::DEFAULT_REDACTION_STR, + ) + .into_owned(), + } + } + + pub(crate) fn kind(&self) -> ErrorKind { + self.kind + } + } + + impl BlobStore { + pub(crate) fn new(builder: impl opendal::Builder) -> Result { + Ok(Self( + Operator::new(builder) + .map_err(BlobStoreError::new)? + .finish(), + )) + } + + /// Metadata for one blob, used by the callers' overwrite guards. + pub(crate) async fn stat(&self, path: &str) -> Result { + self.0.stat(path).await.map_err(BlobStoreError::new) + } + + async fn writer( + &self, + path: &str, + options: WriteOptions, + ) -> Result { + self.0 + .writer_options(path, options) + .await + .map(BlobWriter) + .map_err(BlobStoreError::new) + } + } + + /// A writer that masks its errors, for the same reason [`BlobStore`] does: a + /// failed block upload reports the URL it was sent to. + struct BlobWriter(opendal::Writer); + + impl BlobWriter { + async fn write(&mut self, chunk: tokio_util::bytes::Bytes) -> Result<(), BlobStoreError> { + self.0.write(chunk).await.map_err(BlobStoreError::new) + } + + async fn close(&mut self) -> Result<(), BlobStoreError> { + self.0 + .close() + .await + .map(|_| ()) + .map_err(BlobStoreError::new) + } + + async fn abort(&mut self) -> Result<(), BlobStoreError> { + self.0.abort().await.map_err(BlobStoreError::new) + } + } + /// Size of a single chunk handed to the writer. S3 rejects every multipart /// part but the last below 5 MiB, and Azure Blob bills per block, so both /// backends prefer few large chunks. @@ -144,7 +239,7 @@ pub(crate) mod object_store { /// the backend, which is free to drop it — the caller is responsible for any /// guard it needs on top (see `azure::upload_single_package`). pub(crate) async fn stream_package_to_object_store( - op: &Operator, + store: &BlobStore, target: &BlobUploadTarget, package_file: &Path, destination: &str, @@ -175,7 +270,7 @@ pub(crate) mod object_store { let already_exists = || miette::miette!("Package {destination} already exists. Use --force to overwrite."); - let mut writer = match op.writer_options(target.key(), options).await { + let mut writer = match store.writer(target.key(), options).await { Ok(writer) => writer, Err(e) if e.kind() == ErrorKind::ConditionNotMatch => return Err(already_exists()), Err(e) => return Err(e).into_diagnostic(), @@ -204,7 +299,7 @@ pub(crate) mod object_store { /// Feeds exactly `size` bytes of `reader` to `writer`. opendal buffers them /// into correctly sized parts/blocks and uploads `PART_CONCURRENCY` at a time. async fn stream_chunks( - writer: &mut opendal::Writer, + writer: &mut BlobWriter, reader: &mut (impl AsyncReadExt + Unpin), size: u64, ) -> miette::Result<()> { @@ -226,7 +321,7 @@ pub(crate) mod object_store { /// Uncommitted parts are billed until they are discarded. S3 discards them /// here; azblob's abort is a no-op, so Azure only collects its uncommitted /// blocks after a week without further writes to the blob. - async fn discard_partial_upload(writer: &mut opendal::Writer, destination: &str) { + async fn discard_partial_upload(writer: &mut BlobWriter, destination: &str) { if let Err(e) = writer.abort().await { tracing::warn!("Failed to discard the partial upload of {destination}: {e}"); } @@ -234,8 +329,9 @@ pub(crate) mod object_store { #[cfg(test)] mod test { - use super::hash_file; + use super::{BlobStoreError, hash_file}; use crate::upload::test_utils::test_package_path; + use opendal::ErrorKind; use rattler_digest::{Md5, Sha256, compute_file_digest}; /// The size the upload streams and the hashes it records must come @@ -257,6 +353,27 @@ pub(crate) mod object_store { "recorded md5 must match the file's" ); } + + /// The whole reason `BlobStore` hides its `Operator`: opendal puts the + /// request URL in the error context, and for Azure the SAS is in that URL. + #[test] + fn blob_store_errors_do_not_carry_a_signature() { + let err = BlobStoreError::new( + opendal::Error::new(ErrorKind::NotFound, "blob not found").with_context( + "url", + "https://acct.blob.core.windows.net/c/p?sv=2025-01-05&sig=s3cr3t", + ), + ); + + let message = err.to_string(); + assert!(!message.contains("s3cr3t"), "{message}"); + // Everything that makes the error useful survives. + assert_eq!(err.kind(), ErrorKind::NotFound); + assert!( + message.contains("acct.blob.core.windows.net/c/p"), + "{message}" + ); + } } } diff --git a/crates/rattler_upload/src/upload/s3.rs b/crates/rattler_upload/src/upload/s3.rs index 7c0532004f..83933647de 100644 --- a/crates/rattler_upload/src/upload/s3.rs +++ b/crates/rattler_upload/src/upload/s3.rs @@ -2,12 +2,14 @@ use std::path::{Path, PathBuf}; use futures::StreamExt; use miette::IntoDiagnostic; -use opendal::{Configurator, Operator, services::S3Config}; +use opendal::{Configurator, services::S3Config}; use rattler_s3::ResolvedS3Credentials; use url::Url; use crate::upload::{ - object_store::{BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store}, + object_store::{ + BlobStore, BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store, + }, opt::ForceOverwrite, package::ExtractedPackage, }; @@ -41,7 +43,7 @@ pub async fn upload_package_to_s3( credentials.addressing_style == rattler_s3::S3AddressingStyle::VirtualHost; let builder = s3_config.into_builder(); - let op = Operator::new(builder).into_diagnostic()?.finish(); + let op = BlobStore::new(builder).into_diagnostic()?; // Upload multiple packages concurrently. Each individual package upload also // streams its chunks concurrently (see `upload_single_package`). @@ -64,7 +66,7 @@ pub async fn upload_package_to_s3( /// Uploads a single package file to the S3 bucket via the given operator. async fn upload_single_package( - op: &Operator, + op: &BlobStore, channel: &Url, bucket: &str, package_file: &Path, From e32d10081d8efe28163e23684d638eb6463013e4 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:21:04 +1000 Subject: [PATCH 92/98] test(azure): give the emulator tests teeth and stop them vanishing --- .github/workflows/e2e-azure-tests.yml | 6 ++ crates/rattler_index/Cargo.toml | 6 ++ crates/rattler_index/tests/azure_azurite.rs | 76 +++++++++++++++---- crates/rattler_networking/Cargo.toml | 10 +++ .../tests/azure_azurite_fetch.rs | 37 +++------ .../tests/azure_real_fetch.rs | 1 - crates/rattler_upload/src/upload/azure.rs | 2 +- 7 files changed, 98 insertions(+), 40 deletions(-) diff --git a/.github/workflows/e2e-azure-tests.yml b/.github/workflows/e2e-azure-tests.yml index 8b2b0950ac..06e21c8869 100644 --- a/.github/workflows/e2e-azure-tests.yml +++ b/.github/workflows/e2e-azure-tests.yml @@ -4,9 +4,15 @@ on: pull_request: paths: - crates/rattler_azure/** + - crates/rattler_config/** - crates/rattler_index/** - crates/rattler_networking/** - crates/rattler_upload/** + # An opendal bump touches only the root manifest, and these tests are the + # canary for the opendal bug the upload path works around. + - Cargo.toml + - Cargo.lock + - test-data/** - pixi.toml - pixi.lock - .github/workflows/e2e-azure-tests.yml diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index f7014c62b7..3dcc9ecd59 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -79,6 +79,12 @@ url = { workspace = true } zstd = { workspace = true } retry-policies = { workspace = true } +# Declared rather than auto-discovered, so a feature name that stops matching is a +# cargo error instead of a test binary that silently holds no tests. +[[test]] +name = "azure_azurite" +required-features = ["azure"] + [dev-dependencies] rattler_repodata_gateway = { path = "../rattler_repodata_gateway", default-features = false, features = [ "sparse", diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs index 1538d7b273..24b740dfc8 100644 --- a/crates/rattler_index/tests/azure_azurite.rs +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -31,7 +31,6 @@ //! version with `AuthorizationFailure` rather than `InvalidHeaderValue`, i.e. it //! validates the signature instead of rejecting the version. Add the flag only if //! an older emulator rejects the version outright. -#![cfg(feature = "azure")] use std::{collections::HashMap, path::PathBuf}; @@ -130,25 +129,66 @@ fn verify_operator(prefix: &str) -> Operator { .finish() } -/// Create the channel's container, which Azurite never does implicitly. +/// Create the channel's container and clear this test's prefix inside it. /// -/// This goes through `AzureMiddleware` because it is the one signer already -/// reachable from here: opendal exposes no container-creation operation, and -/// hand-rolling shared-key signing in a test fixture would be more code than the -/// tests it supports. -async fn ensure_container() { +/// The clearing is what makes the assertions below mean anything: the emulator is +/// long-lived, and a blob left by an earlier run answers the same read a correct +/// run would produce — so a derivation that writes to the wrong prefix passes by +/// finding the *previous* run's output. Removal goes through the hand-written +/// operator, not the one under test, for the same reason. +async fn ensure_empty_prefix(prefix: &str) { + ensure_container().await; + verify_operator(prefix) + .delete_with("") + .recursive(true) + .await + .expect("clearing the test prefix failed"); +} + +/// A signing client for the requests opendal cannot make. +/// +/// `AzureMiddleware` is the one signer already reachable from here, and there is no +/// opendal operation for creating a container or reading a block list — hand-rolling +/// shared-key signing in a test fixture would be more code than the tests it +/// supports. +fn azure_client() -> reqwest_middleware::ClientWithMiddleware { let options = HashMap::from([( AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), azurite_options().fetch(), )]); - let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) + reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) .with(rattler_networking::AzureMiddleware::new( reqwest::Client::new(), options, )) - .build(); + .build() +} - let created = client +/// How many blocks a blob was committed from. +/// +/// The one witness that a write took Put Block List and not the single-shot Put +/// Blob path — a single-shot blob has no committed blocks at all, whatever its +/// size. +async fn committed_block_count(prefix: &str, blob: &str) -> usize { + let body = azure_client() + .get(format!( + "az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}/{prefix}/{blob}\ + ?comp=blocklist&blocklisttype=committed" + )) + .send() + .await + .expect("get block list request failed") + .error_for_status() + .expect("get block list was refused") + .text() + .await + .expect("get block list body"); + body.matches("").count() +} + +/// Create the channel's container, which Azurite never does implicitly. +async fn ensure_container() { + let created = azure_client() .put(format!( "az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}?restype=container" )) @@ -206,7 +246,7 @@ fn index_config(channel: AzureChannelUrl) -> IndexAzureConfig { async fn azurite_index_round_trip_through_a_path_style_entry() { with_azurite_credentials(async { const PREFIX: &str = "round-trip"; - ensure_container().await; + ensure_empty_prefix(PREFIX).await; let seeded = verify_operator(PREFIX); seeded @@ -263,7 +303,7 @@ async fn azurite_index_round_trip_through_a_path_style_entry() { async fn azurite_multi_block_write_keeps_cache_control() { with_azurite_credentials(async { const PREFIX: &str = "multi-block-cache-control"; - ensure_container().await; + ensure_empty_prefix(PREFIX).await; let op = production_operator(&channel(PREFIX)); // Two chunks' worth, so `write` is called more than once and the writer @@ -294,6 +334,16 @@ async fn azurite_multi_block_write_keeps_cache_control() { Some(CACHE_CONTROL_REPODATA), "Put Block List should carry x-ms-blob-cache-control through its commit" ); + + // Without this the test still passes if opendal ever takes the single-shot + // path for the whole 5 MiB, and then it asserts nothing about the commit + // path it is named for. 5 MiB in 2 MiB chunks is three blocks. + assert_eq!( + committed_block_count(PREFIX, "noarch/repodata.json").await, + 3, + "a 5 MiB write in 2 MiB chunks must commit three blocks through Put Block List; zero \ + blocks means the single-shot Put Blob path was taken instead" + ); }) .await; } @@ -321,7 +371,7 @@ async fn azurite_multi_block_write_keeps_cache_control() { async fn azurite_if_not_exists_is_dropped_on_the_multi_block_path() { with_azurite_credentials(async { const PREFIX: &str = "overwrite-guard"; - ensure_container().await; + ensure_empty_prefix(PREFIX).await; let op = production_operator(&channel(PREFIX)); // Baseline: below the chunk size, the guard works and opendal reports the diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 3d1d4e8656..7c16e483e8 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -100,6 +100,16 @@ dbus-secret-service-keyring-store = { workspace = true, optional = true, feature windows-native-keyring-store = { workspace = true, optional = true } regex = { workspace = true, optional = true } +# Declared rather than auto-discovered, so a feature name that stops matching is a +# cargo error instead of a test binary that silently holds no tests. +[[test]] +name = "azure_azurite_fetch" +required-features = ["azure"] + +[[test]] +name = "azure_real_fetch" +required-features = ["azure"] + [dev-dependencies] anyhow = { workspace = true } insta = { workspace = true, features = ["json"] } diff --git a/crates/rattler_networking/tests/azure_azurite_fetch.rs b/crates/rattler_networking/tests/azure_azurite_fetch.rs index 3405bde2d5..d0bee7aa22 100644 --- a/crates/rattler_networking/tests/azure_azurite_fetch.rs +++ b/crates/rattler_networking/tests/azure_azurite_fetch.rs @@ -1,11 +1,5 @@ //! Live fetch-path integration tests against a local Azurite emulator. //! -//! These are the read-side half of the answer to "has any of this been tested -//! against a real Azure-compatible backend?". Everything else about the grant -//! model is unit-tested with mocks; only Azurite can show that a granted entry -//! produces a signature a real Azure Blob implementation accepts, and that an -//! ungranted one does not. -//! //! Everything is driven through a single `azure-options` entry, which is the //! point of the exercise — there is no out-of-band account or endpoint //! configuration on the fetch path: @@ -25,13 +19,6 @@ //! cargo nextest run -p rattler_networking --features azure --test azure_azurite_fetch \ //! --run-ignored all //! ``` -//! -//! No `--skipApiVersionCheck` needed: the `x-ms-version` this middleware pins is -//! older than what current Azurite accepts. Verified on 3.36.0, which answers -//! that version with `AuthorizationFailure` rather than `InvalidHeaderValue`, -//! i.e. it validates the signature instead of rejecting the version. Add the flag -//! only if an older emulator rejects the version outright. -#![cfg(feature = "azure")] use std::{ collections::HashMap, @@ -240,7 +227,7 @@ async fn azurite_granted_entry_fetches_repodata() { /// refuses it. This is the core claim of the anonymous-by-default model. /// /// The primary assertion is on the outgoing request, not the status: no -/// `Authorization` header and no SAS `sig` reaches the wire. The 403 stays as a +/// `Authorization` header and no SAS `sig` reaches the wire. The status is a /// secondary check that the container really is private, but on its own it would /// also pass while the account key was being sent with a signature Azurite /// rejected. @@ -255,7 +242,7 @@ async fn azurite_ungranted_entry_is_refused_by_a_private_container() { async { // Seed with a grant, then read without one. The credential is present in // the environment throughout, so a success below would mean the grant - // check leaked it — not that the test was misconfigured. + // check leaked it seed(&client(Auth::DefaultChain)).await; let url = format!("{}/noarch/repodata.json", channel_url()); @@ -279,16 +266,16 @@ async fn azurite_ungranted_entry_is_refused_by_a_private_container() { signature: {sent_url}" ); - let status = resp.status(); - assert!( - // Both statuses are correct answers to an unsigned read of a private - // container: Azurite says 403, real Azure says 404 so that a missing - // grant is indistinguishable from a missing blob. Accepting either - // keeps the assertion about "refused", which is the actual claim. - status == reqwest::StatusCode::FORBIDDEN - || status == reqwest::StatusCode::NOT_FOUND, - "an ungranted read of a private container should be refused, got {status} for \ - {url}" + // 403 exactly, which is Azurite-specific: real Azure answers 404 to an + // unsigned read of a private container so that a missing grant is + // indistinguishable from a missing blob. This test only ever runs against + // the emulator, and accepting 404 too would also admit a request sent to + // the wrong URL — a wrong account segment or a dropped one both 404 here. + assert_eq!( + resp.status(), + reqwest::StatusCode::FORBIDDEN, + "Azurite refuses an unsigned read of a private container with 403; anything else \ + means the request did not reach the blob this URL names: {url}" ); }, ) diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs index 6e9562708a..1dc90ac47b 100644 --- a/crates/rattler_networking/tests/azure_real_fetch.rs +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -16,7 +16,6 @@ //! //! Set `AZURE_TEST_HOST` to target a sovereign cloud or emulator directly //! (overrides the default `{account}.blob.core.windows.net` host). -#![cfg(feature = "azure")] use std::collections::HashMap; diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs index 32fbe83073..d86af92f0e 100644 --- a/crates/rattler_upload/src/upload/azure.rs +++ b/crates/rattler_upload/src/upload/azure.rs @@ -197,7 +197,7 @@ mod test { ) } - /// C2: without `--force`, uploading over an existing blob must error rather + /// without `--force`, uploading over an existing blob must error rather /// than silently overwrite it. The memory backend honours `if_not_exists` on /// every path and the fixture is a single block, so this covers the small-blob /// path only — for the multi-block behaviour see From aee4df479c703d11b9e61b2a08acab319433222c Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:30:50 +1000 Subject: [PATCH 93/98] fix(azure): say which host and grant a failed credential resolution came from --- .../src/azure_middleware.rs | 45 ++++++++++++++++--- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 109d5d0adb..ec02a29c85 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -168,8 +168,9 @@ impl AzureMiddleware { /// public-channel read pay that timeout — and it would pull an ambient /// credential into memory for a host the user never granted. /// - /// Under [`Auth::DefaultChain`] any signing failure is propagated. reqsign - /// collapses "no credential" and "broken credential" into the same + /// Under [`Auth::DefaultChain`] any signing failure is propagated, carrying the + /// host, the grant that required signing and the remedies. reqsign collapses "no + /// credential" and "broken credential" into the same /// [`reqsign_core::ErrorKind::CredentialInvalid`], and since the user asked for /// signing there is no case left where going anonymous is the right answer. async fn sign(&self, req: &mut Request, auth: Auth) -> MiddlewareResult<()> { @@ -209,10 +210,27 @@ impl AzureMiddleware { })?; let (mut parts, ()) = http_req.into_parts(); - self.signer - .sign(&mut parts, None) - .await - .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::anyhow!(e)))?; + // reqsign says only "failed to load signing credential": its chain walks + // past a provider that errors exactly as it walks past one that finds + // nothing, so an expired `az login` and an empty environment arrive here + // indistinguishable, after however long the chain took to give up. The host + // and the grant that asked for signing are both in scope here and nowhere + // further up, so this is where they get attached. + self.signer.sign(&mut parts, None).await.map_err(|e| { + let authority = req.url().authority(); + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "could not resolve an Azure credential for `{authority}`, which \ + `[azure-options.\"{authority}\"] auth = true` requires: {e}\n\ + \n\ + Try one of:\n\ + \x20 - `az login`\n\ + \x20 - `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_ACCOUNT_KEY` in the \ + environment\n\ + \x20 - remove `auth = true` to fetch this host anonymously\n\ + \n\ + Debug logging lists the credential providers that were tried." + )) + })?; *req.headers_mut() = parts.headers; let signed_url = Url::parse(&parts.uri.to_string()).map_err(|e| { @@ -516,6 +534,21 @@ mod tests { req.headers().get(http::header::AUTHORIZATION).is_none(), "a failed signing attempt must not leave a partial Authorization header" ); + + // reqsign's own message names neither the host nor a remedy, and the chain + // hides which provider failed. Everything actionable has to come from here. + let message = result.unwrap_err().to_string(); + for expected in [ + "acct.blob.core.windows.net", + "auth = true", + "az login", + "AZURE_STORAGE_ACCOUNT_KEY", + ] { + assert!( + message.contains(expected), + "the failure must name `{expected}`, got: {message}" + ); + } } /// A URL that already carries a SAS token must not be re-signed even where the From dda4cc7c6e56c8b32ceb552c4a9c92f8a8bbd289 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:59:53 +1000 Subject: [PATCH 94/98] refactor: tighten azure names into types and dedupe the 404 hint --- crates/rattler-bin/src/commands/client.rs | 2 +- crates/rattler_azure/src/lib.rs | 269 +++++++++++++----- crates/rattler_config/src/config/azure.rs | 26 +- .../src/azure_middleware.rs | 92 +++++- crates/rattler_upload/src/upload/mod.rs | 5 + py-rattler/Cargo.lock | 3 + py-rattler/src/networking/client.rs | 5 +- 7 files changed, 297 insertions(+), 105 deletions(-) diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index 01bc98d7db..fc322373fe 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -67,7 +67,7 @@ pub fn create_client_with_middleware( #[cfg(feature = "azure")] let client = client.with(rattler_networking::AzureMiddleware::new( download_client, - HashMap::new(), + [], )); Ok(client.build()) diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index c3fa84e6fb..b9c03bca78 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -8,8 +8,12 @@ //! What a host is *granted* — credentials, wire scheme, addressing style — is //! declared per host in [`options`] and never inferred from the host name, and //! the default grant is [`Auth::Anonymous`], so naming a host in a URL by itself -//! sends nothing to it. Nothing here transmits a credential either: signing lives -//! in `rattler_networking`, and [`account_and_container`] only reads a URL. +//! sends nothing to it. Nothing here signs or sends a request either — that lives +//! in `rattler_networking` — but two functions do handle a credential: +//! `azblob_config` embeds the account key or SAS it is handed into the config it +//! returns, and `mint_user_delegation_sas` spends the user's `az login` session to +//! obtain one. Deriving coordinates from a URL ([`account_and_container`]) +//! touches no credential at all. //! //! Userinfo (`user:pass@host`) is rejected wherever a host is parsed, because //! `az://real.host@evil.example/…` reads as the real host while addressing the @@ -189,17 +193,92 @@ pub enum AzureUrlError { InvalidScheme(String), } +/// A storage account name that has passed Azure's naming rules: 3-24 characters +/// of lowercase letters and digits. +/// +/// Those rules are the only thing that keeps option-shaped text (`--as-user`, +/// `-o`) out of the `az` argv in [`mint_user_delegation_sas`], so the mint takes +/// this type: the guarantee is then carried by what the function accepts rather +/// than by every call site remembering to derive its name through a validating +/// path. The inner `String` is private and [`Self::new`] is the only way to one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountName(String); + +impl AccountName { + /// Check a name against Azure's storage account naming rules. + pub fn new(name: &str) -> Result { + let valid = (3..=24).contains(&name.len()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()); + valid + .then(|| Self(name.to_string())) + .ok_or_else(|| AzureUrlError::InvalidAccountName(name.to_string())) + } + + /// The validated name. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for AccountName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// A blob container name that has passed Azure's naming rules: 3-63 characters of +/// lowercase letters, digits and hyphens, with no leading or trailing hyphen and +/// no consecutive hyphens. +/// +/// Exists for the same reason as [`AccountName`], and is what the container half +/// of the `az` argv is spelled as. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContainerName(String); + +impl ContainerName { + /// Check a name against Azure's blob container naming rules. + pub fn new(name: &str) -> Result { + let valid = (3..=63).contains(&name.len()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + && !name.starts_with('-') + && !name.ends_with('-') + && !name.contains("--"); + valid + .then(|| Self(name.to_string())) + .ok_or_else(|| AzureUrlError::InvalidContainerName(name.to_string())) + } + + /// The validated name. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for ContainerName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + /// The storage account and container an Azure Blob channel URL resolves to. +/// +/// The fields are public because their *types* are the invariant: a +/// `AzureCoordinates` cannot be assembled from unvalidated text, whoever builds +/// it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct AzureCoordinates { /// The storage account name — the first host label under /// [`Addressing::HostStyle`], the first path segment under /// [`Addressing::PathStyle`]. - pub account: String, + pub account: AccountName, /// The blob container name — the first path segment under /// [`Addressing::HostStyle`], the second under [`Addressing::PathStyle`]. - pub container: String, + pub container: ContainerName, } /// Derive the storage account name and container from an Azure Blob channel URL. @@ -267,43 +346,12 @@ pub fn account_and_container( } }; - validate_account(account)?; - validate_container(container)?; - Ok(AzureCoordinates { - account: account.to_string(), - container: container.to_string(), + account: AccountName::new(account)?, + container: ContainerName::new(container)?, }) } -/// Check a derived name against Azure's storage account naming rules. -fn validate_account(name: &str) -> Result<(), AzureUrlError> { - let valid = (3..=24).contains(&name.len()) - && name - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()); - valid - .then_some(()) - .ok_or_else(|| AzureUrlError::InvalidAccountName(name.to_string())) -} - -/// Check a derived name against Azure's blob container naming rules. -/// -/// 3-63 characters of lowercase letters, digits and hyphens, with no leading or -/// trailing hyphen and no consecutive hyphens. -fn validate_container(name: &str) -> Result<(), AzureUrlError> { - let valid = (3..=63).contains(&name.len()) - && name - .chars() - .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') - && !name.starts_with('-') - && !name.ends_with('-') - && !name.contains("--"); - valid - .then_some(()) - .ok_or_else(|| AzureUrlError::InvalidContainerName(name.to_string())) -} - /// A normalized Azure Blob endpoint authority: a host, and its port when one is /// written. #[derive(Clone, PartialEq, Eq, Hash)] @@ -340,10 +388,35 @@ impl AzureHost { let normalized = Self::parse_as(authority, "https")?; let verbatim = Self::parse_as(authority, "az")?; + // `url` reads a bare trailing colon as "no port at all", so `host:` would + // otherwise be accepted as `host` — a different endpoint from the one whose + // port the user was in the middle of writing. Port 0 it keeps, and `wire()` + // then hands out `https://host:0/…`, which no connection can be made to. + let port_reason = match (Self::written_port(authority), verbatim.port()) { + (Some(""), _) => Some("its port is empty"), + (_, Some(0)) => Some("port 0 cannot be connected to"), + _ => None, + }; + if let Some(reason) = port_reason { + return Err(AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: reason.to_string(), + }); + } + let host = normalized.host().ok_or(AzureUrlError::NoHost)?.to_owned(); Self::normalized(host, verbatim.port(), authority) } + /// The port exactly as the authority spells it, when it spells one. + /// + /// An IPv6 literal is bracketed, so a colon inside it is never a port + /// delimiter — only a `]:port` suffix is. + fn written_port(authority: &str) -> Option<&str> { + let (_, port) = authority.rsplit_once(':')?; + (!port.ends_with(']')).then_some(port) + } + /// Parse `://`, reporting a failure against the authority /// text the caller actually wrote. fn parse_as(authority: &str, scheme: &str) -> Result { @@ -355,16 +428,19 @@ impl AzureHost { }) } - /// Apply the two rules the URL host parser does not: strip the DNS root - /// label, reject empty labels. + /// Apply the rules the URL host parser does not: strip the DNS root label, + /// reject empty labels, and hold the name to the 253-character limit DNS puts + /// on one. /// - /// Private, so every route in goes through [`parse`](Self::parse) and neither - /// rule can be skipped. + /// Private, so every route in goes through [`parse`](Self::parse) and no rule + /// can be skipped. fn normalized( host: url::Host, port: Option, authority: &str, ) -> Result { + const DNS_NAME_LIMIT: usize = 253; + let url::Host::Domain(domain) = &host else { // An IP literal is already fully canonical, and has no labels. return Ok(Self { host, port }); @@ -389,6 +465,17 @@ impl AzureHost { reason: "one of its labels is empty".to_string(), }); } + // Measured after IDNA, since the punycode form is what is resolved. + if domain.len() > DNS_NAME_LIMIT { + return Err(AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: format!( + "it is {} characters long, over the {DNS_NAME_LIMIT}-character limit DNS \ + puts on a name", + domain.len() + ), + }); + } } Ok(Self { host, port }) } @@ -545,8 +632,14 @@ pub struct AzureChannelUrl { /// The query, when there is one — a SAS token may be written inline. query: Option, - /// The fragment, when there is one. Meaningless to a channel and never sent - /// on the wire, but dropping it would silently rewrite what the user typed. + /// The fragment, when there is one. + /// + /// Kept so [`canonical`](Self::canonical) spells the channel back the way the + /// user wrote it, which is also the spelling config keys are matched against. + /// It reaches no server: an HTTP request carries only the path and query, and + /// on a signed request it is gone from the URL as well, because + /// `AzureMiddleware::sign` round-trips through `http::Uri`, which has no + /// fragment. fragment: Option, } @@ -847,8 +940,8 @@ pub fn azblob_config( Ok(opendal::services::AzblobConfig { endpoint: Some(endpoint), - account_name: Some(account), - container, + account_name: Some(account.as_str().to_string()), + container: container.as_str().to_string(), root: Some(root), account_key, sas_token, @@ -864,10 +957,6 @@ pub enum AzureCliSasError { #[error("failed to compute the SAS expiry timestamp: {0}")] Expiry(String), - /// The `az` executable could not be found on `PATH`. - #[error("could not find the Azure CLI (`az`) on PATH; install it and run `az login`")] - AzNotFound(#[source] std::io::Error), - /// The `az` executable could not be resolved on `PATH`. #[error("could not resolve the Azure CLI (`az`) on PATH; install it and run `az login`")] AzResolve(#[source] which::Error), @@ -919,8 +1008,8 @@ pub enum AzureCliSasError { /// policy, which this path deliberately does not create. #[cfg(feature = "clap")] pub async fn mint_user_delegation_sas( - account: &str, - container: &str, + account: &AccountName, + container: &ContainerName, permissions: &str, valid_for: std::time::Duration, scheme: AzureScheme, @@ -950,13 +1039,7 @@ pub async fn mint_user_delegation_sas( )) .output() .await - .map_err(|err| { - if err.kind() == std::io::ErrorKind::NotFound { - AzureCliSasError::AzNotFound(err) - } else { - AzureCliSasError::Spawn(err) - } - })?; + .map_err(AzureCliSasError::Spawn)?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); @@ -977,8 +1060,8 @@ pub async fn mint_user_delegation_sas( /// value can be read as anything but one argument. #[cfg(feature = "clap")] fn generate_sas_args<'a>( - account: &'a str, - container: &'a str, + account: &'a AccountName, + container: &'a ContainerName, permissions: &'a str, expiry: &'a str, scheme: AzureScheme, @@ -988,9 +1071,9 @@ fn generate_sas_args<'a>( "container", "generate-sas", "--account-name", - account, + account.as_str(), "--name", - container, + container.as_str(), "--permissions", permissions, "--expiry", @@ -1029,6 +1112,13 @@ mod tests { AzureChannelUrl::parse(url).unwrap_or_else(|err| panic!("{url} should parse: {err}")) } + fn coordinates(account: &str, container: &str) -> AzureCoordinates { + AzureCoordinates { + account: AccountName::new(account).expect("test account name"), + container: ContainerName::new(container).expect("test container name"), + } + } + #[test] fn normal_url_resolves() { assert_eq!( @@ -1037,10 +1127,7 @@ mod tests { Addressing::HostStyle ) .unwrap(), - AzureCoordinates { - account: "acct".to_string(), - container: "general".to_string(), - } + coordinates("acct", "general") ); } @@ -1117,12 +1204,12 @@ mod tests { } } - /// An empty name reaching validation must be rejected by validation itself, - /// not only by a caller remembering to filter it out first. + /// An empty name reaching a constructor must be rejected there, not only by a + /// caller remembering to filter it out first. #[test] fn empty_components_are_rejected() { - assert!(validate_account("").is_err()); - assert!(validate_container("").is_err()); + assert!(AccountName::new("").is_err()); + assert!(ContainerName::new("").is_err()); } #[test] @@ -1142,10 +1229,7 @@ mod tests { Addressing::PathStyle ) .unwrap(), - AzureCoordinates { - account: "devstoreaccount1".to_string(), - container: "general".to_string(), - }, + coordinates("devstoreaccount1", "general"), "path-style derivation failed for {host}" ); } @@ -1182,7 +1266,7 @@ mod tests { Addressing::PathStyle, ) .expect("addressing is the user's call, so this is a warning and not an error"); - assert_eq!(coords.account, "general"); + assert_eq!(coords.account.as_str(), "general"); assert!(logs_contain("path-style = true")); assert!(logs_contain("acct.blob.core.windows.net")); @@ -1194,7 +1278,7 @@ mod tests { Addressing::PathStyle, ) .unwrap(); - assert_eq!(coords.account, "devstoreaccount1"); + assert_eq!(coords.account.as_str(), "devstoreaccount1"); assert!(!logs_contain("acct.blob.example.com")); } @@ -1504,6 +1588,12 @@ mod tests { /// rather than silently reinterpreting. #[test] fn host_rejects_anything_that_is_not_a_bare_authority() { + // A name DNS cannot resolve and a port nothing can connect to: `wire()` + // would otherwise hand out `https://host:0/…`, and a bare `host:` would be + // silently read as the portless host, a different endpoint entirely. + // Labels of 60, so length is the only rule under test. + let label = "a".repeat(60); + let too_long = format!("{}.blob.example", [label.as_str(); 8].join(".")); for authority in [ "acct.blob.core.windows.net/general", "acct.blob.core.windows.net?sv=token", @@ -1511,12 +1601,27 @@ mod tests { "https://acct.blob.core.windows.net", "", "acct.blob.core.windows.net:notaport", + "acct.blob.core.windows.net:", + "acct.blob.core.windows.net:0", + "[::1]:", + "[::1]:0", + &too_long, ] { assert!( AzureHost::parse(authority).is_err(), "expected a rejection for {authority:?}" ); } + + // A name right at the limit still parses, so the check bounds the length + // rather than the number of labels. + let at_limit = format!( + "{}.{}.blob.example", + [label.as_str(); 3].join("."), + "a".repeat(57) + ); + assert_eq!(at_limit.len(), 253); + assert!(AzureHost::parse(&at_limit).is_ok()); } fn hash_of(host: &AzureHost) -> u64 { @@ -1753,8 +1858,16 @@ mod tests { #[cfg(feature = "clap")] #[test] fn https_only_follows_the_configured_scheme() { - let args = - |scheme| generate_sas_args("acct", "general", "cw", "2030-01-01T00:00:00Z", scheme); + let coordinates = coordinates("acct", "general"); + let args = |scheme| { + generate_sas_args( + &coordinates.account, + &coordinates.container, + "cw", + "2030-01-01T00:00:00Z", + scheme, + ) + }; assert!(args(AzureScheme::Https).contains(&"--https-only")); assert!(!args(AzureScheme::Http).contains(&"--https-only")); diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs index 9e6c38601a..2b58f3c64e 100644 --- a/crates/rattler_config/src/config/azure.rs +++ b/crates/rattler_config/src/config/azure.rs @@ -1,5 +1,5 @@ use indexmap::IndexMap; -use rattler_azure::{Auth, AzureEndpointOptions, AzureHost, AzureScheme}; +use rattler_azure::{Auth, AzureEndpointOptions, AzureFetchOptions, AzureHost, AzureScheme}; use serde::{Deserialize, Serialize}; use crate::config::Config; @@ -48,11 +48,6 @@ fn is_local(host: &AzureHost) -> bool { pub struct AzureOptionsMap(IndexMap); impl AzureOptionsMap { - /// Returns `true` if no Azure hosts are configured. - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - /// The options for `host`, or the defaults (anonymous, https, host-style) /// when it has no entry. /// @@ -68,6 +63,14 @@ impl AzureOptionsMap { pub fn hosts(&self) -> impl Iterator { self.0.keys() } + + /// The grants as the fetch path takes them, ready to hand to + /// `AzureMiddleware::new` without a caller rebuilding a map by hand. + pub fn fetch_options(&self) -> impl Iterator { + self.0 + .iter() + .map(|(host, options)| (host.clone(), options.fetch())) + } } /// Reject a document that spells one host two ways. @@ -110,7 +113,7 @@ impl Config for AzureOptionsMap { // A host is granted or not as a whole, so mentioning a host in a // higher-precedence file replaces the lower file's entry outright rather // than merging field-wise the way `repodata-config` does. - let mut merged = self.0.clone(); + let mut merged = self.0; for (key, value) in &other.0 { merged.insert(key.clone(), *value); } @@ -185,6 +188,15 @@ mod tests { let unlisted = map.get(&host("someoneelse.blob.core.windows.net")); assert!(!unlisted.fetch().auth.is_granted()); assert_eq!(unlisted, AzureEndpointOptions::default()); + + // The table feeds the fetch middleware directly, keys and all. + assert_eq!( + map.fetch_options().collect::>(), + vec![ + (host("127.0.0.1:10000"), azurite.fetch()), + (host("mycompany.blob.core.windows.net"), real.fetch()), + ] + ); } /// A grant may only ride cleartext to an endpoint that is not routable off diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index ec02a29c85..6732ff0572 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -82,13 +82,10 @@ pub struct AzureMiddleware { /// separate code path. /// /// A plain `HashMap` rather than `rattler_config::AzureOptionsMap`, mirroring - /// [`crate::S3Middleware`]. No caller has a `rattler_config::Config` in hand - /// today — every one of them passes an empty table — so taking the config type - /// would buy a mandatory `rattler_config` edge on the `azure` feature for zero - /// saved conversions. When a caller does grow one, add a - /// `#[cfg(feature = "rattler_config")]` helper next to - /// [`crate::s3_middleware::compute_s3_config_from_config`] rather than changing - /// this signature. + /// [`crate::S3Middleware`]: taking the config type would put a mandatory + /// `rattler_config` edge on the `azure` feature. The constructors take any + /// iterator of host/options pairs instead, which `AzureOptionsMap` yields + /// directly from its own `fetch_options`. options: HashMap, } @@ -100,9 +97,13 @@ impl AzureMiddleware { /// client — proxy, CA bundle, and TLS settings carry through to those /// requests. /// - /// `options` is the `azure-options` table: the per-host grants. An empty map - /// means every `az://` request is anonymous. - pub fn new(client: Client, options: HashMap) -> Self { + /// `options` is the `azure-options` table: the per-host grants, in any shape + /// that iterates them — `rattler_config::AzureOptionsMap::fetch_options` yields + /// exactly this. An empty iterator means every `az://` request is anonymous. + pub fn new( + client: Client, + options: impl IntoIterator, + ) -> Self { Self::with_credential_provider(client, DefaultCredentialProvider::new(), options) } @@ -115,7 +116,7 @@ impl AzureMiddleware { fn with_credential_provider( client: Client, provider: impl ProvideCredential + 'static, - options: HashMap, + options: impl IntoIterator, ) -> Self { let ctx = Context::new() .with_file_read(TokioFileRead) @@ -123,7 +124,10 @@ impl AzureMiddleware { .with_command_execute(TokioCommandExecute) .with_env(OsEnv); let signer = Signer::new(ctx, provider, RequestSigner::new()); - Self { signer, options } + Self { + signer, + options: options.into_iter().collect(), + } } /// Resolve an `az://` request URL to the channel URL it names and the options @@ -265,9 +269,13 @@ impl Middleware for AzureMiddleware { // Azure answers an unauthorized read of a private container with 404, not // 403, so "no grant" and "no such blob" are the same status on the wire. - // Say so once, naming the config the user would have to write — spelled - // through `AzureHost` so the key printed is the key a lookup arrives with. - if response.status() == http::StatusCode::NOT_FOUND && !options.auth.is_granted() { + // Say so once per host, naming the config the user would have to write — + // spelled through `AzureHost` so the key printed is the key a lookup arrives + // with. + if response.status() == http::StatusCode::NOT_FOUND + && !options.auth.is_granted() + && first_404_for_host(channel.host()) + { // One line, and spelled the way `AzureUrlError::InvalidHost` spells its // fix: a wrapped multi-line hint is harder to grep out of a log, and // the two guided messages should read as the same instruction. @@ -285,6 +293,23 @@ impl Middleware for AzureMiddleware { } } +/// Whether `host` still owes the 404 hint, claiming it if so. +/// +/// A 404 is the *normal* answer to plenty of requests a healthy public channel +/// makes — the repodata gateway probes for a shard index under every subdir it +/// fetches, and a non-sharded channel misses every time — so a hint emitted per +/// response is a security warning printed repeatedly at users whose channel is +/// fine. Once per host per process is enough for the one case it is about: a +/// private container the user forgot to grant. +fn first_404_for_host(host: &AzureHost) -> bool { + static HINTED: std::sync::LazyLock>> = + std::sync::LazyLock::new(Default::default); + HINTED + .lock() + .expect("the 404-hint set is never held across a panic") + .insert(host.clone()) +} + #[cfg(test)] mod tests { use rattler_azure::AzureScheme; @@ -631,6 +656,43 @@ mod tests { assert!(logs_contain("auth = true")); } + /// A public non-sharded channel 404s on every shard-index probe the repodata + /// gateway makes, so a hint per response is a security warning repeated at a + /// user whose channel is healthy. + #[tokio::test] + #[tracing_test::traced_test] + async fn the_404_hint_is_emitted_once_per_host() { + let host = spawn_404_server().await; + let client = reqwest_middleware::ClientBuilder::new(Client::new()) + .with(middleware(options( + &host.to_string(), + emulator_entry(Auth::Anonymous), + ))) + .build(); + + for subdir in ["noarch", "linux-64", "osx-64"] { + let status = client + .get(format!( + "az://{host}/devstoreaccount1/c/{subdir}/repodata_shards.msgpack.zst" + )) + .send() + .await + .expect("request through azure middleware failed") + .status(); + assert_eq!(status, 404); + } + + logs_assert(|lines: &[&str]| { + let hints = lines + .iter() + .filter(|line| line.contains("auth = true")) + .count(); + (hints == 1) + .then_some(()) + .ok_or_else(|| format!("expected exactly one hint, got {hints}")) + }); + } + /// With a grant in place a 404 means what it says, so the hint would be noise. #[tokio::test] #[tracing_test::traced_test] diff --git a/crates/rattler_upload/src/upload/mod.rs b/crates/rattler_upload/src/upload/mod.rs index 29b1b9ab76..f584bca360 100644 --- a/crates/rattler_upload/src/upload/mod.rs +++ b/crates/rattler_upload/src/upload/mod.rs @@ -270,6 +270,11 @@ pub(crate) mod object_store { let already_exists = || miette::miette!("Package {destination} already exists. Use --force to overwrite."); + // `if_not_exists` is not evaluated here on either backend: both build their + // writer with a pure constructor and issue nothing until the first chunk, so + // a lost race always surfaces at `close()` below. This arm covers what + // `writer()` itself can reject — a capability opendal refuses up front — and + // exists so that the answer cannot depend on which backend is in play. let mut writer = match store.writer(target.key(), options).await { Ok(writer) => writer, Err(e) if e.kind() == ErrorKind::ConditionNotMatch => return Err(already_exists()), diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index dddf3e2d82..794600e9eb 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -4120,9 +4120,11 @@ dependencies = [ "jiff", "opendal", "percent-encoding", + "secrecy", "serde", "thiserror 2.0.18", "tokio", + "tracing", "url", "which", ] @@ -4259,6 +4261,7 @@ dependencies = [ "rattler_digest", "rattler_networking", "rattler_package_streaming", + "rattler_redaction", "rattler_s3", "reqwest", "retry-policies", diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index 5e8f2d5f37..9bc5b94a84 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -87,10 +87,7 @@ impl PyClientWithMiddleware { PyMiddleware::Azure(_middleware) => { // Anonymous only: the options table is not exposed to Python // yet. See `PyAzureMiddleware`. - client = client.with(AzureMiddleware::new( - reqwest_client.clone(), - std::collections::HashMap::new(), - )); + client = client.with(AzureMiddleware::new(reqwest_client.clone(), [])); } PyMiddleware::S3(middleware) => { client = client.with(S3Middleware::new( From a4540ece235924565a81f5ddd0f53bff82cf1e3a Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:02:58 +1000 Subject: [PATCH 95/98] feat: let callers write and revoke azure host grants --- crates/rattler_config/src/config/azure.rs | 50 +++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs index 2b58f3c64e..18984dbebf 100644 --- a/crates/rattler_config/src/config/azure.rs +++ b/crates/rattler_config/src/config/azure.rs @@ -64,6 +64,36 @@ impl AzureOptionsMap { self.0.keys() } + /// Whether no host is configured, which is also "every `az://` host is + /// anonymous". Serializers skip the table on this. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Grant `host` these options, returning what it was granted before. + /// + /// Taking an [`AzureHost`] rather than a string is what lets the inner map stay + /// private while still being writable: a caller editing config (`pixi config + /// set azure-options."…"`) has to have parsed its key, so it cannot install an + /// entry a lookup would fail to find. There is no `get_mut`, and none is + /// needed — [`AzureEndpointOptions`] is `Copy`, so editing one field is + /// [`get`](Self::get), change, insert. + pub fn insert( + &mut self, + host: AzureHost, + options: AzureEndpointOptions, + ) -> Option { + self.0.insert(host, options) + } + + /// Revoke `host`'s grant, returning it if there was one. + /// + /// Shift-removes, so the remaining entries keep their relative order and a + /// serialized table does not reshuffle on an unrelated edit. + pub fn remove(&mut self, host: &AzureHost) -> Option { + self.0.shift_remove(host) + } + /// The grants as the fetch path takes them, ready to hand to /// `AzureMiddleware::new` without a caller rebuilding a map by hand. pub fn fetch_options(&self) -> impl Iterator { @@ -157,6 +187,26 @@ mod tests { AzureHost::parse(authority).expect("test host should parse") } + /// A grant can be written and revoked without the inner map being public, and + /// a revoked host falls back to anonymous rather than lingering. + #[test] + fn a_grant_can_be_written_and_revoked() { + let key = host("mycompany.blob.core.windows.net"); + let granted = + AzureEndpointOptions::new(Auth::DefaultChain, rattler_azure::AzureEndpoint::default()); + + let mut map = AzureOptionsMap::default(); + assert!(map.is_empty()); + assert_eq!(map.insert(key.clone(), granted), None); + assert_eq!(map.get(&key), granted); + assert!(!map.is_empty()); + + assert_eq!(map.remove(&key), Some(granted)); + assert!(!map.get(&key).fetch().auth.is_granted()); + assert!(map.is_empty()); + assert_eq!(map.remove(&key), None); + } + /// The table parses in the shape documented for users, and an absent host /// answers with the anonymous defaults rather than requiring a presence check. #[test] From cd7701f39cb392c34eee4adb388bf2adb1e8693c Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:40:07 +1000 Subject: [PATCH 96/98] fix: trust a bare 304 when revalidating a shard index --- .../src/gateway/sharded_subdir/tokio/index.rs | 77 +++++++++++-------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs index fbac0cb7a9..c6a2492cb4 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs @@ -15,7 +15,7 @@ use async_fd_lock::{LockWrite, RwLockWriteGuard}; use bytes::Bytes; use fs_err::tokio as tokio_fs; use futures::{TryFutureExt, future::OptionFuture}; -use http::{HeaderMap, Method, Uri}; +use http::{HeaderMap, Method, StatusCode, Uri}; use http_cache_semantics::{AfterResponse, BeforeRequest, CachePolicy, RequestLike}; use rattler_conda_types::Channel; use rattler_networking::LazyClient; @@ -267,46 +267,55 @@ pub async fn fetch_index( return Err(create_subdir_not_found_error(channel_base_url)); } - match cache_header.policy.after_response( + let after_response = cache_header.policy.after_response( &state_request, &response, SystemTime::now(), - ) { - AfterResponse::NotModified(_policy, _) => { - // The cached file is still valid - match read_shard_index_from_reader(&mut cache_reader).await { - Ok(shard_index) => { - tracing::debug!("shard index cache was not modified"); - if let Some((reporter, index)) = download_reporter { - reporter.on_download_complete(response.url(), index); - } - // If reading the file failed for some reason we'll just - // fetch it again. - return Ok(shard_index); + ); + + // The status is consulted directly rather than left to + // `after_response` alone, which only reports `NotModified` when the + // 304 echoes back the validator it matched. Azure Blob does not: it + // answers a conditional GET with a bare 304 carrying no `etag` and no + // `last-modified`, just `x-ms-error-code: ConditionNotMet`. That + // reads as `Modified`, and a 304 then reaches `from_response`, which + // rejects it for not being a success — so every `az://` sharded + // channel failed on the *second* fetch, once there was a cache entry + // to revalidate. A 304 is only ever sent because the validator we + // ourselves sent matched, so it is trustworthy on its own here, which + // is also how the `repodata.json` path has always read it. + if response.status() == StatusCode::NOT_MODIFIED + || matches!(after_response, AfterResponse::NotModified(..)) + { + // The cached file is still valid + match read_shard_index_from_reader(&mut cache_reader).await { + Ok(shard_index) => { + tracing::debug!("shard index cache was not modified"); + if let Some((reporter, index)) = download_reporter { + reporter.on_download_complete(response.url(), index); } - Err(e) => { - tracing::warn!( - "the cached shard index has been corrupted: {e}" - ); - if let Some((reporter, index)) = download_reporter { - reporter.on_download_complete(response.url(), index); - } + return Ok(shard_index); + } + Err(e) => { + // Fall through to the unconditional fetch below. + tracing::warn!("the cached shard index has been corrupted: {e}"); + if let Some((reporter, index)) = download_reporter { + reporter.on_download_complete(response.url(), index); } } } - AfterResponse::Modified(policy, _) => { - // Close the old file so we can create a new one. - tracing::debug!("shard index cache has become stale"); - return from_response( - cache_reader.into_inner(), - &cache_path, - policy, - response, - download_reporter, - request_permit, - ) - .await; - } + } else if let AfterResponse::Modified(policy, _) = after_response { + // Close the old file so we can create a new one. + tracing::debug!("shard index cache has become stale"); + return from_response( + cache_reader.into_inner(), + &cache_path, + policy, + response, + download_reporter, + request_permit, + ) + .await; } } } From 4d7f17143e1a0f49a93b991ca986015ed5e3bf33 Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:32:00 +1000 Subject: [PATCH 97/98] feat: key azure-options auth grants per container --- Cargo.lock | 1 + crates/rattler_azure/Cargo.toml | 11 +- crates/rattler_azure/src/lib.rs | 196 +++++- crates/rattler_azure/src/options.rs | 300 ++++++++-- crates/rattler_config/src/config/azure.rs | 257 +++++--- .../test-data/compat/kitchen-sink.toml | 12 +- .../test-data/compat/override-layer.toml | 6 + crates/rattler_config/tests/compat.rs | 64 +- ...at__merge__kitchen_sink_plus_override.snap | 10 +- .../compat__parse__kitchen-sink.toml.snap | 15 +- .../compat__parse__override-layer.toml.snap | 15 +- crates/rattler_index/README.md | 9 +- crates/rattler_index/src/main.rs | 9 +- crates/rattler_index/tests/azure_azurite.rs | 16 +- .../src/azure_middleware.rs | 564 +++++++++++++----- .../tests/azure_azurite_fetch.rs | 37 +- .../tests/azure_real_fetch.rs | 19 +- 17 files changed, 1204 insertions(+), 337 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 90df8fad74..f942a0258d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5054,6 +5054,7 @@ name = "rattler_azure" version = "0.1.0" dependencies = [ "clap", + "indexmap 2.14.0", "jiff", "opendal", "percent-encoding", diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml index 6d3301318d..15b1f5b135 100644 --- a/crates/rattler_azure/Cargo.toml +++ b/crates/rattler_azure/Cargo.toml @@ -18,13 +18,16 @@ clap = ["dep:clap", "dep:jiff", "dep:which", "dep:tokio"] # `azblob_config`: build an opendal `AzblobConfig` from a channel URL. opendal = ["dep:opendal"] # Serde derives for the `azure-options` config table: the per-host endpoint -# options and the `AzureHost` used as their map key. The types themselves are -# always compiled; only the derives are gated. Pulls in `serde` alone, so config -# crates can depend on this crate with default features off. -serde = ["dep:serde"] +# options, the `AzureHost` used as their map key and the `ContainerName` keying +# their per-container grants. The types themselves are always compiled; only the +# derives are gated. Pulls in `serde` alone, so config crates can depend on this +# crate with default features off. +serde = ["dep:serde", "indexmap/serde"] [dependencies] clap = { workspace = true, optional = true } +# The per-container grant table, which keeps the order the config file wrote. +indexmap = { workspace = true } jiff = { workspace = true, optional = true } opendal = { workspace = true, default-features = false, features = [ "services-azblob", diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index b9c03bca78..1e4d00c5ee 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -5,10 +5,11 @@ //! //! This crate does not police which hosts are legitimate Azure endpoints: the //! host a channel URL names is taken to be the storage endpoint it says it is. -//! What a host is *granted* — credentials, wire scheme, addressing style — is -//! declared per host in [`options`] and never inferred from the host name, and -//! the default grant is [`Auth::Anonymous`], so naming a host in a URL by itself -//! sends nothing to it. Nothing here signs or sends a request either — that lives +//! What is *granted* — credentials, wire scheme, addressing style — is declared in +//! [`options`] and never inferred from the host name: the wire scheme and the +//! addressing per host, and credentials per *container*, because that is the scope +//! Azure's own RBAC has. The default grant is [`Auth::Anonymous`], so naming a host +//! or a container in a URL by itself sends nothing to it. Nothing here signs or sends a request either — that lives //! in `rattler_networking` — but two functions do handle a credential: //! `azblob_config` embeds the account key or SAS it is handed into the config it //! returns, and `mint_user_delegation_sas` spends the user's `az login` session to @@ -234,7 +235,19 @@ impl std::fmt::Display for AccountName { /// /// Exists for the same reason as [`AccountName`], and is what the container half /// of the `az` argv is spelled as. -#[derive(Debug, Clone, PartialEq, Eq)] +/// +/// It is also the key of an `auth` table in `azure-options`, which is why it is +/// hashable and has the same string serde bridge [`AzureHost`] has: a grant is +/// written per container, and the name a grant is stored under must be the name a +/// lookup arrives with. Azure's rules do the normalizing for free — a container +/// name is lowercase by construction, so unlike a host there is only ever one +/// spelling of one container. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(try_from = "String", into = "String") +)] pub struct ContainerName(String); impl ContainerName { @@ -264,6 +277,32 @@ impl std::fmt::Display for ContainerName { } } +impl std::str::FromStr for ContainerName { + type Err = AzureUrlError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } +} + +/// The serde bridge for using a `ContainerName` as a map key: serde hands map keys +/// over as owned strings, so `serde(try_from = "String")` is what routes a written +/// `auth` key through [`ContainerName::new`] instead of storing it raw. A key Azure +/// would refuse is then a config error at load, not a grant that can never match. +impl TryFrom for ContainerName { + type Error = AzureUrlError; + + fn try_from(value: String) -> Result { + Self::new(&value) + } +} + +impl From for String { + fn from(container: ContainerName) -> Self { + container.0 + } +} + /// The storage account and container an Azure Blob channel URL resolves to. /// /// The fields are public because their *types* are the invariant: a @@ -314,18 +353,15 @@ pub fn account_and_container( addressing: Addressing, ) -> Result { let host = channel.host(); - // Empty segments are never a valid name, so an empty one is a missing one. - let mut segments = channel - .path_segments() - .map(|segment| (!segment.is_empty()).then_some(segment)); - let mut next_segment = || segments.next().flatten(); + let container_segment = + || segment(channel, addressing.container_segment()).ok_or(AzureUrlError::NoContainer); let (account, container) = match addressing { Addressing::HostStyle => { let account = host .account_label() .ok_or_else(|| AzureUrlError::InvalidHost(host.to_string()))?; - (account, next_segment().ok_or(AzureUrlError::NoContainer)?) + (account, container_segment()?) } Addressing::PathStyle => { if host.is_known_azure_blob_endpoint() { @@ -340,8 +376,8 @@ pub fn account_and_container( ); } ( - next_segment().ok_or(AzureUrlError::NoAccount)?, - next_segment().ok_or(AzureUrlError::NoContainer)?, + segment(channel, 0).ok_or(AzureUrlError::NoAccount)?, + container_segment()?, ) } }; @@ -352,6 +388,43 @@ pub fn account_and_container( }) } +/// The container an Azure Blob URL names, when it names one. +/// +/// This is the fetch path's half of [`account_and_container`]: a grant is written +/// per container, so the middleware needs the container and nothing else — no +/// account, which is what keeps a URL on a host that cannot carry an account label +/// (an IP literal read host-style) from failing here where it fetches happily +/// today. +/// +/// The two answers it can give are deliberately different: +/// +/// - `Ok(None)`: the URL has no container segment — the host root, or a path too +/// short to have one under this addressing. There is nothing to attribute a +/// grant to, so the caller sends nothing. +/// - `Err`: the segment is there but is not a name Azure allows for a container. +/// No legitimate blob request can land here, so this is a malformed endpoint +/// rather than an ungranted one, and saying so beats going quietly anonymous and +/// surfacing later as an unexplained 401. +pub fn container( + channel: &AzureChannelUrl, + addressing: Addressing, +) -> Result, AzureUrlError> { + segment(channel, addressing.container_segment()) + .map(ContainerName::new) + .transpose() +} + +/// The `index`-th path segment, or `None` when it is missing or empty. +/// +/// An empty segment is a missing one: no Azure name may be empty, so `//general` +/// has no first segment rather than an unnamed one. +fn segment(channel: &AzureChannelUrl, index: usize) -> Option<&str> { + channel + .path_segments() + .nth(index) + .filter(|segment| !segment.is_empty()) +} + /// A normalized Azure Blob endpoint authority: a host, and its port when one is /// written. #[derive(Clone, PartialEq, Eq, Hash)] @@ -850,8 +923,8 @@ fn strip_az_scheme(value: &str) -> Option<&str> { /// The account name, endpoint, container and root prefix are all derived from the /// channel URL, read the way `options.addressing` says to read it and reached over /// `options.scheme`; the credentials supply only the account key or SAS token. -/// `options.auth` is not consulted — this is the write path, where the credential -/// has already been chosen by the caller. +/// The per-container grants are not part of [`AzureEndpoint`] at all — this is the +/// write path, where the credential has already been chosen by the caller. /// /// Taking the [`AzureChannelUrl`] rather than a wire `Url` is what keeps the /// scheme in the config from disagreeing with the scheme in the endpoint: both @@ -1131,6 +1204,99 @@ mod tests { ); } + /// The fetch path's derivation: it must find the same container + /// `account_and_container` does, under both addressing styles, and it must not + /// inherit that function's account rules — a host-style IP literal has no + /// account label, but it still has a container, and a fetch for it is a request + /// that works today. + #[test] + fn container_is_derived_from_the_addressing() { + for (url, addressing, expected) in [ + ( + "az://acct.blob.core.windows.net/general/noarch", + Addressing::HostStyle, + "general", + ), + ( + "az://127.0.0.1:10000/devstoreaccount1/general/noarch", + Addressing::PathStyle, + "general", + ), + ( + "az://127.0.0.1:10000/general/noarch", + Addressing::HostStyle, + "general", + ), + ] { + assert_eq!( + container(&channel(url), addressing).unwrap(), + Some(ContainerName::new(expected).unwrap()), + "{url}" + ); + } + + // Where both derivations answer, they must answer the same thing: a grant + // looked up for one container and applied to another is a security bug. + for (url, addressing) in [ + ( + "az://acct.blob.core.windows.net/general/noarch", + Addressing::HostStyle, + ), + ( + "az://127.0.0.1:10000/devstoreaccount1/general", + Addressing::PathStyle, + ), + ] { + assert_eq!( + container(&channel(url), addressing).unwrap(), + Some( + account_and_container(&channel(url), addressing) + .unwrap() + .container + ), + "{url}" + ); + } + } + + /// A URL with no container segment is not an error: there is nothing to + /// attribute a grant to, so the fetch path sends nothing and stays total for + /// URLs that are not channel-scoped. + #[test] + fn a_url_without_a_container_names_none() { + for (url, addressing) in [ + ("az://acct.blob.core.windows.net", Addressing::HostStyle), + ("az://acct.blob.core.windows.net/", Addressing::HostStyle), + ( + "az://127.0.0.1:10000/devstoreaccount1", + Addressing::PathStyle, + ), + ("az://127.0.0.1:10000/", Addressing::PathStyle), + ] { + assert_eq!(container(&channel(url), addressing).unwrap(), None, "{url}"); + } + } + + /// A segment that cannot be a container name is a malformed endpoint, not an + /// ungranted one — Azure forbids uppercase, so no legitimate request lands + /// here. Going quietly anonymous would surface as an unexplained 401 instead of + /// naming the fault. + #[test] + fn a_url_with_an_unusable_container_is_an_error() { + for url in [ + "az://acct.blob.core.windows.net/General/noarch", + "az://acct.blob.core.windows.net/ab/noarch", + "az://acct.blob.core.windows.net/a--b/noarch", + ] { + let err = container(&channel(url), Addressing::HostStyle) + .expect_err("an illegal container name must be reported"); + assert!( + matches!(err, AzureUrlError::InvalidContainerName(_)), + "{url}: {err}" + ); + } + } + #[test] fn userinfo_is_rejected() { assert!(matches!( diff --git a/crates/rattler_azure/src/options.rs b/crates/rattler_azure/src/options.rs index cf14734fd3..9dfe905610 100644 --- a/crates/rattler_azure/src/options.rs +++ b/crates/rattler_azure/src/options.rs @@ -1,25 +1,31 @@ //! Per-host endpoint options for Azure Blob channels. //! //! An entry in the `azure-options` config table is the *only* thing that grants -//! a host anything: without one, a channel on that host is fetched anonymously -//! over https in host-style addressing. There is deliberately no hardcoded list -//! of "official" Azure suffixes — since a grant must be written per host, -//! suffix classification carries no security weight, and the absence of the list -//! is what lets custom endpoints and the Azurite emulator work at all. +//! a host or one of its containers anything: without one, a channel on that host +//! is fetched anonymously over https in host-style addressing. There is +//! deliberately no hardcoded list of "official" Azure suffixes — since a grant must +//! be written per host, suffix classification carries no security weight, and the +//! absence of the list is what lets custom endpoints and the Azurite emulator work +//! at all. //! //! # Three types, one table //! -//! [`AzureEndpointOptions`] is the file format. Nothing consumes it directly: -//! the fetch path takes [`AzureFetchOptions`] and the write path takes -//! [`AzureEndpoint`], so `auth` cannot reach a caller that supplies its own -//! credential and [`Addressing`] cannot reach one that never derives an account. -//! A field a consumer would have to ignore reads as a guarantee, and the write -//! path ignoring `auth` looked exactly like a credential gate that was never there. +//! [`AzureEndpointOptions`] is the file format. The write path never sees it: it +//! takes [`AzureEndpoint`], so a grant cannot reach a caller that supplies its own +//! credential — a field a consumer would have to ignore reads as a guarantee, and +//! the write path ignoring a grant looked exactly like a credential gate that was +//! never there. +//! +//! The fetch path holds the entry but never acts on it directly either: per request +//! it narrows to [`AzureFetchOptions`], which carries the grant for the one +//! container the request addresses and no way to address another. It cannot narrow +//! any earlier, because finding that container needs the entry's [`Addressing`] +//! first. //! //! # Why enums for what the config spells as bools //! -//! The TOML surface stays `auth = true` / `path-style = true`, because that is -//! the ergonomic spelling and it keeps the table skimmable. Internally each is an +//! The TOML surface stays ` = true` / `path-style = true`, because that +//! is the ergonomic spelling and it keeps the table skimmable. Internally each is an //! enum, so no call site can mix up two unrelated booleans, and the meaning of a //! value is legible without chasing the field name. The bridge is a //! `serde(from = "bool", into = "bool")` pair with a `From` impl each way, applied @@ -33,10 +39,13 @@ //! the `serde` feature, so a consumer that just wants the URL types does not //! pull serde in. -/// Whether credentials may attach to requests for a host. +use crate::ContainerName; + +/// Whether credentials may attach to requests for a container. /// -/// Defaults to [`Auth::Anonymous`]: a host gets no credentials until a config -/// entry says otherwise. Serializes as the bool `auth` in `azure-options`. +/// Defaults to [`Auth::Anonymous`]: a container gets no credentials until a config +/// entry names it. Serializes as the bool a container is spelled with in an +/// `azure-options` `auth` table. #[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] #[cfg_attr( feature = "serde", @@ -50,7 +59,7 @@ pub enum Auth { #[default] Anonymous, - /// Run the standard Azure credential chain for this host and sign with what + /// Run the standard Azure credential chain for this container and sign with what /// it returns. Because this is an explicit grant, a broken or unusable /// credential is a hard error — never a silent downgrade to anonymous. DefaultChain, @@ -144,6 +153,24 @@ pub enum Addressing { PathStyle, } +impl Addressing { + /// Which path segment holds the container name under this addressing. + /// + /// One number, read by both derivations that need it — + /// [`account_and_container`](crate::account_and_container) and + /// [`container`](crate::container). Two derivations that disagreed about which + /// segment is the container would look a grant up for one container and apply + /// it to another. + pub(crate) fn container_segment(self) -> usize { + match self { + // `.host//…` + Addressing::HostStyle => 0, + // `host///…` + Addressing::PathStyle => 1, + } + } +} + impl From for Addressing { fn from(value: bool) -> Self { if value { @@ -176,13 +203,16 @@ pub struct AzureEndpoint { pub addressing: Addressing, } -/// What the fetch middleware needs to reach one Azure Blob host. +/// What the fetch middleware needs to send one request. /// /// [`Addressing`] is absent rather than ignored: the fetch path forwards a path -/// and never derives an account name from it. +/// and never derives an account name from it. It does read the addressing to find +/// the container a grant is looked up by, but that happens one step earlier, on +/// [`AzureEndpointOptions`] — by the time this exists the grant is already +/// resolved, so there is nothing left here to address. #[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] pub struct AzureFetchOptions { - /// Whether credentials may be sent to this host. + /// Whether credentials may be sent for the container this was resolved for. pub auth: Auth, /// The scheme `az://` is rewritten to for this host. @@ -191,24 +221,49 @@ pub struct AzureFetchOptions { /// One `azure-options` entry, as the config file spells it. /// -/// This is the serde surface and nothing else: the three TOML keys live here, and -/// each consumer takes the narrower view it can actually act on, via -/// [`Self::endpoint`] or [`Self::fetch`]. The fields are private so that view is -/// the only way in. +/// This is the serde surface and nothing else: the TOML keys live here, and each +/// consumer takes the narrower view it can actually act on, via [`Self::endpoint`] +/// or [`Self::fetch`]. The fields are private so that view is the only way in. /// -/// The default value is the no-entry behaviour: anonymous, https, host-style. A +/// The default value is the no-entry behaviour: no grants, https, host-style. A /// host with no config entry behaves exactly as if it had a defaulted entry, so /// callers can look up an absent host and fall back to `default()` rather than /// branching on presence. -#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +/// +/// # Why the grant is per container and the endpoint is per host +/// +/// The two settings have different scopes, and it is not a matter of taste. Azure +/// assigns RBAC per *container*, so one storage account routinely holds a mix of +/// private and anonymous-read containers; a grant that could only be written per +/// host cannot express that account at all — signing the anonymous-read container +/// 403s for any identity holding no role on it, and not signing breaks the private +/// ones. `scheme` and `addressing` describe the *endpoint*: two containers on one +/// account disagreeing about where the account name lives is not a configuration, +/// it is a contradiction. +/// +/// So there is deliberately no host-level `auth` field. Not "defaults to false" — +/// absent from the type, so the one setting whose blast radius would be every +/// container on the account, including containers created after it was written, is +/// unrepresentable rather than merely discouraged. The worst typo here grants one +/// container. +/// +/// ```toml +/// [azure-options."mycompany.blob.core.windows.net"] +/// scheme = "https" +/// path-style = false +/// +/// [azure-options."mycompany.blob.core.windows.net".auth] +/// releases = true +/// staging = true +/// # a container not listed here is fetched anonymously +/// ``` +#[derive(Default, Clone, Debug, PartialEq, Eq)] #[cfg_attr( feature = "serde", derive(serde::Deserialize, serde::Serialize), serde(rename_all = "kebab-case", default) )] pub struct AzureEndpointOptions { - auth: Auth, - scheme: AzureScheme, /// The field is named for what it holds, but the config key stays @@ -216,55 +271,118 @@ pub struct AzureEndpointOptions { /// is what the key means. #[cfg_attr(feature = "serde", serde(rename = "path-style", alias = "path_style"))] addressing: Addressing, + + /// Which containers on this host may be sent credentials. + /// + /// The value is an [`Auth`] rather than a `bool` because that is what + /// [`AzureFetchOptions`] already speaks, so a grant flows from here to the + /// signer without a `if granted { … } else { … }` at every boundary — one less + /// place to invert a condition on a credential decision. The TOML surface is + /// unaffected: the bool bridge keeps it `releases = true`. + /// + /// An explicit `false` is legal and redundant with omission, which is what + /// makes a higher-precedence config file able to *revoke* a grant a lower one + /// made (see [`Self::layered_over`]). + /// + /// Last, because the TOML serializer must emit an entry's scalars before its + /// tables. + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "indexmap::IndexMap::is_empty") + )] + auth: indexmap::IndexMap, } impl AzureEndpointOptions { - /// Build an entry from a grant and the endpoint it applies to. - pub fn new(auth: Auth, endpoint: AzureEndpoint) -> Self { + /// Build an entry from its per-container grants and the endpoint they apply to. + pub fn new( + auth: impl IntoIterator, + endpoint: AzureEndpoint, + ) -> Self { Self { - auth, scheme: endpoint.scheme, addressing: endpoint.addressing, + auth: auth.into_iter().collect(), } } /// How to address this host, for the write path. - pub fn endpoint(self) -> AzureEndpoint { + pub fn endpoint(&self) -> AzureEndpoint { AzureEndpoint { scheme: self.scheme, addressing: self.addressing, } } - /// The grant and wire scheme, for the fetch path. - pub fn fetch(self) -> AzureFetchOptions { + /// The grant and wire scheme for one container, for the fetch path. + /// + /// `container` is an `Option` because a URL need not name one: the host root, + /// or a path too short for the addressing, has nothing to attribute a grant to. + /// Answering that case here rather than at the call site is what keeps "no + /// container" from being spelled two ways — it can only mean anonymous, since + /// there is no entry it could match. + pub fn fetch(&self, container: Option<&ContainerName>) -> AzureFetchOptions { AzureFetchOptions { - auth: self.auth, + auth: container + .and_then(|container| self.auth.get(container)) + .copied() + .unwrap_or_default(), scheme: self.scheme, } } + + /// Every container this entry mentions, and what it grants it. + /// + /// Includes the explicit `false`s: a caller validating or listing the table has + /// to see what the file says, not what it effectively means. + pub fn grants(&self) -> impl Iterator { + self.auth.iter().map(|(container, auth)| (container, *auth)) + } + + /// This entry layered over the one a lower-precedence config file wrote. + /// + /// `scheme` and `addressing` describe the endpoint as a whole, so this entry + /// replaces them outright. The grants merge per container: a file naming one + /// container must not silently drop a grant on a container it never mentions. + /// The merge is not a one-way ratchet, because an explicit `false` is a legal + /// grant — that is how a user file revokes what a system file granted. + pub fn layered_over(&self, lower: &Self) -> Self { + let mut auth = lower.auth.clone(); + auth.extend(self.auth.iter().map(|(c, auth)| (c.clone(), *auth))); + Self { + scheme: self.scheme, + addressing: self.addressing, + auth, + } + } } #[cfg(all(test, feature = "serde"))] mod tests { use super::*; + fn container(name: &str) -> ContainerName { + ContainerName::new(name).expect("test container name") + } + /// The whole point of the bool bridge: the TOML stays boolean while the code /// sees enums, and an absent field takes the safe default. #[test] fn toml_bools_map_to_enums() { let opts: AzureEndpointOptions = toml::from_str( r#" - auth = true scheme = "http" path-style = true + + [auth] + releases = true "#, ) .unwrap(); assert_eq!( opts, AzureEndpointOptions::new( - Auth::DefaultChain, + [(container("releases"), Auth::DefaultChain)], AzureEndpoint { scheme: AzureScheme::Http, addressing: Addressing::PathStyle, @@ -275,14 +393,97 @@ mod tests { // An empty entry is the same as no entry: anonymous, https, host-style. let empty: AzureEndpointOptions = toml::from_str("").unwrap(); assert_eq!(empty, AzureEndpointOptions::default()); - assert_eq!(empty.fetch(), AzureFetchOptions::default()); - assert!(!empty.fetch().auth.is_granted()); + assert_eq!( + empty.fetch(Some(&container("releases"))), + AzureFetchOptions::default() + ); + assert!(!empty.fetch(Some(&container("releases"))).auth.is_granted()); assert_eq!(empty.endpoint(), AzureEndpoint::default()); + } - // `auth = false` is spelled out explicitly by some users; it must not be - // mistaken for a grant. - let denied: AzureEndpointOptions = toml::from_str("auth = false").unwrap(); - assert!(!denied.fetch().auth.is_granted()); + /// A grant applies to the container it names and to no other, which is the + /// whole reason the table is keyed per container: one storage account holds + /// private and anonymous-read containers side by side. + #[test] + fn a_grant_applies_to_one_container_only() { + let opts: AzureEndpointOptions = toml::from_str( + r#" + [auth] + releases = true + public = false + "#, + ) + .unwrap(); + + assert!(opts.fetch(Some(&container("releases"))).auth.is_granted()); + // An explicit `false` and an unlisted container behave identically; the + // spelling exists so a reader can tell "deliberately unsigned" from + // "forgotten", and so a higher-precedence file can revoke a grant. + assert!(!opts.fetch(Some(&container("public"))).auth.is_granted()); + assert!(!opts.fetch(Some(&container("staging"))).auth.is_granted()); + + // A URL naming no container has nothing to attribute a grant to. + assert!(!opts.fetch(None).auth.is_granted()); + + // `grants` reports what the file says, explicit `false` included — in the + // order the document's table iterated (`toml::Table` is a `BTreeMap`, so + // that is byte order, not write order). + assert_eq!( + opts.grants().collect::>(), + vec![ + (&container("public"), Auth::Anonymous), + (&container("releases"), Auth::DefaultChain), + ] + ); + } + + /// The endpoint is host-scoped and replaces wholesale; the grants merge per + /// container, in both directions — a higher file adds one grant without + /// dropping another, and revokes with an explicit `false`. + #[test] + fn layering_replaces_the_endpoint_and_merges_the_grants() { + let lower: AzureEndpointOptions = toml::from_str( + r#" + path-style = true + + [auth] + releases = true + staging = true + "#, + ) + .unwrap(); + let higher: AzureEndpointOptions = toml::from_str( + r#" + scheme = "http" + + [auth] + staging = false + internal = true + "#, + ) + .unwrap(); + + let merged = higher.layered_over(&lower); + + assert_eq!(merged.endpoint(), higher.endpoint()); + assert!( + merged.fetch(Some(&container("releases"))).auth.is_granted(), + "a grant the higher file never mentions must survive" + ); + assert!( + !merged.fetch(Some(&container("staging"))).auth.is_granted(), + "an explicit `false` in the higher file must revoke the lower grant" + ); + assert!(merged.fetch(Some(&container("internal"))).auth.is_granted()); + } + + /// A container name Azure would refuse is a config error at load, not a grant + /// that can never match anything. + #[test] + fn an_unusable_container_key_is_rejected() { + let err = toml::from_str::("[auth]\nReleases = true\n") + .expect_err("uppercase is not a legal container name"); + assert!(err.to_string().contains("Releases"), "{err}"); } /// Round-tripping must preserve the boolean spelling, not leak the enum @@ -290,16 +491,25 @@ mod tests { #[test] fn enums_serialize_back_to_bools() { let toml = toml::to_string(&AzureEndpointOptions::new( - Auth::DefaultChain, + [ + (container("releases"), Auth::DefaultChain), + (container("public"), Auth::Anonymous), + ], AzureEndpoint { scheme: AzureScheme::Http, addressing: Addressing::PathStyle, }, )) .unwrap(); - assert!(toml.contains("auth = true"), "{toml}"); + assert!(toml.contains("releases = true"), "{toml}"); + assert!(toml.contains("public = false"), "{toml}"); assert!(toml.contains("path-style = true"), "{toml}"); assert!(toml.contains(r#"scheme = "http""#), "{toml}"); assert!(!toml.contains("DefaultChain"), "{toml}"); + + // An entry granting nothing writes no `auth` table at all, so a config + // file keeps saying what it said. + let anonymous = toml::to_string(&AzureEndpointOptions::default()).unwrap(); + assert!(!anonymous.contains("auth"), "{anonymous}"); } } diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs index 18984dbebf..a674b0446b 100644 --- a/crates/rattler_config/src/config/azure.rs +++ b/crates/rattler_config/src/config/azure.rs @@ -1,5 +1,5 @@ use indexmap::IndexMap; -use rattler_azure::{Auth, AzureEndpointOptions, AzureFetchOptions, AzureHost, AzureScheme}; +use rattler_azure::{AzureEndpointOptions, AzureHost, AzureScheme}; use serde::{Deserialize, Serialize}; use crate::config::Config; @@ -21,10 +21,16 @@ fn is_local(host: &AzureHost) -> bool { /// Per-host options for Azure Blob channels, keyed by endpoint authority /// (including a port where one is used, e.g. `127.0.0.1:10000`). /// -/// An entry is a *grant*: it is the only way a host gets credentials, a -/// non-default scheme, or path-style addressing. A host with no entry is fetched -/// anonymously over https in host-style addressing, so an empty map is the safe -/// default and [`AzureOptionsMap::get`] can answer for absent hosts too. +/// An entry is a *grant*: it is the only way a container gets credentials, or a +/// host a non-default scheme or path-style addressing. A host with no entry is +/// fetched anonymously over https in host-style addressing, so an empty map is the +/// safe default and [`AzureOptionsMap::get`] can answer for absent hosts too. +/// +/// The grant itself is keyed per container *inside* the entry (see +/// [`AzureEndpointOptions`]), because Azure assigns RBAC per container. Container +/// names need none of the normalization the host key below is about: Azure allows +/// only lowercase in one, so a container has exactly one spelling and two keys that +/// mean the same container cannot be written. /// /// # Why the key is an [`AzureHost`] and not a `String` /// @@ -55,7 +61,7 @@ impl AzureOptionsMap { /// defaulted entry" are defined to behave identically, so branching on /// presence only invites the two paths to drift apart. pub fn get(&self, host: &AzureHost) -> AzureEndpointOptions { - self.0.get(host).copied().unwrap_or_default() + self.0.get(host).cloned().unwrap_or_default() } /// The configured hosts, in the order the document's table iterated them @@ -76,8 +82,9 @@ impl AzureOptionsMap { /// private while still being writable: a caller editing config (`pixi config /// set azure-options."…"`) has to have parsed its key, so it cannot install an /// entry a lookup would fail to find. There is no `get_mut`, and none is - /// needed — [`AzureEndpointOptions`] is `Copy`, so editing one field is - /// [`get`](Self::get), change, insert. + /// needed: editing an entry is [`get`](Self::get), change, insert. That copies + /// the entry's grant table, which is not worth a second mutable path into a + /// private map — config editing happens once per `config set`, not per request. pub fn insert( &mut self, host: AzureHost, @@ -94,19 +101,24 @@ impl AzureOptionsMap { self.0.shift_remove(host) } - /// The grants as the fetch path takes them, ready to hand to + /// The entries as the fetch middleware takes them, ready to hand to /// `AzureMiddleware::new` without a caller rebuilding a map by hand. - pub fn fetch_options(&self) -> impl Iterator { + /// + /// Whole entries, not the narrower `AzureFetchOptions`: the middleware has to + /// read a host's addressing before it can tell which path segment is the + /// container, and only then can it look the container's grant up. The narrowing + /// therefore happens per request, inside the middleware, and not here. + pub fn endpoint_options(&self) -> impl Iterator { self.0 .iter() - .map(|(host, options)| (host.clone(), options.fetch())) + .map(|(host, options)| (host.clone(), options.clone())) } } /// Reject a document that spells one host two ways. /// /// Both spellings reach serde, which keeps whichever the table iterated last — -/// silently overriding an `auth = false` or dropping a grant. TOML's own +/// silently dropping one spelling's whole entry, grants and all. TOML's own /// duplicate-key check runs on the raw text, so it cannot see the collision; this /// has to run while both spellings are still visible. pub(crate) fn ensure_no_colliding_hosts(document: &toml::Table) -> Result<(), String> { @@ -139,28 +151,35 @@ impl Config for AzureOptionsMap { } fn merge_config(self, other: &Self) -> Result { - // Merge the two maps, with `other`'s entries overwriting existing keys. - // A host is granted or not as a whole, so mentioning a host in a - // higher-precedence file replaces the lower file's entry outright rather - // than merging field-wise the way `repodata-config` does. + // Merge the two maps, with `other`'s entries layered over existing keys. + // The host-scoped fields — `scheme`, `path-style` — replace wholesale, but + // the grants merge per container: a higher-precedence file naming one + // container must not discard a grant a lower file made on a different + // container it never mentions. It can still revoke the container it does + // name, because an explicit `false` is a legal grant. let mut merged = self.0; for (key, value) in &other.0 { - merged.insert(key.clone(), *value); + let layered = match merged.get(key) { + Some(lower) => value.layered_over(lower), + None => value.clone(), + }; + merged.insert(key.clone(), layered); } Ok(AzureOptionsMap(merged)) } fn validate(&self) -> Result<(), super::ValidationError> { for (host, options) in &self.0 { - let fetch = options.fetch(); - if fetch.auth == Auth::DefaultChain - && fetch.scheme == AzureScheme::Http - && !is_local(host) - { + if options.endpoint().scheme != AzureScheme::Http || is_local(host) { + continue; + } + // One granted container is enough: the scheme is host-scoped, so its + // requests all ride the same cleartext connection. + if let Some((container, _)) = options.grants().find(|(_, auth)| auth.is_granted()) { return Err(super::ValidationError::Invalid(format!( - "`azure-options.\"{host}\"` grants credentials over cleartext http. A \ - credential may only be sent unencrypted to a local endpoint: use an \ - https scheme, or address the emulator by loopback address." + "`azure-options.\"{host}\".auth` grants credentials to `{container}` over \ + cleartext http. A credential may only be sent unencrypted to a local \ + endpoint: use an https scheme, or address the emulator by loopback address." ))); } } @@ -169,17 +188,33 @@ impl Config for AzureOptionsMap { fn keys(&self) -> Vec { // Quoted, because every Azure authority contains dots and an unquoted key - // is not the TOML path the user must pass to `config set`/`unset`. + // is not the TOML path the user must pass to `config set`/`unset`. A + // container name never needs quoting — Azure's rules leave nothing in one + // that a bare TOML key cannot hold. + // + // The per-container grants are listed as their own keys so each is + // separately unsettable. There is deliberately no `."".auth` key: the + // path exists only as a table of containers, so `config set + // azure-options."".auth true` has nowhere to land — which is the + // point, since that is the one edit whose blast radius would be the whole + // account. self.0 - .keys() - .map(|host| toml::Value::from(host.to_string()).to_string()) + .iter() + .flat_map(|(host, options)| { + let host = toml::Value::from(host.to_string()).to_string(); + let grants = options + .grants() + .map(|(container, _)| format!("{host}.auth.{container}")) + .collect::>(); + std::iter::once(host).chain(grants) + }) .collect() } } #[cfg(test)] mod tests { - use rattler_azure::{Addressing, Auth, AzureScheme}; + use rattler_azure::{Addressing, Auth, AzureScheme, ContainerName}; use super::*; @@ -187,22 +222,33 @@ mod tests { AzureHost::parse(authority).expect("test host should parse") } + fn container(name: &str) -> ContainerName { + ContainerName::new(name).expect("test container name") + } + /// A grant can be written and revoked without the inner map being public, and /// a revoked host falls back to anonymous rather than lingering. #[test] fn a_grant_can_be_written_and_revoked() { let key = host("mycompany.blob.core.windows.net"); - let granted = - AzureEndpointOptions::new(Auth::DefaultChain, rattler_azure::AzureEndpoint::default()); + let granted = AzureEndpointOptions::new( + [(container("releases"), Auth::DefaultChain)], + rattler_azure::AzureEndpoint::default(), + ); let mut map = AzureOptionsMap::default(); assert!(map.is_empty()); - assert_eq!(map.insert(key.clone(), granted), None); + assert_eq!(map.insert(key.clone(), granted.clone()), None); assert_eq!(map.get(&key), granted); assert!(!map.is_empty()); assert_eq!(map.remove(&key), Some(granted)); - assert!(!map.get(&key).fetch().auth.is_granted()); + assert!( + !map.get(&key) + .fetch(Some(&container("releases"))) + .auth + .is_granted() + ); assert!(map.is_empty()); assert_eq!(map.remove(&key), None); } @@ -213,38 +259,53 @@ mod tests { fn table_parses_and_absent_hosts_default() { let map: AzureOptionsMap = toml::from_str( r#" - ["mycompany.blob.core.windows.net"] - auth = true + ["mycompany.blob.core.windows.net".auth] + releases = true ["127.0.0.1:10000"] - auth = true scheme = "http" path-style = true + + ["127.0.0.1:10000".auth] + general = true "#, ) .unwrap(); let real = map.get(&host("mycompany.blob.core.windows.net")); - assert_eq!(real.fetch().auth, Auth::DefaultChain); + assert_eq!( + real.fetch(Some(&container("releases"))).auth, + Auth::DefaultChain + ); assert_eq!(real.endpoint().scheme, AzureScheme::Https); assert_eq!(real.endpoint().addressing, Addressing::HostStyle); let azurite = map.get(&host("127.0.0.1:10000")); - assert_eq!(azurite.fetch().auth, Auth::DefaultChain); + assert_eq!( + azurite.fetch(Some(&container("general"))).auth, + Auth::DefaultChain + ); assert_eq!(azurite.endpoint().scheme, AzureScheme::Http); assert_eq!(azurite.endpoint().addressing, Addressing::PathStyle); - // An unlisted host gets no grant. + // Neither a container the account never granted nor an unlisted host gets + // anything, and the two are the same answer by construction. + assert!(!real.fetch(Some(&container("public"))).auth.is_granted()); let unlisted = map.get(&host("someoneelse.blob.core.windows.net")); - assert!(!unlisted.fetch().auth.is_granted()); + assert!( + !unlisted + .fetch(Some(&container("releases"))) + .auth + .is_granted() + ); assert_eq!(unlisted, AzureEndpointOptions::default()); // The table feeds the fetch middleware directly, keys and all. assert_eq!( - map.fetch_options().collect::>(), + map.endpoint_options().collect::>(), vec![ - (host("127.0.0.1:10000"), azurite.fetch()), - (host("mycompany.blob.core.windows.net"), real.fetch()), + (host("127.0.0.1:10000"), azurite), + (host("mycompany.blob.core.windows.net"), real), ] ); } @@ -255,7 +316,7 @@ mod tests { fn cleartext_grants_are_confined_to_local_endpoints() { for authority in ["127.0.0.1:10000", "[::1]:10000", "azurite:10000"] { let map: AzureOptionsMap = toml::from_str(&format!( - "[\"{authority}\"]\nauth = true\nscheme = \"http\"\n" + "[\"{authority}\"]\nscheme = \"http\"\n[\"{authority}\".auth]\ngeneral = true\n" )) .unwrap(); assert!(map.validate().is_ok(), "{authority} is local"); @@ -263,55 +324,83 @@ mod tests { for authority in ["mycompany.blob.core.windows.net", "internal.example.com"] { let map: AzureOptionsMap = toml::from_str(&format!( - "[\"{authority}\"]\nauth = true\nscheme = \"http\"\n" + "[\"{authority}\"]\nscheme = \"http\"\n[\"{authority}\".auth]\npublic = false\nreleases = true\n" )) .unwrap(); - assert!(map.validate().is_err(), "{authority} is routable"); - - // The same host over https, and the same cleartext scheme without a - // grant, are both fine — it is only the pair that is refused. + let err = map.validate().expect_err("{authority} is routable"); + // The message names the container at fault: the entry may hold many, and + // only the granted ones are the problem. + assert!(err.to_string().contains("releases"), "{err}"); + + // The same host over https, and the same cleartext scheme with no + // container granted, are both fine — it is only the pair that is + // refused, and an explicit `false` is not a grant. let https: AzureOptionsMap = - toml::from_str(&format!("[\"{authority}\"]\nauth = true\n")).unwrap(); + toml::from_str(&format!("[\"{authority}\".auth]\nreleases = true\n")).unwrap(); assert!(https.validate().is_ok()); - let anonymous: AzureOptionsMap = - toml::from_str(&format!("[\"{authority}\"]\nscheme = \"http\"\n")).unwrap(); + let anonymous: AzureOptionsMap = toml::from_str(&format!( + "[\"{authority}\"]\nscheme = \"http\"\n[\"{authority}\".auth]\nreleases = false\n" + )) + .unwrap(); assert!(anonymous.validate().is_ok()); } } /// Two spellings of one host must be refused rather than one silently - /// winning: the loser here is an explicit `auth = false`. + /// winning: the loser here is an explicit `releases = false`. #[test] fn a_document_naming_one_host_twice_is_refused() { let document = r#" -[azure-options."acct.blob.example"] -auth = false +[azure-options."acct.blob.example".auth] +releases = false -[azure-options."ACCT.blob.example."] -auth = true +[azure-options."ACCT.blob.example.".auth] +releases = true "#; let error = ensure_no_colliding_hosts(&document.parse().unwrap()) .expect_err("a collision must be reported"); assert!(error.contains("acct.blob.example"), "{error}"); } - /// A later config file overwrites a host wholesale. It must not be able to - /// keep an earlier `auth = true` while changing only the scheme. + /// A later config file replaces a host's endpoint wholesale. It must not be + /// able to keep an earlier `path-style = true` while changing only the scheme. #[test] - fn merge_replaces_entries_wholesale() { + fn merge_replaces_the_endpoint_wholesale() { let base: AzureOptionsMap = - toml::from_str("[\"host.example\"]\nauth = true\npath-style = true\n").unwrap(); + toml::from_str("[\"host.example\"]\npath-style = true\n").unwrap(); let over: AzureOptionsMap = toml::from_str("[\"host.example\"]\nscheme = \"http\"\n").unwrap(); let merged = base.merge_config(&over).unwrap(); let entry = merged.get(&host("host.example")); assert_eq!(entry.endpoint().scheme, AzureScheme::Http); + assert_eq!(entry.endpoint().addressing, Addressing::HostStyle); + } + + /// The grants, unlike the endpoint, merge per container: a user file naming one + /// container must not silently revoke a grant a system file made on another — + /// and, because an explicit `false` is legal, it can still revoke the one it + /// does name. Without that, per-container merging would be a one-way ratchet. + #[test] + fn merge_layers_grants_per_container() { + let system: AzureOptionsMap = + toml::from_str("[\"host.example\".auth]\nreleases = true\nstaging = true\n").unwrap(); + let user: AzureOptionsMap = + toml::from_str("[\"host.example\".auth]\nstaging = false\ninternal = true\n").unwrap(); + + let entry = system + .merge_config(&user) + .unwrap() + .get(&host("host.example")); assert!( - !entry.fetch().auth.is_granted(), - "overwriting an entry must not inherit the previous grant" + entry.fetch(Some(&container("releases"))).auth.is_granted(), + "a grant the user file never mentions must survive the merge" ); - assert_eq!(entry.endpoint().addressing, Addressing::HostStyle); + assert!( + !entry.fetch(Some(&container("staging"))).auth.is_granted(), + "an explicit `false` must revoke a lower-precedence grant" + ); + assert!(entry.fetch(Some(&container("internal"))).auth.is_granted()); } /// The defect this key type exists to kill: every one of these keys is a @@ -335,20 +424,31 @@ auth = true ("acct.blob.core.windows.net.", "acct.blob.core.windows.net"), ] { let map: AzureOptionsMap = - toml::from_str(&format!("[\"{written}\"]\nauth = true\n")).unwrap(); + toml::from_str(&format!("[\"{written}\".auth]\nreleases = true\n")).unwrap(); assert!( - map.get(&host(looked_up)).fetch().auth.is_granted(), + map.get(&host(looked_up)) + .fetch(Some(&container("releases"))) + .auth + .is_granted(), "the grant written as `{written}` did not apply to `{looked_up}`" ); // The key is stored canonically, so `keys()` reports what a lookup // would need rather than what happened to be typed — quoted, as the - // TOML path it has to be written as — and writing the table back out - // produces a key that parses to the same host. - assert_eq!(map.keys(), vec![format!("\"{looked_up}\"")], "{written}"); + // TOML path it has to be written as, with one key per grant so each is + // separately settable — and writing the table back out produces a key + // that parses to the same host. + assert_eq!( + map.keys(), + vec![ + format!("\"{looked_up}\""), + format!("\"{looked_up}\".auth.releases"), + ], + "{written}" + ); let written_back = toml::to_string(&map).unwrap(); assert!( - written_back.contains(&format!("[\"{looked_up}\"]")), + written_back.contains(&format!("[\"{looked_up}\"")), "{written} was written back as {written_back}" ); } @@ -358,12 +458,23 @@ auth = true /// that silently never matches. #[test] fn an_unparseable_key_is_rejected() { - let err = - toml::from_str::("[\"acct.blob.example/general\"]\nauth = true\n") - .expect_err("a key carrying a path must be rejected"); + let err = toml::from_str::( + "[\"acct.blob.example/general\".auth]\nreleases = true\n", + ) + .expect_err("a key carrying a path must be rejected"); assert!( err.to_string().contains("acct.blob.example/general"), "{err}" ); } + + /// The same rule one level down: a container key Azure would refuse is a config + /// error, since it can never match a request either. + #[test] + fn an_unusable_container_key_is_rejected() { + let err = + toml::from_str::("[\"acct.blob.example\".auth]\nReleases = true\n") + .expect_err("an uppercase container name must be rejected"); + assert!(err.to_string().contains("Releases"), "{err}"); + } } diff --git a/crates/rattler_config/test-data/compat/kitchen-sink.toml b/crates/rattler_config/test-data/compat/kitchen-sink.toml index 31c4416480..b12a3bfbee 100644 --- a/crates/rattler_config/test-data/compat/kitchen-sink.toml +++ b/crates/rattler_config/test-data/compat/kitchen-sink.toml @@ -39,16 +39,20 @@ endpoint-url = "https://s3.example.com" region = "eu-central-1" force-path-style = true -# A private channel on real Azure: the grant is the whole entry. -[azure-options."mycompany.blob.core.windows.net"] -auth = true +# A private channel on real Azure: the grant is written per container, so the +# anonymous-read container on the same account stays anonymous. +[azure-options."mycompany.blob.core.windows.net".auth] +releases = true +public = false # A local Azurite emulator: the entry is also the endpoint trust declaration. [azure-options."127.0.0.1:10000"] -auth = true scheme = "http" path-style = true +[azure-options."127.0.0.1:10000".auth] +general = true + [index-config] write-zst = true write-shards = false diff --git a/crates/rattler_config/test-data/compat/override-layer.toml b/crates/rattler_config/test-data/compat/override-layer.toml index 63404e741e..6f057b5cbd 100644 --- a/crates/rattler_config/test-data/compat/override-layer.toml +++ b/crates/rattler_config/test-data/compat/override-layer.toml @@ -18,3 +18,9 @@ solves = 9 endpoint-url = "https://minio.example.com" region = "auto" force-path-style = false + +# Grants merge per container, unlike every other nested table here: this layer +# adds one and revokes one, and the lower layer's `releases = true` survives both. +[azure-options."mycompany.blob.core.windows.net".auth] +staging = true +public = false diff --git a/crates/rattler_config/tests/compat.rs b/crates/rattler_config/tests/compat.rs index f3d6a78290..12a4f4633f 100644 --- a/crates/rattler_config/tests/compat.rs +++ b/crates/rattler_config/tests/compat.rs @@ -185,10 +185,11 @@ const EDIT_MATRIX: &[(&str, &str)] = &[ "s3-options.some-bucket", r#"{"endpoint-url": "https://s3.example.com", "region": "auto", "force-path-style": true}"#, ), - // Quoted, because every Azure authority contains dots. + // Quoted, because every Azure authority contains dots. The grant is a table + // keyed by container, so there is no host-wide `auth` value to set. ( r#"azure-options."acct.blob.core.windows.net""#, - r#"{"auth": true}"#, + r#"{"auth": {"releases": true}}"#, ), ]; @@ -236,6 +237,52 @@ fn edit_matrix_set_roundtrip_unset() { } } +/// A grant is settable per container, and there is no host-wide switch to set by +/// mistake: `azure-options."".auth` is a table of containers, so a bool +/// aimed at it has nowhere to land. That refusal is the central safety property of +/// the per-container design, and `config set` is where a user would try it. +#[test] +fn edit_grants_one_container_at_a_time() { + let host = rattler_azure::AzureHost::parse("acct.blob.core.windows.net").unwrap(); + let releases = rattler_azure::ContainerName::new("releases").unwrap(); + let public = rattler_azure::ContainerName::new("public").unwrap(); + + let mut config = Config::default(); + for (container, value) in [("releases", "true"), ("public", "false")] { + let key = format!(r#"azure-options."acct.blob.core.windows.net".auth.{container}"#); + config + .set(&key, Some(value.to_string())) + .unwrap_or_else(|e| panic!("{key} must be settable: {e}")); + } + + let entry = config.azure_options.get(&host); + assert!(entry.fetch(Some(&releases)).auth.is_granted()); + assert!(!entry.fetch(Some(&public)).auth.is_granted()); + + // The whole point: no host-level `auth` value exists to grant every container + // on the account at once. + assert!( + config + .set( + r#"azure-options."acct.blob.core.windows.net".auth"#, + Some("true".to_string()) + ) + .is_err(), + "a host-wide grant must not be settable" + ); + + // Revoking one grant leaves the other alone. + config + .set( + r#"azure-options."acct.blob.core.windows.net".auth.releases"#, + None, + ) + .unwrap(); + let entry = config.azure_options.get(&host); + assert!(!entry.fetch(Some(&releases)).auth.is_granted()); + assert_eq!(entry.grants().count(), 1, "`public` must survive"); +} + /// Unknown keys must be rejected by `set` (both set and unset direction). #[test] fn edit_rejects_unknown_keys() { @@ -281,6 +328,19 @@ fn merge_semantics() { let per_channel = &merged.repodata_config.per_channel[&prefix_dev]; assert_eq!(per_channel.disable_sharded, Some(true)); // from layer 1 assert_eq!(per_channel.disable_zstd, Some(true)); // from layer 2 + // `azure-options` grants merge per container: the lower layer's `releases` + // survives a layer that names only `staging` and `public`. + let mycompany = merged + .azure_options + .get(&rattler_azure::AzureHost::parse("mycompany.blob.core.windows.net").unwrap()); + for (container, granted) in [("releases", true), ("staging", true), ("public", false)] { + let container = rattler_azure::ContainerName::new(container).unwrap(); + assert_eq!( + mycompany.fetch(Some(&container)).auth.is_granted(), + granted, + "{container}" + ); + } // concurrency: explicitly set values win over the lower layer. assert_eq!(merged.concurrency.solves, 9); assert_eq!(merged.concurrency.downloads, 12); diff --git a/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap b/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap index 5738e529f3..fe492e5ecd 100644 --- a/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap +++ b/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap @@ -53,15 +53,21 @@ region = "auto" force-path-style = false [azure-options."127.0.0.1:10000"] -auth = true scheme = "http" path-style = true +[azure-options."127.0.0.1:10000".auth] +general = true + [azure-options."mycompany.blob.core.windows.net"] -auth = true scheme = "https" path-style = false +[azure-options."mycompany.blob.core.windows.net".auth] +public = false +releases = true +staging = true + [index-config] write-zst = true write-shards = false diff --git a/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap index 75b8f782b3..74d69a6306 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap @@ -209,14 +209,25 @@ expression: "(unused, normalized(config))" azure_options: AzureOptionsMap( { AzureHost("127.0.0.1:10000"): AzureEndpointOptions { - auth: DefaultChain, scheme: Http, addressing: PathStyle, + auth: { + ContainerName( + "general", + ): DefaultChain, + }, }, AzureHost("mycompany.blob.core.windows.net"): AzureEndpointOptions { - auth: DefaultChain, scheme: Https, addressing: HostStyle, + auth: { + ContainerName( + "public", + ): Anonymous, + ContainerName( + "releases", + ): DefaultChain, + }, }, }, ), diff --git a/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap index 1c889eb8f0..14dc4e1b0f 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap @@ -137,7 +137,20 @@ expression: "(unused, normalized(config))" }, ), azure_options: AzureOptionsMap( - {}, + { + AzureHost("mycompany.blob.core.windows.net"): AzureEndpointOptions { + scheme: Https, + addressing: HostStyle, + auth: { + ContainerName( + "public", + ): Anonymous, + ContainerName( + "staging", + ): DefaultChain, + }, + }, + }, ), index_config: IndexConfig { default: IndexChannelConfig { diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index 6468ffebb9..dddc268098 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -89,17 +89,20 @@ its port when the URL has one: | Field | Type | Description | | --- | --- | --- | -| `auth` | boolean | Whether credentials may be sent to this host **when fetching**. Defaults to `false`, which fetches anonymously, and is the only way a credential attaches to a host on the fetch path — so keep these entries in your user-level config file, never in a checked-in project file. It has no effect on `rattler-index` or `rattler upload`, which take their credentials from the command line. | | `scheme` | string | The scheme `az://` is rewritten to: `"https"` (default) or `"http"`. Use `http` for local emulators only. | | `path-style` | boolean | Where the storage account is found. `false` (default) reads it from the first host label. `true` reads it from the first path segment instead, which is the only form that works for an IP-literal or single-label host. | +| `auth` | table | Which containers on the host may be sent credentials **when fetching**: one ` = true` line each. A container not listed is fetched anonymously, so one account can hold private and anonymous-read containers side by side — which is what Azure's per-container RBAC enforces. There is no host-wide switch, because a grant covering every container on an account, including ones created later, is not something to be able to write by accident. This is the only way a credential attaches on the fetch path, so keep these entries in your user-level config file, never in a checked-in project file. It has no effect on `rattler-index` or `rattler upload`, which take their credentials from the command line. | -Indexing a channel in the Azurite emulator needs the two wire settings (add -`auth = true` as well if the same host is also fetched from): +Indexing a channel in the Azurite emulator needs the two wire settings (add an +`auth` line per container if the same host is also fetched from): ```toml [azure-options."127.0.0.1:10000"] scheme = "http" path-style = true + +[azure-options."127.0.0.1:10000".auth] +general = true ``` ```shell diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index fbf5858b22..4e649f7701 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -292,8 +292,9 @@ async fn main() -> anyhow::Result<()> { /// /// A host without an entry and a host with an empty entry are defined to behave /// identically, so this never has to report which of the two it found. The entry's -/// `auth` grant is not part of the result: indexing signs with the credential its -/// caller supplied, so there is no ambient chain for a grant to gate. +/// per-container grants are not part of the result: indexing signs with the +/// credential its caller supplied, so there is no ambient chain for a grant to +/// gate. #[cfg(feature = "azure")] fn azure_endpoint(config: &Option, host: &AzureHost) -> AzureEndpoint { config @@ -378,9 +379,11 @@ mod tests { let config = config_from( r#" [azure-options."127.0.0.1:10000"] - auth = true scheme = "http" path-style = true + + [azure-options."127.0.0.1:10000".auth] + general = true "#, ); let channel = diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs index 24b740dfc8..468fef1d19 100644 --- a/crates/rattler_index/tests/azure_azurite.rs +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -8,7 +8,6 @@ //! //! ```toml //! [azure-options."127.0.0.1:10000"] -//! auth = true //! scheme = "http" //! path-style = true //! ``` @@ -37,7 +36,7 @@ use std::{collections::HashMap, path::PathBuf}; use opendal::{Configurator, ErrorKind, Operator, services::AzblobConfig}; use rattler_azure::{ Addressing, Auth, AzureChannelUrl, AzureCredentials, AzureEndpoint, AzureEndpointOptions, - AzureHost, AzureScheme, + AzureHost, AzureScheme, ContainerName, }; use rattler_index::{IndexAzureConfig, PackageRevisionAssignment, index_azure}; @@ -83,9 +82,16 @@ fn channel(prefix: &str) -> AzureChannelUrl { /// The `azure-options` entry for the emulator: the only configuration these tests /// hand to the indexer. +/// +/// The grant on `CONTAINER` is for the fetch-side fixture below — `index_azure` +/// signs with the credential it is handed and never reads a grant — and it is +/// written per container because that is the only scope a grant has. fn azurite_options() -> AzureEndpointOptions { AzureEndpointOptions::new( - Auth::DefaultChain, + [( + ContainerName::new(CONTAINER).expect("azurite container name"), + Auth::DefaultChain, + )], AzureEndpoint { scheme: AzureScheme::Http, addressing: Addressing::PathStyle, @@ -154,7 +160,7 @@ async fn ensure_empty_prefix(prefix: &str) { fn azure_client() -> reqwest_middleware::ClientWithMiddleware { let options = HashMap::from([( AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), - azurite_options().fetch(), + azurite_options(), )]); reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) .with(rattler_networking::AzureMiddleware::new( @@ -206,7 +212,7 @@ async fn ensure_container() { /// Run `body` with the emulator credentials in the environment. /// /// reqsign's env provider sits first in its default chain, so a shared key is how -/// an `auth = true` grant resolves against Azurite — it rejects the AAD bearer +/// a container's grant resolves against Azurite — it rejects the AAD bearer /// tokens the rest of the chain produces. The chain itself is left alone. async fn with_azurite_credentials>(body: F) { temp_env::async_with_vars( diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs index 6732ff0572..03754f9641 100644 --- a/crates/rattler_networking/src/azure_middleware.rs +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -2,7 +2,9 @@ use std::collections::HashMap; use async_trait::async_trait; -use rattler_azure::{Auth, AzureChannelUrl, AzureFetchOptions, AzureHost}; +use rattler_azure::{ + Auth, AzureChannelUrl, AzureEndpointOptions, AzureFetchOptions, AzureHost, ContainerName, +}; use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; use reqsign_command_execute_tokio::TokioCommandExecute; use reqsign_core::{Context, OsEnv, ProvideCredential, Signer}; @@ -34,20 +36,29 @@ const X_MS_VERSION: &str = "2021-12-02"; /// credential can leak to a host the user never named, and an anonymous read of a /// public container does not block on the managed-identity / IMDS probe. /// -/// A credential attaches to a host only because an `azure-options` entry -/// for it says [`Auth::DefaultChain`], which comes from the user's `azure-options` -/// config table: +/// A credential attaches to a request only because the user's `azure-options` +/// table grants it to the *container* the request addresses: /// /// ```toml -/// [azure-options."mycompany.blob.core.windows.net"] -/// auth = true +/// [azure-options."mycompany.blob.core.windows.net".auth] +/// releases = true +/// # a container not listed is fetched anonymously, so one account can hold +/// # private and anonymous-read containers side by side — which is what Azure's +/// # per-container RBAC actually enforces. /// /// [azure-options."127.0.0.1:10000"] # Azurite -/// auth = true /// scheme = "http" +/// path-style = true +/// +/// [azure-options."127.0.0.1:10000".auth] +/// general = true /// ``` /// -/// Two consequences of the grant being explicit: +/// There is no host-level grant, by design: a single field meaning "every +/// container on this account, including the ones created later" is exactly the +/// mistake worth making unrepresentable. +/// +/// Three consequences of the grant being explicit: /// /// - **Nothing is inferred from the host name.** There is no allow-list of /// "official" Azure suffixes, and none is needed: a host nobody granted gets @@ -56,6 +67,10 @@ const X_MS_VERSION: &str = "2021-12-02"; /// - **A broken credential is a hard error.** Because the user asked for signing, /// an unusable credential must be reported, not silently downgraded to an /// anonymous request that Azure will answer with a confusing 404. +/// - **A new private container fails closed** until someone adds a line for it. +/// That is the deliberate cost of a per-container grant, and it is why the 404 +/// hint below prints the exact line to write: unhelped, the failure reads as "the +/// channel is broken". /// /// Entries are user-scoped by contract: a project- or workspace-level manifest /// must never be allowed to write one, since that would let a checked-out @@ -76,17 +91,69 @@ pub struct AzureMiddleware { /// reqsign signer; caches the resolved credential internally. signer: Signer, - /// Per-host endpoint options, keyed by the same normalized authority the - /// `azure-options` config table is keyed by. An absent host is *defined* to - /// behave as a defaulted entry (anonymous, https), so a miss is never a - /// separate code path. + /// Whole `azure-options` entries, keyed by the same normalized authority the + /// config table is keyed by. An absent host is *defined* to behave as a + /// defaulted entry (no grants, https), so a miss is never a separate code path. + /// + /// Entries and not the narrower [`AzureFetchOptions`], because resolving a + /// grant is two steps and they are ordered: the host's addressing decides which + /// path segment is the container, and only then can the container's grant be + /// read. The narrowing therefore happens per request, in [`Self::resolve`]. /// /// A plain `HashMap` rather than `rattler_config::AzureOptionsMap`, mirroring /// [`crate::S3Middleware`]: taking the config type would put a mandatory /// `rattler_config` edge on the `azure` feature. The constructors take any - /// iterator of host/options pairs instead, which `AzureOptionsMap` yields - /// directly from its own `fetch_options`. - options: HashMap, + /// iterator of host/entry pairs instead, which `AzureOptionsMap` yields + /// directly from its own `endpoint_options`. + options: HashMap, +} + +/// One `az://` request, resolved against the options table. +/// +/// The container is kept next to the grant it produced, because the message the +/// user needs when a request comes back 404 is the TOML line naming *that* +/// container — a hint naming only the host would be a line that grants the wrong +/// thing. +#[derive(Debug)] +struct Resolved { + /// The channel URL the request names. + channel: AzureChannelUrl, + + /// The container it addresses, when it addresses one. `None` for a URL with no + /// container segment — the host root, or a path too short for the host's + /// addressing — which has nothing to attribute a grant to. + container: Option, + + /// The grant for that container, and the wire scheme for the host. + options: AzureFetchOptions, +} + +/// What a resolved request asks of the signer. +/// +/// [`Signing::Granted`] carries the container whose entry granted it, so "sign +/// this, but for no container" is unrepresentable and the failure message can +/// always quote the line that asked for signing. A grant is only ever read out of a +/// container's entry in an `auth` table, so there is no other way for one to exist. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Signing<'a> { + /// No grant: send unsigned, and resolve no credential. + Anonymous, + + /// `container` is granted, so sign — and fail loudly if that is impossible. + Granted(&'a ContainerName), +} + +impl<'a> Signing<'a> { + /// The signing decision for a resolved request. + fn new(auth: Auth, container: Option<&'a ContainerName>) -> Self { + match (auth, container) { + (Auth::DefaultChain, Some(container)) => Self::Granted(container), + // `DefaultChain` without a container cannot arise — the grant was read + // out of a container's entry — and anonymous is the arm that sends + // nothing, which is the right way for an impossible pair to fall. + (Auth::DefaultChain | Auth::Anonymous, _) => Self::Anonymous, + } + } } impl AzureMiddleware { @@ -97,12 +164,13 @@ impl AzureMiddleware { /// client — proxy, CA bundle, and TLS settings carry through to those /// requests. /// - /// `options` is the `azure-options` table: the per-host grants, in any shape - /// that iterates them — `rattler_config::AzureOptionsMap::fetch_options` yields - /// exactly this. An empty iterator means every `az://` request is anonymous. + /// `options` is the `azure-options` table: the per-host entries carrying the + /// per-container grants, in any shape that iterates them — + /// `rattler_config::AzureOptionsMap::endpoint_options` yields exactly this. An + /// empty iterator means every `az://` request is anonymous. pub fn new( client: Client, - options: impl IntoIterator, + options: impl IntoIterator, ) -> Self { Self::with_credential_provider(client, DefaultCredentialProvider::new(), options) } @@ -116,7 +184,7 @@ impl AzureMiddleware { fn with_credential_provider( client: Client, provider: impl ProvideCredential + 'static, - options: impl IntoIterator, + options: impl IntoIterator, ) -> Self { let ctx = Context::new() .with_file_read(TokioFileRead) @@ -130,27 +198,44 @@ impl AzureMiddleware { } } - /// Resolve an `az://` request URL to the channel URL it names and the options - /// configured for its host. + /// Resolve an `az://` request URL to the channel URL it names, the container it + /// addresses, and the options that apply to it. /// /// Going through [`AzureChannelUrl`] is what keeps this middleware from owning /// a second copy of rules that live in `rattler_azure`: that parser is what /// rejects userinfo, and it normalizes the authority into the exact spelling /// the options table is keyed by, so a grant cannot miss over case, a trailing - /// dot, an IDNA name or an IP literal written oddly. + /// dot, an IDNA name or an IP literal written oddly. The container comes from + /// [`rattler_azure::container`] for the same reason — it is the same derivation + /// the write path's coordinates use, and two derivations that disagreed would + /// look a grant up for one container and send it to another. /// - fn resolve(&self, url: &Url) -> MiddlewareResult<(AzureChannelUrl, AzureFetchOptions)> { + /// The order is forced: the host's entry carries the addressing, the addressing + /// says which path segment is the container, and the container selects the + /// grant. Nothing earlier can know the container. + fn resolve(&self, url: &Url) -> MiddlewareResult { let channel = AzureChannelUrl::parse(url.as_str()).map_err(|e| { // The URL is not echoed back: the one rejection a user hits here is // userinfo, and quoting it would print their password. reqwest_middleware::Error::Middleware(anyhow::Error::from(e)) })?; - let options = self - .options - .get(channel.host()) - .copied() - .unwrap_or_default(); - Ok((channel, options)) + + // An absent host is defined to behave as a defaulted entry, so the fallback + // is a value and not a branch. + let unconfigured = AzureEndpointOptions::default(); + let entry = self.options.get(channel.host()).unwrap_or(&unconfigured); + + // A URL with no container segment resolves to no grant; a container segment + // Azure could never accept is a malformed endpoint, and saying so beats an + // anonymous request that comes back as an unexplained 401. + let container = rattler_azure::container(&channel, entry.endpoint().addressing) + .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::Error::from(e)))?; + + Ok(Resolved { + options: entry.fetch(container.as_ref()), + container, + channel, + }) } /// Whether the URL already carries an explicit SAS token (a `sig` query @@ -159,25 +244,26 @@ impl AzureMiddleware { url.query_pairs().any(|(key, _)| key == "sig") } - /// Sign a reqwest `Request` in place using reqsign, when `auth` grants it. + /// Sign a reqwest `Request` in place using reqsign, when `signing` grants it. /// /// Two cases return without invoking reqsign at all: /// - The URL already carries an explicit SAS (`?...&sig=...`). Signing would /// add an `Authorization` header that Azure prefers over the SAS, silently /// overriding the caller's explicit token. - /// - [`Auth::Anonymous`] — no grant. Crucially the credential is not *resolved* + /// - [`Signing::Anonymous`] — no grant. Crucially the credential is not *resolved* /// either: reqsign would otherwise probe the managed-identity / IMDS endpoint /// and block until it times out (~30s on a machine with no metadata service) /// before we could decide not to use the result, making every anonymous /// public-channel read pay that timeout — and it would pull an ambient /// credential into memory for a host the user never granted. /// - /// Under [`Auth::DefaultChain`] any signing failure is propagated, carrying the - /// host, the grant that required signing and the remedies. reqsign collapses "no + /// Under [`Signing::Granted`] any signing failure is propagated, carrying the + /// host, the container whose grant required signing and the remedies. reqsign + /// collapses "no /// credential" and "broken credential" into the same /// [`reqsign_core::ErrorKind::CredentialInvalid`], and since the user asked for /// signing there is no case left where going anonymous is the right answer. - async fn sign(&self, req: &mut Request, auth: Auth) -> MiddlewareResult<()> { + async fn sign(&self, req: &mut Request, signing: Signing<'_>) -> MiddlewareResult<()> { if Self::has_sas_token(req.url()) { return Ok(()); } @@ -187,8 +273,8 @@ impl AzureMiddleware { .insert("x-ms-version", http::HeaderValue::from_static(X_MS_VERSION)); } - match auth { - Auth::Anonymous => { + let container = match signing { + Signing::Anonymous => { // The authority, not `host_str()`: a message naming a host the user // could act on must carry the port, or it names a host that is not // the one in their config. @@ -198,8 +284,8 @@ impl AzureMiddleware { ); return Ok(()); } - Auth::DefaultChain => {} - } + Signing::Granted(container) => container, + }; let mut builder = http::Request::builder() .method(req.method().clone()) @@ -223,14 +309,14 @@ impl AzureMiddleware { self.signer.sign(&mut parts, None).await.map_err(|e| { let authority = req.url().authority(); reqwest_middleware::Error::Middleware(anyhow::anyhow!( - "could not resolve an Azure credential for `{authority}`, which \ - `[azure-options.\"{authority}\"] auth = true` requires: {e}\n\ + "could not resolve an Azure credential for `{container}` on `{authority}`, which \ + `[azure-options.\"{authority}\".auth]` `{container} = true` requires: {e}\n\ \n\ Try one of:\n\ \x20 - `az login`\n\ \x20 - `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_ACCOUNT_KEY` in the \ environment\n\ - \x20 - remove `auth = true` to fetch this host anonymously\n\ + \x20 - set `{container} = false` to fetch this container anonymously\n\ \n\ Debug logging lists the credential providers that were tried." )) @@ -261,29 +347,40 @@ impl Middleware for AzureMiddleware { return next.run(req, extensions).await; } - let (channel, options) = self.resolve(req.url())?; + let Resolved { + channel, + container, + options, + } = self.resolve(req.url())?; *req.url_mut() = channel.wire(options.scheme); - self.sign(&mut req, options.auth).await?; + self.sign(&mut req, Signing::new(options.auth, container.as_ref())) + .await?; let response = next.run(req, extensions).await?; // Azure answers an unauthorized read of a private container with 404, not // 403, so "no grant" and "no such blob" are the same status on the wire. - // Say so once per host, naming the config the user would have to write — - // spelled through `AzureHost` so the key printed is the key a lookup arrives - // with. - if response.status() == http::StatusCode::NOT_FOUND - && !options.auth.is_granted() - && first_404_for_host(channel.host()) + // Under a per-container grant a newly-created private container fails closed + // until someone writes a line for it, so this hint is what stands between + // that and a user reading "404" as "the channel is broken". Say it once per + // container, naming the config to write — spelled through `AzureHost` and + // `ContainerName` so the key printed is the key a lookup arrives with. + // + // A URL naming no container gets no hint: there is no line that would grant + // it anything. + if let Some(container) = container.filter(|_| { + response.status() == http::StatusCode::NOT_FOUND && !options.auth.is_granted() + }) && first_404_for_container(channel.host(), &container) { // One line, and spelled the way `AzureUrlError::InvalidHost` spells its // fix: a wrapped multi-line hint is harder to grep out of a log, and // the two guided messages should read as the same instruction. tracing::warn!( - "`{}` returned 404 and this host has no `azure-options` auth grant. Azure answers \ - an anonymous read of a *private* container with 404 rather than 403, so a missing \ - grant looks exactly like a missing file. If the container is private, grant it in \ - your user configuration with `[azure-options.\"{}\"]` and `auth = true`.", + "`{}` returned 404 and container `{container}` has no `azure-options` auth grant. \ + Azure answers an anonymous read of a *private* container with 404 rather than \ + 403, so a missing grant looks exactly like a missing file. If the container is \ + private, grant it in your user configuration with \ + `[azure-options.\"{}\".auth]` and `{container} = true`.", channel.canonical(), channel.host() ); @@ -293,56 +390,70 @@ impl Middleware for AzureMiddleware { } } -/// Whether `host` still owes the 404 hint, claiming it if so. +/// Whether this container still owes the 404 hint, claiming it if so. /// /// A 404 is the *normal* answer to plenty of requests a healthy public channel /// makes — the repodata gateway probes for a shard index under every subdir it /// fetches, and a non-sharded channel misses every time — so a hint emitted per /// response is a security warning printed repeatedly at users whose channel is -/// fine. Once per host per process is enough for the one case it is about: a -/// private container the user forgot to grant. -fn first_404_for_host(host: &AzureHost) -> bool { - static HINTED: std::sync::LazyLock>> = - std::sync::LazyLock::new(Default::default); +/// fine. Once per container per process is enough for the one case it is about: a +/// private container the user forgot to grant. Per container and not per host, +/// because the line to add differs per container: silencing a host after its first +/// ungranted container would leave the second one unexplained. +fn first_404_for_container(host: &AzureHost, container: &ContainerName) -> bool { + static HINTED: std::sync::LazyLock< + std::sync::Mutex>, + > = std::sync::LazyLock::new(Default::default); HINTED .lock() .expect("the 404-hint set is never held across a panic") - .insert(host.clone()) + .insert((host.clone(), container.clone())) } #[cfg(test)] mod tests { - use rattler_azure::AzureScheme; + use rattler_azure::{Addressing, AzureEndpoint, AzureScheme}; use super::*; + fn container(name: &str) -> ContainerName { + ContainerName::new(name).expect("test container name") + } + /// The `azure-options` table for one host, as a caller would build it. fn options( authority: &str, - options: AzureFetchOptions, - ) -> HashMap { + options: AzureEndpointOptions, + ) -> HashMap { HashMap::from([(AzureHost::parse(authority).expect("test host"), options)]) } - /// A grant with everything else defaulted: anonymous is the only interesting - /// axis in most of these tests. - fn granted() -> AzureFetchOptions { - AzureFetchOptions { - auth: Auth::DefaultChain, - ..Default::default() - } + /// An entry granting one container, with everything else defaulted: which + /// container is granted is the only interesting axis in most of these tests. + fn granting(container_name: &str) -> AzureEndpointOptions { + AzureEndpointOptions::new( + [(container(container_name), Auth::DefaultChain)], + AzureEndpoint::default(), + ) } - fn middleware(options: HashMap) -> AzureMiddleware { + fn middleware(options: HashMap) -> AzureMiddleware { AzureMiddleware::new(Client::new(), options) } /// Resolve a URL and hand back the wire spelling its options ask for. fn wire_of(middleware: &AzureMiddleware, url: &str) -> String { - let (channel, options) = middleware + let resolved = middleware .resolve(&Url::parse(url).expect("test url")) .expect("url should resolve"); - channel.wire(options.scheme).to_string() + resolved.channel.wire(resolved.options.scheme).to_string() + } + + /// Resolve a URL, or panic with the middleware's rejection. + fn resolve(middleware: &AzureMiddleware, url: &str) -> Resolved { + middleware + .resolve(&Url::parse(url).expect("test url")) + .unwrap_or_else(|err| panic!("{url} should resolve: {err}")) } /// With no entry the scheme defaults to https, and path, query and fragment @@ -360,9 +471,9 @@ mod tests { assert_eq!( wire_of( &middleware, - "az://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc#frag" + "az://acct.blob.core.windows.net/general/x.json?sv=2021&sig=abc#frag" ), - "https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc#frag" + "https://acct.blob.core.windows.net/general/x.json?sv=2021&sig=abc#frag" ); } @@ -371,19 +482,13 @@ mod tests { /// default, but `:443` would be under https and must not be dropped either. #[test] fn rewrites_to_http_for_an_emulator_entry() { - let emulator = middleware(options( - "127.0.0.1:10000", - AzureFetchOptions { - auth: Auth::DefaultChain, - scheme: AzureScheme::Http, - }, - )); + let emulator = middleware(options("127.0.0.1:10000", emulator_entry(["general"]))); assert_eq!( wire_of( &emulator, - "az://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" + "az://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json" ), - "http://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" + "http://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json" ); // The same host with no entry stays on https: an emulator grant must not @@ -391,9 +496,9 @@ mod tests { assert_eq!( wire_of( &middleware(HashMap::new()), - "az://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" + "az://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json" ), - "https://127.0.0.1:10000/devstoreaccount1/noarch/repodata.json" + "https://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json" ); } @@ -402,11 +507,127 @@ mod tests { /// "not authorized". Delegating to `AzureHost` on both sides is what buys this. #[test] fn a_grant_applies_regardless_of_how_the_host_is_spelled() { - let middleware = middleware(options("MyCompany.blob.core.windows.net.", granted())); - let (_, resolved) = middleware - .resolve(&Url::parse("az://mycompany.blob.core.windows.net/c/x.json").unwrap()) - .unwrap(); - assert!(resolved.auth.is_granted()); + let middleware = middleware(options( + "MyCompany.blob.core.windows.net.", + granting("releases"), + )); + assert!( + resolve( + &middleware, + "az://mycompany.blob.core.windows.net/releases/x.json" + ) + .options + .auth + .is_granted() + ); + } + + /// The point of the per-container table: one account holding a private and an + /// anonymous-read container is configurable, because the grant stops at the + /// container it names. Under a host-level grant the second URL here would be + /// signed too, and 403 for any identity holding no role on it. + #[test] + fn a_grant_stops_at_the_container_it_names() { + let middleware = middleware(options( + "mycompany.blob.core.windows.net", + AzureEndpointOptions::new( + [ + (container("releases"), Auth::DefaultChain), + // Redundant with omission, and legal: it says "deliberately + // unsigned" rather than "forgotten". + (container("public"), Auth::Anonymous), + ], + AzureEndpoint::default(), + ), + )); + + for (url, granted) in [ + ("az://mycompany.blob.core.windows.net/releases/x.json", true), + ("az://mycompany.blob.core.windows.net/public/x.json", false), + ("az://mycompany.blob.core.windows.net/staging/x.json", false), + ] { + assert_eq!( + resolve(&middleware, url).options.auth.is_granted(), + granted, + "{url}" + ); + } + } + + /// A container is found where the host's addressing says it is, so a grant on a + /// path-style host applies to the second segment and not the account in the + /// first. + #[test] + fn a_container_is_read_through_the_hosts_addressing() { + let path_style = middleware(options("127.0.0.1:10000", emulator_entry(["general"]))); + let resolved = resolve( + &path_style, + "az://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json", + ); + assert_eq!(resolved.container, Some(container("general"))); + assert!(resolved.options.auth.is_granted()); + + // Host-style on the same URL reads the account segment as the container, so + // the grant does not apply — the addressing is what decides which name a + // grant is even about. + let host_style = middleware(options( + "127.0.0.1:10000", + AzureEndpointOptions::new( + [(container("general"), Auth::DefaultChain)], + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::HostStyle, + }, + ), + )); + let resolved = resolve( + &host_style, + "az://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json", + ); + assert_eq!(resolved.container, Some(container("devstoreaccount1"))); + assert!(!resolved.options.auth.is_granted()); + } + + /// A URL naming no container has nothing to attribute a grant to, so it is + /// anonymous rather than an error: the fetch path stays total for URLs that are + /// not channel-scoped, which is what it is today. + #[test] + fn a_url_without_a_container_is_anonymous() { + let middleware = middleware(options( + "mycompany.blob.core.windows.net", + granting("releases"), + )); + + for url in [ + "az://mycompany.blob.core.windows.net", + "az://mycompany.blob.core.windows.net/", + "az://mycompany.blob.core.windows.net/?comp=list", + ] { + let resolved = resolve(&middleware, url); + assert_eq!(resolved.container, None, "{url}"); + assert!(!resolved.options.auth.is_granted(), "{url}"); + } + } + + /// A container segment Azure could never accept is a malformed endpoint, not an + /// ungranted one: no legitimate request can land here, and going quietly + /// anonymous would surface as an unexplained 401 rather than naming the fault. + #[test] + fn a_url_with_an_unusable_container_is_refused() { + let middleware = middleware(options( + "mycompany.blob.core.windows.net", + granting("releases"), + )); + + for url in [ + "az://mycompany.blob.core.windows.net/Releases/x.json", + "az://mycompany.blob.core.windows.net/ab/x.json", + ] { + let err = middleware + .resolve(&Url::parse(url).unwrap()) + .expect_err("an illegal container name must be refused"); + assert!(err.to_string().contains("container name"), "{url}: {err}"); + } } /// Userinfo is refused before any rewrite or signing: the host is the request @@ -416,8 +637,8 @@ mod tests { fn rejects_userinfo() { let middleware = middleware(HashMap::new()); for url in [ - "az://user:pass@acct.blob.core.windows.net/c/x.json", - "az://user@acct.blob.core.windows.net/c/x.json", + "az://user:pass@acct.blob.core.windows.net/general/x.json", + "az://user@acct.blob.core.windows.net/general/x.json", ] { let err = middleware .resolve(&Url::parse(url).unwrap()) @@ -426,7 +647,7 @@ mod tests { } assert!( middleware - .resolve(&Url::parse("az://acct.blob.core.windows.net/c/x.json").unwrap()) + .resolve(&Url::parse("az://acct.blob.core.windows.net/general/x.json").unwrap()) .is_ok() ); } @@ -446,12 +667,12 @@ mod tests { assert!(result.is_err()); } - /// A host with no grant is sent unsigned, and its credential is never even + /// A container with no grant is sent unsigned, and its credential is never even /// resolved — so nothing blocks on the IMDS probe and no ambient credential is /// pulled into memory for a host the user never named. The provider flips a /// flag if it is ever asked. #[tokio::test] - async fn an_ungranted_host_sends_unsigned_without_resolving_a_credential() { + async fn an_ungranted_container_sends_unsigned_without_resolving_a_credential() { use std::sync::{ Arc, atomic::{AtomicBool, Ordering}, @@ -482,7 +703,7 @@ mod tests { .unwrap(); middleware - .sign(&mut req, Auth::Anonymous) + .sign(&mut req, Signing::Anonymous) .await .expect("an ungranted request must pass through unsigned"); @@ -500,10 +721,10 @@ mod tests { ); } - /// A granted host is actually signed: the credential resolves and the request - /// comes back carrying Shared Key authorization. + /// A granted container is actually signed: the credential resolves and the + /// request comes back carrying Shared Key authorization. #[tokio::test] - async fn a_granted_host_is_signed() { + async fn a_granted_container_is_signed() { use reqsign_azure_storage::StaticCredentialProvider; let middleware = AzureMiddleware::with_credential_provider( @@ -511,19 +732,22 @@ mod tests { // A valid base64 account key, so the provider yields a usable // SharedKey credential. StaticCredentialProvider::new_shared_key("acct", "dGVzdF9rZXk="), - options("acct.blob.core.windows.net", granted()), + options("acct.blob.core.windows.net", granting("releases")), ); let mut req = Client::new() - .get("https://acct.blob.core.windows.net/c/noarch/repodata.json") + .get("https://acct.blob.core.windows.net/releases/noarch/repodata.json") .build() .unwrap(); - middleware.sign(&mut req, Auth::DefaultChain).await.unwrap(); + middleware + .sign(&mut req, Signing::Granted(&container("releases"))) + .await + .unwrap(); let authorization = req .headers() .get(http::header::AUTHORIZATION) - .expect("a granted host must be signed"); + .expect("a granted container must be signed"); assert!( authorization.to_str().unwrap().starts_with("SharedKey "), "{authorization:?}" @@ -536,20 +760,22 @@ mod tests { /// empty provider chain resolves nothing, which reqsign reports the same way it /// reports a broken credential. #[tokio::test] - async fn a_granted_host_with_broken_credentials_is_a_hard_error() { + async fn a_granted_container_with_broken_credentials_is_a_hard_error() { use reqsign_core::ProvideCredentialChain; let middleware = AzureMiddleware::with_credential_provider( Client::new(), ProvideCredentialChain::::new(), - options("acct.blob.core.windows.net", granted()), + options("acct.blob.core.windows.net", granting("releases")), ); let mut req = Client::new() - .get("https://acct.blob.core.windows.net/c/noarch/repodata.json") + .get("https://acct.blob.core.windows.net/releases/noarch/repodata.json") .build() .unwrap(); - let result = middleware.sign(&mut req, Auth::DefaultChain).await; + let result = middleware + .sign(&mut req, Signing::Granted(&container("releases"))) + .await; assert!( result.is_err(), @@ -561,11 +787,13 @@ mod tests { ); // reqsign's own message names neither the host nor a remedy, and the chain - // hides which provider failed. Everything actionable has to come from here. + // hides which provider failed. Everything actionable has to come from here — + // including which container's grant asked for the signing, since the entry + // may hold several and only one line is at fault. let message = result.unwrap_err().to_string(); for expected in [ "acct.blob.core.windows.net", - "auth = true", + "releases = true", "az login", "AZURE_STORAGE_ACCOUNT_KEY", ] { @@ -577,8 +805,8 @@ mod tests { } /// A URL that already carries a SAS token must not be re-signed even where the - /// host is granted: Azure prefers an `Authorization` header over the SAS, so - /// signing would silently override the caller's explicit token. + /// container is granted: Azure prefers an `Authorization` header over the SAS, + /// so signing would silently override the caller's explicit token. #[tokio::test] async fn a_sas_in_the_url_passes_through() { use reqsign_azure_storage::StaticCredentialProvider; @@ -586,14 +814,17 @@ mod tests { let middleware = AzureMiddleware::with_credential_provider( Client::new(), StaticCredentialProvider::new_shared_key("acct", "dGVzdF9rZXk="), - options("acct.blob.core.windows.net", granted()), + options("acct.blob.core.windows.net", granting("releases")), ); let mut req = Client::new() - .get("https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc") + .get("https://acct.blob.core.windows.net/releases/x.json?sv=2021&sig=abc") .build() .unwrap(); - middleware.sign(&mut req, Auth::DefaultChain).await.unwrap(); + middleware + .sign(&mut req, Signing::Granted(&container("releases"))) + .await + .unwrap(); assert!( req.headers().get(http::header::AUTHORIZATION).is_none(), @@ -605,7 +836,7 @@ mod tests { ); assert_eq!( req.url().as_str(), - "https://acct.blob.core.windows.net/c/x.json?sv=2021&sig=abc" + "https://acct.blob.core.windows.net/releases/x.json?sv=2021&sig=abc" ); } @@ -619,21 +850,30 @@ mod tests { AzureHost::parse(&addr.to_string()).unwrap() } - /// An emulator-shaped entry (http, path-style) with the grant taken from the - /// caller, so one server can exercise both sides of the hint. - fn emulator_entry(auth: Auth) -> AzureFetchOptions { - AzureFetchOptions { - auth, - scheme: AzureScheme::Http, - } + /// An emulator-shaped entry (http, path-style) granting whichever containers the + /// caller names, so one server can exercise both sides of the hint. + fn emulator_entry<'a>(granted: impl IntoIterator) -> AzureEndpointOptions { + AzureEndpointOptions::new( + granted + .into_iter() + .map(|name| (container(name), Auth::DefaultChain)), + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) } - async fn get_az(middleware: AzureMiddleware, host: &AzureHost) -> reqwest::StatusCode { + async fn get_az( + middleware: AzureMiddleware, + host: &AzureHost, + container: &str, + ) -> reqwest::StatusCode { reqwest_middleware::ClientBuilder::new(Client::new()) .with(middleware) .build() .get(format!( - "az://{host}/devstoreaccount1/c/noarch/repodata.json" + "az://{host}/devstoreaccount1/{container}/noarch/repodata.json" )) .send() .await @@ -643,70 +883,74 @@ mod tests { /// The 404 hint must name the config block to add, keyed exactly as the table /// is keyed — including the port, which an earlier version of this hint - /// dropped, printing a key that could never match. + /// dropped, printing a key that could never match — and the container, since + /// that is the line the user has to write. #[tokio::test] #[tracing_test::traced_test] - async fn the_404_hint_names_the_config_block_for_an_ungranted_host() { + async fn the_404_hint_names_the_config_block_for_an_ungranted_container() { let host = spawn_404_server().await; - let middleware = middleware(options(&host.to_string(), emulator_entry(Auth::Anonymous))); + let middleware = middleware(options(&host.to_string(), emulator_entry([]))); - assert_eq!(get_az(middleware, &host).await, 404); + assert_eq!(get_az(middleware, &host, "general").await, 404); - assert!(logs_contain(&format!("[azure-options.\"{host}\"]"))); - assert!(logs_contain("auth = true")); + assert!(logs_contain(&format!("[azure-options.\"{host}\".auth]"))); + assert!(logs_contain("general = true")); } /// A public non-sharded channel 404s on every shard-index probe the repodata /// gateway makes, so a hint per response is a security warning repeated at a - /// user whose channel is healthy. + /// user whose channel is healthy. It is silenced per container, though: a second + /// ungranted container needs a different line, so it gets its own hint. #[tokio::test] #[tracing_test::traced_test] - async fn the_404_hint_is_emitted_once_per_host() { + async fn the_404_hint_is_emitted_once_per_container() { let host = spawn_404_server().await; let client = reqwest_middleware::ClientBuilder::new(Client::new()) - .with(middleware(options( - &host.to_string(), - emulator_entry(Auth::Anonymous), - ))) + .with(middleware(options(&host.to_string(), emulator_entry([])))) .build(); - for subdir in ["noarch", "linux-64", "osx-64"] { - let status = client - .get(format!( - "az://{host}/devstoreaccount1/c/{subdir}/repodata_shards.msgpack.zst" - )) - .send() - .await - .expect("request through azure middleware failed") - .status(); - assert_eq!(status, 404); + for container in ["general", "staging"] { + for subdir in ["noarch", "linux-64", "osx-64"] { + let status = client + .get(format!( + "az://{host}/devstoreaccount1/{container}/{subdir}/\ + repodata_shards.msgpack.zst" + )) + .send() + .await + .expect("request through azure middleware failed") + .status(); + assert_eq!(status, 404); + } } - logs_assert(|lines: &[&str]| { - let hints = lines - .iter() - .filter(|line| line.contains("auth = true")) - .count(); - (hints == 1) - .then_some(()) - .ok_or_else(|| format!("expected exactly one hint, got {hints}")) - }); + for container in ["general", "staging"] { + logs_assert(move |lines: &[&str]| { + let hints = lines + .iter() + .filter(|line| line.contains(&format!("{container} = true"))) + .count(); + (hints == 1).then_some(()).ok_or_else(|| { + format!("expected exactly one hint for {container}, got {hints}") + }) + }); + } } /// With a grant in place a 404 means what it says, so the hint would be noise. #[tokio::test] #[tracing_test::traced_test] - async fn the_404_hint_is_silent_for_a_granted_host() { + async fn the_404_hint_is_silent_for_a_granted_container() { use reqsign_azure_storage::StaticCredentialProvider; let host = spawn_404_server().await; let middleware = AzureMiddleware::with_credential_provider( Client::new(), StaticCredentialProvider::new_shared_key("devstoreaccount1", "dGVzdF9rZXk="), - options(&host.to_string(), emulator_entry(Auth::DefaultChain)), + options(&host.to_string(), emulator_entry(["general"])), ); - assert_eq!(get_az(middleware, &host).await, 404); + assert_eq!(get_az(middleware, &host, "general").await, 404); assert!(!logs_contain("azure-options")); } diff --git a/crates/rattler_networking/tests/azure_azurite_fetch.rs b/crates/rattler_networking/tests/azure_azurite_fetch.rs index d0bee7aa22..1786f35dc3 100644 --- a/crates/rattler_networking/tests/azure_azurite_fetch.rs +++ b/crates/rattler_networking/tests/azure_azurite_fetch.rs @@ -6,9 +6,11 @@ //! //! ```toml //! [azure-options."127.0.0.1:10000"] -//! auth = true //! scheme = "http" //! path-style = true +//! +//! [azure-options."127.0.0.1:10000".auth] +//! cli-channel = true //! ``` //! //! Run with: @@ -26,7 +28,9 @@ use std::{ }; use async_trait::async_trait; -use rattler_azure::{Auth, AzureFetchOptions, AzureHost, AzureScheme}; +use rattler_azure::{ + Addressing, Auth, AzureEndpoint, AzureEndpointOptions, AzureHost, AzureScheme, ContainerName, +}; use rattler_networking::AzureMiddleware; use reqwest::{ Request, Response, @@ -79,17 +83,27 @@ fn channel_url() -> String { format!("az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}") } -/// The one `azure-options` entry these tests run on, with `auth` as the only -/// variable. `scheme` and `path-style` stay set even in the ungranted case: the -/// entry is what makes the emulator reachable at all, and keeping it identical +/// The one `azure-options` entry these tests run on, with the container's grant as +/// the only variable. `scheme` and `path-style` stay set even in the ungranted case: +/// the entry is what makes the emulator reachable at all, and keeping it identical /// means the two tests differ in the grant and nothing else. -fn azurite_entry(auth: Auth) -> HashMap { +/// +/// The grant is written for `CONTAINER` specifically, which is also what makes the +/// ungranted case below a real test of the per-container lookup rather than of an +/// empty table: `Auth::Anonymous` here is the container named and *refused*. +fn azurite_entry(auth: Auth) -> HashMap { HashMap::from([( AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), - AzureFetchOptions { - auth, - scheme: AzureScheme::Http, - }, + AzureEndpointOptions::new( + [( + ContainerName::new(CONTAINER).expect("azurite container name"), + auth, + )], + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ), )]) } @@ -223,7 +237,8 @@ async fn azurite_granted_entry_fetches_repodata() { .await; } -/// Without `auth = true` the request goes out unsigned, and a private container +/// Without a grant for this container the request goes out unsigned, and a private +/// container /// refuses it. This is the core claim of the anonymous-by-default model. /// /// The primary assertion is on the outgoing request, not the status: no diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs index 1dc90ac47b..e57dc6d50b 100644 --- a/crates/rattler_networking/tests/azure_real_fetch.rs +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -19,7 +19,7 @@ use std::collections::HashMap; -use rattler_azure::{Auth, AzureFetchOptions, AzureHost}; +use rattler_azure::{Auth, AzureEndpoint, AzureEndpointOptions, AzureHost, ContainerName}; use rattler_networking::AzureMiddleware; use reqwest_middleware::ClientBuilder; @@ -34,15 +34,20 @@ async fn azure_middleware_fetches_real_repodata() { .unwrap_or_else(|_| format!("{account}.blob.core.windows.net")); // The grant is what makes this the AAD test rather than an anonymous read: an - // `azure-options` entry for the host is the only thing that lets the + // `azure-options` entry naming this container is the only thing that lets the // `az login` credential attach to it, and it makes a broken credential fail - // loudly instead of falling through to an unsigned 404. + // loudly instead of falling through to an unsigned 404. Sibling containers on + // the same account are unaffected, which is the point of keying it per + // container. let options = HashMap::from([( AzureHost::parse(&host).expect("AZURE_TEST_HOST is not a valid host[:port]"), - AzureFetchOptions { - auth: Auth::DefaultChain, - ..Default::default() - }, + AzureEndpointOptions::new( + [( + ContainerName::new(&container).expect("AZURE_TEST_CONTAINER is not a valid name"), + Auth::DefaultChain, + )], + AzureEndpoint::default(), + ), )]); let client = ClientBuilder::new(reqwest::Client::new()) From 1ebf8ecca7b37e0fa7e828795d66d7e5d296640b Mon Sep 17 00:00:00 2001 From: tenzinplatter <143778894+TenzinPlatter@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:34:22 +1000 Subject: [PATCH 98/98] docs: correct stale per-host wording in azure grant docs --- crates/rattler_azure/src/lib.rs | 5 +++-- crates/rattler_azure/src/options.rs | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs index 1e4d00c5ee..ae3c335174 100644 --- a/crates/rattler_azure/src/lib.rs +++ b/crates/rattler_azure/src/lib.rs @@ -9,8 +9,9 @@ //! [`options`] and never inferred from the host name: the wire scheme and the //! addressing per host, and credentials per *container*, because that is the scope //! Azure's own RBAC has. The default grant is [`Auth::Anonymous`], so naming a host -//! or a container in a URL by itself sends nothing to it. Nothing here signs or sends a request either — that lives -//! in `rattler_networking` — but two functions do handle a credential: +//! or a container in a URL by itself sends nothing to it. Nothing here signs or +//! sends a request either — that lives in `rattler_networking` — but two functions +//! do handle a credential: //! `azblob_config` embeds the account key or SAS it is handed into the config it //! returns, and `mint_user_delegation_sas` spends the user's `az login` session to //! obtain one. Deriving coordinates from a URL ([`account_and_container`]) diff --git a/crates/rattler_azure/src/options.rs b/crates/rattler_azure/src/options.rs index 9dfe905610..9fdf071720 100644 --- a/crates/rattler_azure/src/options.rs +++ b/crates/rattler_azure/src/options.rs @@ -4,9 +4,9 @@ //! a host or one of its containers anything: without one, a channel on that host //! is fetched anonymously over https in host-style addressing. There is //! deliberately no hardcoded list of "official" Azure suffixes — since a grant must -//! be written per host, suffix classification carries no security weight, and the -//! absence of the list is what lets custom endpoints and the Azurite emulator work -//! at all. +//! be written out for the container it applies to, suffix classification carries no +//! security weight, and the absence of the list is what lets custom endpoints and +//! the Azurite emulator work at all. //! //! # Three types, one table //!