diff --git a/.github/workflows/e2e-azure-tests.yml b/.github/workflows/e2e-azure-tests.yml new file mode 100644 index 0000000000..06e21c8869 --- /dev/null +++ b/.github/workflows/e2e-azure-tests.yml @@ -0,0 +1,71 @@ +on: + push: + branches: [main] + pull_request: + paths: + - crates/rattler_azure/** + - crates/rattler_config/** + - crates/rattler_index/** + - crates/rattler_networking/** + - crates/rattler_upload/** + # An opendal bump touches only the root manifest, and these tests are the + # canary for the opendal bug the upload path works around. + - Cargo.toml + - Cargo.lock + - test-data/** + - pixi.toml + - pixi.lock + - .github/workflows/e2e-azure-tests.yml + +name: E2E Azure Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + RUST_LOG: info + RUST_BACKTRACE: 1 + CARGO_TERM_COLOR: always + +jobs: + e2e-azurite-test: + name: E2E Index/Fetch [Azurite] + runs-on: ubuntu-latest + + env: + # Enable sccache. + # + # This environment variable is picked up by pixi build which will then + # set up the rust build using sccache. + SCCACHE_GHA_ENABLED: "true" + + # No secrets: the emulator's `devstoreaccount1` credentials are published + # constants, spelled out in the tests, and only ever reach loopback. + services: + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.36.0 + ports: + - 10000:10000 + options: >- + --health-cmd "nc -z 127.0.0.1 10000" + --health-interval 2s + --health-timeout 2s + --health-retries 30 + + steps: + - name: Checkout source code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + submodules: recursive + + - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11 # v0.9.6 + with: + environments: default + + - name: Run the Azurite integration tests + run: | + pixi run -vv -- cargo nextest run --run-ignored all \ + --features rattler_networking/azure \ + -p rattler_index --test azure_azurite \ + -p rattler_networking --test azure_azurite_fetch diff --git a/Cargo.lock b/Cargo.lock index 295e4f503a..f942a0258d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4075,6 +4075,7 @@ checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" dependencies = [ "opendal-core", "opendal-layer-retry", + "opendal-service-azblob", "opendal-service-fs", "opendal-service-s3", ] @@ -4118,6 +4119,37 @@ dependencies = [ "opendal-core", ] +[[package]] +name = "opendal-service-azblob" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +dependencies = [ + "base64 0.22.1", + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.39.4", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.11.0", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +dependencies = [ + "http 1.4.2", + "opendal-core", +] + [[package]] name = "opendal-service-fs" version = "0.57.0" @@ -4357,6 +4389,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "pem" version = "3.0.6" @@ -4514,6 +4556,21 @@ dependencies = [ "spki", ] +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -4521,6 +4578,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", + "pkcs5", + "rand_core 0.6.4", "spki", ] @@ -4990,6 +5049,26 @@ dependencies = [ "url", ] +[[package]] +name = "rattler_azure" +version = "0.1.0" +dependencies = [ + "clap", + "indexmap 2.14.0", + "jiff", + "opendal", + "percent-encoding", + "secrecy", + "serde", + "thiserror 2.0.19", + "tokio", + "toml", + "tracing", + "tracing-test", + "url", + "which", +] + [[package]] name = "rattler_cache" version = "0.10.4" @@ -5091,6 +5170,7 @@ dependencies = [ "fs-err", "indexmap 2.14.0", "insta", + "rattler_azure", "rattler_conda_types", "serde", "serde_ignored", @@ -5152,6 +5232,7 @@ version = "0.30.10" dependencies = [ "ahash", "anyhow", + "astral-reqwest-middleware", "bytes", "clap", "clap-verbosity-flag", @@ -5163,11 +5244,13 @@ dependencies = [ "indicatif", "jiff", "opendal", + "rattler_azure", "rattler_conda_types", "rattler_config", "rattler_digest", "rattler_networking", "rattler_package_streaming", + "rattler_redaction", "rattler_repodata_gateway", "rattler_s3", "reqwest", @@ -5177,6 +5260,7 @@ dependencies = [ "serde_json", "sha2 0.11.0", "tar", + "temp-env", "tempfile", "thiserror 2.0.19", "tokio", @@ -5298,8 +5382,14 @@ dependencies = [ "itertools 0.15.0", "keyring-core", "netrc-rs", + "rattler_azure", "rattler_config", "regex", + "reqsign-azure-storage", + "reqsign-command-execute-tokio", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-http-send-reqwest", "reqwest", "retry-policies", "rstest", @@ -5561,6 +5651,7 @@ dependencies = [ "indicatif", "miette", "opendal", + "rattler_azure", "rattler_conda_types", "rattler_config", "rattler_digest", @@ -5737,6 +5828,37 @@ dependencies = [ "sha1 0.11.0", ] +[[package]] +name = "reqsign-azure-storage" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ebd8524185ce9c64063e3095f83968acfa90922f00c601a4a0f3aca15b077e" +dependencies = [ + "anyhow", + "base64 0.22.1", + "bytes", + "form_urlencoded", + "http 1.4.2", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-command-execute-tokio" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc7d03c76a4d849a872d6fee4b6217b5e47a1234d1b8c65833779f200f2c953" +dependencies = [ + "reqsign-core", + "tokio", +] + [[package]] name = "reqsign-core" version = "3.2.0" @@ -5753,6 +5875,9 @@ dependencies = [ "jiff", "log", "percent-encoding", + "rsa", + "serde", + "serde_json", "sha1 0.11.0", "sha2 0.11.0", "windows-sys 0.61.2", @@ -5769,6 +5894,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "reqsign-http-send-reqwest" +version = "4.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d87ab629e40e93184828f462d3e8f444017d4afbf5b0afc04b5b203caf748bc" +dependencies = [ + "anyhow", + "bytes", + "futures-channel", + "http 1.4.2", + "http-body-util", + "reqsign-core", + "reqwest", + "wasm-bindgen-futures", +] + [[package]] name = "reqwest" version = "0.13.4" @@ -5904,6 +6045,7 @@ dependencies = [ "pkcs1", "pkcs8", "rand_core 0.6.4", + "sha2 0.10.9", "signature", "spki", "subtle", @@ -6107,6 +6249,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6155,6 +6306,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sec1" version = "0.7.3" diff --git a/Cargo.toml b/Cargo.toml index 056386e089..9add3b1373 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,11 @@ google-cloud-auth = { version = "1", default-features = false, features = [ aws-config = { version = "1", default-features = false } aws-sdk-s3 = { version = "1", default-features = false } aws-smithy-http-client = { version = "1", default-features = false } +reqsign-core = "3.0.1" +reqsign-azure-storage = "3.0.1" +reqsign-command-execute-tokio = "3.0.1" +reqsign-file-read-tokio = "3.0.1" +reqsign-http-send-reqwest = "4.0.1" aws-credential-types = { version = "1", default-features = false } hashbrown = "0.17" hex = "0.4" @@ -148,6 +153,7 @@ rstest = { version = "0.26" } rstest_reuse = "0.7" rustix = { version = "1.1", default-features = false } simd-json = { version = "0.17", features = ["serde_impl"] } +secrecy = "0.10" self_cell = "1" serde = { version = "1" } serde_bytes = { version = "0.11" } @@ -211,6 +217,7 @@ coalesced_map = { path = "crates/coalesced_map", version = "=0.1.4", default-fea file_url = { path = "crates/file_url", version = "=0.3.2", default-features = false } path_resolver = { path = "crates/path_resolver", version = "=0.2.12", default-features = false } rattler = { path = "crates/rattler", version = "=0.48.0", default-features = false } +rattler_azure = { path = "crates/rattler_azure", version = "=0.1.0", default-features = false } rattler_cache = { path = "crates/rattler_cache", version = "=0.10.4", default-features = false } rattler_conda_types = { path = "crates/rattler_conda_types", version = "=0.49.0", default-features = false } rattler_config = { path = "crates/rattler_config", version = "=0.6.2", default-features = false } diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index b7c44db220..c9a0fa12e3 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -16,13 +16,14 @@ name = "rattler" path = "src/main.rs" [features] -default = ["rustls", "s3", "gcs", "oauth"] +default = ["rustls", "s3", "gcs", "azure", "oauth"] native-tls = [ "reqwest/native-tls", "rattler/native-tls", "rattler_repodata_gateway/native-tls", "rattler_networking/native-tls", "rattler_cache/native-tls", + "rattler_upload/native-tls", ] rustls = [ "reqwest/rustls", @@ -30,9 +31,11 @@ rustls = [ "rattler_repodata_gateway/rustls", "rattler_networking/rustls", "rattler_cache/rustls", + "rattler_upload/rustls", ] s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] +azure = ["rattler_networking/azure", "rattler_upload/azure"] oauth = ["rattler/oauth"] [dependencies] @@ -68,7 +71,9 @@ rattler_menuinst = { workspace = true, default-features = false } rattler_package_streaming = { workspace = true, default-features = false, features = [ "reqwest", ] } -rattler_upload = { workspace = true } +rattler_upload = { workspace = true, default-features = false, features = [ + "sigstore-sign", +] } rattler_shell = { workspace = true, default-features = false } reqwest = { workspace = true, features = ["stream", "query"] } reqwest-middleware = { workspace = true } diff --git a/crates/rattler-bin/src/commands/client.rs b/crates/rattler-bin/src/commands/client.rs index 6897a16d15..fc322373fe 100644 --- a/crates/rattler-bin/src/commands/client.rs +++ b/crates/rattler-bin/src/commands/client.rs @@ -1,4 +1,6 @@ -use std::{collections::HashMap, sync::Arc}; +#[cfg(any(feature = "s3", feature = "azure"))] +use std::collections::HashMap; +use std::sync::Arc; use miette::{Context, IntoDiagnostic}; use rattler_networking::{ @@ -49,7 +51,9 @@ pub fn create_client_with_middleware( ))) .with_arc(Arc::new(AuthChallengeMiddleware::default())); - let client = client.with(rattler_networking::OciMiddleware::new(download_client)); + let client = client.with(rattler_networking::OciMiddleware::new( + download_client.clone(), + )); #[cfg(feature = "s3")] let client = client.with(rattler_networking::S3Middleware::new( HashMap::new(), @@ -57,6 +61,14 @@ pub fn create_client_with_middleware( )); #[cfg(feature = "gcs")] let client = client.with(rattler_networking::GCSMiddleware::default()); + // `az://` URLs carry the full blob endpoint, so the middleware needs no + // configuration to *reach* one — it just swaps the scheme. The empty options + // table means every `az://` fetch here is anonymous. + #[cfg(feature = "azure")] + let client = client.with(rattler_networking::AzureMiddleware::new( + download_client, + [], + )); Ok(client.build()) } diff --git a/crates/rattler_azure/Cargo.toml b/crates/rattler_azure/Cargo.toml new file mode 100644 index 0000000000..15b1f5b135 --- /dev/null +++ b/crates/rattler_azure/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "rattler_azure" +version = "0.1.0" +description = "A crate to streamline interaction with Azure Blob storage for rattler" +categories.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +edition.workspace = true +readme.workspace = true + +[features] +default = [] +# CLI credential options; resolving them may mint a short-lived user-delegation +# SAS by shelling out to the Azure CLI (which needs `jiff` for the expiry and +# `tokio` to run `az` without blocking the async runtime). +clap = ["dep:clap", "dep:jiff", "dep:which", "dep:tokio"] +# `azblob_config`: build an opendal `AzblobConfig` from a channel URL. +opendal = ["dep:opendal"] +# Serde derives for the `azure-options` config table: the per-host endpoint +# options, the `AzureHost` used as their map key and the `ContainerName` keying +# their per-container grants. The types themselves are always compiled; only the +# derives are gated. Pulls in `serde` alone, so config crates can depend on this +# crate with default features off. +serde = ["dep:serde", "indexmap/serde"] + +[dependencies] +clap = { workspace = true, optional = true } +# The per-container grant table, which keeps the order the config file wrote. +indexmap = { workspace = true } +jiff = { workspace = true, optional = true } +opendal = { workspace = true, default-features = false, features = [ + "services-azblob", +], optional = true } +percent-encoding = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true, features = ["derive"], optional = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["process"], optional = true } +tracing = { workspace = true } +url = { workspace = true } +which = { workspace = true, optional = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt"] } +toml = { workspace = true } +tracing-test = { workspace = true } diff --git a/crates/rattler_azure/src/clap.rs b/crates/rattler_azure/src/clap.rs new file mode 100644 index 0000000000..72e7abb45a --- /dev/null +++ b/crates/rattler_azure/src/clap.rs @@ -0,0 +1,415 @@ +use std::time::Duration; + +use clap::Parser; + +use secrecy::{ExposeSecret, SecretString}; + +use crate::{ + AzureChannelUrl, AzureCliSasError, AzureCoordinates, AzureCredentials, AzureEndpoint, + AzureUrlError, account_and_container, mint_user_delegation_sas, +}; + +/// Default lifetime, in minutes, of a SAS minted from an `az login` session. +/// +/// SAS tokens are deliberately short-lived: a SAS cannot be individually revoked, +/// so a short lifetime keeps the blast radius small if one leaks. Thirty minutes +/// comfortably covers a typical index or upload run. +const DEFAULT_AZURE_CLI_SAS_TTL_MINUTES: u64 = 30; + +/// Upper bound, in minutes, accepted for `--azure-cli-sas-ttl-minutes`. +/// +/// A SAS is meant to be short-lived; one week is already generous. Capping the +/// value at the clap layer also keeps `minutes * 60` well clear of overflowing +/// the [`Duration`] arithmetic in [`AzureCredentialsOpts::source`]. +const MAX_AZURE_CLI_SAS_TTL_MINUTES: u64 = 7 * 24 * 60; + +/// Errors that can occur while resolving [`AzureCredentialsOpts`] into +/// [`AzureCredentials`]. +#[derive(Debug, thiserror::Error)] +pub enum AzureCredentialsError { + /// No credential source was supplied. + #[error("no Azure credentials supplied: pass --account-key, --sas-token, or --azure-cli")] + Missing, + + /// The channel URL required to mint a SAS could not be parsed. + #[error(transparent)] + Url(#[from] AzureUrlError), + + /// Minting a SAS via the Azure CLI failed. + #[error("failed to mint a user-delegation SAS from the Azure CLI")] + Cli(#[from] AzureCliSasError), +} + +/// A resolved, unambiguous choice of authentication source. +/// +/// [`AzureCredentialsOpts`] can express several inputs at once (an exported +/// `AZURE_STORAGE_KEY` and an explicit `--azure-cli`, say); this enum is the +/// single winner after precedence is applied, so downstream code never has to +/// reason about combinations. Only [`AzureAuthSource::AzureCli`] carries state +/// (the minting TTL), which is why account/container derivation is needed for +/// that arm alone. +#[derive(Clone, Debug)] +pub enum AzureAuthSource { + /// Use a shared storage account key verbatim. + AccountKey(SecretString), + + /// Use a supplied SAS token verbatim. + SasToken(SecretString), + + /// Mint a short-lived user-delegation SAS from the current `az login` + /// session, valid for `ttl`. + AzureCli { + /// How long the minted SAS should remain valid. + ttl: Duration, + }, +} + +impl AzureAuthSource { + /// Resolve this source into concrete [`AzureCredentials`]. + /// + /// `permissions`, `channel` and `endpoint` are consulted **only** for the + /// [`AzureAuthSource::AzureCli`] arm, which mints a SAS scoped to the channel's + /// container with those permissions. Taking the channel and its endpoint rather + /// than pre-derived coordinates is what keeps the account the SAS is minted for + /// and the scheme it is restricted to from coming from two different places. + pub async fn resolve( + self, + permissions: &str, + channel: &AzureChannelUrl, + endpoint: AzureEndpoint, + ) -> Result { + match self { + AzureAuthSource::AccountKey(key) => Ok(AzureCredentials::AccountKey(key)), + AzureAuthSource::SasToken(token) => Ok(AzureCredentials::SasToken(token)), + AzureAuthSource::AzureCli { ttl } => { + let AzureCoordinates { account, container } = + account_and_container(channel, endpoint.addressing)?; + let token = mint_user_delegation_sas( + &account, + &container, + permissions, + ttl, + endpoint.scheme, + ) + .await?; + Ok(AzureCredentials::SasToken(token)) + } + } + } +} + +/// Manually specified Azure Blob credentials. +/// +/// See [`super::AzureCredentials`] for details on how these credentials are used. +/// Several inputs may be present at once (for example when `AZURE_STORAGE_KEY` is +/// exported *and* `--azure-cli` is passed), so [`AzureCredentialsOpts::source`] +/// applies an explicit precedence rather than treating the combination as an +/// error — see that method for the exact ordering. +#[derive(Clone, Debug, Parser)] +pub struct AzureCredentialsOpts { + /// The Azure Storage account key. + /// + /// Mutually exclusive with `--sas-token`: supplying both is a usage error + /// rather than silently discarding one. `--azure-cli` layers on top of both + /// (see [`AzureCredentialsOpts::source`]). + #[arg( + long, + env = "AZURE_STORAGE_KEY", + conflicts_with = "sas_token", + help_heading = "Azure Credentials", + value_parser = secret + )] + pub account_key: Option, + + /// A shared access signature (SAS) token, with or without a leading `?`. + #[arg( + long, + env = "AZURE_STORAGE_SAS_TOKEN", + help_heading = "Azure Credentials", + value_parser = secret + )] + pub sas_token: Option, + + /// Mint a short-lived user-delegation SAS from the current `az login` + /// session (requires the Azure CLI). + /// + /// Takes precedence over AZURE_STORAGE_KEY / AZURE_STORAGE_SAS_TOKEN, so it + /// can be used to override ambient credentials picked up from the + /// environment. + #[allow(clippy::doc_markdown)] + #[arg(long, help_heading = "Azure Credentials")] + pub azure_cli: bool, + + /// Lifetime, in minutes, of the SAS minted for `--azure-cli`. + /// + /// The default keeps the token short-lived. Raise it for very large index or + /// upload runs: if the SAS expires mid-run, subsequent requests fail with a + /// 403 and the run aborts, potentially leaving a partial index behind. + #[arg( + long, + default_value_t = DEFAULT_AZURE_CLI_SAS_TTL_MINUTES, + value_parser = clap::value_parser!(u64).range(1..=MAX_AZURE_CLI_SAS_TTL_MINUTES), + help_heading = "Azure Credentials" + )] + pub azure_cli_sas_ttl_minutes: u64, +} + +/// Take a command-line or environment value straight into a [`SecretString`], so +/// it is never held as a plain `String` that a `{:?}` could reach. +fn secret(value: &str) -> Result { + Ok(value.into()) +} + +impl PartialEq for AzureCredentialsOpts { + /// Hand-written because [`SecretString`] withholds `PartialEq` — comparing + /// secrets is not constant-time. The containing `UploadOpts` tree derives + /// `PartialEq`, and comparing parsed command lines is not a secrets check. + fn eq(&self, other: &Self) -> bool { + fn same(left: Option<&SecretString>, right: Option<&SecretString>) -> bool { + match (left, right) { + (Some(left), Some(right)) => left.expose_secret() == right.expose_secret(), + (None, None) => true, + _ => false, + } + } + + same(self.account_key.as_ref(), other.account_key.as_ref()) + && same(self.sas_token.as_ref(), other.sas_token.as_ref()) + && self.azure_cli == other.azure_cli + && self.azure_cli_sas_ttl_minutes == other.azure_cli_sas_ttl_minutes + } +} + +impl AzureCredentialsOpts { + /// Collapse the supplied options into a single, unambiguous + /// [`AzureAuthSource`]. + /// + /// When more than one input is present the following precedence applies, + /// highest first: + /// + /// 1. `--azure-cli` — an explicit opt-in, so it wins over anything picked up + /// from the environment. + /// 2. `--sas-token` / `AZURE_STORAGE_SAS_TOKEN`. + /// 3. `--account-key` / `AZURE_STORAGE_KEY`. + /// + /// If none are set, returns [`AzureCredentialsError::Missing`]. + pub fn source(&self) -> Result { + if self.azure_cli { + Ok(AzureAuthSource::AzureCli { + ttl: Duration::from_secs(self.azure_cli_sas_ttl_minutes.saturating_mul(60)), + }) + } else if let Some(sas_token) = &self.sas_token { + Ok(AzureAuthSource::SasToken(sas_token.clone())) + } else if let Some(account_key) = &self.account_key { + Ok(AzureAuthSource::AccountKey(account_key.clone())) + } else { + Err(AzureCredentialsError::Missing) + } + } + + /// Resolve the supplied options into concrete [`AzureCredentials`]. + /// + /// Precedence is applied by [`AzureCredentialsOpts::source`]. `permissions`, + /// `channel` and `endpoint` are consulted only when the winning source is + /// `--azure-cli`; see [`AzureAuthSource::resolve`]. + pub async fn resolve( + self, + permissions: &str, + channel: &AzureChannelUrl, + endpoint: AzureEndpoint, + ) -> Result { + self.source()?.resolve(permissions, channel, endpoint).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn opts( + account_key: Option<&str>, + sas_token: Option<&str>, + azure_cli: bool, + ) -> AzureCredentialsOpts { + AzureCredentialsOpts { + account_key: account_key.map(Into::into), + sas_token: sas_token.map(Into::into), + azure_cli, + azure_cli_sas_ttl_minutes: DEFAULT_AZURE_CLI_SAS_TTL_MINUTES, + } + } + + /// A channel whose coordinates cannot be derived under `endpoint`: resolving a + /// verbatim credential must not need them, and this is what proves it. + fn underivable() -> (AzureChannelUrl, AzureEndpoint) { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + let endpoint = AzureEndpoint::default(); + assert!(account_and_container(&channel, endpoint.addressing).is_err()); + (channel, endpoint) + } + + #[tokio::test] + async fn account_key_resolves() { + let (channel, endpoint) = underivable(); + assert!(matches!( + opts(Some("key"), None, false).resolve("cw", &channel, endpoint).await, + Ok(AzureCredentials::AccountKey(k)) if k.expose_secret() == "key" + )); + } + + #[tokio::test] + async fn sas_token_resolves() { + let (channel, endpoint) = underivable(); + assert!(matches!( + opts(None, Some("sv=..."), false).resolve("cw", &channel, endpoint).await, + Ok(AzureCredentials::SasToken(t)) if t.expose_secret() == "sv=..." + )); + } + + #[tokio::test] + async fn none_is_rejected() { + let (channel, endpoint) = underivable(); + assert!(matches!( + opts(None, None, false) + .resolve("cw", &channel, endpoint) + .await, + Err(AzureCredentialsError::Missing) + )); + } + + #[test] + fn azure_cli_beats_sas_beats_account_key() { + // All three present: `--azure-cli` wins. + assert!(matches!( + opts(Some("key"), Some("sv=..."), true).source(), + Ok(AzureAuthSource::AzureCli { .. }) + )); + // SAS token beats an account key when `--azure-cli` is absent. + assert!(matches!( + opts(Some("key"), Some("sv=..."), false).source(), + Ok(AzureAuthSource::SasToken(t)) if t.expose_secret() == "sv=..." + )); + // Account key is the last resort. + assert!(matches!( + opts(Some("key"), None, false).source(), + Ok(AzureAuthSource::AccountKey(k)) if k.expose_secret() == "key" + )); + } + + #[test] + fn azure_cli_ttl_is_carried_through() { + let mut opts = opts(None, None, true); + opts.azure_cli_sas_ttl_minutes = 45; + assert!(matches!( + opts.source().unwrap(), + AzureAuthSource::AzureCli { ttl } if ttl == Duration::from_secs(45 * 60) + )); + } + + // The `--azure-cli` resolve path shells out to `az`, which isn't available in + // the test environment, so we only assert the flag and its TTL parse through + // clap and that precedence selects the CLI source; the mint itself is not + // exercised here. + #[test] + fn azure_cli_flag_and_ttl_parse() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + creds: AzureCredentialsOpts, + } + + let cli = Cli::try_parse_from(["test", "--azure-cli", "--azure-cli-sas-ttl-minutes", "90"]) + .expect("should parse"); + assert!(cli.creds.azure_cli); + assert_eq!(cli.creds.azure_cli_sas_ttl_minutes, 90); + + let default = Cli::try_parse_from(["test", "--azure-cli"]).expect("should parse"); + assert_eq!( + default.creds.azure_cli_sas_ttl_minutes, + DEFAULT_AZURE_CLI_SAS_TTL_MINUTES + ); + } + + /// `--account-key` and `--sas-token` are mutually exclusive: passing both is + /// a clap error rather than silently discarding one. + #[test] + fn account_key_and_sas_token_conflict() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + creds: AzureCredentialsOpts, + } + + let err = Cli::try_parse_from(["test", "--account-key", "k", "--sas-token", "sv=..."]) + .map(|_| ()) + .expect_err("passing both --account-key and --sas-token must be rejected"); + assert_eq!(err.kind(), clap::error::ErrorKind::ArgumentConflict); + } + + /// Neither the resolved source nor the raw options may print a secret. + #[test] + fn debug_never_prints_secrets() { + let sources = [ + AzureAuthSource::AccountKey("supersecretkey".into()), + AzureAuthSource::SasToken("sig=deadbeef".into()), + ]; + for source in &sources { + let out = format!("{source:?}"); + assert!(out.contains("REDACTED"), "not redacted: {out}"); + assert!(!out.contains("supersecret"), "leaked key: {out}"); + assert!(!out.contains("deadbeef"), "leaked token: {out}"); + } + + // The TTL arm carries no secret, so it stays fully printable. + let cli = AzureAuthSource::AzureCli { + ttl: Duration::from_secs(60), + }; + assert!(format!("{cli:?}").contains("60")); + + let out = format!("{:?}", opts(Some("supersecretkey"), None, false)); + assert!(out.contains("REDACTED"), "not redacted: {out}"); + assert!(!out.contains("supersecret"), "leaked key: {out}"); + + let out = format!("{:?}", opts(None, Some("sig=deadbeef"), false)); + assert!(out.contains("REDACTED"), "not redacted: {out}"); + assert!(!out.contains("deadbeef"), "leaked token: {out}"); + + // Absent secrets print as `None`, so the redaction cannot be mistaken for + // a supplied-but-hidden value. + let out = format!("{:?}", opts(None, None, true)); + assert!(out.contains("account_key: None"), "unexpected: {out}"); + assert!(out.contains("sas_token: None"), "unexpected: {out}"); + } + + /// A zero TTL is rejected, and the maximum is capped so `minutes * 60` + /// cannot overflow. + #[test] + fn ttl_range_is_enforced() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + creds: AzureCredentialsOpts, + } + + assert!( + Cli::try_parse_from(["test", "--azure-cli", "--azure-cli-sas-ttl-minutes", "0"]) + .is_err() + ); + assert!( + Cli::try_parse_from([ + "test", + "--azure-cli", + "--azure-cli-sas-ttl-minutes", + &(MAX_AZURE_CLI_SAS_TTL_MINUTES + 1).to_string(), + ]) + .is_err() + ); + } +} diff --git a/crates/rattler_azure/src/lib.rs b/crates/rattler_azure/src/lib.rs new file mode 100644 index 0000000000..ae3c335174 --- /dev/null +++ b/crates/rattler_azure/src/lib.rs @@ -0,0 +1,2100 @@ +//! Helpers for deriving Azure Blob coordinates from channel URLs and for minting +//! short-lived credentials for them. +//! +//! # Host model +//! +//! This crate does not police which hosts are legitimate Azure endpoints: the +//! host a channel URL names is taken to be the storage endpoint it says it is. +//! What is *granted* — credentials, wire scheme, addressing style — is declared in +//! [`options`] and never inferred from the host name: the wire scheme and the +//! addressing per host, and credentials per *container*, because that is the scope +//! Azure's own RBAC has. The default grant is [`Auth::Anonymous`], so naming a host +//! or a container in a URL by itself sends nothing to it. Nothing here signs or +//! sends a request either — that lives in `rattler_networking` — but two functions +//! do handle a credential: +//! `azblob_config` embeds the account key or SAS it is handed into the config it +//! returns, and `mint_user_delegation_sas` spends the user's `az login` session to +//! obtain one. Deriving coordinates from a URL ([`account_and_container`]) +//! touches no credential at all. +//! +//! Userinfo (`user:pass@host`) is rejected wherever a host is parsed, because +//! `az://real.host@evil.example/…` reads as the real host while addressing the +//! attacker's and provides no real functionality. + +#[cfg(feature = "clap")] +pub mod clap; + +pub mod options; + +pub use options::{ + Addressing, Auth, AzureEndpoint, AzureEndpointOptions, AzureFetchOptions, AzureScheme, +}; + +pub use secrecy::{ExposeSecret, SecretString}; +use url::Url; + +/// Credentials for authenticating to Azure Blob storage. +/// +/// Exactly one authentication method is carried, so the ambiguous "both a key +/// and a SAS token" and "neither" states are unrepresentable. The storage +/// account name, endpoint, and container are not stored here: they are derived +/// by the consumer from the channel URL together with the host's addressing +/// style (see [`account_and_container`]). +/// +/// Both variants hold a [`SecretString`], so `Debug` redacts them, the bytes are +/// zeroized on drop, and every read is a visible `expose_secret()`. The type has +/// no `Serialize`/`Deserialize` either, so it cannot reach disk. +#[derive(Clone, Debug)] +pub enum AzureCredentials { + /// A shared storage account key. + AccountKey(SecretString), + + /// A shared access signature (SAS) token. + SasToken(SecretString), +} + +/// Strip a single leading `?` from a SAS token. +/// +/// `--sas-token` may be supplied with or without a leading `?`, but a SAS minted +/// by [`mint_user_delegation_sas`] never has one. Normalizing at the single point +/// where a token is handed to opendal means both sources behave identically. +pub fn normalize_sas_token(token: &str) -> &str { + token.strip_prefix('?').unwrap_or(token) +} + +/// Errors that can occur while deriving Azure Blob coordinates from a channel +/// URL. +#[derive(Debug, thiserror::Error)] +pub enum AzureUrlError { + /// The URL has no host component. + #[error("no host in Azure blob URL")] + NoHost, + + /// The URL carries userinfo (`user:pass@host`). + #[error( + "Azure blob URL must not contain userinfo (`user:pass@host`): the `user@host` form is a \ + host-spoofing vector that can disguise the real target host, and userinfo is invalid in \ + blob URLs" + )] + UserInfoNotAllowed, + + /// The text handed to [`AzureHost::parse`] is not a usable `host[:port]`. + /// + /// This is what a malformed `azure-options` key produces, so it quotes the + /// text back and says what was expected instead. + #[error("`{authority}` is not a valid Azure host: {reason}; expected `host` or `host:port`")] + InvalidHostAuthority { + /// The offending authority text. + authority: String, + /// Why it was rejected. + reason: String, + }, + + /// Host-style addressing was requested but the host has no account label: it + /// is an IP literal, or a domain with only one label. + /// + /// This is the error an Azurite or custom-endpoint user hits first, and the + /// fix is a config line rather than a URL change, so the message names that + /// line verbatim instead of leaving the user to discover `path-style`. The + /// host is spelled the way [`AzureHost`] spells it, which is the way the + /// config table is keyed — a key copied out of this message matches. + #[error( + "Azure blob URL host `{0}` is not a dotted domain of the form `.blob.`, \ + so its first label cannot be a storage account. Such a host needs path-style addressing, \ + where the storage account is the first path segment instead; that is not selectable from \ + configuration yet, and will be enabled by `[azure-options.\"{0}\"]` with \ + `path-style = true`" + )] + InvalidHost(String), + + /// The URL has no path segment to read the account from (path-style only). + #[error("could not derive account name from Azure blob URL")] + NoAccount, + + /// The URL has no container path segment. + #[error("no container in Azure blob URL")] + NoContainer, + + /// The derived account name is not a legal Azure storage account name. + #[error( + "`{0}` is not a valid Azure storage account name: account names are 3-24 characters of \ + lowercase letters and digits only" + )] + InvalidAccountName(String), + + /// The derived container name is not a legal Azure blob container name. + #[error( + "`{0}` is not a valid Azure blob container name: container names are 3-63 characters of \ + lowercase letters, digits and hyphens, must start and end with a letter or digit, and \ + must not contain consecutive hyphens" + )] + InvalidContainerName(String), + + /// The channel URL string could not be parsed. + #[error("`{value}` is not a valid URL")] + InvalidUrl { + /// The offending input. + value: String, + /// The underlying parse error. + #[source] + source: url::ParseError, + }, + + /// The written path is not the path the URL Standard resolves it to. + /// + /// `..` segments — including percent-encoded ones — are resolved before any + /// segment is validated, so a path that reads as one container (or, path-style, + /// one account) can address another. The resolved form is quoted so the user + /// can see where the URL would actually have gone. + #[error( + "Azure blob channel URL path `{written}` is not the path it resolves to, `{resolved}`; a \ + channel URL must name the location it addresses, so write `{resolved}` if that is the \ + location you mean" + )] + NonCanonicalPath { + /// The path as written. + written: String, + /// The path it resolves to. + resolved: String, + }, + + /// A path segment percent-decodes to bytes that are not UTF-8. + /// + /// Blob names are UTF-8, so there is nothing to send such a segment as. Decoding + /// lossily would substitute U+FFFD and address a different blob than the URL + /// names, silently and without an error at any layer. + #[error( + "Azure blob channel URL segment `{segment}` percent-decodes to bytes that are not UTF-8, \ + so it cannot name a blob" + )] + NonUtf8Path { + /// The segment as written. + segment: String, + /// Where the decoded bytes stop being UTF-8. + #[source] + source: std::str::Utf8Error, + }, + + /// A path segment contains `%2F`, an encoded slash. + /// + /// One segment holding a slash and two segments are different blob paths, and + /// the URL Standard does not resolve `%2F`, so whichever reading we picked would + /// be a place the URL text does not say. Refusing keeps the written path and the + /// blob path the same shape. + #[error( + "Azure blob channel URL segment `{0}` contains an encoded slash (`%2F`); write the path \ + separator as `/` if you mean a new segment" + )] + EncodedSlashInPath(String), + + /// The channel URL does not use the `az://` scheme. + #[error( + "Azure blob channel URL must use the `az://` scheme, e.g. \ + `az://.blob.core.windows.net//...`: got `{0}`" + )] + InvalidScheme(String), +} + +/// A storage account name that has passed Azure's naming rules: 3-24 characters +/// of lowercase letters and digits. +/// +/// Those rules are the only thing that keeps option-shaped text (`--as-user`, +/// `-o`) out of the `az` argv in [`mint_user_delegation_sas`], so the mint takes +/// this type: the guarantee is then carried by what the function accepts rather +/// than by every call site remembering to derive its name through a validating +/// path. The inner `String` is private and [`Self::new`] is the only way to one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AccountName(String); + +impl AccountName { + /// Check a name against Azure's storage account naming rules. + pub fn new(name: &str) -> Result { + let valid = (3..=24).contains(&name.len()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit()); + valid + .then(|| Self(name.to_string())) + .ok_or_else(|| AzureUrlError::InvalidAccountName(name.to_string())) + } + + /// The validated name. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for AccountName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// A blob container name that has passed Azure's naming rules: 3-63 characters of +/// lowercase letters, digits and hyphens, with no leading or trailing hyphen and +/// no consecutive hyphens. +/// +/// Exists for the same reason as [`AccountName`], and is what the container half +/// of the `az` argv is spelled as. +/// +/// It is also the key of an `auth` table in `azure-options`, which is why it is +/// hashable and has the same string serde bridge [`AzureHost`] has: a grant is +/// written per container, and the name a grant is stored under must be the name a +/// lookup arrives with. Azure's rules do the normalizing for free — a container +/// name is lowercase by construction, so unlike a host there is only ever one +/// spelling of one container. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(try_from = "String", into = "String") +)] +pub struct ContainerName(String); + +impl ContainerName { + /// Check a name against Azure's blob container naming rules. + pub fn new(name: &str) -> Result { + let valid = (3..=63).contains(&name.len()) + && name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + && !name.starts_with('-') + && !name.ends_with('-') + && !name.contains("--"); + valid + .then(|| Self(name.to_string())) + .ok_or_else(|| AzureUrlError::InvalidContainerName(name.to_string())) + } + + /// The validated name. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for ContainerName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::str::FromStr for ContainerName { + type Err = AzureUrlError; + + fn from_str(value: &str) -> Result { + Self::new(value) + } +} + +/// The serde bridge for using a `ContainerName` as a map key: serde hands map keys +/// over as owned strings, so `serde(try_from = "String")` is what routes a written +/// `auth` key through [`ContainerName::new`] instead of storing it raw. A key Azure +/// would refuse is then a config error at load, not a grant that can never match. +impl TryFrom for ContainerName { + type Error = AzureUrlError; + + fn try_from(value: String) -> Result { + Self::new(&value) + } +} + +impl From for String { + fn from(container: ContainerName) -> Self { + container.0 + } +} + +/// The storage account and container an Azure Blob channel URL resolves to. +/// +/// The fields are public because their *types* are the invariant: a +/// `AzureCoordinates` cannot be assembled from unvalidated text, whoever builds +/// it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AzureCoordinates { + /// The storage account name — the first host label under + /// [`Addressing::HostStyle`], the first path segment under + /// [`Addressing::PathStyle`]. + pub account: AccountName, + + /// The blob container name — the first path segment under + /// [`Addressing::HostStyle`], the second under [`Addressing::PathStyle`]. + pub container: ContainerName, +} + +/// Derive the storage account name and container from an Azure Blob channel URL. +/// +/// Where the account name lives is decided by `addressing`, which comes from the +/// host's `azure-options` entry — it is not guessable from the URL, because +/// `https://host/a/b` is a valid reading under both styles: +/// +/// - [`Addressing::HostStyle`] (real Azure, the default): account = first label +/// of the host, container = first path segment. The host must be a domain with +/// at least two labels, so IP literals and single-label hosts fail with +/// [`AzureUrlError::InvalidHost`], whose message names the config line that +/// switches to path-style. +/// - [`Addressing::PathStyle`] (Azurite and other emulators): account = first +/// path segment, container = second. On a host under a known Azure Blob suffix +/// this is almost certainly a config mistake — the two styles then disagree +/// about which name is the account while producing identical request URLs, so +/// nothing fails until a mint asks for a delegation SAS on whatever the path +/// spelled. It is only a warning: the list is advisory, cannot cover a +/// proxy or a private endpoint, and choosing the addressing for a host remains +/// the user's call. +/// +/// The host is otherwise trusted verbatim (see the [crate-level docs] for the +/// host model): an honest, arbitrary host is the caller's responsibility. The +/// derived account and container are additionally held to Azure's own naming +/// rules — under *both* addressing styles, since path-style takes the account +/// from user-controlled path text. Those rules reject an empty name, any +/// character outside `[a-z0-9-]`, and a leading `-`. +/// +/// [crate-level docs]: crate +pub fn account_and_container( + channel: &AzureChannelUrl, + addressing: Addressing, +) -> Result { + let host = channel.host(); + let container_segment = + || segment(channel, addressing.container_segment()).ok_or(AzureUrlError::NoContainer); + + let (account, container) = match addressing { + Addressing::HostStyle => { + let account = host + .account_label() + .ok_or_else(|| AzureUrlError::InvalidHost(host.to_string()))?; + (account, container_segment()?) + } + Addressing::PathStyle => { + if host.is_known_azure_blob_endpoint() { + tracing::warn!( + "`path-style = true` is set for `{host}`, which is a real Azure Blob endpoint \ + addressed host-style: its storage account is `{}`, not the first path \ + segment. Requests still come out identical, but anything that needs the \ + account on its own — minting a user-delegation SAS, for one — will use the \ + path segment instead. Remove `path-style = true` from \ + `[azure-options.\"{host}\"]` unless you meant it", + host.account_label().unwrap_or(""), + ); + } + ( + segment(channel, 0).ok_or(AzureUrlError::NoAccount)?, + container_segment()?, + ) + } + }; + + Ok(AzureCoordinates { + account: AccountName::new(account)?, + container: ContainerName::new(container)?, + }) +} + +/// The container an Azure Blob URL names, when it names one. +/// +/// This is the fetch path's half of [`account_and_container`]: a grant is written +/// per container, so the middleware needs the container and nothing else — no +/// account, which is what keeps a URL on a host that cannot carry an account label +/// (an IP literal read host-style) from failing here where it fetches happily +/// today. +/// +/// The two answers it can give are deliberately different: +/// +/// - `Ok(None)`: the URL has no container segment — the host root, or a path too +/// short to have one under this addressing. There is nothing to attribute a +/// grant to, so the caller sends nothing. +/// - `Err`: the segment is there but is not a name Azure allows for a container. +/// No legitimate blob request can land here, so this is a malformed endpoint +/// rather than an ungranted one, and saying so beats going quietly anonymous and +/// surfacing later as an unexplained 401. +pub fn container( + channel: &AzureChannelUrl, + addressing: Addressing, +) -> Result, AzureUrlError> { + segment(channel, addressing.container_segment()) + .map(ContainerName::new) + .transpose() +} + +/// The `index`-th path segment, or `None` when it is missing or empty. +/// +/// An empty segment is a missing one: no Azure name may be empty, so `//general` +/// has no first segment rather than an unnamed one. +fn segment(channel: &AzureChannelUrl, index: usize) -> Option<&str> { + channel + .path_segments() + .nth(index) + .filter(|segment| !segment.is_empty()) +} + +/// A normalized Azure Blob endpoint authority: a host, and its port when one is +/// written. +#[derive(Clone, PartialEq, Eq, Hash)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(try_from = "String", into = "String") +)] +pub struct AzureHost { + host: url::Host, + port: Option, +} + +impl AzureHost { + /// Parse and normalize a bare `host[:port]` authority. + pub fn parse(authority: &str) -> Result { + if authority.contains('@') { + return Err(AzureUrlError::UserInfoNotAllowed); + } + if authority.contains(['/', '\\', '?', '#']) { + return Err(AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: "it carries a path, query or fragment".to_string(), + }); + } + + // Two parses, each for the one thing it is authoritative about, because no + // single scheme gives both. `https` is a special scheme, so it runs the URL + // Standard's host parser: lowercasing, IDNA, and IP literals as typed + // `Ipv4`/`Ipv6` hosts — but it also drops `:443`. `az` is not special, so + // it has no default port to drop, but its opaque-host parsing leaves the + // host unnormalized (`MyCompany.X` stays mixed case, `127.0.0.1` arrives as + // a `Domain`). Host from the first, port from the second. + let normalized = Self::parse_as(authority, "https")?; + let verbatim = Self::parse_as(authority, "az")?; + + // `url` reads a bare trailing colon as "no port at all", so `host:` would + // otherwise be accepted as `host` — a different endpoint from the one whose + // port the user was in the middle of writing. Port 0 it keeps, and `wire()` + // then hands out `https://host:0/…`, which no connection can be made to. + let port_reason = match (Self::written_port(authority), verbatim.port()) { + (Some(""), _) => Some("its port is empty"), + (_, Some(0)) => Some("port 0 cannot be connected to"), + _ => None, + }; + if let Some(reason) = port_reason { + return Err(AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: reason.to_string(), + }); + } + + let host = normalized.host().ok_or(AzureUrlError::NoHost)?.to_owned(); + Self::normalized(host, verbatim.port(), authority) + } + + /// The port exactly as the authority spells it, when it spells one. + /// + /// An IPv6 literal is bracketed, so a colon inside it is never a port + /// delimiter — only a `]:port` suffix is. + fn written_port(authority: &str) -> Option<&str> { + let (_, port) = authority.rsplit_once(':')?; + (!port.ends_with(']')).then_some(port) + } + + /// Parse `://`, reporting a failure against the authority + /// text the caller actually wrote. + fn parse_as(authority: &str, scheme: &str) -> Result { + Url::parse(&format!("{scheme}://{authority}")).map_err(|err| { + AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: err.to_string(), + } + }) + } + + /// Apply the rules the URL host parser does not: strip the DNS root label, + /// reject empty labels, and hold the name to the 253-character limit DNS puts + /// on one. + /// + /// Private, so every route in goes through [`parse`](Self::parse) and no rule + /// can be skipped. + fn normalized( + host: url::Host, + port: Option, + authority: &str, + ) -> Result { + const DNS_NAME_LIMIT: usize = 253; + + let url::Host::Domain(domain) = &host else { + // An IP literal is already fully canonical, and has no labels. + return Ok(Self { host, port }); + }; + + // Re-run the host parser on the trimmed name so there is exactly one + // normalization path rather than a second, hand-rolled one. + let host = url::Host::parse(domain.strip_suffix('.').unwrap_or(domain)).map_err(|err| { + AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: err.to_string(), + } + })?; + if let url::Host::Domain(domain) = &host { + // Only one trailing dot is stripped, so `acct.example..` still has an + // empty label here — as does `acct..example`. Rejecting both is what + // lets `Display` round-trip, and what stops account derivation from + // handing out an empty account name. + if domain.split('.').any(str::is_empty) { + return Err(AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: "one of its labels is empty".to_string(), + }); + } + // Measured after IDNA, since the punycode form is what is resolved. + if domain.len() > DNS_NAME_LIMIT { + return Err(AzureUrlError::InvalidHostAuthority { + authority: authority.to_string(), + reason: format!( + "it is {} characters long, over the {DNS_NAME_LIMIT}-character limit DNS \ + puts on a name", + domain.len() + ), + }); + } + } + Ok(Self { host, port }) + } + + /// The parsed host, without the port. + pub fn host(&self) -> &url::Host { + &self.host + } + + /// The port, when the authority names one. + pub fn port(&self) -> Option { + self.port + } + + /// The storage account label under host-style addressing. + /// + /// `None` whenever the host cannot carry an account name, which the stored + /// [`url::Host`] answers by construction rather than by inspecting text: + /// + /// - an IP literal is an [`url::Host::Ipv4`] or [`url::Host::Ipv6`], so it can + /// never be read as a label — including `127.0.0.1`, which a "does it + /// contain a dot" test would happily split into an account named `127`; + /// - a domain must have at least two labels, so `localhost` is rejected. + /// + /// [`parse`](Self::parse) has already guaranteed no label is empty and there + /// is no trailing dot, so "at least two labels" here means "at least two + /// non-empty labels". + fn account_label(&self) -> Option<&str> { + match &self.host { + url::Host::Domain(domain) => { + let mut labels = domain.split('.'); + let first = labels.next()?; + labels.next().is_some().then_some(first) + } + url::Host::Ipv4(_) | url::Host::Ipv6(_) => None, + } + } + + /// Whether this host sits under a suffix Microsoft operates, where the account + /// is by definition the first label. + /// + /// Advisory only, and deliberately not a security boundary: a grant is written + /// per host, so no behaviour hangs off this answer. It exists to warn about a + /// `path-style = true` that cannot be what the user meant. A proxy or private + /// endpoint in front of real Azure answers `false`, which is why a `false` here + /// is never treated as evidence of anything. + pub fn is_known_azure_blob_endpoint(&self) -> bool { + const SUFFIXES: &[&str] = &[ + "blob.core.windows.net", // global + "blob.core.usgovcloudapi.net", // US Government + "blob.core.chinacloudapi.cn", // China, operated by 21Vianet + ]; + + let url::Host::Domain(domain) = &self.host else { + return false; + }; + SUFFIXES.iter().any(|suffix| { + // The dot has to be part of the match, or `notblob.core.windows.net` + // would pass as `blob.core.windows.net`. + domain + .strip_suffix(suffix) + .is_some_and(|prefix| prefix.ends_with('.')) + }) + } +} + +impl std::fmt::Display for AzureHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // `url::Host`'s own `Display` brackets an IPv6 literal, which is what an + // authority needs. + write!(f, "{}", self.host)?; + if let Some(port) = self.port { + write!(f, ":{port}")?; + } + Ok(()) + } +} + +impl std::fmt::Debug for AzureHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // The canonical text *is* the identity; the host/port split is an + // implementation detail, and printing it would only make a config dump + // harder to read. + write!(f, "AzureHost({:?})", self.to_string()) + } +} + +impl std::str::FromStr for AzureHost { + type Err = AzureUrlError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } +} + +/// The serde bridge for using an `AzureHost` as a map key: serde hands map keys +/// over as owned strings, so `serde(try_from = "String")` is what routes a written +/// `azure-options` key through [`AzureHost::parse`] instead of storing it raw. +impl TryFrom for AzureHost { + type Error = AzureUrlError; + + fn try_from(value: String) -> Result { + Self::parse(&value) + } +} + +impl From for String { + fn from(host: AzureHost) -> Self { + host.to_string() + } +} + +/// A validated Azure Blob **channel** URL, which has two spellings: `az://…` as +/// the user writes it and in configuration, and `http(s)://…` on the wire. +/// +/// # Why the parts are stored, and not a URL +/// +/// The obvious shape is a struct holding both spellings, which can hold a pair +/// that disagrees — a canonical URL for one host and a wire URL for another — and +/// nothing but discipline stops it. The next-obvious shape is one `Url` in the +/// wire form with a fixed scheme, deriving the other spelling from it. That is +/// worse than it looks: a `Url`'s port is scheme-relative, so storing +/// `az://host:443/…` as `https` drops the port on the way in, and +/// [`wire`](Self::wire) then hands out `http://host/…` — port 80, a different +/// endpoint. +/// +/// So the authority is stored as an [`AzureHost`], which holds host and port +/// explicitly and normalizes both without reference to any scheme, next to the +/// already-normalized path, query and fragment. Every spelling is built from those +/// same parts by one private helper, so no two spellings can disagree about host, +/// port, path or query. +/// +/// # Why the scheme is a `wire()` argument and not a field +/// +/// Which scheme a host is reached over comes from its `azure-options` entry, and +/// [`parse`](Self::parse) runs as a clap `value_parser` — before any config file +/// is read. A stored scheme would therefore have to be a guess made at parse time +/// and corrected later, which is exactly the drift this type exists to prevent. +/// Passing it in at call time keeps the choice at the site that makes it. +/// +/// Nothing in the type ties the argument to an options entry. `rattler-index` +/// takes it from the channel host's entry; `rattler_upload` passes the default, +/// because it reads no config file at all (see the note in +/// `rattler_upload::upload_from_args`). +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct AzureChannelUrl { + /// The authority, normalized independently of any scheme. + host: AzureHost, + + /// The path as the URL Standard normalizes it: always a leading `/`, still + /// percent-encoded. + path: String, + + /// The query, when there is one — a SAS token may be written inline. + query: Option, + + /// The fragment, when there is one. + /// + /// Kept so [`canonical`](Self::canonical) spells the channel back the way the + /// user wrote it, which is also the spelling config keys are matched against. + /// It reaches no server: an HTTP request carries only the path and query, and + /// on a signed request it is gone from the URL as well, because + /// `AzureMiddleware::sign` round-trips through `http::Uri`, which has no + /// fragment. + fragment: Option, +} + +impl AzureChannelUrl { + /// Parse and validate an `az://` channel URL. + /// + /// The only accepted spelling is `az:///<…>`. A bare `http(s)://` URL is + /// deliberately *not* accepted: `az://` is the single canonical spelling for an + /// Azure channel, and accepting the wire URL as a second input spelling would + /// only invite confusion about which one is authoritative. + /// + /// Account and container derivation is *not* performed here: it depends on the + /// host's addressing style, which is config that does not exist yet at clap + /// parse time. It happens in [`account_and_container`], which today runs only + /// where an account name is genuinely needed — minting a SAS from an + /// `az login` session, and building the opendal config for a write. The fetch + /// path never calls it. + pub fn parse(value: &str) -> Result { + // URL schemes are case-insensitive and `Url` lowercases them, so `AZ://…` + // reaches every downstream `scheme() == "az"` comparison as `az`. Matching + // case-insensitively here keeps this parser from rejecting what those + // comparisons accept. + let rest = strip_az_scheme(value) + .ok_or_else(|| AzureUrlError::InvalidScheme(value.to_string()))?; + + // The authority runs to the first path, query or fragment delimiter. `\` is + // in the set because the special-scheme parser used below treats it as `/`, + // and splitting on it keeps the authority this type validates equal to the + // authority that parser sees. + let authority_end = rest.find(['/', '\\', '?', '#']).unwrap_or(rest.len()); + let (authority, tail) = rest.split_at(authority_end); + let host = AzureHost::parse(authority)?; + + // Parse the whole thing as `https` for the path, query and fragment: the + // special-scheme parser is what normalizes them, and `wire()` hands them + // straight to an `http(s)` URL, so they have to be normalized its way. + let url = Url::parse(&format!("https://{authority}{tail}")).map_err(|source| { + AzureUrlError::InvalidUrl { + value: value.to_string(), + source, + } + })?; + + // Dot segments — `%2e%2e` as much as `..` — are resolved by that parser + // before anything here has looked at a segment, so a path reading as one + // container (path-style: one *account*) can address another. Nothing needs + // to guess which rewrites are benign: a path that is not already the path + // it resolves to is not the path the user can be assumed to have meant. + let written = match tail.split(['?', '#']).next().unwrap_or_default() { + "" => "/", + path => path, + }; + if written != url.path() { + return Err(AzureUrlError::NonCanonicalPath { + written: written.to_string(), + resolved: url.path().to_string(), + }); + } + + // Every segment must survive the round trip to a blob name. `%2F` is checked + // before decoding, because after it there is no telling it from a `/` the + // user wrote. + for segment in url.path_segments().into_iter().flatten() { + if segment.to_ascii_uppercase().contains("%2F") { + return Err(AzureUrlError::EncodedSlashInPath(segment.to_string())); + } + percent_encoding::percent_decode_str(segment) + .decode_utf8() + .map_err(|source| AzureUrlError::NonUtf8Path { + segment: segment.to_string(), + source, + })?; + } + + Ok(Self { + host, + path: url.path().to_string(), + query: url.query().map(str::to_string), + fragment: url.fragment().map(str::to_string), + }) + } + + /// The `az://host/path` spelling: the channel's identity. + /// + /// This is what users write, what is shown back to them, and what config keys + /// are matched against: `rattler-index` resolves `[index-config."az://…"]` + /// through this spelling, and `[azure-options."…"]` through [`Self::host`]. + /// Matching the wire string instead was reviewer issue 5 — the two spellings + /// exist so a config key never has to guess which one a channel was stored as. + /// A SAS written inline is masked: this spelling is the one that reaches logs + /// and error messages, and [`Self::wire`] is the only way to the signature. + pub fn canonical(&self) -> Url { + self.spelled("az", Sas::Masked) + } + + /// The `http(s)://host/path` spelling used for actual requests, over the + /// scheme the host's options entry asks for. + pub fn wire(&self, scheme: AzureScheme) -> Url { + self.spelled(scheme.as_str(), Sas::Exposed) + } + + /// Build one spelling of this URL. + /// + /// Both public spellings go through here, so they cannot differ in anything + /// but the scheme and whether the signature is masked: the host, port, path, + /// query and fragment they are built from are literally the same values. + fn spelled(&self, scheme: &str, sas: Sas) -> Url { + let mut text = format!("{scheme}://{}{}", self.host, self.path); + if let Some(query) = &self.query { + text.push('?'); + match sas { + Sas::Exposed => text.push_str(query), + Sas::Masked => text.push_str(&mask_sas_signature(query)), + } + } + if let Some(fragment) = &self.fragment { + text.push('#'); + text.push_str(fragment); + } + // Cannot fail: the authority re-serializes to the normalized form it was + // parsed from, and the path, query and fragment are already-encoded output + // of a `Url` parse. Every host shape `AzureHost` can hold (normalized + // domain, IPv4 literal, bracketed IPv6) is valid both to the special-scheme + // host parser and to the opaque-host parser `az://` gets. + Url::parse(&text).expect("a normalized authority, path and query is a valid URL") + } + + /// The host, with its port when the URL carries one. + /// + /// This is the `azure-options` key for the channel, so options can be looked + /// up without a caller re-deriving it from a URL and getting the port handling + /// subtly wrong. + pub fn host(&self) -> &AzureHost { + &self.host + } + + /// The still-encoded path segments, exactly as [`Url::path_segments`] would + /// yield them for the wire spelling. + fn path_segments(&self) -> std::str::Split<'_, char> { + self.path.strip_prefix('/').unwrap_or(&self.path).split('/') + } +} + +impl std::fmt::Display for AzureChannelUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // The canonical spelling is the one users recognize and the one config is + // keyed by, so it is the only sensible thing to print. + write!(f, "{}", self.canonical()) + } +} + +impl std::fmt::Debug for AzureChannelUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Derived, this would print the raw query and hand a `{:?}` on any struct + // holding a channel the signature that `canonical()` exists to withhold. + f.debug_tuple("AzureChannelUrl") + .field(&self.canonical().as_str()) + .finish() + } +} + +/// Whether a spelling of a channel URL may carry the SAS signature. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Sas { + /// For the wire: the signature is what makes the request authentic. + Exposed, + /// For anything a human or a log sees. + Masked, +} + +/// Replace the value of a query's `sig` parameter, leaving the rest intact. +/// +/// The other SAS parameters (`sv`, `se`, `sp`, …) describe the grant and are worth +/// showing; `sig` is the secret that makes it usable. +fn mask_sas_signature(query: &str) -> String { + query + .split('&') + .map(|parameter| match parameter.split_once('=') { + Some((name, _)) if name.eq_ignore_ascii_case("sig") => format!("{name}=REDACTED"), + _ => parameter.to_string(), + }) + .collect::>() + .join("&") +} + +impl std::str::FromStr for AzureChannelUrl { + type Err = AzureUrlError; + + fn from_str(value: &str) -> Result { + Self::parse(value) + } +} + +/// Strip a case-insensitive `az://` prefix, or return `None` when it is absent. +fn strip_az_scheme(value: &str) -> Option<&str> { + const PREFIX: &str = "az://"; + // `get` rather than slicing: a multi-byte leading character would panic on a + // non-char-boundary index. + value + .get(..PREFIX.len()) + .filter(|prefix| prefix.eq_ignore_ascii_case(PREFIX)) + .map(|_| &value[PREFIX.len()..]) +} + +/// Build an opendal [`AzblobConfig`](opendal::services::AzblobConfig) from a +/// channel URL, the endpoint options of its host, and credentials. +/// +/// The account name, endpoint, container and root prefix are all derived from the +/// channel URL, read the way `options.addressing` says to read it and reached over +/// `options.scheme`; the credentials supply only the account key or SAS token. +/// The per-container grants are not part of [`AzureEndpoint`] at all — this is the +/// write path, where the credential has already been chosen by the caller. +/// +/// Taking the [`AzureChannelUrl`] rather than a wire `Url` is what keeps the +/// scheme in the config from disagreeing with the scheme in the endpoint: both +/// come from the same `options`. +/// +/// # The two addressing shapes +/// +/// opendal's azblob core builds every request URI as `{endpoint}/{container}/{path}` +/// and its core struct carries no account field at all, so under path-style the +/// account can only reach the URL through `endpoint`: +/// +/// - [`Addressing::HostStyle`]: `endpoint` is `{scheme}://{host}[:{port}]`, the +/// account is the first host label, and `root` is the path after the container. +/// - [`Addressing::PathStyle`]: `endpoint` is +/// `{scheme}://{host}[:{port}]/{account}`, the account is the first path +/// segment, and `root` is the path after *both* the account and the container. +/// +/// `account_name` is set under both styles, and is mandatory under both: opendal +/// infers it only from three known Azure suffixes and returns `None` — not an +/// error — for anything else, so omitting it from a path-style config makes +/// shared-key signing quietly never engage, and the failure surfaces as a 403 +/// rather than as a config error. +/// +/// Neither endpoint ends in a slash. `AzblobBuilder::endpoint` trims one, but this +/// builds the config struct literally, where nothing does, and a stray slash would +/// yield `//{container}/…`. +#[cfg(feature = "opendal")] +pub fn azblob_config( + credentials: &AzureCredentials, + channel: &AzureChannelUrl, + endpoint_options: AzureEndpoint, +) -> Result { + let AzureCoordinates { account, container } = + account_and_container(channel, endpoint_options.addressing)?; + + // The authority comes from `AzureHost`, not from a wire URL: a `Url` has + // already dropped a port equal to its scheme's default, so reading it back + // would turn a written `:443` into no port at all. + let authority = channel.host(); + let endpoint = match endpoint_options.addressing { + Addressing::HostStyle => format!("{}://{authority}", endpoint_options.scheme), + Addressing::PathStyle => format!("{}://{authority}/{account}", endpoint_options.scheme), + }; + + // Root prefix = the path after the segments the coordinates already consumed: + // the container, plus the account when path-style put it in the path. Skipping + // one too few there leaves the account segment inside `root`, so every blob is + // written one directory deeper than the channel actually lives — silently, and + // in the right container, which is what makes it hard to spot. + let consumed = match endpoint_options.addressing { + Addressing::HostStyle => 1, + Addressing::PathStyle => 2, + }; + + // Percent-decode each segment: `path_segments()` yields still-encoded segments + // and opendal percent-encodes `root + path` again, so passing them through + // verbatim would double-encode a prefix containing a space or a `+`. + // `account_and_container` has already confirmed the consumed segments exist. + let root = format!( + "/{}", + channel + .path_segments() + .skip(consumed) + // Infallible in practice: `AzureChannelUrl::parse` rejects a segment that + // does not decode to UTF-8. Erroring rather than substituting U+FFFD is + // what keeps that a guarantee instead of an assumption. + .map(|segment| { + percent_encoding::percent_decode_str(segment) + .decode_utf8() + .map_err(|source| AzureUrlError::NonUtf8Path { + segment: segment.to_string(), + source, + }) + }) + .collect::, _>>()? + .join("/") + ); + + let (account_key, sas_token) = match credentials { + AzureCredentials::AccountKey(key) => (Some(key.expose_secret().to_string()), None), + AzureCredentials::SasToken(token) => ( + None, + Some(normalize_sas_token(token.expose_secret()).to_string()), + ), + }; + + Ok(opendal::services::AzblobConfig { + endpoint: Some(endpoint), + account_name: Some(account.as_str().to_string()), + container: container.as_str().to_string(), + root: Some(root), + account_key, + sas_token, + ..Default::default() + }) +} + +/// Errors that can occur while minting a user-delegation SAS via the Azure CLI. +#[cfg(feature = "clap")] +#[derive(Debug, thiserror::Error)] +pub enum AzureCliSasError { + /// The SAS expiry timestamp could not be computed. + #[error("failed to compute the SAS expiry timestamp: {0}")] + Expiry(String), + + /// The `az` executable could not be resolved on `PATH`. + #[error("could not resolve the Azure CLI (`az`) on PATH; install it and run `az login`")] + AzResolve(#[source] which::Error), + + /// The `az` process could not be spawned. + #[error("failed to run the Azure CLI (`az`)")] + Spawn(#[source] std::io::Error), + + /// `az` exited with a non-zero status. + #[error("the Azure CLI failed to generate a user-delegation SAS (is `az login` current?): {0}")] + CommandFailed(String), + + /// `az` succeeded but produced no SAS token. + #[error("the Azure CLI returned an empty SAS token")] + EmptyOutput, +} + +/// Mint a short-lived user-delegation SAS for a container by shelling out to the +/// Azure CLI. +/// +/// opendal's azblob backend (used by the index and upload write paths) only +/// accepts a shared account key or a SAS token, not an AAD bearer token. To let +/// users authenticate writes with their `az login` session, this converts that +/// session into a SAS via: +/// +/// ```text +/// az storage container generate-sas --account-name --name +/// --permissions --expiry --auth-mode login --as-user +/// [--https-only] -o tsv +/// ``` +/// +/// `permissions` is the Azure SAS permission string (e.g. `"cw"`). The returned +/// token has no leading `?`. Requires `az` on `PATH` and a prior `az login`. +/// +/// `scheme` is the wire scheme the host's options entry asks for: `--https-only` +/// restricts the SAS to TLS, which would make it unusable against a host reached +/// over http. +/// +/// Runs the `az` process on the tokio runtime; it is meant to be called once at +/// setup time. +/// +/// # Container-scope limitation +/// +/// A user-delegation SAS minted against a flat container is *container-scoped*, +/// not prefix-scoped: it grants its permissions over the whole container, so a +/// SAS for one channel also grants rights over any sibling channels that share +/// the same container. The short TTL requested here bounds the blast radius, but +/// prefix-scoping a flat container is not possible without a stored access +/// policy, which this path deliberately does not create. +#[cfg(feature = "clap")] +pub async fn mint_user_delegation_sas( + account: &AccountName, + container: &ContainerName, + permissions: &str, + valid_for: std::time::Duration, + scheme: AzureScheme, +) -> Result { + /// Slack for a client clock running up to two minutes slow, since the expiry + /// is computed here and evaluated by Azure. + const CLOCK_SKEW_HEADROOM: std::time::Duration = std::time::Duration::from_secs(120); + + let signed = jiff::SignedDuration::try_from(valid_for.saturating_add(CLOCK_SKEW_HEADROOM)) + .map_err(|err| AzureCliSasError::Expiry(err.to_string()))?; + let expiry = jiff::Timestamp::now() + .checked_add(signed) + .map_err(|err| AzureCliSasError::Expiry(err.to_string()))?; + // `az` expects an ISO-8601 UTC timestamp; keep second precision so the window + // is not floored down to the enclosing whole minute. + let expiry = expiry.strftime("%Y-%m-%dT%H:%M:%SZ").to_string(); + + let mut command = az_command()?; + + let output = command + .args(generate_sas_args( + account, + container, + permissions, + &expiry, + scheme, + )) + .output() + .await + .map_err(AzureCliSasError::Spawn)?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + return Err(AzureCliSasError::CommandFailed(stderr)); + } + + let token = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if token.is_empty() { + return Err(AzureCliSasError::EmptyOutput); + } + Ok(token.into()) +} + +/// The argv for the `az storage container generate-sas` call. +/// +/// Split out from the spawn so the argument list is testable without an `az` on +/// `PATH`. It stays a list of separate arguments — never a command line — so no +/// value can be read as anything but one argument. +#[cfg(feature = "clap")] +fn generate_sas_args<'a>( + account: &'a AccountName, + container: &'a ContainerName, + permissions: &'a str, + expiry: &'a str, + scheme: AzureScheme, +) -> Vec<&'a str> { + let mut args = vec![ + "storage", + "container", + "generate-sas", + "--account-name", + account.as_str(), + "--name", + container.as_str(), + "--permissions", + permissions, + "--expiry", + expiry, + "--auth-mode", + "login", + "--as-user", + ]; + if let AzureScheme::Https = scheme { + args.push("--https-only"); + } + args.extend(["-o", "tsv"]); + args +} + +/// Build the [`tokio::process::Command`] used to invoke the Azure CLI. +/// +/// `which` resolves `az` up front so a missing CLI surfaces as [`AzureCliSasError::AzResolve`] +/// rather than an opaque spawn failure. It also matters on Windows, where the CLI +/// is an `az.cmd` batch shim: the process spawner does not honor `PATHEXT`, so a +/// bare `az` fails to resolve, but `which` applies `PATHEXT` to find the real path. +/// The resolved path is invoked directly; routing through the command interpreter +/// (`cmd /C az ...`) is deliberately avoided as an argument-injection vector. +#[cfg(feature = "clap")] +fn az_command() -> Result { + let path = which::which("az").map_err(AzureCliSasError::AzResolve)?; + Ok(tokio::process::Command::new(path)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Every derivation runs off an [`AzureChannelUrl`], so the tests build one. + fn channel(url: &str) -> AzureChannelUrl { + AzureChannelUrl::parse(url).unwrap_or_else(|err| panic!("{url} should parse: {err}")) + } + + fn coordinates(account: &str, container: &str) -> AzureCoordinates { + AzureCoordinates { + account: AccountName::new(account).expect("test account name"), + container: ContainerName::new(container).expect("test container name"), + } + } + + #[test] + fn normal_url_resolves() { + assert_eq!( + account_and_container( + &channel("az://acct.blob.core.windows.net/general/noarch"), + Addressing::HostStyle + ) + .unwrap(), + coordinates("acct", "general") + ); + } + + /// The fetch path's derivation: it must find the same container + /// `account_and_container` does, under both addressing styles, and it must not + /// inherit that function's account rules — a host-style IP literal has no + /// account label, but it still has a container, and a fetch for it is a request + /// that works today. + #[test] + fn container_is_derived_from_the_addressing() { + for (url, addressing, expected) in [ + ( + "az://acct.blob.core.windows.net/general/noarch", + Addressing::HostStyle, + "general", + ), + ( + "az://127.0.0.1:10000/devstoreaccount1/general/noarch", + Addressing::PathStyle, + "general", + ), + ( + "az://127.0.0.1:10000/general/noarch", + Addressing::HostStyle, + "general", + ), + ] { + assert_eq!( + container(&channel(url), addressing).unwrap(), + Some(ContainerName::new(expected).unwrap()), + "{url}" + ); + } + + // Where both derivations answer, they must answer the same thing: a grant + // looked up for one container and applied to another is a security bug. + for (url, addressing) in [ + ( + "az://acct.blob.core.windows.net/general/noarch", + Addressing::HostStyle, + ), + ( + "az://127.0.0.1:10000/devstoreaccount1/general", + Addressing::PathStyle, + ), + ] { + assert_eq!( + container(&channel(url), addressing).unwrap(), + Some( + account_and_container(&channel(url), addressing) + .unwrap() + .container + ), + "{url}" + ); + } + } + + /// A URL with no container segment is not an error: there is nothing to + /// attribute a grant to, so the fetch path sends nothing and stays total for + /// URLs that are not channel-scoped. + #[test] + fn a_url_without_a_container_names_none() { + for (url, addressing) in [ + ("az://acct.blob.core.windows.net", Addressing::HostStyle), + ("az://acct.blob.core.windows.net/", Addressing::HostStyle), + ( + "az://127.0.0.1:10000/devstoreaccount1", + Addressing::PathStyle, + ), + ("az://127.0.0.1:10000/", Addressing::PathStyle), + ] { + assert_eq!(container(&channel(url), addressing).unwrap(), None, "{url}"); + } + } + + /// A segment that cannot be a container name is a malformed endpoint, not an + /// ungranted one — Azure forbids uppercase, so no legitimate request lands + /// here. Going quietly anonymous would surface as an unexplained 401 instead of + /// naming the fault. + #[test] + fn a_url_with_an_unusable_container_is_an_error() { + for url in [ + "az://acct.blob.core.windows.net/General/noarch", + "az://acct.blob.core.windows.net/ab/noarch", + "az://acct.blob.core.windows.net/a--b/noarch", + ] { + let err = container(&channel(url), Addressing::HostStyle) + .expect_err("an illegal container name must be reported"); + assert!( + matches!(err, AzureUrlError::InvalidContainerName(_)), + "{url}: {err}" + ); + } + } + + #[test] + fn userinfo_is_rejected() { + assert!(matches!( + AzureChannelUrl::parse("az://acct.blob.core.windows.net@evil.example/general"), + Err(AzureUrlError::UserInfoNotAllowed) + )); + assert!(matches!( + AzureHost::parse("acct.blob.core.windows.net@evil.example"), + Err(AzureUrlError::UserInfoNotAllowed) + )); + } + + /// Azure's naming rules are what keep injection-shaped values out of the `az` + /// subprocess, so they have to hold under path-style too — where the account + /// comes from user-controlled path text rather than a host label. + #[test] + fn invalid_component_names_are_rejected_under_both_styles() { + assert!(matches!( + account_and_container( + &channel("az://acct.blob.core.windows.net/general;evil/noarch"), + Addressing::HostStyle + ), + Err(AzureUrlError::InvalidContainerName(_)) + )); + + for (path, account_at_fault) in [ + ("az://127.0.0.1:10000/devstore;evil/general", true), + ("az://127.0.0.1:10000/DevStoreAccount1/general", true), + // Azure allows no hyphen at all in an account name. + ("az://127.0.0.1:10000/dev-store/general", true), + // Too short for Azure, whatever the charset says. + ("az://127.0.0.1:10000/ab/general", true), + ("az://127.0.0.1:10000/devstoreaccount1/general;evil", false), + ("az://127.0.0.1:10000/devstoreaccount1/ab", false), + ("az://127.0.0.1:10000/devstoreaccount1/a--b", false), + ("az://127.0.0.1:10000/devstoreaccount1/-general", false), + ("az://127.0.0.1:10000/devstoreaccount1/general-", false), + ] { + let Err(err) = account_and_container(&channel(path), Addressing::PathStyle) else { + panic!("expected a rejection for {path}"); + }; + let matched = if account_at_fault { + matches!(err, AzureUrlError::InvalidAccountName(_)) + } else { + matches!(err, AzureUrlError::InvalidContainerName(_)) + }; + assert!(matched, "unexpected error for {path}: {err}"); + } + } + + /// The docstring on [`account_and_container`] promises option-shaped values can + /// never reach the `az` argv. A charset of `[a-z0-9-]` alone does not deliver + /// that, because a leading `-` is inside it. + #[test] + fn option_shaped_components_are_rejected() { + for (url, account_at_fault) in [ + ("az://--as-user.blob.core.windows.net/general", true), + ("az://-o.blob.core.windows.net/general", true), + ("az://acct.blob.core.windows.net/--https-only/noarch", false), + ("az://acct.blob.core.windows.net/-o/noarch", false), + ] { + let Err(err) = account_and_container(&channel(url), Addressing::HostStyle) else { + panic!("expected a rejection for {url}"); + }; + let matched = if account_at_fault { + matches!(err, AzureUrlError::InvalidAccountName(_)) + } else { + matches!(err, AzureUrlError::InvalidContainerName(_)) + }; + assert!(matched, "unexpected error for {url}: {err}"); + } + } + + /// An empty name reaching a constructor must be rejected there, not only by a + /// caller remembering to filter it out first. + #[test] + fn empty_components_are_rejected() { + assert!(AccountName::new("").is_err()); + assert!(ContainerName::new("").is_err()); + } + + #[test] + fn path_style_derives_account_from_first_segment() { + for host in [ + "127.0.0.1:10000", + "[::1]:10000", + "azurite:10000", + "localhost:10000", + // A bare docker service name, which is the shape a compose file gives. + "azurite", + "localhost", + ] { + assert_eq!( + account_and_container( + &channel(&format!("az://{host}/devstoreaccount1/general/noarch")), + Addressing::PathStyle + ) + .unwrap(), + coordinates("devstoreaccount1", "general"), + "path-style derivation failed for {host}" + ); + } + } + + #[test] + fn path_style_needs_two_segments() { + assert!(matches!( + account_and_container( + &channel("az://127.0.0.1:10000/devstoreaccount1"), + Addressing::PathStyle + ), + Err(AzureUrlError::NoContainer) + )); + + for empty in ["az://127.0.0.1:10000/", "az://127.0.0.1:10000"] { + assert!(matches!( + account_and_container(&channel(empty), Addressing::PathStyle), + Err(AzureUrlError::NoAccount) + )); + } + } + + /// A `path-style = true` entry on a real Azure host is a config mistake with no + /// visible symptom — request URLs come out identical under both styles — right + /// up to a mint asking for a delegation SAS on the account name the *path* + /// happened to spell. Which addressing a host uses is still the user's call, so + /// this warns and proceeds. + #[test] + #[tracing_test::traced_test] + fn path_style_on_a_real_azure_host_warns_and_proceeds() { + let coords = account_and_container( + &channel("az://acct.blob.core.windows.net/general/mychannel"), + Addressing::PathStyle, + ) + .expect("addressing is the user's call, so this is a warning and not an error"); + assert_eq!(coords.account.as_str(), "general"); + + assert!(logs_contain("path-style = true")); + assert!(logs_contain("acct.blob.core.windows.net")); + + // A host that is not a known Azure endpoint gets no warning, however much + // its first label looks like an account name. + let coords = account_and_container( + &channel("az://acct.blob.example.com/devstoreaccount1/general"), + Addressing::PathStyle, + ) + .unwrap(); + assert_eq!(coords.account.as_str(), "devstoreaccount1"); + assert!(!logs_contain("acct.blob.example.com")); + } + + /// The suffix list is advisory, but a sloppy match on it would warn about + /// hosts Microsoft does not operate — and stay silent on ones it does. + #[test] + fn known_azure_endpoints_are_matched_on_a_label_boundary() { + for host in [ + "acct.blob.core.windows.net", + "acct.blob.core.usgovcloudapi.net", + "acct.blob.core.chinacloudapi.cn", + ] { + assert!( + AzureHost::parse(host) + .unwrap() + .is_known_azure_blob_endpoint(), + "{host}" + ); + } + + for host in [ + "notblob.core.windows.net", // no label boundary + "blob.core.windows.net", // the suffix alone carries no account + "acct.blob.core.windows.net.evil.test", // suffix in the middle + "127.0.0.1:10000", + "azurite", + ] { + assert!( + !AzureHost::parse(host) + .unwrap() + .is_known_azure_blob_endpoint(), + "{host}" + ); + } + } + + /// Host-style must keep rejecting hosts it cannot derive an account from — and + /// the rejection must hand the user a config key that would actually match, + /// which means the port has to be in it and the host has to be spelled the way + /// [`AzureHost`] spells it. + #[test] + fn host_style_rejects_undottable_hosts_with_a_guided_error() { + for (host, expected_key) in [ + ("127.0.0.1:10000", "127.0.0.1:10000"), + ("azurite:10000", "azurite:10000"), + ("localhost", "localhost"), + ("[::1]:10000", "[::1]:10000"), + // A trailing dot is the DNS root label, not a second label: this host + // must not sneak past the dotted-domain gate. + ("localhost.", "localhost"), + ("LocalHost", "localhost"), + // A port equal to a wire scheme's default: it is part of the key, and + // only survives because the channel — not a `Url` — is what is read. + ("azurite:443", "azurite:443"), + ("azurite:80", "azurite:80"), + ] { + let channel = channel(&format!("az://{host}/devstoreaccount1/general")); + let err = account_and_container(&channel, Addressing::HostStyle) + .expect_err("host-style must not accept an undottable host"); + assert!(matches!(err, AzureUrlError::InvalidHost(_)), "{err}"); + + let message = err.to_string(); + assert!(message.contains("path-style = true"), "{message}"); + let key = format!("[azure-options.\"{expected_key}\"]"); + assert!(message.contains(&key), "{message}"); + // The key named must be the one an `azure-options` lookup is made + // with, or the entry the user writes cannot ever apply. + assert_eq!(expected_key, channel.host().to_string(), "{host}"); + } + } + + /// An empty host label is never legal, and used to yield an empty first + /// "label" as the account name. + #[test] + fn empty_host_labels_are_rejected() { + for host in [ + "acct..blob.core.windows.net", + "acct.blob.example..", + ".example", + ] { + assert!( + matches!( + AzureHost::parse(host), + Err(AzureUrlError::InvalidHostAuthority { .. }) + ), + "expected a rejection for {host}" + ); + assert!( + matches!( + AzureChannelUrl::parse(&format!("az://{host}/general/noarch")), + Err(AzureUrlError::InvalidHostAuthority { .. }) + ), + "expected a rejection for {host}" + ); + } + } + + #[test] + fn parse_requires_the_az_scheme() { + for input in [ + "https://acct.blob.core.windows.net/general", + "http://acct.blob.core.windows.net/general", + "ftp://acct.blob.core.windows.net/general", + "acct.blob.core.windows.net/general", + ] { + assert!( + matches!( + AzureChannelUrl::parse(input), + Err(AzureUrlError::InvalidScheme(_)) + ), + "expected InvalidScheme for {input}" + ); + } + } + + /// URL schemes are case-insensitive, and the middleware's `scheme() == "az"` + /// test sees an already-lowercased scheme, so it accepts `AZ://`. This parser + /// must not disagree with it. + #[test] + fn parse_accepts_a_scheme_in_any_case() { + for input in [ + "AZ://acct.blob.core.windows.net/general", + "Az://acct.blob.core.windows.net/general", + "aZ://acct.blob.core.windows.net/general", + ] { + let channel = AzureChannelUrl::parse(input) + .unwrap_or_else(|err| panic!("{input} should parse: {err}")); + assert_eq!( + channel.canonical().as_str(), + "az://acct.blob.core.windows.net/general" + ); + } + } + + #[test] + fn canonical_and_wire_round_trip() { + let channel = + AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/noarch").unwrap(); + + assert_eq!( + channel.canonical().as_str(), + "az://acct.blob.core.windows.net/general/noarch" + ); + assert_eq!( + channel.wire(AzureScheme::Https).as_str(), + "https://acct.blob.core.windows.net/general/noarch" + ); + assert_eq!( + channel.wire(AzureScheme::Http).as_str(), + "http://acct.blob.core.windows.net/general/noarch" + ); + assert_eq!(channel.to_string(), channel.canonical().to_string()); + // `FromStr` is the same parser, so the canonical spelling parses back to + // the same value — which is what lets a config key round-trip. + assert_eq!( + channel, + channel + .canonical() + .as_str() + .parse::() + .unwrap() + ); + } + + /// The point of storing the parts: no scheme choice can make the two spellings + /// describe different locations. + #[test] + fn spellings_cannot_disagree() { + for input in [ + "az://acct.blob.core.windows.net/general/noarch", + "az://127.0.0.1:10000/devstoreaccount1/general", + "az://acct.blob.core.windows.net/general/with%20space?sv=token", + // An IPv6 literal is the host shape most likely to break the canonical + // rebuild, since it has to survive being re-parsed as an opaque host. + "az://[::1]:10000/devstoreaccount1/general", + // The scheme-default ports: exactly the spellings a `Url` stored with a + // fixed scheme silently drops. + "az://azurite.local:443/devstoreaccount1/general", + "az://azurite.local:80/devstoreaccount1/general", + ] { + let channel = AzureChannelUrl::parse(input).unwrap(); + let canonical = channel.canonical(); + for scheme in [AzureScheme::Https, AzureScheme::Http] { + let wire = channel.wire(scheme); + assert_eq!(wire.scheme(), scheme.as_str()); + assert_eq!(canonical.host_str(), wire.host_str(), "{input}"); + assert_eq!(canonical.path(), wire.path(), "{input}"); + assert_eq!(canonical.query(), wire.query(), "{input}"); + + // Ports are compared semantically, not textually: `az` has no + // default port so the canonical form always spells one out when the + // URL has one, while a wire URL omits a port equal to its scheme's + // default. An omitted port on `http` *is* 80, so those agree. + let default = match scheme { + AzureScheme::Https => 443, + AzureScheme::Http => 80, + }; + assert_eq!( + wire.port_or_known_default(), + Some(canonical.port().unwrap_or(default)), + "{input} over {scheme}" + ); + } + } + } + + /// The `:443` regression: a wire URL stored with the `https` scheme drops this + /// port, and `wire(Http)` then names a completely different endpoint. + #[test] + fn a_written_default_port_survives() { + let channel = + AzureChannelUrl::parse("az://azurite.local:443/devstoreaccount1/general").unwrap(); + + assert_eq!(channel.host().to_string(), "azurite.local:443"); + assert_eq!(channel.host().port(), Some(443)); + assert_eq!( + channel.canonical().as_str(), + "az://azurite.local:443/devstoreaccount1/general" + ); + assert_eq!( + channel.wire(AzureScheme::Http).as_str(), + "http://azurite.local:443/devstoreaccount1/general" + ); + assert_eq!( + channel.wire(AzureScheme::Https).as_str(), + "https://azurite.local/devstoreaccount1/general" + ); + + // Identity must not be scheme-relative either: a host on 443 is not the + // same endpoint as the same host with no port, because the scheme that + // would make them equal is not known here. + let no_port = + AzureChannelUrl::parse("az://azurite.local/devstoreaccount1/general").unwrap(); + assert_ne!(channel, no_port); + assert_ne!(channel.host(), no_port.host()); + } + + #[test] + fn host_keeps_a_non_default_port() { + let emulator = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + assert_eq!(emulator.host().to_string(), "127.0.0.1:10000"); + assert_eq!( + emulator.wire(AzureScheme::Http).as_str(), + "http://127.0.0.1:10000/devstoreaccount1/general" + ); + assert_eq!( + emulator.canonical().as_str(), + "az://127.0.0.1:10000/devstoreaccount1/general" + ); + + // No port written, none invented. + let azure = AzureChannelUrl::parse("az://acct.blob.core.windows.net/general").unwrap(); + assert_eq!(azure.host().to_string(), "acct.blob.core.windows.net"); + assert_eq!(azure.host().port(), None); + } + + /// Every normalization the URL host parser performs is a way for a written + /// config key and a looked-up host to disagree, unless both go through the same + /// parser. They do: this is that parser, and these are the classes it has to + /// collapse. + #[test] + fn host_normalization_collapses_equivalent_spellings() { + for (written, canonical) in [ + ( + "MyCompany.blob.core.windows.net", + "mycompany.blob.core.windows.net", + ), + ( + "mycompany.blob.core.windows.net:443", + "mycompany.blob.core.windows.net:443", + ), + ("ünï.blob.example", "xn--n-nga1b.blob.example"), + ("xn--n-nga1b.blob.example", "xn--n-nga1b.blob.example"), + ("[0:0:0:0:0:0:0:1]:10000", "[::1]:10000"), + ("[::1]:10000", "[::1]:10000"), + ("0x7f.1", "127.0.0.1"), + ("127.0.0.1", "127.0.0.1"), + ("acct.blob.core.windows.net.", "acct.blob.core.windows.net"), + ("acct.blob.core.windows.net", "acct.blob.core.windows.net"), + ] { + let host = AzureHost::parse(written) + .unwrap_or_else(|err| panic!("{written} should parse: {err}")); + assert_eq!(host.to_string(), canonical, "{written}"); + + // Display and parse round-trip, so a key written out of an `AzureHost` + // parses back to the same host… + let reparsed = AzureHost::parse(canonical).unwrap(); + assert_eq!(reparsed, host, "{written}"); + // …and equal hosts hash equally, so they land on the same map entry. + assert_eq!(hash_of(&host), hash_of(&reparsed), "{written}"); + } + } + + /// A written port is part of the endpoint's identity: nothing here knows the + /// scheme, so nothing here can call 443 or 80 redundant. + #[test] + fn host_equality_is_not_scheme_relative() { + let with_port = AzureHost::parse("azurite.local:443").unwrap(); + let without = AzureHost::parse("azurite.local").unwrap(); + assert_ne!(with_port, without); + assert_ne!(with_port, AzureHost::parse("azurite.local:80").unwrap()); + assert_eq!(with_port.to_string(), "azurite.local:443"); + } + + /// A config key is a bare authority; anything else is a mistake worth naming + /// rather than silently reinterpreting. + #[test] + fn host_rejects_anything_that_is_not_a_bare_authority() { + // A name DNS cannot resolve and a port nothing can connect to: `wire()` + // would otherwise hand out `https://host:0/…`, and a bare `host:` would be + // silently read as the portless host, a different endpoint entirely. + // Labels of 60, so length is the only rule under test. + let label = "a".repeat(60); + let too_long = format!("{}.blob.example", [label.as_str(); 8].join(".")); + for authority in [ + "acct.blob.core.windows.net/general", + "acct.blob.core.windows.net?sv=token", + "acct.blob.core.windows.net#frag", + "https://acct.blob.core.windows.net", + "", + "acct.blob.core.windows.net:notaport", + "acct.blob.core.windows.net:", + "acct.blob.core.windows.net:0", + "[::1]:", + "[::1]:0", + &too_long, + ] { + assert!( + AzureHost::parse(authority).is_err(), + "expected a rejection for {authority:?}" + ); + } + + // A name right at the limit still parses, so the check bounds the length + // rather than the number of labels. + let at_limit = format!( + "{}.{}.blob.example", + [label.as_str(); 3].join("."), + "a".repeat(57) + ); + assert_eq!(at_limit.len(), 253); + assert!(AzureHost::parse(&at_limit).is_ok()); + } + + fn hash_of(host: &AzureHost) -> u64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + host.hash(&mut hasher); + hasher.finish() + } + + /// The path-style recipe, asserted string by string, because every field but + /// `container` differs from host-style and each one fails silently when it is + /// wrong: a missing `account_name` becomes a 403, a trailing slash becomes + /// `//container/…`, and a `root` that skips one segment too few writes the + /// whole channel one directory too deep. + #[cfg(feature = "opendal")] + #[test] + fn azblob_config_under_path_style() { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general/mychannel") + .unwrap(); + let config = azblob_config( + &AzureCredentials::AccountKey("key".into()), + &channel, + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) + .unwrap(); + + assert_eq!( + config.endpoint.as_deref(), + Some("http://127.0.0.1:10000/devstoreaccount1") + ); + assert_eq!(config.account_name.as_deref(), Some("devstoreaccount1")); + assert_eq!(config.container, "general"); + assert_eq!(config.root.as_deref(), Some("/mychannel")); + assert_eq!(config.account_key.as_deref(), Some("key")); + + let endpoint = config.endpoint.unwrap(); + assert!(!endpoint.ends_with('/'), "{endpoint}"); + let root = config.root.unwrap(); + assert!( + !root.contains("general"), + "the container must not appear in the root: {root}" + ); + assert!( + !root.contains("devstoreaccount1"), + "the account must not appear in the root: {root}" + ); + } + + /// A channel that is a bare `account/container` leaves nothing for the root, + /// which must still be `/` and not the empty string opendal would treat as a + /// relative path. + #[cfg(feature = "opendal")] + #[test] + fn azblob_config_path_style_without_a_prefix() { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + let config = azblob_config( + &AzureCredentials::SasToken("?sv=token".into()), + &channel, + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) + .unwrap(); + + assert_eq!(config.root.as_deref(), Some("/")); + assert_eq!(config.container, "general"); + // The leading `?` is stripped exactly once, wherever the token came from. + assert_eq!(config.sas_token.as_deref(), Some("sv=token")); + } + + /// Host-style is the shape every existing caller uses, so honouring + /// path-style must not have moved it. + #[cfg(feature = "opendal")] + #[test] + fn azblob_config_under_host_style_is_unchanged() { + let channel = + AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general/sub/dir") + .unwrap(); + let config = azblob_config( + &AzureCredentials::SasToken("sv=token".into()), + &channel, + AzureEndpoint::default(), + ) + .unwrap(); + + assert_eq!( + config.endpoint.as_deref(), + Some("https://stcondachannel.blob.core.windows.net") + ); + assert_eq!(config.account_name.as_deref(), Some("stcondachannel")); + assert_eq!(config.container, "general"); + assert_eq!(config.root.as_deref(), Some("/sub/dir")); + assert_eq!(config.sas_token.as_deref(), Some("sv=token")); + assert_eq!(config.account_key, None); + } + + /// A prefix with a space arrives here percent-encoded and opendal encodes + /// `root + path` again, so the root has to be handed over decoded. + #[cfg(feature = "opendal")] + #[test] + fn azblob_config_decodes_the_root() { + let channel = + AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/with%20space").unwrap(); + let config = azblob_config( + &AzureCredentials::AccountKey("key".into()), + &channel, + AzureEndpoint::default(), + ) + .unwrap(); + + assert_eq!(config.root.as_deref(), Some("/with space")); + } + + /// Account and container derivation is deliberately *not* part of parsing: the + /// addressing style is config that does not exist yet when clap parses the + /// argument, so an emulator URL must survive parsing and be rejected (or not) + /// later, once its options entry is known. + #[test] + fn parse_defers_account_derivation() { + let channel = channel("az://127.0.0.1:10000/devstoreaccount1/general"); + + assert!(account_and_container(&channel, Addressing::HostStyle).is_err()); + assert!(account_and_container(&channel, Addressing::PathStyle).is_ok()); + } + + /// The rewrite is invisible in the URL a user reads back: under path-style it + /// moves the *account* too, so a channel URL that says `devstoreaccount1` mints + /// a SAS for whatever account the escaped `..` climbs out to. + #[test] + fn a_rewritten_path_is_rejected() { + for (input, resolved) in [ + ( + "az://acct.blob.core.windows.net/general/%2e%2e/%2e%2e/othercontainer/x", + "/othercontainer/x", + ), + ( + "az://127.0.0.1:10000/devstoreaccount1/general/%2e%2e/%2e%2e/otheraccount/othercontainer", + "/otheraccount/othercontainer", + ), + ( + "az://acct.blob.core.windows.net/general/../../othercontainer", + "/othercontainer", + ), + ( + "az://acct.blob.core.windows.net/general/./noarch", + "/general/noarch", + ), + ] { + let Err(err) = AzureChannelUrl::parse(input) else { + panic!("expected a rejection for {input}"); + }; + assert!( + matches!(&err, AzureUrlError::NonCanonicalPath { resolved: got, .. } if got == resolved), + "unexpected error for {input}: {err}" + ); + // Both spellings are in the message, so the user can see where the URL + // they wrote would have gone. + let message = err.to_string(); + assert!(message.contains(resolved), "{message}"); + assert!(message.contains("/general/"), "{message}"); + } + } + + /// The paths that must keep parsing: nothing about rejecting rewrites may + /// narrow what an ordinary channel URL can say. + #[test] + fn unrewritten_paths_still_parse() { + for (input, path) in [ + ( + "az://acct.blob.core.windows.net/general/prefix", + "/general/prefix", + ), + ("az://acct.blob.core.windows.net/general/", "/general/"), + ("az://acct.blob.core.windows.net/", "/"), + ("az://acct.blob.core.windows.net", "/"), + ( + "az://acct.blob.core.windows.net/general/with%20space", + "/general/with%20space", + ), + ( + "az://acct.blob.core.windows.net/general/p?sv=token#frag", + "/general/p", + ), + // A dot inside a segment is not a dot segment. + ( + "az://acct.blob.core.windows.net/general/..hidden/...", + "/general/..hidden/...", + ), + ] { + assert_eq!(channel(input).canonical().path(), path, "{input}"); + } + } + + /// A segment that cannot become a blob name is refused, rather than becoming a + /// different blob name than the URL says. + #[test] + fn segments_that_cannot_name_a_blob_are_rejected() { + assert!(matches!( + AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/%ff"), + Err(AzureUrlError::NonUtf8Path { .. }) + )); + + // Both spellings: `url` normalizes the hex digits' case but not the escape. + for input in [ + "az://acct.blob.core.windows.net/general/a%2Fb", + "az://acct.blob.core.windows.net/general/a%2fb", + ] { + assert!( + matches!( + AzureChannelUrl::parse(input), + Err(AzureUrlError::EncodedSlashInPath(_)) + ), + "{input}" + ); + } + + // A percent escape that is valid UTF-8 is still a legitimate segment. + assert_eq!( + channel("az://acct.blob.core.windows.net/general/caf%C3%A9") + .canonical() + .path(), + "/general/caf%C3%A9" + ); + } + + /// `--https-only` restricts the SAS to TLS, so a host configured for http would + /// get a token it cannot use. Nothing else about the grant may move with it. + #[cfg(feature = "clap")] + #[test] + fn https_only_follows_the_configured_scheme() { + let coordinates = coordinates("acct", "general"); + let args = |scheme| { + generate_sas_args( + &coordinates.account, + &coordinates.container, + "cw", + "2030-01-01T00:00:00Z", + scheme, + ) + }; + + assert!(args(AzureScheme::Https).contains(&"--https-only")); + assert!(!args(AzureScheme::Http).contains(&"--https-only")); + + for scheme in [AzureScheme::Https, AzureScheme::Http] { + let args = args(scheme); + assert!(args.windows(2).any(|pair| pair == ["--permissions", "cw"])); + assert!( + args.windows(2) + .any(|pair| pair == ["--expiry", "2030-01-01T00:00:00Z"]) + ); + assert!(args.contains(&"--as-user")); + assert!(args.windows(2).any(|pair| pair == ["--auth-mode", "login"])); + } + } +} + +#[cfg(test)] +mod debug_redaction_tests { + use super::*; + + #[test] + fn debug_never_prints_secret() { + for creds in [ + AzureCredentials::AccountKey("supersecretkey".into()), + AzureCredentials::SasToken("sig=deadbeef".into()), + ] { + let out = format!("{creds:?}"); + assert!(out.contains("REDACTED"), "not redacted: {out}"); + assert!(!out.contains("supersecret")); + assert!(!out.contains("deadbeef")); + } + } + + /// An inline SAS reaches the wire and nothing else. Every other spelling of the + /// channel is a log line or an error message waiting to happen. + #[test] + fn only_the_wire_spelling_carries_the_signature() { + let channel = AzureChannelUrl::parse( + "az://acct.blob.core.windows.net/general/p?sv=2024-11-04&sig=SECRETSIG&se=z", + ) + .unwrap(); + + for shown in [ + channel.canonical().to_string(), + channel.to_string(), + format!("{channel:?}"), + ] { + assert!(!shown.contains("SECRETSIG"), "signature leaked: {shown}"); + // The rest of the grant is not secret and is worth showing. + assert!(shown.contains("sv=2024-11-04"), "over-redacted: {shown}"); + assert!(shown.contains("se=z"), "over-redacted: {shown}"); + } + + assert!( + channel + .wire(AzureScheme::Https) + .to_string() + .contains("sig=SECRETSIG"), + "the wire spelling must keep the signature that authenticates the request" + ); + } +} diff --git a/crates/rattler_azure/src/options.rs b/crates/rattler_azure/src/options.rs new file mode 100644 index 0000000000..9fdf071720 --- /dev/null +++ b/crates/rattler_azure/src/options.rs @@ -0,0 +1,515 @@ +//! Per-host endpoint options for Azure Blob channels. +//! +//! An entry in the `azure-options` config table is the *only* thing that grants +//! a host or one of its containers anything: without one, a channel on that host +//! is fetched anonymously over https in host-style addressing. There is +//! deliberately no hardcoded list of "official" Azure suffixes — since a grant must +//! be written out for the container it applies to, suffix classification carries no +//! security weight, and the absence of the list is what lets custom endpoints and +//! the Azurite emulator work at all. +//! +//! # Three types, one table +//! +//! [`AzureEndpointOptions`] is the file format. The write path never sees it: it +//! takes [`AzureEndpoint`], so a grant cannot reach a caller that supplies its own +//! credential — a field a consumer would have to ignore reads as a guarantee, and +//! the write path ignoring a grant looked exactly like a credential gate that was +//! never there. +//! +//! The fetch path holds the entry but never acts on it directly either: per request +//! it narrows to [`AzureFetchOptions`], which carries the grant for the one +//! container the request addresses and no way to address another. It cannot narrow +//! any earlier, because finding that container needs the entry's [`Addressing`] +//! first. +//! +//! # Why enums for what the config spells as bools +//! +//! The TOML surface stays ` = true` / `path-style = true`, because that +//! is the ergonomic spelling and it keeps the table skimmable. Internally each is an +//! enum, so no call site can mix up two unrelated booleans, and the meaning of a +//! value is legible without chasing the field name. The bridge is a +//! `serde(from = "bool", into = "bool")` pair with a `From` impl each way, applied +//! through `cfg_attr` so the serde attributes stay behind the `serde` feature +//! along with the derives they configure. +//! +//! The types themselves are always available — [`Addressing`] decides how +//! [`account_and_container`](crate::account_and_container) reads a URL and +//! [`AzureScheme`] is what [`AzureChannelUrl::wire`](crate::AzureChannelUrl::wire) +//! is spelled in, neither of which involves serde. Only the derives are behind +//! the `serde` feature, so a consumer that just wants the URL types does not +//! pull serde in. + +use crate::ContainerName; + +/// Whether credentials may attach to requests for a container. +/// +/// Defaults to [`Auth::Anonymous`]: a container gets no credentials until a config +/// entry names it. Serializes as the bool a container is spelled with in an +/// `azure-options` `auth` table. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(from = "bool", into = "bool") +)] +pub enum Auth { + /// Send requests unsigned. No credential is resolved, so no ambient + /// credential can be exfiltrated to this host, and nothing blocks on the + /// managed-identity / IMDS probe. + #[default] + Anonymous, + + /// Run the standard Azure credential chain for this container and sign with what + /// it returns. Because this is an explicit grant, a broken or unusable + /// credential is a hard error — never a silent downgrade to anonymous. + DefaultChain, +} + +impl From for Auth { + fn from(value: bool) -> Self { + if value { + Auth::DefaultChain + } else { + Auth::Anonymous + } + } +} + +impl From for bool { + fn from(value: Auth) -> Self { + matches!(value, Auth::DefaultChain) + } +} + +impl Auth { + /// Whether this grant permits a credential to be sent. + pub fn is_granted(self) -> bool { + matches!(self, Auth::DefaultChain) + } +} + +/// The wire scheme an `az://` channel URL is rewritten to when a request is sent. +/// +/// Named for the crate rather than spelled bare `Scheme`, because this crate also +/// depends on `opendal`, whose own `Scheme` names a storage service — two very +/// different things one import away from each other. +/// +/// Defaults to [`AzureScheme::Https`]. `Http` exists for local emulators such as +/// Azurite; choosing it is an explicit, per-host decision in config, so a plain +/// `az://` URL can never be silently downgraded to cleartext. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "lowercase") +)] +pub enum AzureScheme { + /// Send requests over TLS. + #[default] + Https, + + /// Send requests in cleartext. For local emulators only. + Http, +} + +impl AzureScheme { + /// The scheme as it appears in a URL, without the `://`. + pub fn as_str(self) -> &'static str { + match self { + AzureScheme::Https => "https", + AzureScheme::Http => "http", + } + } +} + +impl std::fmt::Display for AzureScheme { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Where the storage account name is found in a blob URL. +/// +/// Defaults to [`Addressing::HostStyle`], which is how real Azure addresses +/// accounts. Serializes as the bool `path-style` in `azure-options`; `s3-options` +/// spells its equivalent `force-path-style`, also a bool. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(from = "bool", into = "bool") +)] +pub enum Addressing { + /// The account is the first label of the host, as in + /// `.blob.core.windows.net/`. Requires a domain with at + /// least two labels, so IP literals and single-label hosts cannot be + /// addressed this way. + #[default] + HostStyle, + + /// The account is the first path segment, as in + /// `//`. This is what Azurite and other emulators + /// use, and it is the only form that works for an IP or single-label host. + PathStyle, +} + +impl Addressing { + /// Which path segment holds the container name under this addressing. + /// + /// One number, read by both derivations that need it — + /// [`account_and_container`](crate::account_and_container) and + /// [`container`](crate::container). Two derivations that disagreed about which + /// segment is the container would look a grant up for one container and apply + /// it to another. + pub(crate) fn container_segment(self) -> usize { + match self { + // `.host//…` + Addressing::HostStyle => 0, + // `host///…` + Addressing::PathStyle => 1, + } + } +} + +impl From for Addressing { + fn from(value: bool) -> Self { + if value { + Addressing::PathStyle + } else { + Addressing::HostStyle + } + } +} + +impl From for bool { + fn from(value: Addressing) -> Self { + matches!(value, Addressing::PathStyle) + } +} + +/// How to address one Azure Blob host. Carries no grant. +/// +/// This is what the write path takes. Splitting it out is what stops +/// [`Auth`] from reaching a consumer that cannot act on it: `azblob_config` and +/// the SAS mint are handed a material credential by their caller, so there is no +/// ambient chain for a grant to gate, and a grant they could read would be a +/// promise nothing keeps. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +pub struct AzureEndpoint { + /// The scheme `az://` is rewritten to for this host. + pub scheme: AzureScheme, + + /// Where the account name is found in the URL for this host. + pub addressing: Addressing, +} + +/// What the fetch middleware needs to send one request. +/// +/// [`Addressing`] is absent rather than ignored: the fetch path forwards a path +/// and never derives an account name from it. It does read the addressing to find +/// the container a grant is looked up by, but that happens one step earlier, on +/// [`AzureEndpointOptions`] — by the time this exists the grant is already +/// resolved, so there is nothing left here to address. +#[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] +pub struct AzureFetchOptions { + /// Whether credentials may be sent for the container this was resolved for. + pub auth: Auth, + + /// The scheme `az://` is rewritten to for this host. + pub scheme: AzureScheme, +} + +/// One `azure-options` entry, as the config file spells it. +/// +/// This is the serde surface and nothing else: the TOML keys live here, and each +/// consumer takes the narrower view it can actually act on, via [`Self::endpoint`] +/// or [`Self::fetch`]. The fields are private so that view is the only way in. +/// +/// The default value is the no-entry behaviour: no grants, https, host-style. A +/// host with no config entry behaves exactly as if it had a defaulted entry, so +/// callers can look up an absent host and fall back to `default()` rather than +/// branching on presence. +/// +/// # Why the grant is per container and the endpoint is per host +/// +/// The two settings have different scopes, and it is not a matter of taste. Azure +/// assigns RBAC per *container*, so one storage account routinely holds a mix of +/// private and anonymous-read containers; a grant that could only be written per +/// host cannot express that account at all — signing the anonymous-read container +/// 403s for any identity holding no role on it, and not signing breaks the private +/// ones. `scheme` and `addressing` describe the *endpoint*: two containers on one +/// account disagreeing about where the account name lives is not a configuration, +/// it is a contradiction. +/// +/// So there is deliberately no host-level `auth` field. Not "defaults to false" — +/// absent from the type, so the one setting whose blast radius would be every +/// container on the account, including containers created after it was written, is +/// unrepresentable rather than merely discouraged. The worst typo here grants one +/// container. +/// +/// ```toml +/// [azure-options."mycompany.blob.core.windows.net"] +/// scheme = "https" +/// path-style = false +/// +/// [azure-options."mycompany.blob.core.windows.net".auth] +/// releases = true +/// staging = true +/// # a container not listed here is fetched anonymously +/// ``` +#[derive(Default, Clone, Debug, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "kebab-case", default) +)] +pub struct AzureEndpointOptions { + scheme: AzureScheme, + + /// The field is named for what it holds, but the config key stays + /// `path-style`: that is the spelling users have written, and the bool bridge + /// is what the key means. + #[cfg_attr(feature = "serde", serde(rename = "path-style", alias = "path_style"))] + addressing: Addressing, + + /// Which containers on this host may be sent credentials. + /// + /// The value is an [`Auth`] rather than a `bool` because that is what + /// [`AzureFetchOptions`] already speaks, so a grant flows from here to the + /// signer without a `if granted { … } else { … }` at every boundary — one less + /// place to invert a condition on a credential decision. The TOML surface is + /// unaffected: the bool bridge keeps it `releases = true`. + /// + /// An explicit `false` is legal and redundant with omission, which is what + /// makes a higher-precedence config file able to *revoke* a grant a lower one + /// made (see [`Self::layered_over`]). + /// + /// Last, because the TOML serializer must emit an entry's scalars before its + /// tables. + #[cfg_attr( + feature = "serde", + serde(skip_serializing_if = "indexmap::IndexMap::is_empty") + )] + auth: indexmap::IndexMap, +} + +impl AzureEndpointOptions { + /// Build an entry from its per-container grants and the endpoint they apply to. + pub fn new( + auth: impl IntoIterator, + endpoint: AzureEndpoint, + ) -> Self { + Self { + scheme: endpoint.scheme, + addressing: endpoint.addressing, + auth: auth.into_iter().collect(), + } + } + + /// How to address this host, for the write path. + pub fn endpoint(&self) -> AzureEndpoint { + AzureEndpoint { + scheme: self.scheme, + addressing: self.addressing, + } + } + + /// The grant and wire scheme for one container, for the fetch path. + /// + /// `container` is an `Option` because a URL need not name one: the host root, + /// or a path too short for the addressing, has nothing to attribute a grant to. + /// Answering that case here rather than at the call site is what keeps "no + /// container" from being spelled two ways — it can only mean anonymous, since + /// there is no entry it could match. + pub fn fetch(&self, container: Option<&ContainerName>) -> AzureFetchOptions { + AzureFetchOptions { + auth: container + .and_then(|container| self.auth.get(container)) + .copied() + .unwrap_or_default(), + scheme: self.scheme, + } + } + + /// Every container this entry mentions, and what it grants it. + /// + /// Includes the explicit `false`s: a caller validating or listing the table has + /// to see what the file says, not what it effectively means. + pub fn grants(&self) -> impl Iterator { + self.auth.iter().map(|(container, auth)| (container, *auth)) + } + + /// This entry layered over the one a lower-precedence config file wrote. + /// + /// `scheme` and `addressing` describe the endpoint as a whole, so this entry + /// replaces them outright. The grants merge per container: a file naming one + /// container must not silently drop a grant on a container it never mentions. + /// The merge is not a one-way ratchet, because an explicit `false` is a legal + /// grant — that is how a user file revokes what a system file granted. + pub fn layered_over(&self, lower: &Self) -> Self { + let mut auth = lower.auth.clone(); + auth.extend(self.auth.iter().map(|(c, auth)| (c.clone(), *auth))); + Self { + scheme: self.scheme, + addressing: self.addressing, + auth, + } + } +} + +#[cfg(all(test, feature = "serde"))] +mod tests { + use super::*; + + fn container(name: &str) -> ContainerName { + ContainerName::new(name).expect("test container name") + } + + /// The whole point of the bool bridge: the TOML stays boolean while the code + /// sees enums, and an absent field takes the safe default. + #[test] + fn toml_bools_map_to_enums() { + let opts: AzureEndpointOptions = toml::from_str( + r#" + scheme = "http" + path-style = true + + [auth] + releases = true + "#, + ) + .unwrap(); + assert_eq!( + opts, + AzureEndpointOptions::new( + [(container("releases"), Auth::DefaultChain)], + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) + ); + + // An empty entry is the same as no entry: anonymous, https, host-style. + let empty: AzureEndpointOptions = toml::from_str("").unwrap(); + assert_eq!(empty, AzureEndpointOptions::default()); + assert_eq!( + empty.fetch(Some(&container("releases"))), + AzureFetchOptions::default() + ); + assert!(!empty.fetch(Some(&container("releases"))).auth.is_granted()); + assert_eq!(empty.endpoint(), AzureEndpoint::default()); + } + + /// A grant applies to the container it names and to no other, which is the + /// whole reason the table is keyed per container: one storage account holds + /// private and anonymous-read containers side by side. + #[test] + fn a_grant_applies_to_one_container_only() { + let opts: AzureEndpointOptions = toml::from_str( + r#" + [auth] + releases = true + public = false + "#, + ) + .unwrap(); + + assert!(opts.fetch(Some(&container("releases"))).auth.is_granted()); + // An explicit `false` and an unlisted container behave identically; the + // spelling exists so a reader can tell "deliberately unsigned" from + // "forgotten", and so a higher-precedence file can revoke a grant. + assert!(!opts.fetch(Some(&container("public"))).auth.is_granted()); + assert!(!opts.fetch(Some(&container("staging"))).auth.is_granted()); + + // A URL naming no container has nothing to attribute a grant to. + assert!(!opts.fetch(None).auth.is_granted()); + + // `grants` reports what the file says, explicit `false` included — in the + // order the document's table iterated (`toml::Table` is a `BTreeMap`, so + // that is byte order, not write order). + assert_eq!( + opts.grants().collect::>(), + vec![ + (&container("public"), Auth::Anonymous), + (&container("releases"), Auth::DefaultChain), + ] + ); + } + + /// The endpoint is host-scoped and replaces wholesale; the grants merge per + /// container, in both directions — a higher file adds one grant without + /// dropping another, and revokes with an explicit `false`. + #[test] + fn layering_replaces_the_endpoint_and_merges_the_grants() { + let lower: AzureEndpointOptions = toml::from_str( + r#" + path-style = true + + [auth] + releases = true + staging = true + "#, + ) + .unwrap(); + let higher: AzureEndpointOptions = toml::from_str( + r#" + scheme = "http" + + [auth] + staging = false + internal = true + "#, + ) + .unwrap(); + + let merged = higher.layered_over(&lower); + + assert_eq!(merged.endpoint(), higher.endpoint()); + assert!( + merged.fetch(Some(&container("releases"))).auth.is_granted(), + "a grant the higher file never mentions must survive" + ); + assert!( + !merged.fetch(Some(&container("staging"))).auth.is_granted(), + "an explicit `false` in the higher file must revoke the lower grant" + ); + assert!(merged.fetch(Some(&container("internal"))).auth.is_granted()); + } + + /// A container name Azure would refuse is a config error at load, not a grant + /// that can never match anything. + #[test] + fn an_unusable_container_key_is_rejected() { + let err = toml::from_str::("[auth]\nReleases = true\n") + .expect_err("uppercase is not a legal container name"); + assert!(err.to_string().contains("Releases"), "{err}"); + } + + /// Round-tripping must preserve the boolean spelling, not leak the enum + /// variant names into a written config file. + #[test] + fn enums_serialize_back_to_bools() { + let toml = toml::to_string(&AzureEndpointOptions::new( + [ + (container("releases"), Auth::DefaultChain), + (container("public"), Auth::Anonymous), + ], + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + )) + .unwrap(); + assert!(toml.contains("releases = true"), "{toml}"); + assert!(toml.contains("public = false"), "{toml}"); + assert!(toml.contains("path-style = true"), "{toml}"); + assert!(toml.contains(r#"scheme = "http""#), "{toml}"); + assert!(!toml.contains("DefaultChain"), "{toml}"); + + // An entry granting nothing writes no `auth` table at all, so a config + // file keeps saying what it said. + let anonymous = toml::to_string(&AzureEndpointOptions::default()).unwrap(); + assert!(!anonymous.contains("auth"), "{anonymous}"); + } +} diff --git a/crates/rattler_config/Cargo.toml b/crates/rattler_config/Cargo.toml index cc115124df..ca7b49ec2d 100644 --- a/crates/rattler_config/Cargo.toml +++ b/crates/rattler_config/Cargo.toml @@ -23,6 +23,11 @@ thiserror = { workspace = true } dirs = { workspace = true } fs-err = { workspace = true } indexmap = { workspace = true } +# Default features off: this pulls in only `serde`, so the config crate stays +# free of the networking/opendal/CLI weight behind the azure feature flags. +rattler_azure = { workspace = true, default-features = false, features = [ + "serde", +] } rattler_conda_types = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_ignored = { workspace = true } diff --git a/crates/rattler_config/src/config.rs b/crates/rattler_config/src/config.rs index d514a3ae6a..67d3ae556b 100644 --- a/crates/rattler_config/src/config.rs +++ b/crates/rattler_config/src/config.rs @@ -20,12 +20,14 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use thiserror::Error; use url::Url; +use crate::config::azure::AzureOptionsMap; use crate::config::s3::S3OptionsMap; use crate::config::{ build::BuildConfig, concurrency::ConcurrencyConfig, index::IndexConfig, proxy::ProxyConfig, repodata_config::RepodataConfig, run_post_link_scripts::RunPostLinkScripts, }; +pub mod azure; pub mod build; pub mod channel_config; pub mod concurrency; @@ -194,6 +196,14 @@ pub struct CommonConfig { #[serde(skip_serializing_if = "S3OptionsMap::is_default")] pub s3_options: S3OptionsMap, + /// Configuration for Azure Blob. + /// + /// User-scoped by contract: an entry grants a host credentials, so this must + /// never be read from a project- or workspace-level manifest. + #[serde(default)] + #[serde(skip_serializing_if = "AzureOptionsMap::is_default")] + pub azure_options: AzureOptionsMap, + /// Per-channel configuration for `rattler-index`. #[serde(default, skip_serializing_if = "IndexConfig::is_empty")] pub index_config: IndexConfig, @@ -251,6 +261,7 @@ impl Default for CommonConfig { concurrency: ConcurrencyConfig::default(), proxy_config: ProxyConfig::default(), s3_options: S3OptionsMap::default(), + azure_options: AzureOptionsMap::default(), index_config: IndexConfig::default(), run_post_link_scripts: None, allow_symbolic_links: None, @@ -305,6 +316,7 @@ impl Config for CommonConfig { concurrency: self.concurrency.merge_config(&other.concurrency)?, proxy_config: self.proxy_config.merge_config(&other.proxy_config)?, s3_options: self.s3_options.merge_config(&other.s3_options)?, + azure_options: self.azure_options.merge_config(&other.azure_options)?, index_config: self.index_config.merge_config(&other.index_config)?, run_post_link_scripts: other .run_post_link_scripts @@ -322,6 +334,7 @@ impl Config for CommonConfig { self.concurrency.validate()?; self.proxy_config.validate()?; self.s3_options.validate()?; + self.azure_options.validate()?; self.index_config.validate()?; Ok(()) } @@ -338,6 +351,7 @@ impl Config for CommonConfig { "allow-hard-links".to_string(), "allow-ref-links".to_string(), "s3-options".to_string(), + "azure-options".to_string(), "index-config".to_string(), ]; keys.extend(prefixed_keys("build", self.build.keys())); @@ -348,6 +362,7 @@ impl Config for CommonConfig { keys.extend(prefixed_keys("concurrency", self.concurrency.keys())); keys.extend(prefixed_keys("proxy-config", self.proxy_config.keys())); keys.extend(prefixed_keys("s3-options", self.s3_options.keys())); + keys.extend(prefixed_keys("azure-options", self.azure_options.keys())); keys } } @@ -432,6 +447,9 @@ where /// should surface these to the user as warnings (they are typos or /// keys of other tools). pub fn from_toml_str(input: &str) -> Result<(Self, BTreeSet), toml::de::Error> { + azure::ensure_no_colliding_hosts(&input.parse()?) + .map_err(serde::de::Error::custom::)?; + // The document is deserialized twice: once into the common // configuration and once into the extension. Each pass records the // keys it did not recognize; only keys unknown to *both* passes are diff --git a/crates/rattler_config/src/config/azure.rs b/crates/rattler_config/src/config/azure.rs new file mode 100644 index 0000000000..a674b0446b --- /dev/null +++ b/crates/rattler_config/src/config/azure.rs @@ -0,0 +1,480 @@ +use indexmap::IndexMap; +use rattler_azure::{AzureEndpointOptions, AzureHost, AzureScheme}; +use serde::{Deserialize, Serialize}; + +use crate::config::Config; + +/// Whether a credential may cross this host's network unencrypted. +/// +/// A single-label name (`localhost`, a `docker compose` service) has no public DNS +/// resolution, so it counts as local; anything with a dot does not. +fn is_local(host: &AzureHost) -> bool { + match host.host() { + url::Host::Domain(domain) => !domain.contains('.'), + url::Host::Ipv4(ip) => { + ip.is_loopback() || ip.is_private() || ip.is_link_local() || ip.is_unspecified() + } + url::Host::Ipv6(ip) => ip.is_loopback() || ip.is_unspecified(), + } +} + +/// Per-host options for Azure Blob channels, keyed by endpoint authority +/// (including a port where one is used, e.g. `127.0.0.1:10000`). +/// +/// An entry is a *grant*: it is the only way a container gets credentials, or a +/// host a non-default scheme or path-style addressing. A host with no entry is +/// fetched anonymously over https in host-style addressing, so an empty map is the +/// safe default and [`AzureOptionsMap::get`] can answer for absent hosts too. +/// +/// The grant itself is keyed per container *inside* the entry (see +/// [`AzureEndpointOptions`]), because Azure assigns RBAC per container. Container +/// names need none of the normalization the host key below is about: Azure allows +/// only lowercase in one, so a container has exactly one spelling and two keys that +/// mean the same container cannot be written. +/// +/// # Why the key is an [`AzureHost`] and not a `String` +/// +/// A silently-missed grant is the worst failure this table has: Azure answers an +/// unauthorized request for a private container with a 404, so the user is told +/// "not found" rather than "not authorized". Keyed by raw TOML text, every host +/// normalization is such a miss — `MyCompany.blob…` , `host:443`, `ünï.blob…`, +/// `[0:0:0:0:0:0:0:1]:10000` and `0x7f.1` are all spellings a lookup would arrive +/// with in a different form. Keying by [`AzureHost`] deletes the class: the key is +/// deserialized through the same parser that produces the lookup value, so the two +/// cannot disagree. The inner map is private for the same reason — a key that did +/// not go through that parser must be unrepresentable, not merely discouraged. +/// +/// # Scope +/// +/// Entries are **user-scoped by contract**. A tool must never read this table +/// from a project- or workspace-level manifest: doing so would let a checked-out +/// repository name a host and have the user's ambient Azure credentials sent to +/// it. Keep it to user- and system-level config files. +#[derive(Default, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct AzureOptionsMap(IndexMap); + +impl AzureOptionsMap { + /// The options for `host`, or the defaults (anonymous, https, host-style) + /// when it has no entry. + /// + /// Callers should prefer this over indexing the map: "no entry" and "a + /// defaulted entry" are defined to behave identically, so branching on + /// presence only invites the two paths to drift apart. + pub fn get(&self, host: &AzureHost) -> AzureEndpointOptions { + self.0.get(host).cloned().unwrap_or_default() + } + + /// The configured hosts, in the order the document's table iterated them + /// (`toml::Table` is a `BTreeMap`, so that is byte order, not write order). + pub fn hosts(&self) -> impl Iterator { + self.0.keys() + } + + /// Whether no host is configured, which is also "every `az://` host is + /// anonymous". Serializers skip the table on this. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Grant `host` these options, returning what it was granted before. + /// + /// Taking an [`AzureHost`] rather than a string is what lets the inner map stay + /// private while still being writable: a caller editing config (`pixi config + /// set azure-options."…"`) has to have parsed its key, so it cannot install an + /// entry a lookup would fail to find. There is no `get_mut`, and none is + /// needed: editing an entry is [`get`](Self::get), change, insert. That copies + /// the entry's grant table, which is not worth a second mutable path into a + /// private map — config editing happens once per `config set`, not per request. + pub fn insert( + &mut self, + host: AzureHost, + options: AzureEndpointOptions, + ) -> Option { + self.0.insert(host, options) + } + + /// Revoke `host`'s grant, returning it if there was one. + /// + /// Shift-removes, so the remaining entries keep their relative order and a + /// serialized table does not reshuffle on an unrelated edit. + pub fn remove(&mut self, host: &AzureHost) -> Option { + self.0.shift_remove(host) + } + + /// The entries as the fetch middleware takes them, ready to hand to + /// `AzureMiddleware::new` without a caller rebuilding a map by hand. + /// + /// Whole entries, not the narrower `AzureFetchOptions`: the middleware has to + /// read a host's addressing before it can tell which path segment is the + /// container, and only then can it look the container's grant up. The narrowing + /// therefore happens per request, inside the middleware, and not here. + pub fn endpoint_options(&self) -> impl Iterator { + self.0 + .iter() + .map(|(host, options)| (host.clone(), options.clone())) + } +} + +/// Reject a document that spells one host two ways. +/// +/// Both spellings reach serde, which keeps whichever the table iterated last — +/// silently dropping one spelling's whole entry, grants and all. TOML's own +/// duplicate-key check runs on the raw text, so it cannot see the collision; this +/// has to run while both spellings are still visible. +pub(crate) fn ensure_no_colliding_hosts(document: &toml::Table) -> Result<(), String> { + let Some(table) = document + .get("azure-options") + .and_then(toml::Value::as_table) + else { + return Ok(()); + }; + + let mut seen: IndexMap = IndexMap::new(); + for written in table.keys() { + // An unparseable key is serde's error to report, not ours. + let Ok(host) = AzureHost::parse(written) else { + continue; + }; + if let Some(first) = seen.insert(host.clone(), written) { + return Err(format!( + "`azure-options` names one host twice: \"{first}\" and \"{written}\" are both \ + `{host}`" + )); + } + } + Ok(()) +} + +impl Config for AzureOptionsMap { + fn is_default(&self) -> bool { + self.0.is_empty() + } + + fn merge_config(self, other: &Self) -> Result { + // Merge the two maps, with `other`'s entries layered over existing keys. + // The host-scoped fields — `scheme`, `path-style` — replace wholesale, but + // the grants merge per container: a higher-precedence file naming one + // container must not discard a grant a lower file made on a different + // container it never mentions. It can still revoke the container it does + // name, because an explicit `false` is a legal grant. + let mut merged = self.0; + for (key, value) in &other.0 { + let layered = match merged.get(key) { + Some(lower) => value.layered_over(lower), + None => value.clone(), + }; + merged.insert(key.clone(), layered); + } + Ok(AzureOptionsMap(merged)) + } + + fn validate(&self) -> Result<(), super::ValidationError> { + for (host, options) in &self.0 { + if options.endpoint().scheme != AzureScheme::Http || is_local(host) { + continue; + } + // One granted container is enough: the scheme is host-scoped, so its + // requests all ride the same cleartext connection. + if let Some((container, _)) = options.grants().find(|(_, auth)| auth.is_granted()) { + return Err(super::ValidationError::Invalid(format!( + "`azure-options.\"{host}\".auth` grants credentials to `{container}` over \ + cleartext http. A credential may only be sent unencrypted to a local \ + endpoint: use an https scheme, or address the emulator by loopback address." + ))); + } + } + Ok(()) + } + + fn keys(&self) -> Vec { + // Quoted, because every Azure authority contains dots and an unquoted key + // is not the TOML path the user must pass to `config set`/`unset`. A + // container name never needs quoting — Azure's rules leave nothing in one + // that a bare TOML key cannot hold. + // + // The per-container grants are listed as their own keys so each is + // separately unsettable. There is deliberately no `."".auth` key: the + // path exists only as a table of containers, so `config set + // azure-options."".auth true` has nowhere to land — which is the + // point, since that is the one edit whose blast radius would be the whole + // account. + self.0 + .iter() + .flat_map(|(host, options)| { + let host = toml::Value::from(host.to_string()).to_string(); + let grants = options + .grants() + .map(|(container, _)| format!("{host}.auth.{container}")) + .collect::>(); + std::iter::once(host).chain(grants) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use rattler_azure::{Addressing, Auth, AzureScheme, ContainerName}; + + use super::*; + + fn host(authority: &str) -> AzureHost { + AzureHost::parse(authority).expect("test host should parse") + } + + fn container(name: &str) -> ContainerName { + ContainerName::new(name).expect("test container name") + } + + /// A grant can be written and revoked without the inner map being public, and + /// a revoked host falls back to anonymous rather than lingering. + #[test] + fn a_grant_can_be_written_and_revoked() { + let key = host("mycompany.blob.core.windows.net"); + let granted = AzureEndpointOptions::new( + [(container("releases"), Auth::DefaultChain)], + rattler_azure::AzureEndpoint::default(), + ); + + let mut map = AzureOptionsMap::default(); + assert!(map.is_empty()); + assert_eq!(map.insert(key.clone(), granted.clone()), None); + assert_eq!(map.get(&key), granted); + assert!(!map.is_empty()); + + assert_eq!(map.remove(&key), Some(granted)); + assert!( + !map.get(&key) + .fetch(Some(&container("releases"))) + .auth + .is_granted() + ); + assert!(map.is_empty()); + assert_eq!(map.remove(&key), None); + } + + /// The table parses in the shape documented for users, and an absent host + /// answers with the anonymous defaults rather than requiring a presence check. + #[test] + fn table_parses_and_absent_hosts_default() { + let map: AzureOptionsMap = toml::from_str( + r#" + ["mycompany.blob.core.windows.net".auth] + releases = true + + ["127.0.0.1:10000"] + scheme = "http" + path-style = true + + ["127.0.0.1:10000".auth] + general = true + "#, + ) + .unwrap(); + + let real = map.get(&host("mycompany.blob.core.windows.net")); + assert_eq!( + real.fetch(Some(&container("releases"))).auth, + Auth::DefaultChain + ); + assert_eq!(real.endpoint().scheme, AzureScheme::Https); + assert_eq!(real.endpoint().addressing, Addressing::HostStyle); + + let azurite = map.get(&host("127.0.0.1:10000")); + assert_eq!( + azurite.fetch(Some(&container("general"))).auth, + Auth::DefaultChain + ); + assert_eq!(azurite.endpoint().scheme, AzureScheme::Http); + assert_eq!(azurite.endpoint().addressing, Addressing::PathStyle); + + // Neither a container the account never granted nor an unlisted host gets + // anything, and the two are the same answer by construction. + assert!(!real.fetch(Some(&container("public"))).auth.is_granted()); + let unlisted = map.get(&host("someoneelse.blob.core.windows.net")); + assert!( + !unlisted + .fetch(Some(&container("releases"))) + .auth + .is_granted() + ); + assert_eq!(unlisted, AzureEndpointOptions::default()); + + // The table feeds the fetch middleware directly, keys and all. + assert_eq!( + map.endpoint_options().collect::>(), + vec![ + (host("127.0.0.1:10000"), azurite), + (host("mycompany.blob.core.windows.net"), real), + ] + ); + } + + /// A grant may only ride cleartext to an endpoint that is not routable off + /// the machine or its LAN. + #[test] + fn cleartext_grants_are_confined_to_local_endpoints() { + for authority in ["127.0.0.1:10000", "[::1]:10000", "azurite:10000"] { + let map: AzureOptionsMap = toml::from_str(&format!( + "[\"{authority}\"]\nscheme = \"http\"\n[\"{authority}\".auth]\ngeneral = true\n" + )) + .unwrap(); + assert!(map.validate().is_ok(), "{authority} is local"); + } + + for authority in ["mycompany.blob.core.windows.net", "internal.example.com"] { + let map: AzureOptionsMap = toml::from_str(&format!( + "[\"{authority}\"]\nscheme = \"http\"\n[\"{authority}\".auth]\npublic = false\nreleases = true\n" + )) + .unwrap(); + let err = map.validate().expect_err("{authority} is routable"); + // The message names the container at fault: the entry may hold many, and + // only the granted ones are the problem. + assert!(err.to_string().contains("releases"), "{err}"); + + // The same host over https, and the same cleartext scheme with no + // container granted, are both fine — it is only the pair that is + // refused, and an explicit `false` is not a grant. + let https: AzureOptionsMap = + toml::from_str(&format!("[\"{authority}\".auth]\nreleases = true\n")).unwrap(); + assert!(https.validate().is_ok()); + let anonymous: AzureOptionsMap = toml::from_str(&format!( + "[\"{authority}\"]\nscheme = \"http\"\n[\"{authority}\".auth]\nreleases = false\n" + )) + .unwrap(); + assert!(anonymous.validate().is_ok()); + } + } + + /// Two spellings of one host must be refused rather than one silently + /// winning: the loser here is an explicit `releases = false`. + #[test] + fn a_document_naming_one_host_twice_is_refused() { + let document = r#" +[azure-options."acct.blob.example".auth] +releases = false + +[azure-options."ACCT.blob.example.".auth] +releases = true +"#; + let error = ensure_no_colliding_hosts(&document.parse().unwrap()) + .expect_err("a collision must be reported"); + assert!(error.contains("acct.blob.example"), "{error}"); + } + + /// A later config file replaces a host's endpoint wholesale. It must not be + /// able to keep an earlier `path-style = true` while changing only the scheme. + #[test] + fn merge_replaces_the_endpoint_wholesale() { + let base: AzureOptionsMap = + toml::from_str("[\"host.example\"]\npath-style = true\n").unwrap(); + let over: AzureOptionsMap = + toml::from_str("[\"host.example\"]\nscheme = \"http\"\n").unwrap(); + + let merged = base.merge_config(&over).unwrap(); + let entry = merged.get(&host("host.example")); + assert_eq!(entry.endpoint().scheme, AzureScheme::Http); + assert_eq!(entry.endpoint().addressing, Addressing::HostStyle); + } + + /// The grants, unlike the endpoint, merge per container: a user file naming one + /// container must not silently revoke a grant a system file made on another — + /// and, because an explicit `false` is legal, it can still revoke the one it + /// does name. Without that, per-container merging would be a one-way ratchet. + #[test] + fn merge_layers_grants_per_container() { + let system: AzureOptionsMap = + toml::from_str("[\"host.example\".auth]\nreleases = true\nstaging = true\n").unwrap(); + let user: AzureOptionsMap = + toml::from_str("[\"host.example\".auth]\nstaging = false\ninternal = true\n").unwrap(); + + let entry = system + .merge_config(&user) + .unwrap() + .get(&host("host.example")); + assert!( + entry.fetch(Some(&container("releases"))).auth.is_granted(), + "a grant the user file never mentions must survive the merge" + ); + assert!( + !entry.fetch(Some(&container("staging"))).auth.is_granted(), + "an explicit `false` must revoke a lower-precedence grant" + ); + assert!(entry.fetch(Some(&container("internal"))).auth.is_granted()); + } + + /// The defect this key type exists to kill: every one of these keys is a + /// spelling a lookup arrives with in normalized form, and with a `String` key + /// each was a silent miss — an anonymous fetch, a 404, and a user told "not + /// found" instead of "not authorized". + #[test] + fn keys_are_normalized_the_same_way_lookups_are() { + for (written, looked_up) in [ + ( + "MyCompany.blob.core.windows.net", + "mycompany.blob.core.windows.net", + ), + ( + "mycompany.blob.core.windows.net:443", + "mycompany.blob.core.windows.net:443", + ), + ("ünï.blob.example", "xn--n-nga1b.blob.example"), + ("[0:0:0:0:0:0:0:1]:10000", "[::1]:10000"), + ("0x7f.1", "127.0.0.1"), + ("acct.blob.core.windows.net.", "acct.blob.core.windows.net"), + ] { + let map: AzureOptionsMap = + toml::from_str(&format!("[\"{written}\".auth]\nreleases = true\n")).unwrap(); + + assert!( + map.get(&host(looked_up)) + .fetch(Some(&container("releases"))) + .auth + .is_granted(), + "the grant written as `{written}` did not apply to `{looked_up}`" + ); + // The key is stored canonically, so `keys()` reports what a lookup + // would need rather than what happened to be typed — quoted, as the + // TOML path it has to be written as, with one key per grant so each is + // separately settable — and writing the table back out produces a key + // that parses to the same host. + assert_eq!( + map.keys(), + vec![ + format!("\"{looked_up}\""), + format!("\"{looked_up}\".auth.releases"), + ], + "{written}" + ); + let written_back = toml::to_string(&map).unwrap(); + assert!( + written_back.contains(&format!("[\"{looked_up}\"")), + "{written} was written back as {written_back}" + ); + } + } + + /// A key that cannot be a host is a config error worth naming, not an entry + /// that silently never matches. + #[test] + fn an_unparseable_key_is_rejected() { + let err = toml::from_str::( + "[\"acct.blob.example/general\".auth]\nreleases = true\n", + ) + .expect_err("a key carrying a path must be rejected"); + assert!( + err.to_string().contains("acct.blob.example/general"), + "{err}" + ); + } + + /// The same rule one level down: a container key Azure would refuse is a config + /// error, since it can never match a request either. + #[test] + fn an_unusable_container_key_is_rejected() { + let err = + toml::from_str::("[\"acct.blob.example\".auth]\nReleases = true\n") + .expect_err("an uppercase container name must be rejected"); + assert!(err.to_string().contains("Releases"), "{err}"); + } +} diff --git a/crates/rattler_config/test-data/compat/kitchen-sink.toml b/crates/rattler_config/test-data/compat/kitchen-sink.toml index 28ab3955b6..b12a3bfbee 100644 --- a/crates/rattler_config/test-data/compat/kitchen-sink.toml +++ b/crates/rattler_config/test-data/compat/kitchen-sink.toml @@ -39,6 +39,20 @@ endpoint-url = "https://s3.example.com" region = "eu-central-1" force-path-style = true +# A private channel on real Azure: the grant is written per container, so the +# anonymous-read container on the same account stays anonymous. +[azure-options."mycompany.blob.core.windows.net".auth] +releases = true +public = false + +# A local Azurite emulator: the entry is also the endpoint trust declaration. +[azure-options."127.0.0.1:10000"] +scheme = "http" +path-style = true + +[azure-options."127.0.0.1:10000".auth] +general = true + [index-config] write-zst = true write-shards = false diff --git a/crates/rattler_config/test-data/compat/override-layer.toml b/crates/rattler_config/test-data/compat/override-layer.toml index 63404e741e..6f057b5cbd 100644 --- a/crates/rattler_config/test-data/compat/override-layer.toml +++ b/crates/rattler_config/test-data/compat/override-layer.toml @@ -18,3 +18,9 @@ solves = 9 endpoint-url = "https://minio.example.com" region = "auto" force-path-style = false + +# Grants merge per container, unlike every other nested table here: this layer +# adds one and revokes one, and the lower layer's `releases = true` survives both. +[azure-options."mycompany.blob.core.windows.net".auth] +staging = true +public = false diff --git a/crates/rattler_config/tests/compat.rs b/crates/rattler_config/tests/compat.rs index f929084c0d..12a4f4633f 100644 --- a/crates/rattler_config/tests/compat.rs +++ b/crates/rattler_config/tests/compat.rs @@ -185,6 +185,12 @@ const EDIT_MATRIX: &[(&str, &str)] = &[ "s3-options.some-bucket", r#"{"endpoint-url": "https://s3.example.com", "region": "auto", "force-path-style": true}"#, ), + // Quoted, because every Azure authority contains dots. The grant is a table + // keyed by container, so there is no host-wide `auth` value to set. + ( + r#"azure-options."acct.blob.core.windows.net""#, + r#"{"auth": {"releases": true}}"#, + ), ]; /// Every key in the matrix can be set on a fully populated config, the @@ -231,6 +237,52 @@ fn edit_matrix_set_roundtrip_unset() { } } +/// A grant is settable per container, and there is no host-wide switch to set by +/// mistake: `azure-options."".auth` is a table of containers, so a bool +/// aimed at it has nowhere to land. That refusal is the central safety property of +/// the per-container design, and `config set` is where a user would try it. +#[test] +fn edit_grants_one_container_at_a_time() { + let host = rattler_azure::AzureHost::parse("acct.blob.core.windows.net").unwrap(); + let releases = rattler_azure::ContainerName::new("releases").unwrap(); + let public = rattler_azure::ContainerName::new("public").unwrap(); + + let mut config = Config::default(); + for (container, value) in [("releases", "true"), ("public", "false")] { + let key = format!(r#"azure-options."acct.blob.core.windows.net".auth.{container}"#); + config + .set(&key, Some(value.to_string())) + .unwrap_or_else(|e| panic!("{key} must be settable: {e}")); + } + + let entry = config.azure_options.get(&host); + assert!(entry.fetch(Some(&releases)).auth.is_granted()); + assert!(!entry.fetch(Some(&public)).auth.is_granted()); + + // The whole point: no host-level `auth` value exists to grant every container + // on the account at once. + assert!( + config + .set( + r#"azure-options."acct.blob.core.windows.net".auth"#, + Some("true".to_string()) + ) + .is_err(), + "a host-wide grant must not be settable" + ); + + // Revoking one grant leaves the other alone. + config + .set( + r#"azure-options."acct.blob.core.windows.net".auth.releases"#, + None, + ) + .unwrap(); + let entry = config.azure_options.get(&host); + assert!(!entry.fetch(Some(&releases)).auth.is_granted()); + assert_eq!(entry.grants().count(), 1, "`public` must survive"); +} + /// Unknown keys must be rejected by `set` (both set and unset direction). #[test] fn edit_rejects_unknown_keys() { @@ -276,6 +328,19 @@ fn merge_semantics() { let per_channel = &merged.repodata_config.per_channel[&prefix_dev]; assert_eq!(per_channel.disable_sharded, Some(true)); // from layer 1 assert_eq!(per_channel.disable_zstd, Some(true)); // from layer 2 + // `azure-options` grants merge per container: the lower layer's `releases` + // survives a layer that names only `staging` and `public`. + let mycompany = merged + .azure_options + .get(&rattler_azure::AzureHost::parse("mycompany.blob.core.windows.net").unwrap()); + for (container, granted) in [("releases", true), ("staging", true), ("public", false)] { + let container = rattler_azure::ContainerName::new(container).unwrap(); + assert_eq!( + mycompany.fetch(Some(&container)).auth.is_granted(), + granted, + "{container}" + ); + } // concurrency: explicitly set values win over the lower layer. assert_eq!(merged.concurrency.solves, 9); assert_eq!(merged.concurrency.downloads, 12); diff --git a/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap b/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap index 49b41a3210..fe492e5ecd 100644 --- a/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap +++ b/crates/rattler_config/tests/snapshots/compat__merge__kitchen_sink_plus_override.snap @@ -52,6 +52,22 @@ endpoint-url = "https://minio.example.com/" region = "auto" force-path-style = false +[azure-options."127.0.0.1:10000"] +scheme = "http" +path-style = true + +[azure-options."127.0.0.1:10000".auth] +general = true + +[azure-options."mycompany.blob.core.windows.net"] +scheme = "https" +path-style = false + +[azure-options."mycompany.blob.core.windows.net".auth] +public = false +releases = true +staging = true + [index-config] write-zst = true write-shards = false diff --git a/crates/rattler_config/tests/snapshots/compat__parse__deprecated-and-unknown.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__deprecated-and-unknown.toml.snap index e0a41e88ba..1f22ee1fdd 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__deprecated-and-unknown.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__deprecated-and-unknown.toml.snap @@ -88,6 +88,9 @@ expression: "(unused, normalized(config))" s3_options: S3OptionsMap( {}, ), + azure_options: AzureOptionsMap( + {}, + ), index_config: IndexConfig { default: IndexChannelConfig { write_zst: None, diff --git a/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap index c16deeeba4..74d69a6306 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__kitchen-sink.toml.snap @@ -206,6 +206,31 @@ expression: "(unused, normalized(config))" }, }, ), + azure_options: AzureOptionsMap( + { + AzureHost("127.0.0.1:10000"): AzureEndpointOptions { + scheme: Http, + addressing: PathStyle, + auth: { + ContainerName( + "general", + ): DefaultChain, + }, + }, + AzureHost("mycompany.blob.core.windows.net"): AzureEndpointOptions { + scheme: Https, + addressing: HostStyle, + auth: { + ContainerName( + "public", + ): Anonymous, + ContainerName( + "releases", + ): DefaultChain, + }, + }, + }, + ), index_config: IndexConfig { default: IndexChannelConfig { write_zst: Some( diff --git a/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap index f0d2caef56..14dc4e1b0f 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__override-layer.toml.snap @@ -136,6 +136,22 @@ expression: "(unused, normalized(config))" }, }, ), + azure_options: AzureOptionsMap( + { + AzureHost("mycompany.blob.core.windows.net"): AzureEndpointOptions { + scheme: Https, + addressing: HostStyle, + auth: { + ContainerName( + "public", + ): Anonymous, + ContainerName( + "staging", + ): DefaultChain, + }, + }, + }, + ), index_config: IndexConfig { default: IndexChannelConfig { write_zst: None, diff --git a/crates/rattler_config/tests/snapshots/compat__parse__snake-case-aliases.toml.snap b/crates/rattler_config/tests/snapshots/compat__parse__snake-case-aliases.toml.snap index 1d94eb42e0..f5834d1339 100644 --- a/crates/rattler_config/tests/snapshots/compat__parse__snake-case-aliases.toml.snap +++ b/crates/rattler_config/tests/snapshots/compat__parse__snake-case-aliases.toml.snap @@ -68,6 +68,9 @@ expression: "(unused, normalized(config))" s3_options: S3OptionsMap( {}, ), + azure_options: AzureOptionsMap( + {}, + ), index_config: IndexConfig { default: IndexChannelConfig { write_zst: None, diff --git a/crates/rattler_git/tests/lfs.rs b/crates/rattler_git/tests/lfs.rs index c4ba0bd97b..84efb5ebd4 100644 --- a/crates/rattler_git/tests/lfs.rs +++ b/crates/rattler_git/tests/lfs.rs @@ -71,6 +71,8 @@ impl LfsFixture { git(&["config", "user.name", "Test"]); // Disable signing so a global `commit.gpgSign = true` can't interfere. git(&["config", "commit.gpgsign", "false"]); + // Ignore the developer's global gitignore; e.g. `*.bin` would drop `data.bin`. + git(&["config", "core.excludesFile", ""]); git(&["lfs", "install", "--local"]); fs_err::write( diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index fa142ad175..3dcc9ecd59 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -13,7 +13,7 @@ readme.workspace = true default-run = "rattler-index" [features] -default = ["rustls", "s3"] +default = ["rustls", "s3", "azure"] native-tls = [ "reqwest/native-tls", "rattler_package_streaming/native-tls", @@ -28,6 +28,7 @@ rustls = [ "opendal/reqwest-rustls-tls", ] s3 = ["opendal/services-s3", "dep:rattler_s3"] +azure = ["opendal/services-azblob", "dep:rattler_azure", "rattler_azure/opendal"] [[bin]] name = "rattler-index" @@ -56,8 +57,10 @@ rattler_networking = { workspace = true, default-features = false, features = [ ] } rattler_conda_types = { workspace = true, default-features = false } rattler_digest = { workspace = true, default-features = false } +rattler_redaction = { workspace = true, default-features = false } rattler_package_streaming = { workspace = true, default-features = false } hex = { workspace = true } +rattler_azure = { workspace = true, optional = true, features = ["clap"] } rattler_s3 = { workspace = true, optional = true, features = ["clap"] } reqwest = { workspace = true, default-features = false, features = [ "http2", @@ -76,9 +79,26 @@ url = { workspace = true } zstd = { workspace = true } retry-policies = { workspace = true } +# Declared rather than auto-discovered, so a feature name that stops matching is a +# cargo error instead of a test binary that silently holds no tests. +[[test]] +name = "azure_azurite" +required-features = ["azure"] + [dev-dependencies] rattler_repodata_gateway = { path = "../rattler_repodata_gateway", default-features = false, features = [ "sparse", ] } tempfile = { workspace = true } tools = { path = "../tools", default-features = false } +# `AzureMiddleware` is the only shared-key signer reachable from a test here, and +# the Azurite tests need one to create a container: opendal has no +# container-creation operation. +rattler_networking = { workspace = true, default-features = false, features = [ + "system-integration", + "azure", +] } +reqwest-middleware = { workspace = true } +# Puts Azurite's development credentials in the environment for reqsign's env +# provider without an unsafe `set_var`. +temp-env = { workspace = true } diff --git a/crates/rattler_index/README.md b/crates/rattler_index/README.md index 6dbe16d017..dddc268098 100644 --- a/crates/rattler_index/README.md +++ b/crates/rattler_index/README.md @@ -2,7 +2,10 @@ `rattler_index` creates or updates conda channel indexes by writing `repodata.json`, optional compressed repodata, and optional sharded repodata for -packages stored on a local filesystem or in S3. +packages stored on a local filesystem, in S3, or in Azure Blob Storage. + +S3 support requires the `s3` feature; Azure Blob Storage support requires the +`azure` feature. ## CLI Usage @@ -18,14 +21,99 @@ Index an S3 channel: rattler-index --config ./rattler-config.toml s3 s3://my-bucket/my-channel ``` +Index an Azure Blob Storage channel: + +```shell +rattler-index --config ./rattler-config.toml az \ + az://my-storage-account.blob.core.windows.net/my-container/my-channel \ + --azure-cli +``` + +Indexing writes to the container through opendal, which only accepts a storage +account key or a shared access signature (SAS) token — it cannot use an `az +login` AAD bearer token directly. Supply one of: + +- `--azure-cli`: mint a short-lived user-delegation SAS from the current `az + login` session automatically. Requires the Azure CLI (`az`) on `PATH` and a + prior `az login`. The minted SAS is scoped to the target container, granted + only the permissions indexing needs, and expires after 30 minutes by default + (`--azure-cli-sas-ttl-minutes`, plus two minutes of clock-skew headroom). A SAS + cannot be individually revoked, so it is kept short-lived on purpose. +- `--account-key` / `AZURE_STORAGE_KEY`: a storage account key. +- `--sas-token` / `AZURE_STORAGE_SAS_TOKEN`: a SAS token you supply yourself. + +To mint a SAS manually instead of using `--azure-cli` (for example, to reuse it +across several commands), generate one from your `az login` session and pass it +via `--sas-token`: + +```shell +export AZURE_STORAGE_SAS_TOKEN=$(az storage container generate-sas \ + --account-name my-storage-account --name my-container \ + --permissions rwlc --expiry "$(date -u -d '+30 minutes' +%Y-%m-%dT%H:%MZ)" \ + --auth-mode login --as-user --https-only -o tsv) +rattler-index --config ./rattler-config.toml az \ + az://my-storage-account.blob.core.windows.net/my-container/my-channel +``` + The `--config` flag points at the same TOML configuration file used by pixi. It -configures S3 credentials, concurrency, and per-channel index options under the -`[index-config]` section. +configures S3 credentials, Azure endpoint options, concurrency, and per-channel +index options under the `[index-config]` section. When `--config` is omitted, `rattler-index` falls back to its built-in defaults (`write-zst = true`, `write-shards = true`, no advertised repodata revisions, `from-index-json` revision assignment, no channel metadata). +## Remote storage credentials + +S3 endpoint and region settings live under `[s3-options.]`, keyed by +bucket name: + +```toml +[s3-options.my-bucket] +endpoint-url = "https://my-bucket.s3.amazonaws.com" +region = "eu-central-1" +force-path-style = false +``` + +Azure Blob channels need no block for the common case: the account, container and +endpoint (including sovereign clouds) are all read from the channel URL +`az://.blob.core.windows.net//`. The `az://` scheme +is required and is rewritten to `https://` for the request; a bare `https://` URL +is rejected. Credentials are never stored in the config — they are resolved at +runtime from `--account-key` / `--sas-token`, an `az login` session +(`--azure-cli`), or the `DefaultCredentialProvider` chain. + +A host whose first label is not the storage account, or that is not reached over +https, needs an entry under `[azure-options.""]`. The key is the host with +its port when the URL has one: + +| Field | Type | Description | +| --- | --- | --- | +| `scheme` | string | The scheme `az://` is rewritten to: `"https"` (default) or `"http"`. Use `http` for local emulators only. | +| `path-style` | boolean | Where the storage account is found. `false` (default) reads it from the first host label. `true` reads it from the first path segment instead, which is the only form that works for an IP-literal or single-label host. | +| `auth` | table | Which containers on the host may be sent credentials **when fetching**: one ` = true` line each. A container not listed is fetched anonymously, so one account can hold private and anonymous-read containers side by side — which is what Azure's per-container RBAC enforces. There is no host-wide switch, because a grant covering every container on an account, including ones created later, is not something to be able to write by accident. This is the only way a credential attaches on the fetch path, so keep these entries in your user-level config file, never in a checked-in project file. It has no effect on `rattler-index` or `rattler upload`, which take their credentials from the command line. | + +Indexing a channel in the Azurite emulator needs the two wire settings (add an +`auth` line per container if the same host is also fetched from): + +```toml +[azure-options."127.0.0.1:10000"] +scheme = "http" +path-style = true + +[azure-options."127.0.0.1:10000".auth] +general = true +``` + +```shell +export AZURE_STORAGE_KEY= +rattler-index --config ./rattler-config.toml az \ + az://127.0.0.1:10000/devstoreaccount1/general/my-channel +``` + +Without the entry, that URL fails: host-style addressing cannot read an account +name out of `127.0.0.1`, and the error tells you which line to add. + ## Per-channel index configuration Index options live in `[index-config]` and follow the same shape as diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index be5c547bb2..be590c08da 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -23,11 +23,13 @@ use fs_err::{self as fs}; use futures::{StreamExt, stream::FuturesUnordered}; use indexmap::IndexMap; use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; -#[cfg(feature = "s3")] +#[cfg(any(feature = "s3", feature = "azure"))] use opendal::layers::RetryLayer; #[cfg(feature = "s3")] use opendal::services::S3Config; use opendal::{Configurator, Operator, services::FsConfig}; +#[cfg(feature = "azure")] +use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpoint}; use rattler_conda_types::{ ChannelInfo, ChannelRelations, PackageRecord, PatchInstructions, Platform, RepoData, Shard, ShardedRepodata, ShardedSubdirInfo, UrlOrPath, V3Packages, WhlPackageRecord, @@ -1513,6 +1515,112 @@ pub async fn index_s3_with_channel_metadata( .map(|_| ()) } +/// Configuration for `index_azure` +#[cfg(feature = "azure")] +pub struct IndexAzureConfig { + /// The channel to index, as an `az://` Azure Blob channel URL. + pub channel: AzureChannelUrl, + /// The credentials to use for Azure Blob access. + pub credentials: AzureCredentials, + /// How to address the channel's host: the wire scheme, and whether the account + /// is read from the host or the path. The defaults (https, host-style) describe + /// real Azure. + pub endpoint: AzureEndpoint, + /// The target platform to index. + pub target_platform: Option, + /// The path to a repodata patch to apply to the index. + pub repodata_patch: Option, + /// Whether to write the repodata as a zstd-compressed file. + pub write_zst: bool, + /// Whether to write the repodata shards. + pub write_shards: bool, + /// Repodata revisions to advertise in generated repodata. + pub repodata_revisions: Vec, + /// How packages are assigned to repodata revisions. + pub package_revision_assignment: PackageRevisionAssignment, + /// Whether to force the index to be written. + pub force: bool, + /// The maximum number of parallel tasks to run. + pub max_parallel: usize, + /// The multi-progress bar to use for the index. + pub multi_progress: Option, + // NOTE: no `precondition_checks` field. opendal's azblob service supports + // `if_not_exists` and conditional reads but not conditional (`if_match`) + // writes, so any index of an already-populated channel fails under `Enabled`. + // The Azure path hardcodes `Disabled` rather than exposing a knob that only + // works on a first index; the conditional read is lost along with it. +} + +/// Create a new `repodata.json` for all packages in the channel at the given +/// Azure Blob URL. +#[cfg(feature = "azure")] +pub async fn index_azure(config: IndexAzureConfig) -> anyhow::Result<()> { + index_azure_with_channel_metadata(config, ChannelMetadata::default()).await +} + +/// Create a new `repodata.json` for all packages in the channel at the given +/// Azure Blob URL and write channel metadata into the generated repodata. +#[cfg(feature = "azure")] +pub async fn index_azure_with_channel_metadata( + IndexAzureConfig { + channel, + credentials, + endpoint, + target_platform, + repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force, + max_parallel, + multi_progress, + }: IndexAzureConfig, + channel_metadata: ChannelMetadata, +) -> anyhow::Result<()> { + let azblob_config = rattler_azure::azblob_config(&credentials, &channel, endpoint)?; + let builder = azblob_config.into_builder(); + // opendal's default retry interceptor logs the error with its `url` context, + // and for a SAS the credential is *in* that URL — once per retry, at warn + // level. Same message, signature masked. + let op = Operator::new(builder)? + .layer( + RetryLayer::new().with_notify(|event: opendal::layers::RetryEvent<'_>| { + tracing::warn!( + target: "opendal::layers::retry", + "will retry {:?} (attempt {}) after {}s because: {}", + event.op, + event.attempt, + event.retry_after.as_secs_f64(), + rattler_redaction::redact_signatures_in_text( + &format!("{:?}", event.err), + rattler_redaction::DEFAULT_REDACTION_STR, + ), + ); + }), + ) + .finish(); + + index_with_channel_metadata( + target_platform, + op, + repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force, + max_parallel, + multi_progress, + // opendal's azblob service can't do conditional writes, so preconditions + // must be disabled (matching the filesystem backend). + PreconditionChecks::Disabled, + channel_metadata, + ) + .await + .map(|_| ()) +} + /// Create a new `repodata.json` for all packages in the given operator's root. /// /// If `target_platform` is `Some`, only that specific subdir is indexed. @@ -1814,6 +1922,71 @@ mod tests { use super::*; + #[cfg(feature = "azure")] + #[test] + fn azblob_config_derives_fields_from_url() { + let channel = + AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general/sub/dir") + .unwrap(); + let credentials = AzureCredentials::SasToken("sv=token".into()); + + let config = + rattler_azure::azblob_config(&credentials, &channel, AzureEndpoint::default()).unwrap(); + + assert_eq!( + config.endpoint.as_deref(), + Some("https://stcondachannel.blob.core.windows.net") + ); + assert_eq!(config.account_name.as_deref(), Some("stcondachannel")); + assert_eq!(config.container, "general"); + assert_eq!(config.root.as_deref(), Some("/sub/dir")); + assert_eq!(config.sas_token.as_deref(), Some("sv=token")); + assert_eq!(config.account_key, None); + } + + #[cfg(feature = "azure")] + #[test] + fn azblob_config_container_only_url() { + let channel = + AzureChannelUrl::parse("az://stcondachannel.blob.core.windows.net/general").unwrap(); + let credentials = AzureCredentials::AccountKey("key".into()); + + let config = + rattler_azure::azblob_config(&credentials, &channel, AzureEndpoint::default()).unwrap(); + + assert_eq!(config.container, "general"); + assert_eq!(config.root.as_deref(), Some("/")); + assert_eq!(config.account_key.as_deref(), Some("key")); + assert_eq!(config.sas_token, None); + } + + #[cfg(feature = "azure")] + #[test] + fn azblob_config_preserves_non_default_port() { + let channel = + AzureChannelUrl::parse("az://devstoreaccount1.blob.localhost:10000/testcontainer/ch") + .unwrap(); + let credentials = AzureCredentials::AccountKey("key".into()); + + let config = rattler_azure::azblob_config( + &credentials, + &channel, + AzureEndpoint { + scheme: rattler_azure::AzureScheme::Http, + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!( + config.endpoint.as_deref(), + Some("http://devstoreaccount1.blob.localhost:10000") + ); + assert_eq!(config.account_name.as_deref(), Some("devstoreaccount1")); + assert_eq!(config.container, "testcontainer"); + assert_eq!(config.root.as_deref(), Some("/ch")); + } + #[test] fn package_records_from_repodata_preserves_v3_wheels() { let identifier = ArchiveIdentifier::from_str("demo-1.0-py_0").unwrap(); diff --git a/crates/rattler_index/src/main.rs b/crates/rattler_index/src/main.rs index ba09ef2a5e..4e649f7701 100644 --- a/crates/rattler_index/src/main.rs +++ b/crates/rattler_index/src/main.rs @@ -1,17 +1,24 @@ use std::path::PathBuf; +#[cfg(feature = "s3")] use anyhow::Context; use clap::{Parser, Subcommand}; use clap_verbosity_flag::Verbosity; +#[cfg(feature = "azure")] +use rattler_azure::{AzureChannelUrl, AzureEndpoint, AzureHost}; use rattler_conda_types::Platform; use rattler_config::config::{ concurrency::default_max_concurrent_solves, index::IndexChannelConfig, }; +#[cfg(feature = "s3")] +use rattler_index::PreconditionChecks; use rattler_index::{ ChannelMetadata, IndexFsConfig, PackageRevisionAssignment, index_fs_with_channel_metadata, }; +#[cfg(feature = "azure")] +use rattler_index::{IndexAzureConfig, index_azure_with_channel_metadata}; #[cfg(feature = "s3")] -use rattler_index::{IndexS3Config, PreconditionChecks, index_s3_with_channel_metadata}; +use rattler_index::{IndexS3Config, index_s3_with_channel_metadata}; #[cfg(feature = "s3")] use rattler_networking::AuthenticationStorage; #[cfg(feature = "s3")] @@ -31,6 +38,12 @@ fn parse_s3_url(value: &str) -> Result { } } +/// SAS permissions requested when minting a user-delegation SAS for indexing. +/// Indexing does a read-modify-write of repodata and lists/reads packages, so it +/// needs read, write, list, and create (`r` + `w` + `l` + `c`). +#[cfg(feature = "azure")] +const AZURE_INDEX_SAS_PERMISSIONS: &str = "rwlc"; + /// The `rattler-index` CLI. #[derive(Parser)] #[command(name = "rattler-index", version, about, long_about = None)] @@ -99,6 +112,23 @@ enum Commands { #[clap(flatten)] credentials: rattler_s3::clap::S3CredentialsOpts, }, + + /// Index a channel stored in an Azure Blob container. + #[cfg(feature = "azure")] + #[command(name = "az")] + Azblob { + /// The Azure Blob channel URL, e.g. + /// `az://.blob.core.windows.net//`. + /// + /// Parsed into an [`AzureChannelUrl`] rather than a wire `Url`: the wire + /// scheme comes from the host's `azure-options` entry, which is not read + /// until after clap has run, and the `az://` spelling is what + /// `[index-config."…"]` keys are matched against. + channel: AzureChannelUrl, + + #[clap(flatten)] + credentials: rattler_azure::clap::AzureCredentialsOpts, + }, } /// The configuration type for rattler-index - just extends rattler config and @@ -213,11 +243,66 @@ async fn main() -> anyhow::Result<()> { ) .await } + #[cfg(feature = "azure")] + Commands::Azblob { + channel, + credentials, + } => { + // `canonical()`, not the wire URL: `[index-config."az://…"]` is how a + // user keys an Azure channel, and matching the https spelling meant + // such a key never applied to anything. + let target = channel.canonical().to_string(); + let resolved = resolve_index_channel_config(&config, &target); + let (write_zst, write_shards, repodata_revisions, package_revision_assignment) = + effective_index_options(&resolved); + let channel_metadata = ChannelMetadata::from_index_config(&resolved); + + let endpoint = azure_endpoint(&config, channel.host()); + + let credentials = credentials + .resolve(AZURE_INDEX_SAS_PERMISSIONS, &channel, endpoint) + .await?; + + index_azure_with_channel_metadata( + IndexAzureConfig { + channel, + credentials, + endpoint, + target_platform: cli.target_platform, + repodata_patch: cli.repodata_patch, + write_zst, + write_shards, + repodata_revisions, + package_revision_assignment, + force: cli.force, + max_parallel, + multi_progress: Some(multi_progress), + }, + channel_metadata, + ) + .await + } }?; println!("Finished indexing channel."); Ok(()) } +/// How to address a channel's host, from its `[azure-options.""]` entry, or +/// the https host-style defaults when there is no config file or no entry. +/// +/// A host without an entry and a host with an empty entry are defined to behave +/// identically, so this never has to report which of the two it found. The entry's +/// per-container grants are not part of the result: indexing signs with the +/// credential its caller supplied, so there is no ambient chain for a grant to +/// gate. +#[cfg(feature = "azure")] +fn azure_endpoint(config: &Option, host: &AzureHost) -> AzureEndpoint { + config + .as_ref() + .map(|config| config.azure_options.get(host).endpoint()) + .unwrap_or_default() +} + fn resolve_index_channel_config(config: &Option, target: &str) -> IndexChannelConfig { config .as_ref() @@ -244,3 +329,107 @@ fn effective_index_options( package_revision_assignment, ) } + +#[cfg(all(test, feature = "azure"))] +mod tests { + use rattler_azure::{Addressing, AzureCredentials, AzureScheme}; + + use super::*; + + /// Load a config from TOML the way `--config` does, through a real file, so + /// the test exercises the same deserialization the CLI does. + fn config_from(toml: &str) -> Option { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("rattler-config.toml"); + std::fs::write(&path, toml).expect("write config"); + Some(Config::load_from_files(vec![path]).expect("config should load")) + } + + /// Reviewer issue 5: `[index-config."az://…"]` is the only spelling a user + /// would write for an Azure channel, and matching the https wire URL meant it + /// never applied to anything. + #[test] + fn index_config_is_keyed_by_the_canonical_az_url() { + let config = config_from( + r#" + [index-config."az://acct.blob.core.windows.net/general"] + write-shards = false + "#, + ); + let channel = + AzureChannelUrl::parse("az://acct.blob.core.windows.net/general/mychannel").unwrap(); + + let resolved = resolve_index_channel_config(&config, channel.canonical().as_str()); + assert_eq!(resolved.write_shards, Some(false)); + + // The spelling this used to match against, kept as the negative half of + // the proof: had the key been written in wire form it would still be dead. + let wire = channel.wire(AzureScheme::Https).to_string(); + assert_eq!( + resolve_index_channel_config(&config, &wire).write_shards, + None + ); + } + + /// An Azurite entry has to carry all the way to the opendal config, because + /// every one of these four fields is derived differently under path-style and + /// a wrong one fails silently. + #[test] + fn a_path_style_entry_drives_the_azurite_index_config() { + let config = config_from( + r#" + [azure-options."127.0.0.1:10000"] + scheme = "http" + path-style = true + + [azure-options."127.0.0.1:10000".auth] + general = true + "#, + ); + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general/mychannel") + .unwrap(); + + let endpoint = azure_endpoint(&config, channel.host()); + assert_eq!(endpoint.scheme, AzureScheme::Http); + assert_eq!(endpoint.addressing, Addressing::PathStyle); + + let azblob = rattler_azure::azblob_config( + &AzureCredentials::AccountKey("key".into()), + &channel, + endpoint, + ) + .expect("an Azurite channel must build an opendal config"); + + assert_eq!( + azblob.endpoint.as_deref(), + Some("http://127.0.0.1:10000/devstoreaccount1") + ); + assert_eq!(azblob.account_name.as_deref(), Some("devstoreaccount1")); + assert_eq!(azblob.container, "general"); + assert_eq!(azblob.root.as_deref(), Some("/mychannel")); + } + + /// Without an entry the same URL is an error, not a silently different + /// endpoint: host-style cannot read an account out of an IP literal, and the + /// error names the config line that fixes it. + #[test] + fn an_emulator_host_without_an_entry_is_a_guided_error() { + let channel = + AzureChannelUrl::parse("az://127.0.0.1:10000/devstoreaccount1/general").unwrap(); + let endpoint = azure_endpoint(&None, channel.host()); + + let err = rattler_azure::azblob_config( + &AzureCredentials::AccountKey("key".into()), + &channel, + endpoint, + ) + .expect_err("host-style cannot address an IP literal"); + let message = err.to_string(); + assert!( + message.contains("[azure-options.\"127.0.0.1:10000\"]"), + "{message}" + ); + assert!(message.contains("path-style = true"), "{message}"); + } +} diff --git a/crates/rattler_index/tests/azure_azurite.rs b/crates/rattler_index/tests/azure_azurite.rs new file mode 100644 index 0000000000..468fef1d19 --- /dev/null +++ b/crates/rattler_index/tests/azure_azurite.rs @@ -0,0 +1,456 @@ +//! Live write-path integration tests against a local Azurite emulator. +//! +//! `index_azure` builds its opendal config from a channel URL plus one +//! `azure-options` entry, and under `path-style = true` the account moves from the +//! host into the endpoint path. Unit tests can only assert the strings that +//! construction produces; whether a real Azure Blob implementation accepts them is +//! a different question, and this is where it gets answered: +//! +//! ```toml +//! [azure-options."127.0.0.1:10000"] +//! scheme = "http" +//! path-style = true +//! ``` +//! +//! The rest of the file covers two opendal behaviours that the production code +//! deliberately works around, and that only a real server can demonstrate: the +//! multi-block write path silently ignores `if_not_exists`, and it does carry +//! `Cache-Control` through its commit. +//! +//! Run with: +//! +//! ```text +//! docker run --rm -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ +//! azurite-blob --blobHost 0.0.0.0 +//! cargo nextest run -p rattler_index --test azure_azurite --run-ignored all +//! ``` +//! +//! No `--skipApiVersionCheck` needed: the `x-ms-version` opendal pins is older +//! than what current Azurite accepts. Verified on 3.36.0, which answers that +//! version with `AuthorizationFailure` rather than `InvalidHeaderValue`, i.e. it +//! validates the signature instead of rejecting the version. Add the flag only if +//! an older emulator rejects the version outright. + +use std::{collections::HashMap, path::PathBuf}; + +use opendal::{Configurator, ErrorKind, Operator, services::AzblobConfig}; +use rattler_azure::{ + Addressing, Auth, AzureChannelUrl, AzureCredentials, AzureEndpoint, AzureEndpointOptions, + AzureHost, AzureScheme, ContainerName, +}; +use rattler_index::{IndexAzureConfig, PackageRevisionAssignment, index_azure}; + +/// Azurite's development account and its fixed key. Not a secret: both are +/// published constants of the emulator, hardcoded in opendal's own source, and +/// they only ever address a loopback port. +const ACCOUNT: &str = "devstoreaccount1"; +const ACCOUNT_KEY: &str = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + +/// The authority, which is also the exact `azure-options` table key. An IP with a +/// port is precisely the host shape host-style addressing cannot read an account +/// out of, so it only works through a `path-style = true` entry. +const AUTHORITY: &str = "127.0.0.1:10000"; + +const CONTAINER: &str = "test-channel"; + +/// The `Cache-Control` `rattler_index` writes on repodata (`lib.rs`'s +/// `CACHE_CONTROL_REPODATA`), duplicated because the constant is private. +const CACHE_CONTROL_REPODATA: &str = "public, max-age=300"; + +const MIB: usize = 1024 * 1024; + +/// `rattler_upload`'s `DESIRED_CHUNK_SIZE`, which is what decides whether a +/// package upload takes opendal's single-shot or multi-block path. +const UPLOAD_CHUNK_SIZE: usize = 10 * MIB; + +const PACKAGE: &str = "empty-0.1.0-h4616a5c_0.conda"; + +fn package_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../test-data/packages") + .join(PACKAGE) +} + +/// The channel as a user would write it: the account is the first path segment, +/// which is what `path-style = true` means. `prefix` keeps each test in its own +/// subtree so they can run in parallel. +fn channel(prefix: &str) -> AzureChannelUrl { + AzureChannelUrl::parse(&format!("az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}/{prefix}")) + .expect("azurite channel url") +} + +/// The `azure-options` entry for the emulator: the only configuration these tests +/// hand to the indexer. +/// +/// The grant on `CONTAINER` is for the fetch-side fixture below — `index_azure` +/// signs with the credential it is handed and never reads a grant — and it is +/// written per container because that is the only scope a grant has. +fn azurite_options() -> AzureEndpointOptions { + AzureEndpointOptions::new( + [( + ContainerName::new(CONTAINER).expect("azurite container name"), + Auth::DefaultChain, + )], + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) +} + +/// An operator built exactly the way `index_azure` builds one, so the opendal-level +/// tests below run against the config the production path derives rather than a +/// hand-written stand-in. +fn production_operator(channel: &AzureChannelUrl) -> Operator { + let config = rattler_azure::azblob_config( + &AzureCredentials::AccountKey(ACCOUNT_KEY.into()), + channel, + azurite_options().endpoint(), + ) + .expect("azblob config for an azurite path-style channel"); + Operator::new(config.into_builder()) + .expect("azblob operator") + .finish() +} + +/// An operator written out by hand, *not* derived from the code under test. +/// +/// This is what makes the round-trip assertion mean something: if +/// `azblob_config`'s path-style derivation put the blobs somewhere else, this +/// operator would not find them. It is the verbatim shape the emulator wants — the +/// account appears both inside `endpoint` and in `account_name`, `container` is +/// separate, and `root` is the channel prefix without repeating the container. +fn verify_operator(prefix: &str) -> Operator { + let config = AzblobConfig { + endpoint: Some(format!("http://{AUTHORITY}/{ACCOUNT}")), + account_name: Some(ACCOUNT.to_string()), + account_key: Some(ACCOUNT_KEY.to_string()), + container: CONTAINER.to_string(), + root: Some(format!("/{prefix}")), + ..Default::default() + }; + Operator::new(config.into_builder()) + .expect("azblob operator") + .finish() +} + +/// Create the channel's container and clear this test's prefix inside it. +/// +/// The clearing is what makes the assertions below mean anything: the emulator is +/// long-lived, and a blob left by an earlier run answers the same read a correct +/// run would produce — so a derivation that writes to the wrong prefix passes by +/// finding the *previous* run's output. Removal goes through the hand-written +/// operator, not the one under test, for the same reason. +async fn ensure_empty_prefix(prefix: &str) { + ensure_container().await; + verify_operator(prefix) + .delete_with("") + .recursive(true) + .await + .expect("clearing the test prefix failed"); +} + +/// A signing client for the requests opendal cannot make. +/// +/// `AzureMiddleware` is the one signer already reachable from here, and there is no +/// opendal operation for creating a container or reading a block list — hand-rolling +/// shared-key signing in a test fixture would be more code than the tests it +/// supports. +fn azure_client() -> reqwest_middleware::ClientWithMiddleware { + let options = HashMap::from([( + AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), + azurite_options(), + )]); + reqwest_middleware::ClientBuilder::new(reqwest::Client::new()) + .with(rattler_networking::AzureMiddleware::new( + reqwest::Client::new(), + options, + )) + .build() +} + +/// How many blocks a blob was committed from. +/// +/// The one witness that a write took Put Block List and not the single-shot Put +/// Blob path — a single-shot blob has no committed blocks at all, whatever its +/// size. +async fn committed_block_count(prefix: &str, blob: &str) -> usize { + let body = azure_client() + .get(format!( + "az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}/{prefix}/{blob}\ + ?comp=blocklist&blocklisttype=committed" + )) + .send() + .await + .expect("get block list request failed") + .error_for_status() + .expect("get block list was refused") + .text() + .await + .expect("get block list body"); + body.matches("").count() +} + +/// Create the channel's container, which Azurite never does implicitly. +async fn ensure_container() { + let created = azure_client() + .put(format!( + "az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}?restype=container" + )) + .send() + .await + .expect("container create request failed"); + assert!( + // 409 is `ContainerAlreadyExists` — a re-run, not a failure. + created.status().is_success() || created.status() == reqwest::StatusCode::CONFLICT, + "could not create container {CONTAINER}: {}", + created.status() + ); +} + +/// Run `body` with the emulator credentials in the environment. +/// +/// reqsign's env provider sits first in its default chain, so a shared key is how +/// a container's grant resolves against Azurite — it rejects the AAD bearer +/// tokens the rest of the chain produces. The chain itself is left alone. +async fn with_azurite_credentials>(body: F) { + temp_env::async_with_vars( + [ + ("AZURE_STORAGE_ACCOUNT_NAME", Some(ACCOUNT)), + ("AZURE_STORAGE_ACCOUNT_KEY", Some(ACCOUNT_KEY)), + ], + body, + ) + .await; +} + +fn index_config(channel: AzureChannelUrl) -> IndexAzureConfig { + IndexAzureConfig { + channel, + credentials: AzureCredentials::AccountKey(ACCOUNT_KEY.into()), + endpoint: azurite_options().endpoint(), + target_platform: None, + repodata_patch: None, + write_zst: false, + write_shards: false, + repodata_revisions: Vec::new(), + package_revision_assignment: PackageRevisionAssignment::default(), + force: true, + max_parallel: 4, + multi_progress: None, + } +} + +/// The round trip: seed a package into a path-style Azurite channel, index it +/// through an `azure-options` entry, and read the written `repodata.json` back. +/// +/// The read side uses the hand-written operator, so this checks that the indexer +/// wrote to the blob the URL names — not merely that it reported success. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_index_round_trip_through_a_path_style_entry() { + with_azurite_credentials(async { + const PREFIX: &str = "round-trip"; + ensure_empty_prefix(PREFIX).await; + + let seeded = verify_operator(PREFIX); + seeded + .write( + &format!("noarch/{PACKAGE}"), + fs_err::read(package_path()).expect("test package"), + ) + .await + .expect("seeding the package failed"); + + index_azure(index_config(channel(PREFIX))) + .await + .expect("indexing an azurite channel failed"); + + let repodata = seeded + .read("noarch/repodata.json") + .await + .expect("the indexer wrote no repodata.json where the channel URL points"); + let json: serde_json::Value = + serde_json::from_slice(&repodata.to_vec()).expect("repodata was not valid json"); + assert!( + json["packages.conda"] + .as_object() + .is_some_and(|packages| packages.contains_key(PACKAGE)), + "repodata should list the seeded package: {json}" + ); + assert_eq!( + json["info"]["subdir"], "noarch", + "repodata should describe the subdir it was written for: {json}" + ); + + // The same write path that ros-recipes' cache-header sweep exists to + // override. Live evidence that opendal's azblob backend honours it at all. + let metadata = seeded + .stat("noarch/repodata.json") + .await + .expect("stat repodata.json"); + assert_eq!(metadata.cache_control(), Some(CACHE_CONTROL_REPODATA)); + }) + .await; +} + +/// A write larger than the chunk size commits through Put Block List, and +/// `Cache-Control` survives that commit. +/// +/// Nothing in production reaches this yet. opendal's azblob backend declares no +/// `write_multi_min_size`, so `Operator::write_with` hands the whole buffer over in +/// a single `write` call and always takes the single-shot Put Blob path, however +/// large the repodata gets; only an explicit `chunk` splits it. So this locks the +/// header down for the commit path a chunked repodata write would take, which is +/// the gap that made the live `Cache-Control` evidence incomplete. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_multi_block_write_keeps_cache_control() { + with_azurite_credentials(async { + const PREFIX: &str = "multi-block-cache-control"; + ensure_empty_prefix(PREFIX).await; + let op = production_operator(&channel(PREFIX)); + + // Two chunks' worth, so `write` is called more than once and the writer + // switches from `write_once` to staging blocks. + let mut writer = op + .writer_with("noarch/repodata.json") + .chunk(2 * MIB) + .cache_control(CACHE_CONTROL_REPODATA) + .await + .expect("opening a chunked writer failed"); + writer + .write(vec![b'{'; 5 * MIB]) + .await + .expect("chunked write failed"); + writer.close().await.expect("Put Block List commit failed"); + + let metadata = op + .stat("noarch/repodata.json") + .await + .expect("stat the multi-block blob"); + assert!( + metadata.content_length() == (5 * MIB) as u64, + "the blob should have committed all blocks, got {} bytes", + metadata.content_length() + ); + assert_eq!( + metadata.cache_control(), + Some(CACHE_CONTROL_REPODATA), + "Put Block List should carry x-ms-blob-cache-control through its commit" + ); + + // Without this the test still passes if opendal ever takes the single-shot + // path for the whole 5 MiB, and then it asserts nothing about the commit + // path it is named for. 5 MiB in 2 MiB chunks is three blocks. + assert_eq!( + committed_block_count(PREFIX, "noarch/repodata.json").await, + 3, + "a 5 MiB write in 2 MiB chunks must commit three blocks through Put Block List; zero \ + blocks means the single-shot Put Blob path was taken instead" + ); + }) + .await; +} + +/// A canary for an opendal bug, so it asserts the *broken* behaviour and fails when +/// upstream fixes it. Read the failure message before touching the assertions. +/// +/// The bug: `azblob_complete_put_block_list_request` never sets `IF_NONE_MATCH`, so +/// `if_not_exists` is silently dropped above the 10 MiB chunk size, while the +/// sibling copy-block-list function does set it. That is the gap +/// `upload_package_to_azure`'s pre-write `stat` exists to close. +/// +/// TODO: revisit once merges — it +/// carries changes in this area. +/// +/// A package over `rattler_upload`'s 10 MiB chunk size is the only way to reach the +/// multi-block path, and this is as close as a test can currently get to that +/// upload: `rattler_upload` reads no config file, so it always builds a host-style +/// endpoint and cannot be pointed at an emulator at all. What it can share is the +/// operator, built here through the same `azblob_config`, and the uploader's chunk +/// size and overwrite guard — so the behaviour being reproduced is the uploader's, +/// even though the entry point is not. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_if_not_exists_is_dropped_on_the_multi_block_path() { + with_azurite_credentials(async { + const PREFIX: &str = "overwrite-guard"; + ensure_empty_prefix(PREFIX).await; + let op = production_operator(&channel(PREFIX)); + + // Baseline: below the chunk size, the guard works and opendal reports the + // conflict the uploader turns into "already exists, use --force". + let small = "noarch/small.conda"; + op.write(small, b"first".to_vec()) + .await + .expect("small write failed"); + let refused = op + .write_with(small, b"second".to_vec()) + .if_not_exists(true) + .await + .expect_err("if_not_exists should refuse to overwrite a small blob"); + assert_eq!(refused.kind(), ErrorKind::ConditionNotMatch); + + // Over the chunk size, the same option is accepted and then ignored: the + // guarded second write goes through and replaces the first payload. + let large = "noarch/large.conda"; + let first = vec![0u8; UPLOAD_CHUNK_SIZE + 2 * MIB]; + let second = vec![1u8; first.len()]; + write_chunked(&op, large, &first, false) + .await + .expect("unguarded multi-block write failed"); + write_chunked(&op, large, &second, true) + .await + .unwrap_or_else(|e| { + panic!( + "opendal now honours `if_not_exists` on the multi-block path ({e}). This is \ + the upstream fix this test was waiting for — do not adjust the assertion. \ + Delete `upload_package_to_azure`'s pre-write `stat` guard (and its TOCTOU \ + window) in crates/rattler_upload/src/upload/azure.rs, then delete this test." + ) + }); + let stored = op + .read(large) + .await + .expect("read back the large blob") + .to_vec(); + assert!( + stored == second, + "the guarded multi-block write should have clobbered the first payload" + ); + + // Which is why the uploader stats first. That check does see the blob, so + // the guard holds for large packages despite opendal dropping the option. + assert!( + op.stat(large).await.is_ok(), + "the pre-write stat must see an existing large blob, since if_not_exists does not" + ); + }) + .await; +} + +/// Write `payload` the way `upload_single_package` does. +/// +/// `guard` is that function's `if_not_exists(!force)`. The commit result is handed +/// back rather than unwrapped: whether a guarded commit over an existing blob +/// succeeds is the thing under test, so the caller owns that message. +async fn write_chunked( + op: &Operator, + path: &str, + payload: &[u8], + guard: bool, +) -> opendal::Result<()> { + let mut writer = op + .writer_with(path) + .chunk(UPLOAD_CHUNK_SIZE) + .if_not_exists(guard) + .await + .expect("opening a chunked writer failed"); + writer + .write(payload.to_vec()) + .await + .expect("chunked write failed"); + writer.close().await.map(|_| ()) +} diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 0ae710bfdf..7c16e483e8 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -17,6 +17,15 @@ native-tls = ["reqwest/native-tls", "ambient-id/native-tls"] rustls = ["reqwest/rustls", "ambient-id/rustls"] gcs = ["google-cloud-auth", "tokio/sync"] s3 = ["aws-config", "aws-sdk-s3", "aws-smithy-http-client"] +azure = [ + "dep:rattler_azure", + "dep:reqsign-core", + "dep:reqsign-azure-storage", + "dep:reqsign-command-execute-tokio", + "dep:reqsign-file-read-tokio", + "dep:reqsign-http-send-reqwest", + "tokio", +] system-integration = ["keyring", "netrc-rs", "dirs"] keyring = [ "keyring-core", @@ -27,7 +36,7 @@ keyring = [ ] [package.metadata.docs.rs] -features = ["gcs", "s3"] +features = ["gcs", "s3", "azure"] [dependencies] ambient-id = { workspace = true, features = ["reqwest-middleware"] } @@ -67,7 +76,13 @@ thiserror = { workspace = true } tokio = { workspace = true, optional = true } tracing = { workspace = true } url = { workspace = true } +rattler_azure = { workspace = true, optional = true } rattler_config = { workspace = true, optional = true } +reqsign-core = { workspace = true, optional = true } +reqsign-azure-storage = { workspace = true, optional = true } +reqsign-command-execute-tokio = { workspace = true, optional = true } +reqsign-file-read-tokio = { workspace = true, optional = true } +reqsign-http-send-reqwest = { workspace = true, optional = true } [target.'cfg( target_arch = "wasm32" )'.dependencies] getrandom = { workspace = true, features = ["wasm_js"] } @@ -85,6 +100,16 @@ dbus-secret-service-keyring-store = { workspace = true, optional = true, feature windows-native-keyring-store = { workspace = true, optional = true } regex = { workspace = true, optional = true } +# Declared rather than auto-discovered, so a feature name that stops matching is a +# cargo error instead of a test binary that silently holds no tests. +[[test]] +name = "azure_azurite_fetch" +required-features = ["azure"] + +[[test]] +name = "azure_real_fetch" +required-features = ["azure"] + [dev-dependencies] anyhow = { workspace = true } insta = { workspace = true, features = ["json"] } diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 272db69f0f..88ca981f3a 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -35,6 +35,13 @@ impl Middleware for AuthenticationMiddleware { return next.run(req, extensions).await; } + // Entries here are keyed by host alone, so without this gate a + // `*.blob.core.windows.net` token would attach to `az://` too. Those schemes + // carry their own signing middleware and their own grant model. + if !matches!(req.url().scheme(), "http" | "https") { + return next.run(req, extensions).await; + } + let url = req.url().clone(); match self.auth_storage.get_by_url_with_host(url) { Err(_) => { @@ -431,6 +438,36 @@ mod tests { Ok(()) } + #[cfg(feature = "keyring")] + #[tokio::test] + async fn non_http_schemes_are_left_unauthenticated() -> anyhow::Result<()> { + let tdir = tempdir()?; + let mut storage = AuthenticationStorage::empty(); + storage.add_backend(Arc::from(FileStorage::from_path( + tdir.path().to_path_buf().join("auth.json"), + )?)); + storage.store( + "*.blob.core.windows.net", + &Authentication::BearerToken("xyztokytoken".to_string()), + )?; + + let (client, mut captured_rx) = make_client_harness(&storage); + let request = client + .get("az://acct.blob.core.windows.net/channel/noarch/repodata.json") + .build()?; + let _ = client.execute(request).await; + + let captured_request = captured_rx.recv().await.unwrap(); + assert_eq!( + captured_request + .headers() + .get(reqwest::header::AUTHORIZATION), + None + ); + + Ok(()) + } + #[test] fn test_host_wildcard_expansion() -> anyhow::Result<()> { for (host, should_succeed) in [ diff --git a/crates/rattler_networking/src/azure_middleware.rs b/crates/rattler_networking/src/azure_middleware.rs new file mode 100644 index 0000000000..03754f9641 --- /dev/null +++ b/crates/rattler_networking/src/azure_middleware.rs @@ -0,0 +1,957 @@ +//! Middleware to handle `az://` URLs to pull artifacts from Azure Blob Storage. +use std::collections::HashMap; + +use async_trait::async_trait; +use rattler_azure::{ + Auth, AzureChannelUrl, AzureEndpointOptions, AzureFetchOptions, AzureHost, ContainerName, +}; +use reqsign_azure_storage::{Credential, DefaultCredentialProvider, RequestSigner}; +use reqsign_command_execute_tokio::TokioCommandExecute; +use reqsign_core::{Context, OsEnv, ProvideCredential, Signer}; +use reqsign_file_read_tokio::TokioFileRead; +use reqsign_http_send_reqwest::ReqwestHttpSend; +use reqwest::{Client, Request, Response}; +use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; +use url::Url; + +/// The Azure Storage REST API version sent on every signed request. A URL that +/// already carries a SAS returns before this is attached, and the write path pins +/// its own version inside opendal. +const X_MS_VERSION: &str = "2021-12-02"; + +/// Middleware that rewrites `az://` URLs to their wire form and, where a host is +/// granted credentials, signs them. +/// +/// The `az://` URL carries the full blob endpoint in its host, so rewriting is a +/// plain scheme swap: `az://{host}/{path}` → `https://{host}/{path}`. A conda +/// channel is therefore addressed the same way it is on the wire, e.g. +/// `az://myaccount.blob.core.windows.net/mycontainer` — no separate account or +/// endpoint configuration is needed. Sovereign clouds work with no configuration +/// at all; an emulator needs only the `scheme = "http"` line below. +/// +/// # Trust model +/// +/// **Anonymous by default.** With no entry for a host, its requests are sent +/// unsigned and *no credential is resolved at all* — so no ambient Azure +/// credential can leak to a host the user never named, and an anonymous read of a +/// public container does not block on the managed-identity / IMDS probe. +/// +/// A credential attaches to a request only because the user's `azure-options` +/// table grants it to the *container* the request addresses: +/// +/// ```toml +/// [azure-options."mycompany.blob.core.windows.net".auth] +/// releases = true +/// # a container not listed is fetched anonymously, so one account can hold +/// # private and anonymous-read containers side by side — which is what Azure's +/// # per-container RBAC actually enforces. +/// +/// [azure-options."127.0.0.1:10000"] # Azurite +/// scheme = "http" +/// path-style = true +/// +/// [azure-options."127.0.0.1:10000".auth] +/// general = true +/// ``` +/// +/// There is no host-level grant, by design: a single field meaning "every +/// container on this account, including the ones created later" is exactly the +/// mistake worth making unrepresentable. +/// +/// Three consequences of the grant being explicit: +/// +/// - **Nothing is inferred from the host name.** There is no allow-list of +/// "official" Azure suffixes, and none is needed: a host nobody granted gets +/// nothing regardless of what it is called, and an entry for a custom host *is* +/// the declaration that the user trusts that endpoint. +/// - **A broken credential is a hard error.** Because the user asked for signing, +/// an unusable credential must be reported, not silently downgraded to an +/// anonymous request that Azure will answer with a confusing 404. +/// - **A new private container fails closed** until someone adds a line for it. +/// That is the deliberate cost of a per-container grant, and it is why the 404 +/// hint below prints the exact line to write: unhelped, the failure reads as "the +/// channel is broken". +/// +/// Entries are user-scoped by contract: a project- or workspace-level manifest +/// must never be allowed to write one, since that would let a checked-out +/// repository name a host and receive the user's credentials. +/// +/// `az://user:pass@host/...` is refused outright. The host becomes the request +/// target verbatim, so userinfo is a host-spoofing vector — the real authority can +/// hide behind it — and userinfo is invalid in a blob URL anyway. +/// +/// Granted credentials are resolved by reqsign's [`DefaultCredentialProvider`] +/// chain, in its order: environment variables, the Azure CLI (`az login`), +/// client certificate, client secret, pipelines, workload identity, IMDS. +/// rattler's [`crate::AuthenticationStorage`] is not consulted — it has no Azure +/// variant, and [`crate::AuthenticationMiddleware`] handles only `http`/`https`, +/// so its host-keyed entries cannot reach an `az://` request either. +#[derive(Clone)] +pub struct AzureMiddleware { + /// reqsign signer; caches the resolved credential internally. + signer: Signer, + + /// Whole `azure-options` entries, keyed by the same normalized authority the + /// config table is keyed by. An absent host is *defined* to behave as a + /// defaulted entry (no grants, https), so a miss is never a separate code path. + /// + /// Entries and not the narrower [`AzureFetchOptions`], because resolving a + /// grant is two steps and they are ordered: the host's addressing decides which + /// path segment is the container, and only then can the container's grant be + /// read. The narrowing therefore happens per request, in [`Self::resolve`]. + /// + /// A plain `HashMap` rather than `rattler_config::AzureOptionsMap`, mirroring + /// [`crate::S3Middleware`]: taking the config type would put a mandatory + /// `rattler_config` edge on the `azure` feature. The constructors take any + /// iterator of host/entry pairs instead, which `AzureOptionsMap` yields + /// directly from its own `endpoint_options`. + options: HashMap, +} + +/// One `az://` request, resolved against the options table. +/// +/// The container is kept next to the grant it produced, because the message the +/// user needs when a request comes back 404 is the TOML line naming *that* +/// container — a hint naming only the host would be a line that grants the wrong +/// thing. +#[derive(Debug)] +struct Resolved { + /// The channel URL the request names. + channel: AzureChannelUrl, + + /// The container it addresses, when it addresses one. `None` for a URL with no + /// container segment — the host root, or a path too short for the host's + /// addressing — which has nothing to attribute a grant to. + container: Option, + + /// The grant for that container, and the wire scheme for the host. + options: AzureFetchOptions, +} + +/// What a resolved request asks of the signer. +/// +/// [`Signing::Granted`] carries the container whose entry granted it, so "sign +/// this, but for no container" is unrepresentable and the failure message can +/// always quote the line that asked for signing. A grant is only ever read out of a +/// container's entry in an `auth` table, so there is no other way for one to exist. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Signing<'a> { + /// No grant: send unsigned, and resolve no credential. + Anonymous, + + /// `container` is granted, so sign — and fail loudly if that is impossible. + Granted(&'a ContainerName), +} + +impl<'a> Signing<'a> { + /// The signing decision for a resolved request. + fn new(auth: Auth, container: Option<&'a ContainerName>) -> Self { + match (auth, container) { + (Auth::DefaultChain, Some(container)) => Self::Granted(container), + // `DefaultChain` without a container cannot arise — the grant was read + // out of a container's entry — and anonymous is the arm that sends + // nothing, which is the right way for an impossible pair to fall. + (Auth::DefaultChain | Auth::Anonymous, _) => Self::Anonymous, + } + } +} + +impl AzureMiddleware { + /// Create a new Azure middleware. + /// + /// `client` is used for reqsign's credential resolution (IMDS / managed + /// identity / AAD token fetches), so it must be the caller's configured + /// client — proxy, CA bundle, and TLS settings carry through to those + /// requests. + /// + /// `options` is the `azure-options` table: the per-host entries carrying the + /// per-container grants, in any shape that iterates them — + /// `rattler_config::AzureOptionsMap::endpoint_options` yields exactly this. An + /// empty iterator means every `az://` request is anonymous. + pub fn new( + client: Client, + options: impl IntoIterator, + ) -> Self { + Self::with_credential_provider(client, DefaultCredentialProvider::new(), options) + } + + /// Build the middleware around an explicit credential provider. + /// + /// [`AzureMiddleware::new`] wires up the [`DefaultCredentialProvider`] chain; + /// tests use this seam to inject a deterministic provider (an empty chain + /// standing in for a broken credential, or a static key) without touching the + /// ambient environment. + fn with_credential_provider( + client: Client, + provider: impl ProvideCredential + 'static, + options: impl IntoIterator, + ) -> Self { + let ctx = Context::new() + .with_file_read(TokioFileRead) + .with_http_send(ReqwestHttpSend::new(client)) + .with_command_execute(TokioCommandExecute) + .with_env(OsEnv); + let signer = Signer::new(ctx, provider, RequestSigner::new()); + Self { + signer, + options: options.into_iter().collect(), + } + } + + /// Resolve an `az://` request URL to the channel URL it names, the container it + /// addresses, and the options that apply to it. + /// + /// Going through [`AzureChannelUrl`] is what keeps this middleware from owning + /// a second copy of rules that live in `rattler_azure`: that parser is what + /// rejects userinfo, and it normalizes the authority into the exact spelling + /// the options table is keyed by, so a grant cannot miss over case, a trailing + /// dot, an IDNA name or an IP literal written oddly. The container comes from + /// [`rattler_azure::container`] for the same reason — it is the same derivation + /// the write path's coordinates use, and two derivations that disagreed would + /// look a grant up for one container and send it to another. + /// + /// The order is forced: the host's entry carries the addressing, the addressing + /// says which path segment is the container, and the container selects the + /// grant. Nothing earlier can know the container. + fn resolve(&self, url: &Url) -> MiddlewareResult { + let channel = AzureChannelUrl::parse(url.as_str()).map_err(|e| { + // The URL is not echoed back: the one rejection a user hits here is + // userinfo, and quoting it would print their password. + reqwest_middleware::Error::Middleware(anyhow::Error::from(e)) + })?; + + // An absent host is defined to behave as a defaulted entry, so the fallback + // is a value and not a branch. + let unconfigured = AzureEndpointOptions::default(); + let entry = self.options.get(channel.host()).unwrap_or(&unconfigured); + + // A URL with no container segment resolves to no grant; a container segment + // Azure could never accept is a malformed endpoint, and saying so beats an + // anonymous request that comes back as an unexplained 401. + let container = rattler_azure::container(&channel, entry.endpoint().addressing) + .map_err(|e| reqwest_middleware::Error::Middleware(anyhow::Error::from(e)))?; + + Ok(Resolved { + options: entry.fetch(container.as_ref()), + container, + channel, + }) + } + + /// Whether the URL already carries an explicit SAS token (a `sig` query + /// parameter). Such a URL is self-authenticating and must not be re-signed. + fn has_sas_token(url: &Url) -> bool { + url.query_pairs().any(|(key, _)| key == "sig") + } + + /// Sign a reqwest `Request` in place using reqsign, when `signing` grants it. + /// + /// Two cases return without invoking reqsign at all: + /// - The URL already carries an explicit SAS (`?...&sig=...`). Signing would + /// add an `Authorization` header that Azure prefers over the SAS, silently + /// overriding the caller's explicit token. + /// - [`Signing::Anonymous`] — no grant. Crucially the credential is not *resolved* + /// either: reqsign would otherwise probe the managed-identity / IMDS endpoint + /// and block until it times out (~30s on a machine with no metadata service) + /// before we could decide not to use the result, making every anonymous + /// public-channel read pay that timeout — and it would pull an ambient + /// credential into memory for a host the user never granted. + /// + /// Under [`Signing::Granted`] any signing failure is propagated, carrying the + /// host, the container whose grant required signing and the remedies. reqsign + /// collapses "no + /// credential" and "broken credential" into the same + /// [`reqsign_core::ErrorKind::CredentialInvalid`], and since the user asked for + /// signing there is no case left where going anonymous is the right answer. + async fn sign(&self, req: &mut Request, signing: Signing<'_>) -> MiddlewareResult<()> { + if Self::has_sas_token(req.url()) { + return Ok(()); + } + + if !req.headers().contains_key("x-ms-version") { + req.headers_mut() + .insert("x-ms-version", http::HeaderValue::from_static(X_MS_VERSION)); + } + + let container = match signing { + Signing::Anonymous => { + // The authority, not `host_str()`: a message naming a host the user + // could act on must carry the port, or it names a host that is not + // the one in their config. + tracing::debug!( + "no `azure-options` auth grant for `{}`; sending `az://` request unsigned", + req.url().authority() + ); + return Ok(()); + } + Signing::Granted(container) => container, + }; + + let mut builder = http::Request::builder() + .method(req.method().clone()) + .uri(req.url().as_str()); + for (name, value) in req.headers() { + builder = builder.header(name, value); + } + let http_req = builder.body(()).map_err(|e| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "failed to build http request for signing: {e}" + )) + })?; + let (mut parts, ()) = http_req.into_parts(); + + // reqsign says only "failed to load signing credential": its chain walks + // past a provider that errors exactly as it walks past one that finds + // nothing, so an expired `az login` and an empty environment arrive here + // indistinguishable, after however long the chain took to give up. The host + // and the grant that asked for signing are both in scope here and nowhere + // further up, so this is where they get attached. + self.signer.sign(&mut parts, None).await.map_err(|e| { + let authority = req.url().authority(); + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "could not resolve an Azure credential for `{container}` on `{authority}`, which \ + `[azure-options.\"{authority}\".auth]` `{container} = true` requires: {e}\n\ + \n\ + Try one of:\n\ + \x20 - `az login`\n\ + \x20 - `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_ACCOUNT_KEY` in the \ + environment\n\ + \x20 - set `{container} = false` to fetch this container anonymously\n\ + \n\ + Debug logging lists the credential providers that were tried." + )) + })?; + + *req.headers_mut() = parts.headers; + let signed_url = Url::parse(&parts.uri.to_string()).map_err(|e| { + reqwest_middleware::Error::Middleware(anyhow::anyhow!( + "failed to parse signed azure URL '{}': {e}", + parts.uri + )) + })?; + *req.url_mut() = signed_url; + Ok(()) + } +} + +#[async_trait] +impl Middleware for AzureMiddleware { + async fn handle( + &self, + mut req: Request, + extensions: &mut http::Extensions, + next: Next<'_>, + ) -> MiddlewareResult { + // Only intercept `az://` requests. + if req.url().scheme() != "az" { + return next.run(req, extensions).await; + } + + let Resolved { + channel, + container, + options, + } = self.resolve(req.url())?; + *req.url_mut() = channel.wire(options.scheme); + self.sign(&mut req, Signing::new(options.auth, container.as_ref())) + .await?; + + let response = next.run(req, extensions).await?; + + // Azure answers an unauthorized read of a private container with 404, not + // 403, so "no grant" and "no such blob" are the same status on the wire. + // Under a per-container grant a newly-created private container fails closed + // until someone writes a line for it, so this hint is what stands between + // that and a user reading "404" as "the channel is broken". Say it once per + // container, naming the config to write — spelled through `AzureHost` and + // `ContainerName` so the key printed is the key a lookup arrives with. + // + // A URL naming no container gets no hint: there is no line that would grant + // it anything. + if let Some(container) = container.filter(|_| { + response.status() == http::StatusCode::NOT_FOUND && !options.auth.is_granted() + }) && first_404_for_container(channel.host(), &container) + { + // One line, and spelled the way `AzureUrlError::InvalidHost` spells its + // fix: a wrapped multi-line hint is harder to grep out of a log, and + // the two guided messages should read as the same instruction. + tracing::warn!( + "`{}` returned 404 and container `{container}` has no `azure-options` auth grant. \ + Azure answers an anonymous read of a *private* container with 404 rather than \ + 403, so a missing grant looks exactly like a missing file. If the container is \ + private, grant it in your user configuration with \ + `[azure-options.\"{}\".auth]` and `{container} = true`.", + channel.canonical(), + channel.host() + ); + } + + Ok(response) + } +} + +/// Whether this container still owes the 404 hint, claiming it if so. +/// +/// A 404 is the *normal* answer to plenty of requests a healthy public channel +/// makes — the repodata gateway probes for a shard index under every subdir it +/// fetches, and a non-sharded channel misses every time — so a hint emitted per +/// response is a security warning printed repeatedly at users whose channel is +/// fine. Once per container per process is enough for the one case it is about: a +/// private container the user forgot to grant. Per container and not per host, +/// because the line to add differs per container: silencing a host after its first +/// ungranted container would leave the second one unexplained. +fn first_404_for_container(host: &AzureHost, container: &ContainerName) -> bool { + static HINTED: std::sync::LazyLock< + std::sync::Mutex>, + > = std::sync::LazyLock::new(Default::default); + HINTED + .lock() + .expect("the 404-hint set is never held across a panic") + .insert((host.clone(), container.clone())) +} + +#[cfg(test)] +mod tests { + use rattler_azure::{Addressing, AzureEndpoint, AzureScheme}; + + use super::*; + + fn container(name: &str) -> ContainerName { + ContainerName::new(name).expect("test container name") + } + + /// The `azure-options` table for one host, as a caller would build it. + fn options( + authority: &str, + options: AzureEndpointOptions, + ) -> HashMap { + HashMap::from([(AzureHost::parse(authority).expect("test host"), options)]) + } + + /// An entry granting one container, with everything else defaulted: which + /// container is granted is the only interesting axis in most of these tests. + fn granting(container_name: &str) -> AzureEndpointOptions { + AzureEndpointOptions::new( + [(container(container_name), Auth::DefaultChain)], + AzureEndpoint::default(), + ) + } + + fn middleware(options: HashMap) -> AzureMiddleware { + AzureMiddleware::new(Client::new(), options) + } + + /// Resolve a URL and hand back the wire spelling its options ask for. + fn wire_of(middleware: &AzureMiddleware, url: &str) -> String { + let resolved = middleware + .resolve(&Url::parse(url).expect("test url")) + .expect("url should resolve"); + resolved.channel.wire(resolved.options.scheme).to_string() + } + + /// Resolve a URL, or panic with the middleware's rejection. + fn resolve(middleware: &AzureMiddleware, url: &str) -> Resolved { + middleware + .resolve(&Url::parse(url).expect("test url")) + .unwrap_or_else(|err| panic!("{url} should resolve: {err}")) + } + + /// With no entry the scheme defaults to https, and path, query and fragment + /// survive the rewrite untouched. + #[test] + fn rewrites_to_https_without_an_entry() { + let middleware = middleware(HashMap::new()); + assert_eq!( + wire_of( + &middleware, + "az://myacct.blob.core.windows.net/mychannel/noarch/repodata.json" + ), + "https://myacct.blob.core.windows.net/mychannel/noarch/repodata.json" + ); + assert_eq!( + wire_of( + &middleware, + "az://acct.blob.core.windows.net/general/x.json?sv=2021&sig=abc#frag" + ), + "https://acct.blob.core.windows.net/general/x.json?sv=2021&sig=abc#frag" + ); + } + + /// An emulator entry is the only thing that can send an `az://` URL in + /// cleartext, and the port has to survive — `:10000` is not any scheme's + /// default, but `:443` would be under https and must not be dropped either. + #[test] + fn rewrites_to_http_for_an_emulator_entry() { + let emulator = middleware(options("127.0.0.1:10000", emulator_entry(["general"]))); + assert_eq!( + wire_of( + &emulator, + "az://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json" + ), + "http://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json" + ); + + // The same host with no entry stays on https: an emulator grant must not + // generalize to a scheme downgrade for anyone else. + assert_eq!( + wire_of( + &middleware(HashMap::new()), + "az://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json" + ), + "https://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json" + ); + } + + /// A grant written in any spelling of a host must apply to a request for that + /// host: a silent miss reads as a 404, i.e. "not found" for what is really + /// "not authorized". Delegating to `AzureHost` on both sides is what buys this. + #[test] + fn a_grant_applies_regardless_of_how_the_host_is_spelled() { + let middleware = middleware(options( + "MyCompany.blob.core.windows.net.", + granting("releases"), + )); + assert!( + resolve( + &middleware, + "az://mycompany.blob.core.windows.net/releases/x.json" + ) + .options + .auth + .is_granted() + ); + } + + /// The point of the per-container table: one account holding a private and an + /// anonymous-read container is configurable, because the grant stops at the + /// container it names. Under a host-level grant the second URL here would be + /// signed too, and 403 for any identity holding no role on it. + #[test] + fn a_grant_stops_at_the_container_it_names() { + let middleware = middleware(options( + "mycompany.blob.core.windows.net", + AzureEndpointOptions::new( + [ + (container("releases"), Auth::DefaultChain), + // Redundant with omission, and legal: it says "deliberately + // unsigned" rather than "forgotten". + (container("public"), Auth::Anonymous), + ], + AzureEndpoint::default(), + ), + )); + + for (url, granted) in [ + ("az://mycompany.blob.core.windows.net/releases/x.json", true), + ("az://mycompany.blob.core.windows.net/public/x.json", false), + ("az://mycompany.blob.core.windows.net/staging/x.json", false), + ] { + assert_eq!( + resolve(&middleware, url).options.auth.is_granted(), + granted, + "{url}" + ); + } + } + + /// A container is found where the host's addressing says it is, so a grant on a + /// path-style host applies to the second segment and not the account in the + /// first. + #[test] + fn a_container_is_read_through_the_hosts_addressing() { + let path_style = middleware(options("127.0.0.1:10000", emulator_entry(["general"]))); + let resolved = resolve( + &path_style, + "az://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json", + ); + assert_eq!(resolved.container, Some(container("general"))); + assert!(resolved.options.auth.is_granted()); + + // Host-style on the same URL reads the account segment as the container, so + // the grant does not apply — the addressing is what decides which name a + // grant is even about. + let host_style = middleware(options( + "127.0.0.1:10000", + AzureEndpointOptions::new( + [(container("general"), Auth::DefaultChain)], + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::HostStyle, + }, + ), + )); + let resolved = resolve( + &host_style, + "az://127.0.0.1:10000/devstoreaccount1/general/noarch/repodata.json", + ); + assert_eq!(resolved.container, Some(container("devstoreaccount1"))); + assert!(!resolved.options.auth.is_granted()); + } + + /// A URL naming no container has nothing to attribute a grant to, so it is + /// anonymous rather than an error: the fetch path stays total for URLs that are + /// not channel-scoped, which is what it is today. + #[test] + fn a_url_without_a_container_is_anonymous() { + let middleware = middleware(options( + "mycompany.blob.core.windows.net", + granting("releases"), + )); + + for url in [ + "az://mycompany.blob.core.windows.net", + "az://mycompany.blob.core.windows.net/", + "az://mycompany.blob.core.windows.net/?comp=list", + ] { + let resolved = resolve(&middleware, url); + assert_eq!(resolved.container, None, "{url}"); + assert!(!resolved.options.auth.is_granted(), "{url}"); + } + } + + /// A container segment Azure could never accept is a malformed endpoint, not an + /// ungranted one: no legitimate request can land here, and going quietly + /// anonymous would surface as an unexplained 401 rather than naming the fault. + #[test] + fn a_url_with_an_unusable_container_is_refused() { + let middleware = middleware(options( + "mycompany.blob.core.windows.net", + granting("releases"), + )); + + for url in [ + "az://mycompany.blob.core.windows.net/Releases/x.json", + "az://mycompany.blob.core.windows.net/ab/x.json", + ] { + let err = middleware + .resolve(&Url::parse(url).unwrap()) + .expect_err("an illegal container name must be refused"); + assert!(err.to_string().contains("container name"), "{url}: {err}"); + } + } + + /// Userinfo is refused before any rewrite or signing: the host is the request + /// target verbatim, so `user:pass@real.host` can hide the real authority. + /// (Rejection lives in `AzureHost::parse`, so there is one copy of the rule.) + #[test] + fn rejects_userinfo() { + let middleware = middleware(HashMap::new()); + for url in [ + "az://user:pass@acct.blob.core.windows.net/general/x.json", + "az://user@acct.blob.core.windows.net/general/x.json", + ] { + let err = middleware + .resolve(&Url::parse(url).unwrap()) + .expect_err("userinfo must be refused"); + assert!(err.to_string().contains("userinfo"), "{err}"); + } + assert!( + middleware + .resolve(&Url::parse("az://acct.blob.core.windows.net/general/x.json").unwrap()) + .is_ok() + ); + } + + #[tokio::test] + async fn passes_through_non_az_schemes_unchanged() { + use reqwest_middleware::ClientBuilder; + let client = ClientBuilder::new(Client::new()) + .with(middleware(HashMap::new())) + .build(); + // A non-`az` request must not be rewritten; it should be attempted as-is + // (and fail on DNS), proving the middleware left it untouched. + let result = client + .get("https://this-host-does-not-exist.invalid/x") + .send() + .await; + assert!(result.is_err()); + } + + /// A container with no grant is sent unsigned, and its credential is never even + /// resolved — so nothing blocks on the IMDS probe and no ambient credential is + /// pulled into memory for a host the user never named. The provider flips a + /// flag if it is ever asked. + #[tokio::test] + async fn an_ungranted_container_sends_unsigned_without_resolving_a_credential() { + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; + + #[derive(Debug)] + struct RecordingProvider(Arc); + impl ProvideCredential for RecordingProvider { + type Credential = Credential; + async fn provide_credential( + &self, + _ctx: &Context, + ) -> reqsign_core::Result> { + self.0.store(true, Ordering::SeqCst); + Ok(None) + } + } + + let probed = Arc::new(AtomicBool::new(false)); + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + RecordingProvider(probed.clone()), + HashMap::new(), + ); + let mut req = Client::new() + .get("https://acct.blob.core.windows.net/pub/noarch/repodata.json") + .build() + .unwrap(); + + middleware + .sign(&mut req, Signing::Anonymous) + .await + .expect("an ungranted request must pass through unsigned"); + + assert!( + !probed.load(Ordering::SeqCst), + "credential provider must not be probed without a grant" + ); + assert!( + req.headers().get(http::header::AUTHORIZATION).is_none(), + "unsigned request must not carry an Authorization header" + ); + assert!( + !req.url().query_pairs().any(|(k, _)| k == "sig"), + "unsigned request must not gain a SAS query parameter" + ); + } + + /// A granted container is actually signed: the credential resolves and the + /// request comes back carrying Shared Key authorization. + #[tokio::test] + async fn a_granted_container_is_signed() { + use reqsign_azure_storage::StaticCredentialProvider; + + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + // A valid base64 account key, so the provider yields a usable + // SharedKey credential. + StaticCredentialProvider::new_shared_key("acct", "dGVzdF9rZXk="), + options("acct.blob.core.windows.net", granting("releases")), + ); + let mut req = Client::new() + .get("https://acct.blob.core.windows.net/releases/noarch/repodata.json") + .build() + .unwrap(); + + middleware + .sign(&mut req, Signing::Granted(&container("releases"))) + .await + .unwrap(); + + let authorization = req + .headers() + .get(http::header::AUTHORIZATION) + .expect("a granted container must be signed"); + assert!( + authorization.to_str().unwrap().starts_with("SharedKey "), + "{authorization:?}" + ); + } + + /// The inversion this design turns on: with a grant, an unusable credential is + /// a hard error. It must never degrade to an anonymous request, which Azure + /// would answer with a 404 the user has no way to read as "auth failed". An + /// empty provider chain resolves nothing, which reqsign reports the same way it + /// reports a broken credential. + #[tokio::test] + async fn a_granted_container_with_broken_credentials_is_a_hard_error() { + use reqsign_core::ProvideCredentialChain; + + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + ProvideCredentialChain::::new(), + options("acct.blob.core.windows.net", granting("releases")), + ); + let mut req = Client::new() + .get("https://acct.blob.core.windows.net/releases/noarch/repodata.json") + .build() + .unwrap(); + + let result = middleware + .sign(&mut req, Signing::Granted(&container("releases"))) + .await; + + assert!( + result.is_err(), + "a granted-but-failing credential must be a hard error, not unsigned" + ); + assert!( + req.headers().get(http::header::AUTHORIZATION).is_none(), + "a failed signing attempt must not leave a partial Authorization header" + ); + + // reqsign's own message names neither the host nor a remedy, and the chain + // hides which provider failed. Everything actionable has to come from here — + // including which container's grant asked for the signing, since the entry + // may hold several and only one line is at fault. + let message = result.unwrap_err().to_string(); + for expected in [ + "acct.blob.core.windows.net", + "releases = true", + "az login", + "AZURE_STORAGE_ACCOUNT_KEY", + ] { + assert!( + message.contains(expected), + "the failure must name `{expected}`, got: {message}" + ); + } + } + + /// A URL that already carries a SAS token must not be re-signed even where the + /// container is granted: Azure prefers an `Authorization` header over the SAS, + /// so signing would silently override the caller's explicit token. + #[tokio::test] + async fn a_sas_in_the_url_passes_through() { + use reqsign_azure_storage::StaticCredentialProvider; + + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + StaticCredentialProvider::new_shared_key("acct", "dGVzdF9rZXk="), + options("acct.blob.core.windows.net", granting("releases")), + ); + let mut req = Client::new() + .get("https://acct.blob.core.windows.net/releases/x.json?sv=2021&sig=abc") + .build() + .unwrap(); + + middleware + .sign(&mut req, Signing::Granted(&container("releases"))) + .await + .unwrap(); + + assert!( + req.headers().get(http::header::AUTHORIZATION).is_none(), + "a URL carrying an explicit SAS must not be re-signed" + ); + assert!( + !req.headers().contains_key("x-ms-version"), + "a self-authenticating SAS URL is left untouched" + ); + assert_eq!( + req.url().as_str(), + "https://acct.blob.core.windows.net/releases/x.json?sv=2021&sig=abc" + ); + } + + /// Serve 404 for everything, over http on localhost, standing in for a private + /// container answering an anonymous read. + async fn spawn_404_server() -> AzureHost { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let router = axum::Router::new().fallback(axum::http::StatusCode::NOT_FOUND); + tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + AzureHost::parse(&addr.to_string()).unwrap() + } + + /// An emulator-shaped entry (http, path-style) granting whichever containers the + /// caller names, so one server can exercise both sides of the hint. + fn emulator_entry<'a>(granted: impl IntoIterator) -> AzureEndpointOptions { + AzureEndpointOptions::new( + granted + .into_iter() + .map(|name| (container(name), Auth::DefaultChain)), + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ) + } + + async fn get_az( + middleware: AzureMiddleware, + host: &AzureHost, + container: &str, + ) -> reqwest::StatusCode { + reqwest_middleware::ClientBuilder::new(Client::new()) + .with(middleware) + .build() + .get(format!( + "az://{host}/devstoreaccount1/{container}/noarch/repodata.json" + )) + .send() + .await + .expect("request through azure middleware failed") + .status() + } + + /// The 404 hint must name the config block to add, keyed exactly as the table + /// is keyed — including the port, which an earlier version of this hint + /// dropped, printing a key that could never match — and the container, since + /// that is the line the user has to write. + #[tokio::test] + #[tracing_test::traced_test] + async fn the_404_hint_names_the_config_block_for_an_ungranted_container() { + let host = spawn_404_server().await; + let middleware = middleware(options(&host.to_string(), emulator_entry([]))); + + assert_eq!(get_az(middleware, &host, "general").await, 404); + + assert!(logs_contain(&format!("[azure-options.\"{host}\".auth]"))); + assert!(logs_contain("general = true")); + } + + /// A public non-sharded channel 404s on every shard-index probe the repodata + /// gateway makes, so a hint per response is a security warning repeated at a + /// user whose channel is healthy. It is silenced per container, though: a second + /// ungranted container needs a different line, so it gets its own hint. + #[tokio::test] + #[tracing_test::traced_test] + async fn the_404_hint_is_emitted_once_per_container() { + let host = spawn_404_server().await; + let client = reqwest_middleware::ClientBuilder::new(Client::new()) + .with(middleware(options(&host.to_string(), emulator_entry([])))) + .build(); + + for container in ["general", "staging"] { + for subdir in ["noarch", "linux-64", "osx-64"] { + let status = client + .get(format!( + "az://{host}/devstoreaccount1/{container}/{subdir}/\ + repodata_shards.msgpack.zst" + )) + .send() + .await + .expect("request through azure middleware failed") + .status(); + assert_eq!(status, 404); + } + } + + for container in ["general", "staging"] { + logs_assert(move |lines: &[&str]| { + let hints = lines + .iter() + .filter(|line| line.contains(&format!("{container} = true"))) + .count(); + (hints == 1).then_some(()).ok_or_else(|| { + format!("expected exactly one hint for {container}, got {hints}") + }) + }); + } + } + + /// With a grant in place a 404 means what it says, so the hint would be noise. + #[tokio::test] + #[tracing_test::traced_test] + async fn the_404_hint_is_silent_for_a_granted_container() { + use reqsign_azure_storage::StaticCredentialProvider; + + let host = spawn_404_server().await; + let middleware = AzureMiddleware::with_credential_provider( + Client::new(), + StaticCredentialProvider::new_shared_key("devstoreaccount1", "dGVzdF9rZXk="), + options(&host.to_string(), emulator_entry(["general"])), + ); + + assert_eq!(get_az(middleware, &host, "general").await, 404); + + assert!(!logs_contain("azure-options")); + } +} diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs index c11d21fdd6..b24a557056 100644 --- a/crates/rattler_networking/src/lib.rs +++ b/crates/rattler_networking/src/lib.rs @@ -21,6 +21,11 @@ pub mod s3_middleware; #[cfg(feature = "s3")] pub use s3_middleware::S3Middleware; +#[cfg(feature = "azure")] +pub mod azure_middleware; +#[cfg(feature = "azure")] +pub use azure_middleware::AzureMiddleware; + pub mod authentication_middleware; pub mod authentication_storage; pub mod challenge_middleware; diff --git a/crates/rattler_networking/tests/azure_azurite_fetch.rs b/crates/rattler_networking/tests/azure_azurite_fetch.rs new file mode 100644 index 0000000000..1786f35dc3 --- /dev/null +++ b/crates/rattler_networking/tests/azure_azurite_fetch.rs @@ -0,0 +1,298 @@ +//! Live fetch-path integration tests against a local Azurite emulator. +//! +//! Everything is driven through a single `azure-options` entry, which is the +//! point of the exercise — there is no out-of-band account or endpoint +//! configuration on the fetch path: +//! +//! ```toml +//! [azure-options."127.0.0.1:10000"] +//! scheme = "http" +//! path-style = true +//! +//! [azure-options."127.0.0.1:10000".auth] +//! cli-channel = true +//! ``` +//! +//! Run with: +//! +//! ```text +//! docker run --rm -p 10000:10000 mcr.microsoft.com/azure-storage/azurite \ +//! azurite-blob --blobHost 0.0.0.0 +//! cargo nextest run -p rattler_networking --features azure --test azure_azurite_fetch \ +//! --run-ignored all +//! ``` + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use async_trait::async_trait; +use rattler_azure::{ + Addressing, Auth, AzureEndpoint, AzureEndpointOptions, AzureHost, AzureScheme, ContainerName, +}; +use rattler_networking::AzureMiddleware; +use reqwest::{ + Request, Response, + header::{AUTHORIZATION, HeaderMap}, +}; +use reqwest_middleware::{ + ClientBuilder, ClientWithMiddleware, Middleware, Next, Result as MiddlewareResult, +}; +use url::Url; + +/// Azurite's development account and its fixed key. Not a secret: both are +/// published constants of the emulator, hardcoded in opendal's own source, and +/// they only ever address a loopback port. +const ACCOUNT: &str = "devstoreaccount1"; +const ACCOUNT_KEY: &str = + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + +/// The authority, which is also the exact `azure-options` table key. An IP with a +/// port is precisely the host shape that host-style addressing cannot read an +/// account out of, so it only works through a `path-style = true` entry. +const AUTHORITY: &str = "127.0.0.1:10000"; + +/// Container name from the test this one restores. Azurite creates containers as +/// private, which is what makes the ungranted case below meaningful. +const CONTAINER: &str = "cli-channel"; + +/// Minimal but structurally real repodata, so the assertions can be about +/// content rather than just a status code. +const REPODATA: &str = r#"{ + "info": { "subdir": "noarch" }, + "packages": {}, + "packages.conda": { + "empty-0.1.0-h4616a5c_0.conda": { + "build": "h4616a5c_0", + "build_number": 0, + "depends": [], + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "name": "empty", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "size": 1538, + "subdir": "noarch", + "version": "0.1.0" + } + } +}"#; + +/// The channel as a user would write it: the account is the first path segment, +/// which is what `path-style = true` means. +fn channel_url() -> String { + format!("az://{AUTHORITY}/{ACCOUNT}/{CONTAINER}") +} + +/// The one `azure-options` entry these tests run on, with the container's grant as +/// the only variable. `scheme` and `path-style` stay set even in the ungranted case: +/// the entry is what makes the emulator reachable at all, and keeping it identical +/// means the two tests differ in the grant and nothing else. +/// +/// The grant is written for `CONTAINER` specifically, which is also what makes the +/// ungranted case below a real test of the per-container lookup rather than of an +/// empty table: `Auth::Anonymous` here is the container named and *refused*. +fn azurite_entry(auth: Auth) -> HashMap { + HashMap::from([( + AzureHost::parse(AUTHORITY).expect("azurite authority is a valid host:port"), + AzureEndpointOptions::new( + [( + ContainerName::new(CONTAINER).expect("azurite container name"), + auth, + )], + AzureEndpoint { + scheme: AzureScheme::Http, + addressing: Addressing::PathStyle, + }, + ), + )]) +} + +/// The last request to leave the stack, captured *after* `AzureMiddleware` ran. +/// +/// Azurite answers 403 to any request it cannot authenticate, so a response status +/// cannot tell an unsigned request from a signed one whose signature was rejected — +/// a leak with a bad signature reads exactly like a refusal. The claim being tested +/// is about what goes on the wire, so that is what gets asserted. +#[derive(Clone, Default)] +struct SentRequest(Arc>>); + +impl SentRequest { + fn recorded(&self) -> (Url, HeaderMap) { + self.0 + .lock() + .expect("recorder mutex") + .clone() + .expect("no request reached the recorder") + } +} + +#[async_trait] +impl Middleware for SentRequest { + async fn handle( + &self, + req: Request, + extensions: &mut http::Extensions, + next: Next<'_>, + ) -> MiddlewareResult { + *self.0.lock().expect("recorder mutex") = Some((req.url().clone(), req.headers().clone())); + next.run(req, extensions).await + } +} + +fn recording_client(auth: Auth) -> (ClientWithMiddleware, SentRequest) { + let sent = SentRequest::default(); + let client = ClientBuilder::new(reqwest::Client::new()) + .with(AzureMiddleware::new( + reqwest::Client::new(), + azurite_entry(auth), + )) + .with(sent.clone()) + .build(); + (client, sent) +} + +fn client(auth: Auth) -> ClientWithMiddleware { + recording_client(auth).0 +} + +/// Create the container and put a `noarch/repodata.json` in it. +/// +/// Seeding runs through the granted middleware rather than a separate SDK, which +/// keeps the test dependency-free and doubles as proof that the signature works +/// for writes and for a request carrying a query string (`?restype=container` +/// participates in the canonicalized signing resource, so a wrong signature +/// fails here first). +async fn seed(client: &ClientWithMiddleware) { + let created = client + .put(format!("{}?restype=container", channel_url())) + .send() + .await + .expect("container create request failed"); + assert!( + // 409 is `ContainerAlreadyExists` — a re-run, not a failure. + created.status().is_success() || created.status() == reqwest::StatusCode::CONFLICT, + "could not create container {CONTAINER}: {}", + created.status() + ); + + // `Content-Length` is set by hand because shared-key signing covers it, and + // reqwest only materializes the header inside hyper at send time — after the + // middleware has already signed. That gap is invisible to production, where the + // middleware only ever carries bodyless `az://` reads, but a seeding PUT walks + // straight into it and gets a 403 for a length mismatch. + let put = client + .put(format!("{}/noarch/repodata.json", channel_url())) + .header("x-ms-blob-type", "BlockBlob") + .header(reqwest::header::CONTENT_LENGTH, REPODATA.len()) + .body(REPODATA) + .send() + .await + .expect("blob upload request failed"); + assert!( + put.status().is_success(), + "could not seed repodata.json: {}", + put.status() + ); +} + +/// A granted, `http`, path-style entry fetches a blob out of a private Azurite +/// container — the whole fetch path end to end, with the entry as the only +/// configuration. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_granted_entry_fetches_repodata() { + // reqsign's env provider sits first in the default chain, so the shared key + // is how a grant resolves against the emulator. Azurite rejects the AAD + // bearer tokens the rest of the chain produces, so this is the only credential + // shape that can work here — the chain itself is untouched. + temp_env::async_with_vars( + [ + ("AZURE_STORAGE_ACCOUNT_NAME", Some(ACCOUNT)), + ("AZURE_STORAGE_ACCOUNT_KEY", Some(ACCOUNT_KEY)), + ], + async { + let client = client(Auth::DefaultChain); + seed(&client).await; + + let url = format!("{}/noarch/repodata.json", channel_url()); + let resp = client + .get(&url) + .send() + .await + .expect("request through azure middleware failed"); + let status = resp.status(); + let body = resp.bytes().await.expect("failed to read body"); + assert!(status.is_success(), "unexpected status {status} for {url}"); + + let json: serde_json::Value = + serde_json::from_slice(&body).expect("fetched repodata was not valid json"); + assert!( + json["packages.conda"] + .as_object() + .is_some_and(|packages| packages.contains_key("empty-0.1.0-h4616a5c_0.conda")), + "repodata fetched via az:// should list the seeded package: {json}" + ); + }, + ) + .await; +} + +/// Without a grant for this container the request goes out unsigned, and a private +/// container +/// refuses it. This is the core claim of the anonymous-by-default model. +/// +/// The primary assertion is on the outgoing request, not the status: no +/// `Authorization` header and no SAS `sig` reaches the wire. The status is a +/// secondary check that the container really is private, but on its own it would +/// also pass while the account key was being sent with a signature Azurite +/// rejected. +#[tokio::test] +#[ignore = "requires a running Azurite emulator; see the module docs"] +async fn azurite_ungranted_entry_is_refused_by_a_private_container() { + temp_env::async_with_vars( + [ + ("AZURE_STORAGE_ACCOUNT_NAME", Some(ACCOUNT)), + ("AZURE_STORAGE_ACCOUNT_KEY", Some(ACCOUNT_KEY)), + ], + async { + // Seed with a grant, then read without one. The credential is present in + // the environment throughout, so a success below would mean the grant + // check leaked it + seed(&client(Auth::DefaultChain)).await; + + let url = format!("{}/noarch/repodata.json", channel_url()); + let (client, sent) = recording_client(Auth::Anonymous); + let resp = client + .get(&url) + .send() + .await + .expect("request through azure middleware failed"); + + let (sent_url, sent_headers) = sent.recorded(); + assert!( + !sent_headers.contains_key(AUTHORIZATION), + "an ungranted request must carry no credential, but it went out with an \ + Authorization header: {:?}", + sent_headers.get(AUTHORIZATION) + ); + assert!( + !sent_url.query_pairs().any(|(key, _)| key == "sig"), + "an ungranted request must carry no credential, but its URL went out with a SAS \ + signature: {sent_url}" + ); + + // 403 exactly, which is Azurite-specific: real Azure answers 404 to an + // unsigned read of a private container so that a missing grant is + // indistinguishable from a missing blob. This test only ever runs against + // the emulator, and accepting 404 too would also admit a request sent to + // the wrong URL — a wrong account segment or a dropped one both 404 here. + assert_eq!( + resp.status(), + reqwest::StatusCode::FORBIDDEN, + "Azurite refuses an unsigned read of a private container with 403; anything else \ + means the request did not reach the blob this URL names: {url}" + ); + }, + ) + .await; +} diff --git a/crates/rattler_networking/tests/azure_real_fetch.rs b/crates/rattler_networking/tests/azure_real_fetch.rs new file mode 100644 index 0000000000..e57dc6d50b --- /dev/null +++ b/crates/rattler_networking/tests/azure_real_fetch.rs @@ -0,0 +1,79 @@ +//! Live read-path test against a *real* Azure Blob Storage account. +//! +//! Exercises the AAD / token path of reqsign's `DefaultCredentialProvider` — +//! i.e. the credential `az login` provides — against a real +//! `*.blob.core.windows.net` endpoint. It is purely a read; it never writes. +//! +//! Configure via env and run (requires `az login` with at least +//! `Storage Blob Data Reader` on the container): +//! +//! ```text +//! AZURE_TEST_ACCOUNT=stcondachannel \ +//! AZURE_TEST_CONTAINER=general \ +//! AZURE_TEST_PATH=noarch/repodata.json \ +//! cargo test -p rattler_networking --features azure --test azure_real_fetch -- --ignored --nocapture +//! ``` +//! +//! Set `AZURE_TEST_HOST` to target a sovereign cloud or emulator directly +//! (overrides the default `{account}.blob.core.windows.net` host). + +use std::collections::HashMap; + +use rattler_azure::{Auth, AzureEndpoint, AzureEndpointOptions, AzureHost, ContainerName}; +use rattler_networking::AzureMiddleware; +use reqwest_middleware::ClientBuilder; + +#[tokio::test] +#[ignore = "requires az login and AZURE_TEST_* env vars pointing at a real account"] +async fn azure_middleware_fetches_real_repodata() { + let account = std::env::var("AZURE_TEST_ACCOUNT").expect("AZURE_TEST_ACCOUNT"); + let container = std::env::var("AZURE_TEST_CONTAINER").expect("AZURE_TEST_CONTAINER"); + let path = + std::env::var("AZURE_TEST_PATH").unwrap_or_else(|_| "noarch/repodata.json".to_string()); + let host = std::env::var("AZURE_TEST_HOST") + .unwrap_or_else(|_| format!("{account}.blob.core.windows.net")); + + // The grant is what makes this the AAD test rather than an anonymous read: an + // `azure-options` entry naming this container is the only thing that lets the + // `az login` credential attach to it, and it makes a broken credential fail + // loudly instead of falling through to an unsigned 404. Sibling containers on + // the same account are unaffected, which is the point of keying it per + // container. + let options = HashMap::from([( + AzureHost::parse(&host).expect("AZURE_TEST_HOST is not a valid host[:port]"), + AzureEndpointOptions::new( + [( + ContainerName::new(&container).expect("AZURE_TEST_CONTAINER is not a valid name"), + Auth::DefaultChain, + )], + AzureEndpoint::default(), + ), + )]); + + let client = ClientBuilder::new(reqwest::Client::new()) + .with(AzureMiddleware::new(reqwest::Client::new(), options)) + .build(); + + // The `az://` host carries the full blob endpoint — same form used in a + // channel URL, e.g. `az://stcondachannel.blob.core.windows.net/general`. + let url = format!("az://{host}/{container}/{path}"); + println!("fetching {url}"); + let resp = client + .get(&url) + .send() + .await + .expect("request through azure middleware failed"); + + let status = resp.status(); + let body = resp.bytes().await.expect("failed to read body"); + println!("status={status} bytes={}", body.len()); + assert!(status.is_success(), "unexpected status {status}: {url}"); + + let json: serde_json::Value = + serde_json::from_slice(&body).expect("real repodata was not valid json"); + assert!( + json.get("info").is_some() || json.get("packages").is_some(), + "fetched body does not look like repodata: {json}" + ); + println!("info = {}", json["info"]); +} diff --git a/crates/rattler_redaction/src/lib.rs b/crates/rattler_redaction/src/lib.rs index bef24b14fd..b9bf34691e 100644 --- a/crates/rattler_redaction/src/lib.rs +++ b/crates/rattler_redaction/src/lib.rs @@ -1,8 +1,61 @@ +use std::borrow::Cow; + use url::Url; /// A default string to use for redaction. pub const DEFAULT_REDACTION_STR: &str = "********"; +/// Query parameters whose value is the signature of a pre-signed URL, and is +/// therefore the credential itself: `sig` is Azure's SAS signature and +/// `x-amz-signature` is the S3/SigV4 equivalent. Everything else in such a URL +/// (the validity window, the permissions) is inert without them. +const SIGNATURE_PARAMS: &[&str] = &["sig", "x-amz-signature"]; + +/// Mask the signature of a pre-signed URL wherever one appears in `text`. +/// +/// Takes text rather than a [`Url`] because that is the shape the leak has: a +/// storage backend quotes the request URL inside an error message, and for a SAS +/// the credential is *in* that URL, so the message must be scrubbed before it is +/// logged or shown. A `?`/`&` and a `=` are all that is needed to find the value; +/// anything that cannot appear in a query value ends it. +pub fn redact_signatures_in_text<'a>(text: &'a str, redaction: &str) -> Cow<'a, str> { + let mut out = String::new(); + // Also the "nothing was masked" flag: a masked value always starts past 0. + let mut written = 0; + + for (separator, _) in text.char_indices().filter(|(_, c)| *c == '?' || *c == '&') { + let pair = &text[separator + 1..]; + let Some(equals) = pair.find('=') else { + continue; + }; + if !SIGNATURE_PARAMS + .iter() + .any(|param| pair[..equals].eq_ignore_ascii_case(param)) + { + continue; + } + + let value = separator + 1 + equals + 1; + let end = value + + text[value..] + .find(|c: char| c == '&' || c.is_whitespace() || "\"',)]}".contains(c)) + .unwrap_or(text.len() - value); + if end == value || value < written { + continue; + } + + out.push_str(&text[written..value]); + out.push_str(redaction); + written = end; + } + + if written == 0 { + return Cow::Borrowed(text); + } + out.push_str(&text[written..]); + Cow::Owned(out) +} + /// Anaconda channels are not always publicly available. This function checks if a URL contains a /// secret by identifying whether it contains certain patterns. If it does, the function returns a /// modified URL where any secret has been masked. @@ -28,6 +81,14 @@ pub fn redact_known_secrets_from_url(url: &Url, redaction: &str) -> Option url.set_password(Some(redaction)).ok()?; } + // A pre-signed URL carries its credential in the query, so a URL that reached + // here from an error or a log line has to lose it. + if let Some(query) = url.query() + && let Cow::Owned(masked) = redact_signatures_in_text(&format!("?{query}"), redaction) + { + url.set_query(Some(&masked[1..])); + } + let mut segments = url.path_segments()?; match (segments.next(), segments.next()) { (Some("t"), Some(_)) => { @@ -95,6 +156,52 @@ mod test { use super::*; use std::str::FromStr; + /// The signature is the credential; the rest of a SAS is inert without it and + /// is worth keeping, because the account, container and expiry are what make + /// the error message useful. + #[test] + fn test_redact_signatures_in_text() { + let message = "unexpected status code 403, url=https://acct.blob.core.windows.net/c/p?sv=2025-01-05&se=2026-08-05T00%3A00Z&sig=aBcD%2Fefg%3D, op=stat"; + assert_eq!( + redact_signatures_in_text(message, DEFAULT_REDACTION_STR), + format!( + "unexpected status code 403, url=https://acct.blob.core.windows.net/c/p?sv=2025-01-05&se=2026-08-05T00%3A00Z&sig={DEFAULT_REDACTION_STR}, op=stat" + ) + ); + + // Presigned S3, and a signature that runs to the end of the text. + assert_eq!( + redact_signatures_in_text( + "https://b.s3.amazonaws.com/k?X-Amz-Credential=AK&X-Amz-Signature=deadbeef", + "X" + ), + "https://b.s3.amazonaws.com/k?X-Amz-Credential=AK&X-Amz-Signature=X" + ); + + // Text with nothing to mask is borrowed, not rebuilt. + assert!(matches!( + redact_signatures_in_text("https://prefix.dev/conda-forge?a=b", "X"), + Cow::Borrowed(_) + )); + + // A query param that merely ends in `sig` is not the signature. + assert_eq!( + redact_signatures_in_text("https://h/p?design=keep&sig=drop", "X"), + "https://h/p?design=keep&sig=X" + ); + + // And the same through the `Url` entry point every existing caller uses. + assert_eq!( + Url::from_str("https://acct.blob.core.windows.net/c/p?sv=2025-01-05&sig=secret") + .unwrap() + .redact() + .to_string(), + format!( + "https://acct.blob.core.windows.net/c/p?sv=2025-01-05&sig={DEFAULT_REDACTION_STR}" + ) + ); + } + #[test] fn test_remove_known_secrets_from_url() { assert_eq!( diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs index fbac0cb7a9..c6a2492cb4 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/index.rs @@ -15,7 +15,7 @@ use async_fd_lock::{LockWrite, RwLockWriteGuard}; use bytes::Bytes; use fs_err::tokio as tokio_fs; use futures::{TryFutureExt, future::OptionFuture}; -use http::{HeaderMap, Method, Uri}; +use http::{HeaderMap, Method, StatusCode, Uri}; use http_cache_semantics::{AfterResponse, BeforeRequest, CachePolicy, RequestLike}; use rattler_conda_types::Channel; use rattler_networking::LazyClient; @@ -267,46 +267,55 @@ pub async fn fetch_index( return Err(create_subdir_not_found_error(channel_base_url)); } - match cache_header.policy.after_response( + let after_response = cache_header.policy.after_response( &state_request, &response, SystemTime::now(), - ) { - AfterResponse::NotModified(_policy, _) => { - // The cached file is still valid - match read_shard_index_from_reader(&mut cache_reader).await { - Ok(shard_index) => { - tracing::debug!("shard index cache was not modified"); - if let Some((reporter, index)) = download_reporter { - reporter.on_download_complete(response.url(), index); - } - // If reading the file failed for some reason we'll just - // fetch it again. - return Ok(shard_index); + ); + + // The status is consulted directly rather than left to + // `after_response` alone, which only reports `NotModified` when the + // 304 echoes back the validator it matched. Azure Blob does not: it + // answers a conditional GET with a bare 304 carrying no `etag` and no + // `last-modified`, just `x-ms-error-code: ConditionNotMet`. That + // reads as `Modified`, and a 304 then reaches `from_response`, which + // rejects it for not being a success — so every `az://` sharded + // channel failed on the *second* fetch, once there was a cache entry + // to revalidate. A 304 is only ever sent because the validator we + // ourselves sent matched, so it is trustworthy on its own here, which + // is also how the `repodata.json` path has always read it. + if response.status() == StatusCode::NOT_MODIFIED + || matches!(after_response, AfterResponse::NotModified(..)) + { + // The cached file is still valid + match read_shard_index_from_reader(&mut cache_reader).await { + Ok(shard_index) => { + tracing::debug!("shard index cache was not modified"); + if let Some((reporter, index)) = download_reporter { + reporter.on_download_complete(response.url(), index); } - Err(e) => { - tracing::warn!( - "the cached shard index has been corrupted: {e}" - ); - if let Some((reporter, index)) = download_reporter { - reporter.on_download_complete(response.url(), index); - } + return Ok(shard_index); + } + Err(e) => { + // Fall through to the unconditional fetch below. + tracing::warn!("the cached shard index has been corrupted: {e}"); + if let Some((reporter, index)) = download_reporter { + reporter.on_download_complete(response.url(), index); } } } - AfterResponse::Modified(policy, _) => { - // Close the old file so we can create a new one. - tracing::debug!("shard index cache has become stale"); - return from_response( - cache_reader.into_inner(), - &cache_path, - policy, - response, - download_reporter, - request_permit, - ) - .await; - } + } else if let AfterResponse::Modified(policy, _) = after_response { + // Close the old file so we can create a new one. + tracing::debug!("shard index cache has become stale"); + return from_response( + cache_reader.into_inner(), + &cache_path, + policy, + response, + download_reporter, + request_permit, + ) + .await; } } } diff --git a/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs b/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs index 112f530cdf..a749167005 100644 --- a/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs +++ b/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs @@ -55,6 +55,7 @@ impl<'g> SubdirBuilder<'g> { || url.scheme() == "gcs" || url.scheme() == "oci" || url.scheme() == "s3" + || url.scheme() == "az" { let source_config = self.gateway.channel_config.get(&self.channel.base_url); diff --git a/crates/rattler_upload/Cargo.toml b/crates/rattler_upload/Cargo.toml index 30199ceacf..ce7981448a 100644 --- a/crates/rattler_upload/Cargo.toml +++ b/crates/rattler_upload/Cargo.toml @@ -14,7 +14,7 @@ license.workspace = true readme.workspace = true [features] -default = ["rustls", "s3", "sigstore-sign"] +default = ["rustls", "s3", "azure", "sigstore-sign"] rustls = [ "reqwest/rustls", "rattler_networking/rustls", @@ -28,7 +28,8 @@ native-tls = [ "rattler_package_streaming/native-tls", "sigstore-sign?/native-tls", ] -s3 = ["rattler_networking/s3", "rattler_s3", "dep:opendal"] +s3 = ["rattler_networking/s3", "rattler_s3", "dep:opendal", "opendal/services-s3"] +azure = ["dep:rattler_azure", "rattler_azure/opendal", "dep:opendal", "opendal/services-azblob"] sigstore-sign = ["dep:sigstore-sign", "dep:sigstore-trust-root"] [package.metadata.docs.rs] @@ -44,13 +45,13 @@ rattler_package_streaming = { workspace = true, default-features = false } rattler_config = { workspace = true, default-features = false } rattler_solve = { workspace = true } rattler_s3 = { workspace = true, optional = true, features = ["clap"] } +rattler_azure = { workspace = true, optional = true, features = ["clap"] } miette = { workspace = true, features = ["fancy"] } clap = { workspace = true, features = ["env"] } fs-err = { workspace = true, features = ["tokio"] } futures = { workspace = true } indicatif = { workspace = true } opendal = { workspace = true, optional = true, default-features = false, features = [ - "services-s3", "executors-tokio", ] } reqwest-retry = { workspace = true } @@ -81,3 +82,6 @@ sha2 = { workspace = true } [dev-dependencies] axum = { workspace = true } +opendal = { workspace = true, default-features = false, features = [ + "services-memory", +] } diff --git a/crates/rattler_upload/src/lib.rs b/crates/rattler_upload/src/lib.rs index b7ad07b200..41850bbe75 100644 --- a/crates/rattler_upload/src/lib.rs +++ b/crates/rattler_upload/src/lib.rs @@ -81,6 +81,32 @@ pub async fn upload_from_args(args: UploadOpts) -> miette::Result<()> { ) .await } + #[cfg(feature = "azure")] + ServerType::Azure(azure_opts) => { + let channel = azure_opts.channel; + // The https host-style defaults, because `upload_from_args` reads no + // configuration file at all — it only opens the auth store — so there is + // nowhere for an `[azure-options.""]` entry to come + // from. The ceiling is https plus host-style addressing — any host + // whose first label is the account, Azure or not; an http or + // path-style endpoint (Azurite as normally run) is unreachable. + // Lifting it means giving `rattler_upload` a `--config` of its own, + // which neither `rattler upload` nor rattler-build passes today. + let endpoint = rattler_azure::AzureEndpoint::default(); + let credentials = azure_opts + .credentials + .resolve(upload::AZURE_UPLOAD_SAS_PERMISSIONS, &channel, endpoint) + .await + .into_diagnostic()?; + upload::upload_package_to_azure( + channel, + credentials, + endpoint, + &args.package_files, + azure_opts.force, + ) + .await + } ServerType::CondaForge(conda_forge_opts) => { let conda_forge_data = CondaForgeData::from(conda_forge_opts); upload::conda_forge::upload_packages_to_conda_forge( diff --git a/crates/rattler_upload/src/upload/azure.rs b/crates/rattler_upload/src/upload/azure.rs new file mode 100644 index 0000000000..d86af92f0e --- /dev/null +++ b/crates/rattler_upload/src/upload/azure.rs @@ -0,0 +1,287 @@ +use std::{ + fmt::Write as _, + path::{Path, PathBuf}, + sync::Mutex, +}; + +use futures::{StreamExt, TryStreamExt}; +use miette::IntoDiagnostic; +use opendal::{Configurator, ErrorKind}; +use rattler_azure::{AzureChannelUrl, AzureCredentials, AzureEndpoint}; + +use crate::upload::{ + object_store::{ + BlobStore, BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store, + }, + opt::ForceOverwrite, + package::ExtractedPackage, +}; + +/// SAS permissions requested when minting a user-delegation SAS for uploads. +/// Creating and writing blobs needs `c` + `w`; `r` is required on top of those +/// because the overwrite guard `stat`s each blob before writing it, and a +/// `stat` (HEAD Blob) is a read. The SAS stays container-scoped and short-lived. +pub(crate) const AZURE_UPLOAD_SAS_PERMISSIONS: &str = "rcw"; + +/// What became of one package in a concurrent upload run. +enum PackageOutcome { + Uploaded, + Failed(String), +} + +/// Uploads packages to a channel in an Azure Blob Storage container. +/// +/// The account name, endpoint, container and root prefix are all derived from the +/// channel URL together with `endpoint` (see `azblob_config`), so a path-style entry +/// is what makes an IP, single-label or emulator (Azurite) endpoint uploadable. The +/// [`AzureCredentials`] supply only the account key or SAS token. +pub async fn upload_package_to_azure( + channel: AzureChannelUrl, + credentials: AzureCredentials, + endpoint: AzureEndpoint, + package_files: &[PathBuf], + force: ForceOverwrite, +) -> miette::Result<()> { + let config = + rattler_azure::azblob_config(&credentials, &channel, endpoint).into_diagnostic()?; + + let builder = config.into_builder(); + let op = BlobStore::new(builder).into_diagnostic()?; + + // Upload multiple packages concurrently. Each package is written to its own + // key, so the individual uploads are independent. The first failure aborts + // the remaining uploads rather than letting them run to completion, so the + // outcomes are recorded as they land and summarised below — otherwise a run + // that failed halfway would report one package and stay silent about the + // rest. + let outcomes = Mutex::new(Vec::new()); + let result = futures::stream::iter(package_files.iter()) + .map(Ok) + .try_for_each_concurrent(PACKAGE_CONCURRENCY, |package_file| { + let op = op.clone(); + let channel = &channel; + let outcomes = &outcomes; + async move { + let result = upload_single_package(&op, channel, package_file, force).await; + let outcome = match &result { + Ok(()) => PackageOutcome::Uploaded, + Err(e) => PackageOutcome::Failed(e.to_string()), + }; + outcomes + .lock() + .expect("upload outcome mutex poisoned") + .push((package_file.clone(), outcome)); + result + } + }) + .await; + + let outcomes = outcomes + .into_inner() + .expect("upload outcome mutex poisoned"); + let summary = summarize(&outcomes, package_files.len()); + match result { + Ok(()) => { + tracing::info!("{summary}"); + Ok(()) + } + // Logged rather than attached to the error, so the counts are visible + // next to the failure that stopped the run without repeating it. + Err(e) => { + tracing::error!("{summary}"); + Err(e) + } + } +} + +/// Renders the per-package outcomes of a run. Packages without an outcome were +/// dropped mid-upload by the fail-fast stream or never started. +fn summarize(outcomes: &[(PathBuf, PackageOutcome)], total: usize) -> String { + let failed: Vec<_> = outcomes + .iter() + .filter_map(|(path, outcome)| match outcome { + PackageOutcome::Failed(message) => Some((path, message)), + PackageOutcome::Uploaded => None, + }) + .collect(); + let uploaded = outcomes.len() - failed.len(); + let not_attempted = total - outcomes.len(); + + let mut summary = format!( + "Azure upload summary: uploaded {uploaded} / failed {} / not attempted {not_attempted}", + failed.len() + ); + for (path, message) in failed { + let _ = write!(summary, "\n failed: {}: {message}", path.display()); + } + if not_attempted > 0 { + summary.push_str("\n not attempted: cancelled mid-upload or never started"); + } + summary +} + +/// Uploads a single package file to the Azure Blob container via the given operator. +async fn upload_single_package( + op: &BlobStore, + channel: &AzureChannelUrl, + package_file: &Path, + force: ForceOverwrite, +) -> miette::Result<()> { + let package = ExtractedPackage::from_package_file(package_file)?; + let target = BlobUploadTarget::from_package(&package)?; + + // The blob's address as the user wrote the channel, used only for + // diagnostics. The canonical spelling already carries `//`, + // so the key is appended to it; do not prepend the container again. Query and + // fragment go first, or an inline SAS would land in the middle of the path. + let blob_url = { + let mut channel_url = channel.canonical(); + channel_url.set_query(None); + channel_url.set_fragment(None); + format!("{channel_url}/{}", target.key()) + }; + + // Guard against overwriting an existing blob when `--force` was not passed. + // opendal 0.57 only honours `if_not_exists` on the single-shot Put Blob path, + // never the multi-block Put Block List path used for packages larger than a + // single block, so the writer-level `if_not_exists` silently does nothing for + // large uploads. An explicit `stat` closes that gap for a blob that already + // exists, but not for two writers racing to create one: above the chunk size + // both stat as absent and the second commit wins silently. Concurrent uploads + // of the same package are unsafe until opendal carries `if_none_match` onto + // the Put Block List path. + // + // TODO: delete this guard and its TOCTOU window once + // merges — it carries changes + // in this area. + if !force.is_enabled() { + match op.stat(target.key()).await { + Ok(_) => { + miette::bail!("Package {blob_url} already exists. Use --force to overwrite."); + } + Err(e) if e.kind() == ErrorKind::NotFound => {} + Err(e) => return Err(e).into_diagnostic(), + } + } + + stream_package_to_object_store(op, &target, package_file, &blob_url, force).await +} + +#[cfg(test)] +mod test { + use std::path::PathBuf; + + use opendal::services::Memory; + use rattler_azure::AzureChannelUrl; + + use super::{BlobStore, PackageOutcome, summarize, upload_single_package}; + use crate::upload::opt::ForceOverwrite; + use crate::upload::package::ExtractedPackage; + use crate::upload::test_utils::test_package_path; + + fn memory_operator() -> BlobStore { + BlobStore::new(Memory::default()).unwrap() + } + + fn test_channel() -> AzureChannelUrl { + AzureChannelUrl::parse("az://account.blob.core.windows.net/container/prefix").unwrap() + } + + fn package_key() -> String { + let path = test_package_path(); + let package = ExtractedPackage::from_package_file(&path).unwrap(); + format!( + "{}/{}", + package.subdir().unwrap(), + package.filename().unwrap() + ) + } + + /// without `--force`, uploading over an existing blob must error rather + /// than silently overwrite it. The memory backend honours `if_not_exists` on + /// every path and the fixture is a single block, so this covers the small-blob + /// path only — for the multi-block behaviour see + /// `rattler_index/tests/azure_azurite.rs::azurite_if_not_exists_is_dropped_on_the_multi_block_path`. + #[tokio::test] + async fn test_existing_blob_without_force_errors() { + let op = memory_operator(); + let channel = test_channel(); + let package = test_package_path(); + + // Seed the target blob so the next upload finds it already present. + upload_single_package(&op, &channel, &package, ForceOverwrite(true)) + .await + .expect("initial force upload should succeed"); + + let err = upload_single_package(&op, &channel, &package, ForceOverwrite(false)) + .await + .expect_err("upload over an existing blob without --force must fail"); + assert!( + err.to_string().contains("already exists"), + "unexpected error: {err}" + ); + } + + /// A non-forced upload into an empty container succeeds. + #[tokio::test] + async fn test_upload_into_empty_container_succeeds() { + let op = memory_operator(); + upload_single_package( + &op, + &test_channel(), + &test_package_path(), + ForceOverwrite(false), + ) + .await + .expect("upload into an empty container should succeed"); + + let meta = op.stat(&package_key()).await.unwrap(); + let expected_size = std::fs::metadata(test_package_path()).unwrap().len(); + assert_eq!(meta.content_length(), expected_size); + } + + /// The upload carries the same download metadata as its S3 twin. Only + /// the content disposition can be asserted here — the memory backend ignores + /// user metadata, as azblob does above a single block. + #[tokio::test] + async fn test_upload_sets_content_disposition() { + let op = memory_operator(); + upload_single_package( + &op, + &test_channel(), + &test_package_path(), + ForceOverwrite(false), + ) + .await + .expect("upload should succeed"); + + let path = test_package_path(); + let package = ExtractedPackage::from_package_file(&path).unwrap(); + let meta = op.stat(&package_key()).await.unwrap(); + assert_eq!( + meta.content_disposition(), + Some(format!("attachment; filename={}", package.filename().unwrap()).as_str()) + ); + } + + /// A run that stops early must still say what landed, what failed and + /// how many uploads never got a verdict. + #[test] + fn test_summary_counts_and_names_outcomes() { + let outcomes = vec![ + (PathBuf::from("a.conda"), PackageOutcome::Uploaded), + ( + PathBuf::from("b.conda"), + PackageOutcome::Failed("Package b already exists".to_string()), + ), + ]; + + let summary = summarize(&outcomes, 4); + assert!( + summary.contains("uploaded 1 / failed 1 / not attempted 2"), + "unexpected summary: {summary}" + ); + assert!(summary.contains("failed: b.conda: Package b already exists")); + assert!(summary.contains("not attempted: cancelled mid-upload or never started")); + } +} diff --git a/crates/rattler_upload/src/upload/mod.rs b/crates/rattler_upload/src/upload/mod.rs index e3de54bea4..f584bca360 100644 --- a/crates/rattler_upload/src/upload/mod.rs +++ b/crates/rattler_upload/src/upload/mod.rs @@ -28,6 +28,12 @@ use crate::upload::package::{ExtractedPackage, sha256_sum}; pub(crate) mod test_utils; mod anaconda; +#[cfg(feature = "azure")] +mod azure; +#[cfg(feature = "azure")] +pub(crate) use azure::AZURE_UPLOAD_SAS_PERMISSIONS; +#[cfg(feature = "azure")] +pub use azure::upload_package_to_azure; #[cfg(feature = "sigstore-sign")] pub mod attestation; mod cloudsmith; @@ -44,6 +50,338 @@ pub use anaconda::AnacondaError; pub use cloudsmith::CloudsmithError; pub use prefix::{PrefixUploadError, upload_package_to_prefix}; +/// The streaming upload shared by the object-store backends (S3 and Azure Blob), +/// which both drive an opendal writer. +#[cfg(any(feature = "s3", feature = "azure"))] +pub(crate) mod object_store { + use std::{collections::HashMap, path::Path}; + + use miette::IntoDiagnostic; + use opendal::{ErrorKind, Operator, options::WriteOptions}; + use rattler_digest::{HashingReader, Md5, Md5Hash, Sha256, Sha256Hash}; + use tokio::io::{AsyncReadExt, AsyncSeekExt}; + use tokio_util::bytes::BytesMut; + + use crate::upload::{opt::ForceOverwrite, package::ExtractedPackage}; + + /// An object store whose errors cannot carry a credential. + /// + /// opendal stamps the request URL into the context of every HTTP error it + /// builds, and prints that context from both `Display` and `Debug`. For Azure + /// the SAS *is* in the URL, so any opendal error that escapes unmasked is a + /// leaked credential — into a log, a `miette` report, or a CI transcript. + /// + /// The inner [`Operator`] is private and the only error type out is + /// [`BlobStoreError`], which is built by masking. Reaching for an opendal + /// method this does not have means adding it here, where leaving the masking + /// out is a visible omission rather than a silent leak. + #[derive(Clone)] + pub(crate) struct BlobStore(Operator); + + /// An opendal error with any pre-signed signature masked out of its text. + /// + /// Carries the [`ErrorKind`] separately because callers branch on it — a + /// `NotFound` from the overwrite guard, a `ConditionNotMatch` from a write that + /// lost a race — and must not have to read the message to do so. + #[derive(Debug, thiserror::Error)] + #[error("{message}")] + pub(crate) struct BlobStoreError { + kind: ErrorKind, + message: String, + } + + impl BlobStoreError { + fn new(err: opendal::Error) -> Self { + Self { + kind: err.kind(), + // `Debug` rather than `Display`: it is the spelling that keeps the + // source chain and the operation, and it is also the one the leak + // was found in. + message: rattler_redaction::redact_signatures_in_text( + &format!("{err:?}"), + rattler_redaction::DEFAULT_REDACTION_STR, + ) + .into_owned(), + } + } + + pub(crate) fn kind(&self) -> ErrorKind { + self.kind + } + } + + impl BlobStore { + pub(crate) fn new(builder: impl opendal::Builder) -> Result { + Ok(Self( + Operator::new(builder) + .map_err(BlobStoreError::new)? + .finish(), + )) + } + + /// Metadata for one blob, used by the callers' overwrite guards. + pub(crate) async fn stat(&self, path: &str) -> Result { + self.0.stat(path).await.map_err(BlobStoreError::new) + } + + async fn writer( + &self, + path: &str, + options: WriteOptions, + ) -> Result { + self.0 + .writer_options(path, options) + .await + .map(BlobWriter) + .map_err(BlobStoreError::new) + } + } + + /// A writer that masks its errors, for the same reason [`BlobStore`] does: a + /// failed block upload reports the URL it was sent to. + struct BlobWriter(opendal::Writer); + + impl BlobWriter { + async fn write(&mut self, chunk: tokio_util::bytes::Bytes) -> Result<(), BlobStoreError> { + self.0.write(chunk).await.map_err(BlobStoreError::new) + } + + async fn close(&mut self) -> Result<(), BlobStoreError> { + self.0 + .close() + .await + .map(|_| ()) + .map_err(BlobStoreError::new) + } + + async fn abort(&mut self) -> Result<(), BlobStoreError> { + self.0.abort().await.map_err(BlobStoreError::new) + } + } + + /// Size of a single chunk handed to the writer. S3 rejects every multipart + /// part but the last below 5 MiB, and Azure Blob bills per block, so both + /// backends prefer few large chunks. + /// + /// Peak buffered bytes across a run are `PACKAGE_CONCURRENCY * + /// PART_CONCURRENCY * DESIRED_CHUNK_SIZE` = 160 MiB. + const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; + + /// Number of chunks of a single package that are uploaded concurrently. + const PART_CONCURRENCY: usize = 4; + + /// Number of packages that are uploaded concurrently. + pub(crate) const PACKAGE_CONCURRENCY: usize = 4; + + /// A package resolved to the channel-relative key it is stored under. Holding + /// the key and the filename together keeps the two from disagreeing. + pub(crate) struct BlobUploadTarget { + key: String, + filename: String, + } + + impl BlobUploadTarget { + /// Resolves `/` from the package's own `index.json`. + pub(crate) fn from_package(package: &ExtractedPackage<'_>) -> miette::Result { + let subdir = package + .subdir() + .ok_or_else(|| miette::miette!("Failed to get subdir"))?; + let filename = package + .filename() + .ok_or_else(|| miette::miette!("Failed to get filename"))?; + Ok(Self { + key: format!("{subdir}/{filename}"), + filename: filename.to_string(), + }) + } + + /// The channel-relative key the package is written to. + pub(crate) fn key(&self) -> &str { + &self.key + } + } + + /// A file measured and hashed by one pass over a single handle, rewound and + /// ready to be read again. Size and hashes describe the same bytes, so the + /// upload cannot publish a length a concurrent writer changed after a `stat`. + struct HashedFile { + reader: R, + size: u64, + sha256: Sha256Hash, + md5: Md5Hash, + } + + async fn hash_file( + path: &Path, + ) -> miette::Result> { + let file = + tokio::io::BufReader::new(fs_err::tokio::File::open(path).await.into_diagnostic()?); + let sha256_reader = HashingReader::<_, Sha256>::new(file); + let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); + let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) + .await + .into_diagnostic()?; + let (sha256_reader, md5) = md5_reader.finalize(); + let (mut reader, sha256) = sha256_reader.finalize(); + reader.rewind().await.into_diagnostic()?; + Ok(HashedFile { + reader, + size, + sha256, + md5, + }) + } + + /// Streams `package_file` to `target`'s key through `op`. + /// + /// `destination` is the blob as the user addressed it and appears in the + /// success log and in the "already exists" error. `if_not_exists` is asked of + /// the backend, which is free to drop it — the caller is responsible for any + /// guard it needs on top (see `azure::upload_single_package`). + pub(crate) async fn stream_package_to_object_store( + store: &BlobStore, + target: &BlobUploadTarget, + package_file: &Path, + destination: &str, + force: ForceOverwrite, + ) -> miette::Result<()> { + let HashedFile { + mut reader, + size, + sha256, + md5, + } = hash_file(package_file).await?; + + // S3 honours both. azblob never sends content-disposition, and drops user + // metadata on its Put Block List commit, so a package above + // `DESIRED_CHUNK_SIZE` lands there with neither. + let options = WriteOptions { + chunk: Some(DESIRED_CHUNK_SIZE), + concurrent: PART_CONCURRENCY, + content_disposition: Some(format!("attachment; filename={}", target.filename)), + user_metadata: Some(HashMap::from([ + (String::from("package-sha256"), hex::encode(sha256)), + (String::from("package-md5"), hex::encode(md5)), + ])), + if_not_exists: !force.is_enabled(), + ..WriteOptions::default() + }; + + let already_exists = + || miette::miette!("Package {destination} already exists. Use --force to overwrite."); + + // `if_not_exists` is not evaluated here on either backend: both build their + // writer with a pure constructor and issue nothing until the first chunk, so + // a lost race always surfaces at `close()` below. This arm covers what + // `writer()` itself can reject — a capability opendal refuses up front — and + // exists so that the answer cannot depend on which backend is in play. + let mut writer = match store.writer(target.key(), options).await { + Ok(writer) => writer, + Err(e) if e.kind() == ErrorKind::ConditionNotMatch => return Err(already_exists()), + Err(e) => return Err(e).into_diagnostic(), + }; + + if let Err(e) = stream_chunks(&mut writer, &mut reader, size).await { + discard_partial_upload(&mut writer, destination).await; + return Err(e); + } + + match writer.close().await { + Ok(_) => { + tracing::info!("Uploaded package to {destination}"); + Ok(()) + } + Err(e) => { + discard_partial_upload(&mut writer, destination).await; + if e.kind() == ErrorKind::ConditionNotMatch { + return Err(already_exists()); + } + Err(e).into_diagnostic() + } + } + } + + /// Feeds exactly `size` bytes of `reader` to `writer`. opendal buffers them + /// into correctly sized parts/blocks and uploads `PART_CONCURRENCY` at a time. + async fn stream_chunks( + writer: &mut BlobWriter, + reader: &mut (impl AsyncReadExt + Unpin), + size: u64, + ) -> miette::Result<()> { + let mut remaining_size = size as usize; + while remaining_size > 0 { + let chunk_size = remaining_size.min(DESIRED_CHUNK_SIZE); + let mut chunk = BytesMut::zeroed(chunk_size); + + let bytes_read = reader.read_exact(&mut chunk[..]).await.into_diagnostic()?; + debug_assert_eq!(bytes_read, chunk.len()); + + writer.write(chunk.freeze()).await.into_diagnostic()?; + + remaining_size = remaining_size.saturating_sub(bytes_read); + } + Ok(()) + } + + /// Uncommitted parts are billed until they are discarded. S3 discards them + /// here; azblob's abort is a no-op, so Azure only collects its uncommitted + /// blocks after a week without further writes to the blob. + async fn discard_partial_upload(writer: &mut BlobWriter, destination: &str) { + if let Err(e) = writer.abort().await { + tracing::warn!("Failed to discard the partial upload of {destination}: {e}"); + } + } + + #[cfg(test)] + mod test { + use super::{BlobStoreError, hash_file}; + use crate::upload::test_utils::test_package_path; + use opendal::ErrorKind; + use rattler_digest::{Md5, Sha256, compute_file_digest}; + + /// The size the upload streams and the hashes it records must come + /// from the same pass, so they always describe the same bytes. + #[tokio::test] + async fn test_hash_file_size_and_hashes_agree_with_the_file() { + let path = test_package_path(); + let hashed = hash_file(&path).await.expect("hashing the package failed"); + + assert_eq!(hashed.size, std::fs::metadata(&path).unwrap().len()); + assert_eq!( + hashed.sha256, + compute_file_digest::(&path).unwrap(), + "recorded sha256 must match the file's" + ); + assert_eq!( + hashed.md5, + compute_file_digest::(&path).unwrap(), + "recorded md5 must match the file's" + ); + } + + /// The whole reason `BlobStore` hides its `Operator`: opendal puts the + /// request URL in the error context, and for Azure the SAS is in that URL. + #[test] + fn blob_store_errors_do_not_carry_a_signature() { + let err = BlobStoreError::new( + opendal::Error::new(ErrorKind::NotFound, "blob not found").with_context( + "url", + "https://acct.blob.core.windows.net/c/p?sv=2025-01-05&sig=s3cr3t", + ), + ); + + let message = err.to_string(); + assert!(!message.contains("s3cr3t"), "{message}"); + // Everything that makes the error useful survives. + assert_eq!(err.kind(), ErrorKind::NotFound); + assert!( + message.contains("acct.blob.core.windows.net/c/p"), + "{message}" + ); + } + } +} + /// Returns the style to use for a progress bar that is currently in progress. fn default_bytes_style() -> Result { Ok(indicatif::ProgressStyle::default_bar() diff --git a/crates/rattler_upload/src/upload/opt.rs b/crates/rattler_upload/src/upload/opt.rs index 1cc5dae935..e962e8051f 100644 --- a/crates/rattler_upload/src/upload/opt.rs +++ b/crates/rattler_upload/src/upload/opt.rs @@ -1,7 +1,7 @@ //! Command-line options. use std::path::PathBuf; -use clap::Parser; +use clap::{Parser, builder::TypedValueParser as _}; use rattler_conda_types::utils::url_with_trailing_slash::UrlWithTrailingSlash; use rattler_networking::AuthenticationStorage; use url::Url; @@ -123,6 +123,9 @@ pub enum ServerType { Cloudsmith(CloudsmithOpts), #[cfg(feature = "s3")] S3(S3Opts), + #[cfg(feature = "azure")] + #[command(name = "az")] + Azure(AzureOpts), #[clap(hide = true)] CondaForge(CondaForgeOpts), } @@ -416,6 +419,34 @@ pub struct S3Opts { pub force: bool, } +/// Options for uploading to Azure Blob Storage. +/// +/// Authentication is supplied with either an account key or a shared access +/// signature (SAS) token; the two are mutually exclusive. +#[cfg(feature = "azure")] +#[derive(Clone, Debug, PartialEq, Parser)] +pub struct AzureOpts { + /// The channel URL in the Azure Blob container to upload the package to, + /// e.g., `az://myaccount.blob.core.windows.net/my-container/my-channel` + /// + /// Kept as an [`AzureChannelUrl`](rattler_azure::AzureChannelUrl) rather than + /// a wire `Url`, so the scheme the request goes out over is chosen where the + /// endpoint options are known instead of being fixed at parse time. + #[arg(short, long, env = "AZURE_CHANNEL")] + pub channel: rattler_azure::AzureChannelUrl, + + #[clap(flatten)] + pub credentials: rattler_azure::clap::AzureCredentialsOpts, + + /// Replace files if it already exists. + #[arg( + long, + action = clap::ArgAction::SetTrue, + value_parser = clap::builder::BoolishValueParser::new().map(ForceOverwrite) + )] + pub force: ForceOverwrite, +} + #[derive(Debug)] #[allow(missing_docs)] pub struct AnacondaData { @@ -603,3 +634,20 @@ impl CondaForgeData { } } } + +#[cfg(all(test, feature = "azure"))] +mod test { + use super::{AzureOpts, ForceOverwrite}; + use clap::Parser; + + /// `--force` stays a flag even though it parses into a newtype. + #[test] + fn test_azure_force_parses_as_a_flag() { + let args = ["az", "--channel", "az://account.blob.core.windows.net/c"]; + let opts = AzureOpts::try_parse_from(args).unwrap(); + assert_eq!(opts.force, ForceOverwrite(false)); + + let opts = AzureOpts::try_parse_from(args.iter().chain(["--force"].iter())).unwrap(); + assert_eq!(opts.force, ForceOverwrite(true)); + } +} diff --git a/crates/rattler_upload/src/upload/s3.rs b/crates/rattler_upload/src/upload/s3.rs index 98202f8bcd..83933647de 100644 --- a/crates/rattler_upload/src/upload/s3.rs +++ b/crates/rattler_upload/src/upload/s3.rs @@ -2,24 +2,17 @@ use std::path::{Path, PathBuf}; use futures::StreamExt; use miette::IntoDiagnostic; -use opendal::{Configurator, ErrorKind, Operator, services::S3Config}; -use rattler_digest::{HashingReader, Md5, Sha256}; +use opendal::{Configurator, services::S3Config}; use rattler_s3::ResolvedS3Credentials; -use tokio::io::{AsyncReadExt, AsyncSeekExt}; -use tokio_util::bytes::BytesMut; use url::Url; -use crate::upload::package::ExtractedPackage; - -/// Size of a single multipart chunk. S3 requires every part except the last to -/// be at least 5 MiB; we use a larger value to keep the number of parts low. -const DESIRED_CHUNK_SIZE: usize = 1024 * 1024 * 10; - -/// Number of chunks of a single package that are uploaded concurrently. -const PART_CONCURRENCY: usize = 4; - -/// Number of packages that are uploaded concurrently. -const PACKAGE_CONCURRENCY: usize = 4; +use crate::upload::{ + object_store::{ + BlobStore, BlobUploadTarget, PACKAGE_CONCURRENCY, stream_package_to_object_store, + }, + opt::ForceOverwrite, + package::ExtractedPackage, +}; /// Uploads a package to a channel in an S3 bucket. /// @@ -50,7 +43,7 @@ pub async fn upload_package_to_s3( credentials.addressing_style == rattler_s3::S3AddressingStyle::VirtualHost; let builder = s3_config.into_builder(); - let op = Operator::new(builder).into_diagnostic()?.finish(); + let op = BlobStore::new(builder).into_diagnostic()?; // Upload multiple packages concurrently. Each individual package upload also // streams its chunks concurrently (see `upload_single_package`). @@ -58,7 +51,9 @@ pub async fn upload_package_to_s3( .map(|package_file| { let op = op.clone(); let channel = &channel; - async move { upload_single_package(&op, channel, bucket, package_file, force).await } + async move { + upload_single_package(&op, channel, bucket, package_file, force.into()).await + } }) .buffer_unordered(PACKAGE_CONCURRENCY) .collect::>() @@ -71,107 +66,15 @@ pub async fn upload_package_to_s3( /// Uploads a single package file to the S3 bucket via the given operator. async fn upload_single_package( - op: &Operator, + op: &BlobStore, channel: &Url, bucket: &str, package_file: &Path, - force: bool, + force: ForceOverwrite, ) -> miette::Result<()> { let package = ExtractedPackage::from_package_file(package_file)?; - let subdir = package - .subdir() - .ok_or_else(|| miette::miette!("Failed to get subdir"))?; - let filename = package - .filename() - .ok_or_else(|| miette::miette!("Failed to get filename"))?; - let key = format!("{subdir}/{filename}"); + let target = BlobUploadTarget::from_package(&package)?; + let destination = format!("s3://{bucket}{}/{}", channel.path(), target.key()); - // Compute the hash of the package by streaming its content. - let file = tokio::io::BufReader::new( - fs_err::tokio::File::open(package_file) - .await - .into_diagnostic()?, - ); - let sha256_reader = HashingReader::<_, Sha256>::new(file); - let mut md5_reader = HashingReader::<_, Md5>::new(sha256_reader); - let size = tokio::io::copy(&mut md5_reader, &mut tokio::io::sink()) - .await - .into_diagnostic()?; - let (sha256_reader, md5hash) = md5_reader.finalize(); - let (mut file, sha256hash) = sha256_reader.finalize(); - - // Rewind the file to the beginning. - file.rewind().await.into_diagnostic()?; - - // Construct a writer for the package. Setting `chunk` and `concurrent` - // enables opendal's concurrent multipart upload: data is buffered into - // `DESIRED_CHUNK_SIZE` parts and up to `PART_CONCURRENCY` parts are uploaded - // in parallel. - let mut writer = match op - .writer_with(&key) - .chunk(DESIRED_CHUNK_SIZE) - .concurrent(PART_CONCURRENCY) - .content_disposition(&format!("attachment; filename={filename}")) - .if_not_exists(!force) - .user_metadata([ - (String::from("package-sha256"), hex::encode(sha256hash)), - (String::from("package-md5"), hex::encode(md5hash)), - ]) - .await - { - Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { - miette::bail!( - "Package s3://{bucket}{}/{key} already exists. Use --force to overwrite.", - channel.path().to_string() - ); - } - Ok(writer) => writer, - Err(e) => { - return Err(e).into_diagnostic(); - } - }; - - // Stream the file to the writer in `DESIRED_CHUNK_SIZE` chunks. opendal takes - // care of buffering these into correctly sized parts and uploading them - // concurrently. - let mut remaining_size = size as usize; - while remaining_size > 0 { - // Allocate memory for this chunk. - let chunk_size = remaining_size.min(DESIRED_CHUNK_SIZE); - let mut chunk = BytesMut::with_capacity(chunk_size); - // SAFE: because we do not care about the bytes that are currently in the buffer - unsafe { chunk.set_len(chunk_size) }; - - // Fill the chunk with data. This reads exactly the number of bytes we want. No - // more, no less. - let bytes_read = file.read_exact(&mut chunk[..]).await.into_diagnostic()?; - debug_assert_eq!(bytes_read, chunk.len()); - - // Hand the chunk to the writer. With concurrent writes enabled this returns - // as soon as the chunk is queued rather than fully uploaded. - writer.write(chunk.freeze()).await.into_diagnostic()?; - - // Update the number of remaining bytes. - remaining_size = remaining_size.saturating_sub(bytes_read); - } - - match writer.close().await { - Err(e) if e.kind() == ErrorKind::ConditionNotMatch => { - miette::bail!( - "Package s3://{bucket}{}/{key} already exists. Use --force to overwrite.", - channel.path().to_string() - ); - } - Ok(_) => { - tracing::info!( - "Uploaded package to s3://{bucket}{}/{key}", - channel.path().to_string() - ); - } - Err(e) => { - return Err(e).into_diagnostic(); - } - } - - Ok(()) + stream_package_to_object_store(op, &target, package_file, &destination, force).await } diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index 2522d56abd..794600e9eb 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -1281,7 +1281,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" dependencies = [ "digest 0.10.7", - "spin", + "spin 0.10.1", ] [[package]] @@ -2873,6 +2873,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin 0.9.9", +] [[package]] name = "libbz2-rs-sys" @@ -3116,6 +3119,22 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.7", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -3168,6 +3187,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3200,6 +3220,7 @@ checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" dependencies = [ "opendal-core", "opendal-layer-retry", + "opendal-service-azblob", "opendal-service-fs", "opendal-service-s3", ] @@ -3243,6 +3264,37 @@ dependencies = [ "opendal-core", ] +[[package]] +name = "opendal-service-azblob" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" +dependencies = [ + "base64", + "bytes", + "http 1.4.2", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.39.4", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "sha2 0.11.0", + "uuid", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" +dependencies = [ + "http 1.4.2", + "opendal-core", +] + [[package]] name = "opendal-service-fs" version = "0.57.0" @@ -3416,6 +3468,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3510,6 +3582,32 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -3517,6 +3615,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ "der", + "pkcs5", + "rand_core 0.6.4", "spki", ] @@ -4012,6 +4112,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "rattler_azure" +version = "0.1.0" +dependencies = [ + "clap", + "jiff", + "opendal", + "percent-encoding", + "secrecy", + "serde", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "which", +] + [[package]] name = "rattler_cache" version = "0.10.4" @@ -4092,6 +4209,7 @@ dependencies = [ "dirs", "fs-err", "indexmap 2.14.0", + "rattler_azure", "rattler_conda_types", "serde", "serde_ignored", @@ -4137,11 +4255,13 @@ dependencies = [ "indicatif", "jiff", "opendal", + "rattler_azure", "rattler_conda_types", "rattler_config", "rattler_digest", "rattler_networking", "rattler_package_streaming", + "rattler_redaction", "rattler_s3", "reqwest", "retry-policies", @@ -4247,7 +4367,13 @@ dependencies = [ "itertools 0.15.0", "keyring-core", "netrc-rs", + "rattler_azure", "regex", + "reqsign-azure-storage", + "reqsign-command-execute-tokio", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-http-send-reqwest", "reqwest", "retry-policies", "serde", @@ -4579,6 +4705,37 @@ dependencies = [ "sha1 0.11.0", ] +[[package]] +name = "reqsign-azure-storage" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee8b9e5d0fc927551a6ac25ba5dc6518860c54ddb592fecf685523e528e77bd" +dependencies = [ + "anyhow", + "base64", + "bytes", + "form_urlencoded", + "http 1.4.2", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-command-execute-tokio" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecf28fcf0aef694448cb235d393fbec7b5543d25050eacc0ccc2118596618c52" +dependencies = [ + "reqsign-core", + "tokio", +] + [[package]] name = "reqsign-core" version = "3.1.0" @@ -4596,6 +4753,9 @@ dependencies = [ "jiff", "log", "percent-encoding", + "rsa", + "serde", + "serde_json", "sha1 0.11.0", "sha2 0.11.0", "windows-sys 0.61.2", @@ -4612,6 +4772,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "reqsign-http-send-reqwest" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b4509b12b73289cdeed349ba494553404e33782abc09bc0ae4e09f741807cb" +dependencies = [ + "anyhow", + "bytes", + "futures-channel", + "http 1.4.2", + "http-body-util", + "reqsign-core", + "reqwest", + "wasm-bindgen-futures", +] + [[package]] name = "reqwest" version = "0.13.4" @@ -4728,6 +4904,27 @@ dependencies = [ "serde", ] +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -4879,6 +5076,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -4927,6 +5133,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + [[package]] name = "sec1" version = "0.7.3" @@ -5305,6 +5522,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "spin" version = "0.10.1" diff --git a/py-rattler/Cargo.toml b/py-rattler/Cargo.toml index 16f1870c22..00d3067452 100644 --- a/py-rattler/Cargo.toml +++ b/py-rattler/Cargo.toml @@ -44,6 +44,7 @@ rattler_digest = { path = "../crates/rattler_digest" } rattler_networking = { path = "../crates/rattler_networking", default-features = false, features = [ "gcs", "s3", + "azure", "system-integration", ] } rattler_shell = { path = "../crates/rattler_shell", default-features = false } diff --git a/py-rattler/rattler/networking/__init__.py b/py-rattler/rattler/networking/__init__.py index 580072acda..81897aabf2 100644 --- a/py-rattler/rattler/networking/__init__.py +++ b/py-rattler/rattler/networking/__init__.py @@ -3,6 +3,7 @@ from rattler.networking.middleware import ( AddHeadersMiddleware, AuthenticationMiddleware, + AzureMiddleware, GCSMiddleware, MirrorMiddleware, OciMiddleware, @@ -22,4 +23,5 @@ "OciMiddleware", "S3Middleware", "GCSMiddleware", + "AzureMiddleware", ] diff --git a/py-rattler/rattler/networking/client.py b/py-rattler/rattler/networking/client.py index e18d08f982..d6e98b0e6e 100644 --- a/py-rattler/rattler/networking/client.py +++ b/py-rattler/rattler/networking/client.py @@ -3,6 +3,7 @@ from rattler.networking.middleware import ( AddHeadersMiddleware, AuthenticationMiddleware, + AzureMiddleware, GCSMiddleware, MirrorMiddleware, OciMiddleware, @@ -27,6 +28,7 @@ def __init__( | MirrorMiddleware | OciMiddleware | GCSMiddleware + | AzureMiddleware | S3Middleware ] | None @@ -74,7 +76,7 @@ def default_client( ) -> Client: """ Returns a client with the standard middleware stack: retry, - authentication, OCI, GCS and S3. + authentication, OCI, GCS, Azure and S3. Args: max_retries: Maximum retry attempts for transient errors (default 3). @@ -96,6 +98,7 @@ def default_client( AuthenticationMiddleware(), OciMiddleware(), GCSMiddleware(), + AzureMiddleware(), S3Middleware(), ], headers=headers, diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index 6a791de516..821cbc90e1 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -5,6 +5,7 @@ from rattler.rattler import ( PyAddHeadersMiddleware, PyAuthenticationMiddleware, + PyAzureMiddleware, PyGCSMiddleware, PyMirrorMiddleware, PyOciMiddleware, @@ -156,6 +157,37 @@ def __repr__(self) -> str: return f"{type(self).__name__}()" +class AzureMiddleware: + """ + Middleware to work with az:// URLs. + + Fetches are anonymous. The per-host `azure-options` table (credential grants, + `scheme`, `path-style`) is not exposed to Python yet — unlike `S3Config`, which + does expose per-bucket configuration — so a private container cannot be read + from Python, and an emulator or custom endpoint cannot be reached at all. Azure + answers an unauthorized read of a private container with a 404, so that is what + surfaces. Follow-up work. + + Examples + -------- + ```python + >>> from rattler.networking import Client + >>> middleware = AzureMiddleware() + >>> middleware + AzureMiddleware() + >>> Client([middleware]) + Client() + >>> + ``` + """ + + def __init__(self) -> None: + self._middleware = PyAzureMiddleware() + + def __repr__(self) -> str: + return f"{type(self).__name__}()" + + class S3Config: """ Middleware to work with s3:// URLs diff --git a/py-rattler/src/lib.rs b/py-rattler/src/lib.rs index a8c386aa90..df26e81a44 100644 --- a/py-rattler/src/lib.rs +++ b/py-rattler/src/lib.rs @@ -59,8 +59,8 @@ use match_spec::PyMatchSpec; use meta::get_rattler_version; use nameless_match_spec::PyNamelessMatchSpec; use networking::middleware::{ - PyAddHeadersMiddleware, PyAuthenticationMiddleware, PyGCSMiddleware, PyMirrorMiddleware, - PyOciMiddleware, PyRetryMiddleware, PyS3Config, PyS3Middleware, + PyAddHeadersMiddleware, PyAuthenticationMiddleware, PyAzureMiddleware, PyGCSMiddleware, + PyMirrorMiddleware, PyOciMiddleware, PyRetryMiddleware, PyS3Config, PyS3Middleware, }; use networking::{client::PyClientWithMiddleware, py_fetch_repo_data}; use no_arch_type::PyNoArchType; @@ -119,6 +119,7 @@ fn rattler<'py>(py: Python<'py>, m: Bound<'py, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index 1c6580cb4e..9bc5b94a84 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -4,8 +4,8 @@ use crate::{ }; use pyo3::{PyResult, pyclass, pymethods}; use rattler_networking::{ - AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, LazyClient, MirrorMiddleware, - OciMiddleware, S3Middleware, + AuthenticationMiddleware, AuthenticationStorage, AzureMiddleware, GCSMiddleware, LazyClient, + MirrorMiddleware, OciMiddleware, S3Middleware, }; use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use reqwest_middleware::ClientWithMiddleware; @@ -84,6 +84,11 @@ impl PyClientWithMiddleware { PyMiddleware::Gcs(middleware) => { client = client.with(GCSMiddleware::from(middleware)); } + PyMiddleware::Azure(_middleware) => { + // Anonymous only: the options table is not exposed to Python + // yet. See `PyAzureMiddleware`. + client = client.with(AzureMiddleware::new(reqwest_client.clone(), [])); + } PyMiddleware::S3(middleware) => { client = client.with(S3Middleware::new( middleware diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index 92b12d14ac..cbc4ac12b5 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -20,6 +20,7 @@ pub enum PyMiddleware { Retry(PyRetryMiddleware), Oci(PyOciMiddleware), Gcs(PyGCSMiddleware), + Azure(PyAzureMiddleware), S3(PyS3Middleware), AddHeaders(PyAddHeadersMiddleware), } @@ -124,6 +125,24 @@ impl From for GCSMiddleware { } } +/// Azure middleware with no `azure-options` entries, i.e. every `az://` fetch is +/// anonymous over https. +/// +/// Exposing the per-host options table to Python (grants, `scheme`, `path-style`) +/// mirrors what `PyS3Config` does for buckets, and is follow-up work. +#[pyclass(from_py_object)] +#[repr(transparent)] +#[derive(Clone)] +pub struct PyAzureMiddleware {} + +#[pymethods] +impl PyAzureMiddleware { + #[new] + pub fn __init__() -> Self { + Self {} + } +} + #[derive(Clone)] #[pyclass(from_py_object)] pub struct PyS3Config { diff --git a/py-rattler/tests/unit/test_client.py b/py-rattler/tests/unit/test_client.py new file mode 100644 index 0000000000..13aa19e888 --- /dev/null +++ b/py-rattler/tests/unit/test_client.py @@ -0,0 +1,47 @@ +# type: ignore +from __future__ import annotations + +import rattler.networking.client as client_module +from rattler.networking import Client +from rattler.networking.middleware import ( + AuthenticationMiddleware, + AzureMiddleware, + GCSMiddleware, + OciMiddleware, + RetryMiddleware, + S3Middleware, +) + + +def test_default_client_stack_includes_azure(monkeypatch) -> None: + """The default client's middleware stack must include every cloud backend.""" + constructed: list[type] = [] + + for name in ( + "RetryMiddleware", + "AuthenticationMiddleware", + "OciMiddleware", + "GCSMiddleware", + "AzureMiddleware", + "S3Middleware", + ): + original = getattr(client_module, name) + + def record(*args, _original=original, **kwargs): + constructed.append(_original) + return _original(*args, **kwargs) + + monkeypatch.setattr(client_module, name, record) + + client = Client.default_client() + + assert isinstance(client, Client) + for middleware in ( + RetryMiddleware, + AuthenticationMiddleware, + OciMiddleware, + GCSMiddleware, + AzureMiddleware, + S3Middleware, + ): + assert middleware in constructed diff --git a/py-rattler/tests/unit/test_package_streaming.py b/py-rattler/tests/unit/test_package_streaming.py index a9797637c7..12ef2a391d 100644 --- a/py-rattler/tests/unit/test_package_streaming.py +++ b/py-rattler/tests/unit/test_package_streaming.py @@ -2,7 +2,7 @@ import pytest from pathlib import Path -from rattler.networking.middleware import MirrorMiddleware, OciMiddleware, GCSMiddleware +from rattler.networking.middleware import AzureMiddleware, MirrorMiddleware, OciMiddleware, GCSMiddleware from rattler.package_streaming import ( download_and_extract, download_bytes, @@ -139,3 +139,7 @@ async def test_download_from_oci(tmpdir: Path) -> None: def test_instantiate_gcs_middleware() -> None: _client = Client([GCSMiddleware()]) + + +def test_instantiate_azure_middleware() -> None: + _client = Client([AzureMiddleware()])