From 649faacd08eaf8d8b78b2256cdb2b5e0fb73f253 Mon Sep 17 00:00:00 2001 From: Tobias Hunger Date: Thu, 30 Jul 2026 13:07:45 +0200 Subject: [PATCH] feat(repodata_gateway): read virtual package plugin registrations Channels can register virtual package detection plugins in `info.virtual_package_plugins`, mapping the name of the package providing the plugin to the virtual packages it provides. Keying by plugin lets one `cuda-detect` cover both `__cuda` and `__cuda_arch` instead of registering the same detector twice. Parsed from both `repodata.json` and the sharded index; names are deserialized unchecked so one malformed entry cannot make a channel unusable. The gateway exposes the registrations two ways: - `Gateway::virtual_package_plugins(channel, platform)` mirrors `Gateway::channel_relations` and needs no specs, which matters because the plugin package names only exist inside the metadata being fetched. - `RepoDataQueryOutput::virtual_package_plugins` lists them per subdir in resolved channel-priority order, so channels a solve discovers through CEP-42 are covered too. Registrations are reported exactly as declared: duplicate claims on the same virtual package are left for the caller to resolve, and no plugin is fetched or executed. Registrations stay per subdir rather than collapsed per channel, matching how CEP-42 relations are handled. `rattler virtual-packages -c ` prints them for manual testing against the new `test-data/channels/virtual-package-plugins` fixture. The design, including the parts that are not implemented yet -- the plugin protocol, execution, caching and the trust model -- is written up in `crates/rattler_repodata_gateway/docs/virtual-package-plugins.md`. All of it sits behind the `experimental-virtual-package-plugins` feature; with the feature off the public API and the serialized output are unchanged. Cargo features are additive, so every crate constructing `ChannelInfo` or `ShardedSubdirInfo` must enable it in lockstep, hence the forwarding entries in `rattler_index` and `rattler-bin`. Also switches the sharded-index test mock to `rmp_serde::to_vec_named`, matching what the indexer writes. Positional encoding misaligns as soon as a skipped field precedes a present one. --- crates/rattler-bin/Cargo.toml | 4 + .../src/commands/virtual_packages.rs | 98 ++++- crates/rattler-bin/src/main.rs | 4 +- crates/rattler_conda_types/Cargo.toml | 1 + crates/rattler_conda_types/src/lib.rs | 2 + .../rattler_conda_types/src/repo_data/mod.rs | 117 ++++++ .../src/repo_data/sharded.rs | 49 +++ crates/rattler_index/Cargo.toml | 4 + crates/rattler_index/src/lib.rs | 8 + crates/rattler_repodata_gateway/Cargo.toml | 1 + .../docs/virtual-package-plugins.md | 289 +++++++++++++++ .../src/gateway/local_subdir.rs | 7 + .../src/gateway/mod.rs | 347 ++++++++++++++++++ .../src/gateway/query.rs | 63 ++++ .../src/gateway/remote_subdir/mod.rs | 7 + .../src/gateway/sharded_subdir/mod.rs | 51 ++- .../src/gateway/sharded_subdir/tokio/mod.rs | 7 + .../src/gateway/sharded_subdir/wasm/mod.rs | 7 + .../src/gateway/subdir.rs | 27 ++ crates/rattler_repodata_gateway/src/lib.rs | 2 + .../src/sparse/mod.rs | 20 + .../linux-64/repodata.json | 26 ++ .../noarch/repodata.json | 12 + 23 files changed, 1149 insertions(+), 4 deletions(-) create mode 100644 crates/rattler_repodata_gateway/docs/virtual-package-plugins.md create mode 100644 test-data/channels/virtual-package-plugins/linux-64/repodata.json create mode 100644 test-data/channels/virtual-package-plugins/noarch/repodata.json diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index b7c44db220..a16ecaedf0 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -34,6 +34,10 @@ rustls = [ s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] oauth = ["rattler/oauth"] +experimental-virtual-package-plugins = [ + "rattler_repodata_gateway/experimental-virtual-package-plugins", + "rattler_index/experimental-virtual-package-plugins", +] [dependencies] anyhow = { workspace = true } diff --git a/crates/rattler-bin/src/commands/virtual_packages.rs b/crates/rattler-bin/src/commands/virtual_packages.rs index a1b6fe19a3..f7736316b0 100644 --- a/crates/rattler-bin/src/commands/virtual_packages.rs +++ b/crates/rattler-bin/src/commands/virtual_packages.rs @@ -1,17 +1,111 @@ use miette::IntoDiagnostic; use rattler_conda_types::GenericVirtualPackage; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::Platform; use rattler_virtual_packages::VirtualPackageOverrides; /// Print detected virtual packages. #[derive(Debug, clap::Parser)] -pub struct Opt {} +#[cfg_attr( + feature = "experimental-virtual-package-plugins", + clap(after_help = r#"Examples: + rattler virtual-packages + rattler virtual-packages -c ./test-data/channels/virtual-package-plugins"#) +)] +pub struct Opt { + /// Channels to list registered virtual package plugins for + #[cfg(feature = "experimental-virtual-package-plugins")] + #[clap(short, long)] + channels: Vec, -pub fn virtual_packages(_opt: Opt) -> miette::Result<()> { + /// Platforms to read registrations for [default: current and noarch] + #[cfg(feature = "experimental-virtual-package-plugins")] + #[clap(short, long)] + platforms: Vec, +} + +pub async fn virtual_packages(opt: Opt, offline: bool) -> miette::Result<()> { let virtual_packages = rattler_virtual_packages::VirtualPackage::detect(&VirtualPackageOverrides::from_env()) .into_diagnostic()?; for package in virtual_packages { println!("{}", GenericVirtualPackage::from(package.clone())); } + + #[cfg(feature = "experimental-virtual-package-plugins")] + print_plugins(&opt.channels, &opt.platforms, offline).await?; + + #[cfg(not(feature = "experimental-virtual-package-plugins"))] + let _ = (opt, offline); + + Ok(()) +} + +/// Prints the plugin registrations declared by each `(channel, platform)` +/// subdirectory, in the order the channels were given. +#[cfg(feature = "experimental-virtual-package-plugins")] +async fn print_plugins( + channels: &[String], + platforms: &[Platform], + offline: bool, +) -> miette::Result<()> { + use std::{collections::HashMap, env}; + + use itertools::Itertools; + use rattler_conda_types::{Channel, ChannelConfig, PackageName}; + use rattler_repodata_gateway::{Gateway, SourceConfig}; + + if channels.is_empty() { + return Ok(()); + } + + let channel_config = + ChannelConfig::default_with_root_dir(env::current_dir().into_diagnostic()?); + let channels = channels + .iter() + .map(|channel| Channel::from_str(channel, &channel_config)) + .collect::, _>>() + .into_diagnostic()?; + + let platforms = if platforms.is_empty() { + vec![Platform::current(), Platform::NoArch] + } else { + platforms.to_vec() + }; + + let gateway = Gateway::builder() + .with_client(super::client::create_client_with_middleware(offline)?) + .with_channel_config(rattler_repodata_gateway::ChannelConfig { + default: SourceConfig { + cache_action: super::client::repodata_cache_action(offline), + ..SourceConfig::default() + }, + per_channel: HashMap::new(), + }) + .finish(); + + for channel in &channels { + for platform in &platforms { + let plugins = gateway + .virtual_package_plugins(channel, *platform) + .await + .into_diagnostic()?; + if plugins.is_empty() { + continue; + } + println!( + "\nvirtual package plugins in {} [{platform}]:", + channel.canonical_name() + ); + for (plugin, provided) in &plugins { + println!( + " {} provides {}", + plugin.as_source(), + provided.iter().map(PackageName::as_source).join(", ") + ); + } + } + } + Ok(()) } diff --git a/crates/rattler-bin/src/main.rs b/crates/rattler-bin/src/main.rs index 141db8d627..0deceab30f 100644 --- a/crates/rattler-bin/src/main.rs +++ b/crates/rattler-bin/src/main.rs @@ -124,7 +124,9 @@ async fn async_main() -> miette::Result<()> { Command::Solve(opts) => commands::solve::solve(opts, offline).await, Command::List(opts) => commands::list::list(opts).await, Command::ShellHook(opts) => commands::shell_hook::shell_hook(opts).await, - Command::VirtualPackages(opts) => commands::virtual_packages::virtual_packages(opts), + Command::VirtualPackages(opts) => { + commands::virtual_packages::virtual_packages(opts, offline).await + } Command::InstallMenu(opts) => commands::menu::install_menu(opts).await, Command::RemoveMenu(opts) => commands::menu::remove_menu(opts).await, Command::Run(opts) => commands::run::run(opts).await, diff --git a/crates/rattler_conda_types/Cargo.toml b/crates/rattler_conda_types/Cargo.toml index 923c416982..1042f0e31c 100644 --- a/crates/rattler_conda_types/Cargo.toml +++ b/crates/rattler_conda_types/Cargo.toml @@ -13,6 +13,7 @@ readme.workspace = true [features] default = ["rayon"] +experimental-virtual-package-plugins = [] [package.metadata.docs.rs] all-features = true diff --git a/crates/rattler_conda_types/src/lib.rs b/crates/rattler_conda_types/src/lib.rs index 49b39f503f..57fc49e3bb 100644 --- a/crates/rattler_conda_types/src/lib.rs +++ b/crates/rattler_conda_types/src/lib.rs @@ -64,6 +64,8 @@ pub use platform::{Arch, ParseArchError, ParsePlatformError, Platform}; pub use prefix_data::PrefixData; pub use prefix_record::PrefixRecord; pub use record_traits::HasArtifactIdentificationRefs; +#[cfg(feature = "experimental-virtual-package-plugins")] +pub use repo_data::VirtualPackagePlugins; pub use repo_data::{ ChannelInfo, ChannelRelations, ConvertSubdirError, PackageRecord, RecordFromPath, RepoData, RepodataRevision, RepodataRevisionInfo, RepodataRevisionMetadata, RepodataRevisions, diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 85d81d2156..7e20e9e841 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -107,8 +107,26 @@ pub struct ChannelInfo { /// [CEP-42](https://github.com/conda/ceps/blob/main/cep-0042.md). #[serde(default, skip_serializing_if = "ChannelRelations::is_none_or_empty")] pub channel_relations: Option, + + /// Virtual package detection plugins registered by the channel. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[serde_as( + deserialize_as = "IndexMap>" + )] + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub virtual_package_plugins: VirtualPackagePlugins, } +/// Virtual package detection plugins registered by a channel: the name of the +/// package providing the plugin, mapped to the virtual packages it provides. +/// +/// One plugin may provide several virtual packages, e.g. a `cuda-detect` +/// providing both `__cuda` and `__cuda_arch`. The executable to run is named +/// after the plugin package. Inverting the map is left to the caller: the +/// reverse direction is many-to-many. +#[cfg(feature = "experimental-virtual-package-plugins")] +pub type VirtualPackagePlugins = IndexMap>; + /// Repodata revisions keyed by revision, mirroring the `vN` dictionary of the /// CEP draft . Keying encodes /// uniqueness; insertion order is preserved. @@ -1084,6 +1102,8 @@ mod test { package::DistArchiveIdentifier, repo_data::{compute_package_url, determine_subdir}, }; + #[cfg(feature = "experimental-virtual-package-plugins")] + use crate::{PackageName, repo_data::VirtualPackagePlugins}; // isl-0.12.2-1.tar.bz2 // gmp-5.1.2-6.tar.bz2 @@ -1166,6 +1186,8 @@ mod test { base: Some("../conda-forge".to_string()), overrides: None, }), + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }), packages: IndexMap::default(), conda_packages: IndexMap::default(), @@ -1188,6 +1210,8 @@ mod test { base_url: None, repodata_revisions: IndexMap::default(), channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }), packages: IndexMap::default(), conda_packages: IndexMap::default(), @@ -1199,6 +1223,99 @@ mod test { } } + #[cfg(feature = "experimental-virtual-package-plugins")] + #[test] + fn test_virtual_package_plugins() { + // A single plugin may provide several virtual packages; the order of + // both the plugins and their virtual packages is preserved. + let raw = r#"{ + "info": { + "subdir": "linux-64", + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"], + "rocm-detect": ["__rocm"] + } + }, + "packages": {}, + "packages.conda": {} + }"#; + let repodata: RepoData = serde_json::from_str(raw).unwrap(); + let plugins = &repodata.info.as_ref().unwrap().virtual_package_plugins; + + assert_eq!( + plugins + .keys() + .map(PackageName::as_source) + .collect::>(), + ["cuda-detect", "rocm-detect"] + ); + assert_eq!( + plugins[&PackageName::new_unchecked("cuda-detect")] + .iter() + .map(PackageName::as_source) + .collect::>(), + ["__cuda", "__cuda_arch"] + ); + assert_eq!( + plugins[&PackageName::new_unchecked("rocm-detect")] + .iter() + .map(PackageName::as_source) + .collect::>(), + ["__rocm"] + ); + + let json = serde_json::to_string(&repodata).unwrap(); + assert!(json.contains("\"virtual_package_plugins\"")); + assert_eq!(serde_json::from_str::(&json).unwrap(), repodata); + + let without = RepoData { + version: Some(2), + info: Some(ChannelInfo { + subdir: Some("linux-64".to_string()), + base_url: None, + repodata_revisions: IndexMap::default(), + channel_relations: None, + virtual_package_plugins: VirtualPackagePlugins::default(), + }), + packages: IndexMap::default(), + conda_packages: IndexMap::default(), + v3: V3Packages::default(), + removed: ahash::HashSet::default(), + }; + assert!( + !serde_json::to_string(&without) + .unwrap() + .contains("virtual_package_plugins") + ); + } + + /// Names are deserialized unchecked, so a channel publishing a malformed + /// name does not render the entire repodata unusable. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[test] + fn test_virtual_package_plugins_malformed_name() { + let raw = r#"{ + "info": { + "subdir": "linux-64", + "virtual_package_plugins": { "invalid$plugin": ["__rocm"] } + }, + "packages": {}, + "packages.conda": {} + }"#; + let repodata: RepoData = serde_json::from_str(raw).unwrap(); + assert_eq!( + repodata + .info + .as_ref() + .unwrap() + .virtual_package_plugins + .keys() + .map(PackageName::as_source) + .collect::>(), + ["invalid$plugin"] + ); + } + #[test] fn test_repodata_revisions() { let raw = r#"{ diff --git a/crates/rattler_conda_types/src/repo_data/sharded.rs b/crates/rattler_conda_types/src/repo_data/sharded.rs index e0776c968c..b4af813e6a 100644 --- a/crates/rattler_conda_types/src/repo_data/sharded.rs +++ b/crates/rattler_conda_types/src/repo_data/sharded.rs @@ -2,7 +2,11 @@ use crate::PackageRecord; use crate::package::DistArchiveIdentifier; +#[cfg(feature = "experimental-virtual-package-plugins")] +use crate::repo_data::VirtualPackagePlugins; use crate::repo_data::{ChannelRelations, RepodataRevisions, V3Packages}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use crate::utils::serde::DeserializeFromStrUnchecked; use crate::utils::serde::{sort_index_map_alphabetically, sort_set_alphabetically}; use indexmap::IndexMap; use jiff::Timestamp; @@ -60,6 +64,14 @@ pub struct ShardedSubdirInfo { /// [CEP-42](https://github.com/conda/ceps/blob/main/cep-0042.md). #[serde(default, skip_serializing_if = "ChannelRelations::is_none_or_empty")] pub channel_relations: Option, + + /// Virtual package detection plugins registered by the channel. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[serde_as( + deserialize_as = "IndexMap>" + )] + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub virtual_package_plugins: VirtualPackagePlugins, } #[cfg(test)] @@ -131,11 +143,48 @@ mod tests { created_at: None, repodata_revisions: IndexMap::default(), channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }; let json = serde_json::to_string(&info).unwrap(); assert!(!json.contains("channel_relations")); } } + + #[cfg(feature = "experimental-virtual-package-plugins")] + #[test] + fn test_sharded_subdir_info_virtual_package_plugins() { + let raw = r#"{ + "subdir": "linux-64", + "base_url": "./", + "shards_base_url": "./shards/", + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"] + } + }"#; + let info: ShardedSubdirInfo = serde_json::from_str(raw).unwrap(); + assert_eq!( + info.virtual_package_plugins[&PackageName::new_unchecked("cuda-detect")] + .iter() + .map(PackageName::as_source) + .collect::>(), + ["__cuda", "__cuda_arch"] + ); + + let json = serde_json::to_string(&info).unwrap(); + assert!(json.contains("\"virtual_package_plugins\"")); + + // Omitted entirely when no plugins are registered. + let info = ShardedSubdirInfo { + virtual_package_plugins: VirtualPackagePlugins::default(), + ..info + }; + assert!( + !serde_json::to_string(&info) + .unwrap() + .contains("virtual_package_plugins") + ); + } } /// An individual shard that contains repodata for a single package name. diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index fa142ad175..ca71aae2e4 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -28,6 +28,10 @@ rustls = [ "opendal/reqwest-rustls-tls", ] s3 = ["opendal/services-s3", "dep:rattler_s3"] +experimental-virtual-package-plugins = [ + "rattler_conda_types/experimental-virtual-package-plugins", + "rattler_repodata_gateway/experimental-virtual-package-plugins", +] [[bin]] name = "rattler-index" diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index be5c547bb2..c65d0c03c0 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -28,6 +28,8 @@ use opendal::layers::RetryLayer; #[cfg(feature = "s3")] use opendal::services::S3Config; use opendal::{Configurator, Operator, services::FsConfig}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ ChannelInfo, ChannelRelations, PackageRecord, PatchInstructions, Platform, RepoData, Shard, ShardedRepodata, ShardedSubdirInfo, UrlOrPath, V3Packages, WhlPackageRecord, @@ -931,6 +933,8 @@ async fn index_subdir_inner( base_url: channel_metadata.base_url, repodata_revisions: repodata_revisions_for_packages(&repodata_revisions, &v3), channel_relations: channel_metadata.channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }), packages, conda_packages, @@ -1281,6 +1285,8 @@ pub async fn write_repodata( created_at: Some(jiff::Timestamp::now()), repodata_revisions: sharded_repodata_revisions, channel_relations: sharded_channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }, shards: shards .iter() @@ -1719,6 +1725,8 @@ pub async fn ensure_channel_initialized_with_channel_metadata( base_url: channel_metadata.base_url, repodata_revisions: RepodataRevisions::new(), channel_relations: channel_metadata.channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }), packages: IndexMap::default(), conda_packages: IndexMap::default(), diff --git a/crates/rattler_repodata_gateway/Cargo.toml b/crates/rattler_repodata_gateway/Cargo.toml index 5b0c0bec85..41543b833e 100644 --- a/crates/rattler_repodata_gateway/Cargo.toml +++ b/crates/rattler_repodata_gateway/Cargo.toml @@ -110,6 +110,7 @@ native-tls = ['reqwest/native-tls', 'rattler_networking/native-tls', 'rattler_ca rustls = ['reqwest/rustls', 'rattler_networking/rustls', 'rattler_cache/rustls', 'rattler_redaction/rustls'] sparse = ["rattler_conda_types", "memmap2", "self_cell", "superslice", "itertools", "serde_json/raw_value"] gateway = ["sparse", "http", "http-cache-semantics", "parking_lot", "async-trait", "rattler_package_streaming"] +experimental-virtual-package-plugins = ["rattler_conda_types?/experimental-virtual-package-plugins"] [package.metadata.docs.rs] features = ["sparse", "gateway"] diff --git a/crates/rattler_repodata_gateway/docs/virtual-package-plugins.md b/crates/rattler_repodata_gateway/docs/virtual-package-plugins.md new file mode 100644 index 0000000000..78a7b99fab --- /dev/null +++ b/crates/rattler_repodata_gateway/docs/virtual-package-plugins.md @@ -0,0 +1,289 @@ +# User-Specified Virtual Packages + +## Proposal for Conda Channel-Defined Virtual Package Plugins + +### Status + +The **metadata half** of this proposal is implemented in rattler behind the +`experimental-virtual-package-plugins` cargo feature: the registration is parsed from repodata and +handed to callers. Nothing is visible unless that feature is enabled, and no plugin is fetched or +executed by rattler today. + +| Part | State | +| --- | --- | +| `info.virtual_package_plugins` parsing (`repodata.json` and sharded index) | Implemented | +| `Gateway::virtual_package_plugins(channel, platform)` accessor | Implemented | +| Registrations on `RepoDataQueryOutput`, per channel subdir | Implemented | +| `rattler virtual-packages -c ` for manual inspection | Implemented | +| Conflict resolution across channels | Deliberately not done -- reported as declared, caller decides | +| Plugin protocol types, execution, result caching | Not implemented | +| Solver injection, `CONDA_OVERRIDE_*`, lockfile representation | Not implemented | +| Trust / opt-in model | Open, blocks execution | +| prefix.dev upload validation | Not implemented (server side) | + +`rattler_index` does not yet propagate the field: with the feature off it drops +`info.virtual_package_plugins` on a repodata round-trip, and with the feature on it writes an empty +map. Only a channel server publishing the field directly exercises the path today. + +### Problem + +Today, virtual packages like `__cuda` are hardcoded in the solver client. This made sense when NVIDIA +was the only accelerator that mattered, but the hardware landscape is diversifying fast. AMD ROCm, +Intel oneAPI, and other accelerator stacks each have their own driver versions, runtime libraries, and +capability matrices. Hardcoding detection logic for every new accelerator in every client release +doesn't scale. Channel operators who ship packages targeting these accelerators need a way to define +virtual packages like `__rocm` or `__oneapi` without waiting for upstream client changes. + +### Proposal + +We introduce a plugin-based virtual package system where channel operators on prefix.dev define custom +virtual packages backed by detection plugins. The solver treats them identically to built-in virtual +packages -- packages can depend on `__rocm >= 6.0` or `__oneapi >= 2025.1` the same way they depend on +`__cuda` today. + +The system has two parts: + +1. **Channel-side: plugin registration and validation** +2. **Client-side: plugin execution and caching** + +--- + +### 1. Channel-Side: Plugin Registration + +Channel operators register virtual package plugins as part of their channel configuration on +prefix.dev. Each registration names a conda package containing the detection logic and lists the +virtual packages that plugin provides. + +During package upload, prefix.dev validates that any virtual package dependency declared in a +package's metadata has a corresponding plugin registered in the channel. Uploads referencing undefined +virtual packages are rejected. + +The registration is published in the channel's `repodata.json` under a new `info.virtual_package_plugins` +field, keyed by **plugin package name**: + +```json +{ + "info": { + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"], + "rocm-detect": ["__rocm"] + } + }, + "packages": { ... } +} +``` + +Keying by plugin rather than by virtual package is deliberate. The reverse direction -- +`{"__cuda": "cuda-detect", "__cuda_arch": "cuda-detect"}` -- registers the same detector twice and +gives the client no way to know the two entries are one program doing one piece of work. Keying by +plugin makes "several virtual packages from one plugin" the ordinary case, which is what `__cuda` and +`__cuda_arch` actually need. + +The client resolves the plugin package from the same channel, picking the latest available version. No +version constraint is expressible in the registration. + +The `virtual_package -> plugin` mapping is *derived* by the client if it needs it. That inversion is +many-to-many: two plugins in one channel, or plugins in different channels, may each claim `__rocm`. +Nothing in the metadata prevents it and the client must resolve it. + +The same field is published in the sharded repodata index (`repodata_shards.msgpack.zst`) under +`info`, so sharded channels carry the registration too. + +**Per-subdir, not channel-wide.** `info` lives in each subdir's repodata, so the registration must be +repeated in every subdir of a channel, and different subdirs *may* declare different registrations. +Consumers see one entry per subdir and may union them. A channel-wide location would be better and +needs a CEP. + +**Lenient parsing.** Plugin and virtual package names are parsed without validation, so a channel +publishing a malformed name does not make the whole `repodata.json` unusable. + +### 2. Client-Side: Plugin Execution and Caching + +*Not implemented. This section is the remaining proposal; the metadata above constrains it as noted.* + +When pixi resolves an environment and encounters a dependency on a virtual package provided by a +registered plugin, it: + +1. **Fetches and installs the plugin package** into an isolated, internal environment (separate from + the user's env, cached across solves). + + The plugin environment must be solved using **built-in virtual packages only**. Resolving a + plugin's own dependencies is itself a solve against a channel whose plugin data is not yet + available; restricting that solve to built-ins is what stops the recursion. + +2. **Executes the plugin once**, and the plugin reports on every virtual package it was registered + for. It inspects the local system (checks for driver files, queries `rocm-smi`, reads `/sys/` + entries, etc.) and returns a JSON array: + +```json +{ + "virtual_packages": [ + { "name": "__cuda", "version": "12.4" }, + { "name": "__cuda_arch", "version": "0", "build_string": "sm_89" } + ], + "cache": { + "ttl_seconds": 86400, + "watch_paths": [ + "/opt/rocm/lib/libamdhip64.so", + "/sys/module/amdgpu/version" + ] + } +} +``` + + The array shape follows from one entry point per plugin package: a single `cuda-detect` run has to + be able to report both `__cuda` and `__cuda_arch`. `build_string` is optional and exists because + `__archspec` and `__cuda_arch` carry their information in the build string rather than the version; + without it those cannot be expressed as plugins at all. + +3. **Caches the result** according to the plugin's cache policy: + - **`ttl_seconds`**: how long the cached value is valid. + - **`watch_paths`**: file globs to monitor. If any file's existence or modification time changes, + the cache is invalidated and the plugin re-runs. This handles driver upgrades between solves + without requiring TTL expiry. + + Caches must be keyed on **(channel, plugin package name)**, not the package name alone: names are + unique within a channel, but two channels may each ship a different `cuda-detect`. + +4. **Injects the detected virtual packages** into the solver's virtual package set alongside the + standard ones (`__cuda`, `__glibc`, etc.). A plugin may only inject virtual packages the channel + registered for it; anything else is discarded. Virtual packages the plugin omits are treated as + absent -- the solver simply won't select packages that require them. + +### Plugin Interface + +Plugins are simple executables. **The entry point is the plugin package name**: package `cuda-detect` +ships an executable `cuda-detect`. Package names are unique within a channel and a JSON object cannot +repeat a key, so the entry point needs no separate metadata field, and conda already puts executables +on the environment's `PATH` (`bin/`, `Scripts/`) so no path needs declaring either. + +The contract: + +- **stdin**: empty +- **stdout**: JSON object as shown above +- **stderr**: diagnostic output (logged by pixi at debug level) +- **exit 0**: the plugin ran; `virtual_packages` lists what it detected, and may be empty +- **exit non-zero**: plugin failure (pixi logs a warning, treats all of the plugin's virtual packages + as absent) + +This replaces the draft's earlier three-way exit code contract (`0` present / `1` absent / `2+` +failure). With several virtual packages per plugin, presence is per-entry in the output array and can +no longer be carried by a single exit status: `__cuda` may be present while `__cuda_arch` is not. +**This needs sign-off** -- it is the one part of the interface the implemented metadata forced to +change. + +Plugins can be compiled binaries, shell scripts, or anything else that fits in a conda package. +Keeping the interface this simple means detection for a new accelerator is a single small package with +a shell script that checks a few paths. + +### Gateway Integration + +Implemented. The repodata gateway parses `info.virtual_package_plugins` and reports it; it does not +execute plugins -- it doesn't know what hardware the client has -- and it does not resolve conflicts. + +There are two ways to read the registrations: + +`Gateway::virtual_package_plugins(channel, platform)` returns the map for one subdirectory. It takes +no specs, which is the point: the plugin package names only exist inside the metadata being fetched, +so there is nothing to query for until it has been read. It mirrors `Gateway::channel_relations`, +reusing the internal subdir cache, and yields an empty map for a subdirectory that registers none or +does not exist. + +`RepoDataQueryOutput::virtual_package_plugins` returns one entry per channel subdir that declared a +registration, carrying the channel, the subdir platform, and the plugin-to-virtual-packages map, +ordered by resolved channel priority (including any CEP-42 relation-derived ordering). This is the +view a solve sees, so it also covers channels discovered through CEP-42 that the caller never named. + +Duplicate claims are preserved verbatim in both: two channels each claiming `__rocm`, or two plugins +within one channel each claiming `__rocm`, all come back, and no warning is raised. Deciding which +plugin wins is the caller's job. + +For manual inspection, `rattler virtual-packages -c ` prints the registrations a channel +declares. `test-data/channels/virtual-package-plugins` is a local fixture to point it at, since no +channel publishes the field yet. + +All of this is behind the `experimental-virtual-package-plugins` feature. With the feature off the +gateway's public API and its serialized output are unchanged. + +### Example: Supporting AMD ROCm + +A channel operator shipping packages compiled against ROCm: + +1. Creates a `rocm-detect` conda package containing a shell script named `rocm-detect` that checks for + `/opt/rocm/.info/version` and parses the ROCm version. +2. Registers `rocm-detect -> ["__rocm"]` in their channel config on prefix.dev. +3. Uploads packages with `__rocm >= 6.0` in their run dependencies. +4. When a user with ROCm 6.1.2 installed runs `pixi install`, pixi fetches the plugin, runs it, + discovers ROCm 6.1.2, and the solver selects the appropriate package variants. +5. A user without ROCm gets packages built for CPU fallback (or an unsatisfiable error if no fallback + exists). + +The same pattern works for Intel oneAPI, custom FPGA toolchains, or any other hardware capability that +packages need to select against. + +### Example: One Plugin, Several Virtual Packages + +A `cuda-detect` package registered as `cuda-detect -> ["__cuda", "__cuda_arch"]` queries the driver +once and reports both the driver version and the compute capability: + +```json +{ + "virtual_packages": [ + { "name": "__cuda", "version": "12.4" }, + { "name": "__cuda_arch", "version": "0", "build_string": "sm_89" } + ] +} +``` + +On a machine with no NVIDIA driver the same plugin exits 0 with an empty `virtual_packages` array. +Under the draft's original one-plugin-per-virtual-package scheme this needed two packages, or one +package with two entry points repeating the same driver query. + +### Settled Decisions + +1. **Registration is keyed by plugin package name**, mapping to the list of virtual packages it + provides. +2. **The entry point is the plugin package name.** No entry-point field in the metadata; uniqueness + within a channel comes for free. +3. **No package-record changes.** The registration lives entirely in `info`; `PackageRecord` and + `index.json` are untouched, so a client learns what a plugin provides without fetching the plugin's + record first. +4. **No version constraints in the registration.** Bare package name, latest version. +5. **The gateway reports, it does not decide.** Registrations come back per subdir in channel-priority + order with duplicates intact. +6. **Plugin identity is (channel, package name)** for caching and conflict resolution. +7. **Everything is behind an experimental cargo feature** and invisible when it is off. + +### Open Questions + +1. **Trust and governance.** Execution runs channel-supplied code during solve. Users should have to + opt in, and the shape of that opt-in is unsettled: a single global switch, or a per-channel + allowlist. The nearest precedent in rattler is `run_post_link_scripts`, a two-state setting whose + opt-in value is named `insecure` and which defaults to off. This blocks the executor. +2. **Cache invalidation of the plugin environment.** `ttl_seconds` and `watch_paths` cover the + detection *result*. Setting up the isolated environment is the expensive part, and knowing when + that environment is stale is a separate question. +3. **Cross-installing for another platform.** Detection is inherently host-only. Built-in virtual + packages have `detect_for_platform` with documented cross-compilation defaults; plugins have no + equivalent, and it is not clear what running a host plugin means when solving for a different + target. +4. **Overrides and opt-out.** Users should be able to override or disable a specific virtual package + (e.g. skip detection and assert `__rocm 6.1.2`). Built-ins use `CONDA_OVERRIDE_*`; the naming for + plugin-provided packages, especially with sub-keys like `__cuda_arch` and with several channels + registering the same name, is undecided. +5. **Reproducibility and lockfiles.** Whether the plugin version that produced a detection is recorded + in the lock file, and when plugins are updated. Current leaning: always use the latest available and + do not lock it, but this is unresolved. +6. **Channel-wide storage.** `info` is per-subdir, so the registration is duplicated across subdirs. + A channel-wide metadata location would fix this and needs a CEP. +7. **Channel relations and overriding.** Whether a channel may register a plugin for a virtual package + its base channel already covers (e.g. a private channel overriding `__glibc`), and whether such an + override should affect the base channel. CEP-42 relations already give the gateway a priority + order; the policy question is untouched. +8. **Plugin dependencies.** Detection plugins should be self-contained, but if one needs a shared + library to query a driver API, those deps are resolved from the same channel. Solving the plugin + environment with built-in virtual packages only (see above) breaks the bootstrap recursion; the + remaining risk is ordinary dependency conflict. +9. **Versioning semantics.** Virtual package versions should follow conda version ordering so that + constraints like `__rocm >= 6.0, < 7` work as expected. +10. **wheelnext.** Worth looking at closely -- they are solving essentially the same problem. diff --git a/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs b/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs index ad1d32bdd0..881ef9e269 100644 --- a/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs +++ b/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs @@ -1,5 +1,7 @@ use std::{path::Path, sync::Arc}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{Channel, ChannelRelations, PackageName, RepodataRevisions}; use crate::{ @@ -115,4 +117,9 @@ impl SubdirClient for LocalSubdirClient { fn channel_relations(&self) -> Option<&ChannelRelations> { self.sparse.channel_relations() } + + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + self.sparse.virtual_package_plugins() + } } diff --git a/crates/rattler_repodata_gateway/src/gateway/mod.rs b/crates/rattler_repodata_gateway/src/gateway/mod.rs index c29aa5ab3c..d87540123c 100644 --- a/crates/rattler_repodata_gateway/src/gateway/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/mod.rs @@ -32,9 +32,13 @@ use coalesced_map::{CoalescedGetError, CoalescedMap}; pub use error::GatewayError; #[cfg(feature = "indicatif")] pub use indicatif::{IndicatifReporter, IndicatifReporterBuilder}; +#[cfg(feature = "experimental-virtual-package-plugins")] +pub use query::SubdirVirtualPackagePlugins; pub use query::{NamesQuery, NamesQueryOutput, RepoDataQuery, RepoDataQueryOutput}; #[cfg(not(target_arch = "wasm32"))] use rattler_cache::package_cache::PackageCache; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{Channel, ChannelRelations, MatchSpec, Platform, RepoDataRecord}; use rattler_networking::LazyClient; pub use repo_data::RepoData; @@ -216,6 +220,35 @@ impl Gateway { } } + /// Returns the virtual package detection plugins registered by the + /// given `(channel, platform)` subdirectory, keyed by the name of the + /// package providing the plugin. Empty if the subdirectory registers + /// none or doesn't exist. + /// + /// Needs no specs, unlike [`Gateway::query`], because the plugin + /// packages cannot be named until this metadata has been read. + /// + /// Reuses the internal subdir cache: if the pair has already been + /// fetched by a [`Gateway::query`] this is free. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub async fn virtual_package_plugins( + &self, + channel: &Channel, + platform: Platform, + ) -> Result { + match self + .inner + .get_or_create_subdir(channel, platform, None) + .await + { + Ok(subdir) => Ok(subdir.virtual_package_plugins().clone()), + // As above: noarch reports its absence as an error rather + // than `NotFound`, so an empty map holds for all platforms. + Err(GatewayError::SubdirNotFoundError(_)) => Ok(VirtualPackagePlugins::default()), + Err(err) => Err(err), + } + } + /// Ensure that given repodata records contain `RunExportsJson`. pub async fn ensure_run_exports( &self, @@ -2617,6 +2650,238 @@ mod test { ); } + /// Writes a linux-64 subdir whose `info.virtual_package_plugins` is the + /// given JSON object body, e.g. `"cuda-detect": ["__cuda"]`. + #[cfg(feature = "experimental-virtual-package-plugins")] + fn write_test_subdir_with_plugins(root: &std::path::Path, pkg: &str, plugins: &str) { + let subdir = root.join("linux-64"); + std::fs::create_dir_all(&subdir).unwrap(); + let json = format!( + r#"{{ + "info": {{ + "subdir": "linux-64", + "virtual_package_plugins": {{{plugins}}} + }}, + "packages.conda": {{ + "{pkg}-1.0.0-0.conda": {{ + "build": "0", + "build_number": 0, + "depends": [], + "md5": "00000000000000000000000000000000", + "name": "{pkg}", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "size": 1000, + "subdir": "linux-64", + "timestamp": 1700000000000, + "version": "1.0.0" + }} + }} +}}"# + ); + std::fs::write(subdir.join("repodata.json"), json).unwrap(); + } + + /// One reported registration as + /// `(channel_suffix, platform, [(plugin, [virtual packages])])`. + #[cfg(feature = "experimental-virtual-package-plugins")] + type FlatPlugins = Vec<(String, Platform, Vec<(String, Vec)>)>; + + /// Flattens the reported registrations for order-sensitive comparison. + #[cfg(feature = "experimental-virtual-package-plugins")] + fn flatten_plugins(output: &crate::RepoDataQueryOutput) -> FlatPlugins { + output + .virtual_package_plugins + .iter() + .map(|entry| { + ( + entry.channel.url().path().trim_matches('/').to_string(), + entry.platform, + entry + .plugins + .iter() + .map(|(plugin, provided)| { + ( + plugin.as_source().to_string(), + provided.iter().map(|v| v.as_source().to_string()).collect(), + ) + }) + .collect(), + ) + }) + .collect() + } + + /// A channel registering one plugin that provides several virtual packages + /// is reported with its channel and subdir. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_reported_per_subdir() { + let dir = tempfile::tempdir().unwrap(); + let a = dir.path().join("a"); + write_test_subdir_with_plugins(&a, "shared", r#""cuda-detect": ["__cuda", "__cuda_arch"]"#); + + let server = SimpleChannelServer::new(dir.path()).await; + let output = Gateway::new() + .query( + vec![Channel::from_url(server.url().join("a/").unwrap())], + vec![Platform::Linux64], + vec![MatchSpec::from_str("shared", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + assert_eq!( + flatten_plugins(&output), + vec![( + "a".to_string(), + Platform::Linux64, + vec![( + "cuda-detect".to_string(), + vec!["__cuda".to_string(), "__cuda_arch".to_string()], + )], + )] + ); + assert!(output.warnings.is_empty(), "{:?}", output.warnings); + } + + /// Two channels claiming the same virtual package are both reported, in + /// channel-priority order. Resolving the conflict is the caller's job, so + /// the gateway must not drop either one or warn about it. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_conflicting_channels_both_reported() { + let dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + &dir.path().join("a"), + "shared", + r#""a-detect": ["__rocm"]"#, + ); + write_test_subdir_with_plugins( + &dir.path().join("b"), + "shared", + r#""b-detect": ["__rocm"]"#, + ); + + let server = SimpleChannelServer::new(dir.path()).await; + let a = Channel::from_url(server.url().join("a/").unwrap()); + let b = Channel::from_url(server.url().join("b/").unwrap()); + + let output = Gateway::new() + .query( + vec![a, b], + vec![Platform::Linux64], + vec![MatchSpec::from_str("shared", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + let reported: Vec = flatten_plugins(&output) + .into_iter() + .map(|(channel, _, plugins)| format!("{channel}:{}", plugins[0].0)) + .collect(); + assert_eq!(reported, vec!["a:a-detect", "b:b-detect"]); + assert!(output.warnings.is_empty(), "{:?}", output.warnings); + } + + /// Two plugins in one channel may claim the same virtual package; both + /// survive because inverting the mapping is the caller's job. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_duplicate_claim_within_channel() { + let dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + &dir.path().join("a"), + "shared", + r#""first-detect": ["__rocm"], "second-detect": ["__rocm"]"#, + ); + + let server = SimpleChannelServer::new(dir.path()).await; + let output = Gateway::new() + .query( + vec![Channel::from_url(server.url().join("a/").unwrap())], + vec![Platform::Linux64], + vec![MatchSpec::from_str("shared", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + let plugins = flatten_plugins(&output); + assert_eq!( + plugins[0].2, + vec![ + ("first-detect".to_string(), vec!["__rocm".to_string()]), + ("second-detect".to_string(), vec!["__rocm".to_string()]), + ] + ); + } + + /// Registrations are read off the subdir itself, so a spec that matches no + /// record still reports them. Callers cannot query for the plugin packages + /// by name: the registration is the only place those names come from. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_reported_for_unmatched_spec() { + let dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + &dir.path().join("a"), + "shared", + r#""cuda-detect": ["__cuda"]"#, + ); + + let server = SimpleChannelServer::new(dir.path()).await; + let output = Gateway::new() + .query( + vec![Channel::from_url(server.url().join("a/").unwrap())], + vec![Platform::Linux64], + vec![MatchSpec::from_str("no-such-package", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + assert!( + output.iter().all(RepoData::is_empty), + "the spec must not match any record" + ); + assert_eq!( + flatten_plugins(&output), + vec![( + "a".to_string(), + Platform::Linux64, + vec![("cuda-detect".to_string(), vec!["__cuda".to_string()])], + )] + ); + } + + /// A channel without the metadata reports nothing. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_absent() { + let dir = tempfile::tempdir().unwrap(); + write_test_subdir(&dir.path().join("a"), "shared", "1.0.0", None, None); + + let server = SimpleChannelServer::new(dir.path()).await; + let output = Gateway::new() + .query( + vec![Channel::from_url(server.url().join("a/").unwrap())], + vec![Platform::Linux64], + vec![MatchSpec::from_str("shared", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + assert!(output.virtual_package_plugins.is_empty()); + } + /// Repodata with CEP-42 `channel_relations` in `info`. fn make_repodata_with_relations( name: &str, @@ -2740,6 +3005,88 @@ mod test { } } + /// `Gateway::virtual_package_plugins` round-trips the registration + /// without any spec, which is the point: the plugin package names only + /// exist inside the metadata being fetched. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_gateway_virtual_package_plugins_roundtrip() { + let channel_dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + channel_dir.path(), + "testpkg", + r#""cuda-detect": ["__cuda", "__cuda_arch"], "rocm-detect": ["__rocm"]"#, + ); + + let server = SimpleChannelServer::new(channel_dir.path()).await; + let plugins = Gateway::new() + .virtual_package_plugins(&server.channel(), Platform::Linux64) + .await + .unwrap(); + + assert_eq!( + plugins + .iter() + .map(|(plugin, provided)| ( + plugin.as_source(), + provided + .iter() + .map(PackageName::as_source) + .collect::>() + )) + .collect::>(), + vec![ + ("cuda-detect", vec!["__cuda", "__cuda_arch"]), + ("rocm-detect", vec!["__rocm"]), + ] + ); + } + + /// A channel registering no plugins yields an empty map, not an error. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_gateway_virtual_package_plugins_absent() { + let channel_dir = tempfile::tempdir().unwrap(); + let subdir_path = channel_dir.path().join("linux-64"); + std::fs::create_dir_all(&subdir_path).unwrap(); + std::fs::write( + subdir_path.join("repodata.json"), + make_repodata("testpkg", "1.0.0"), + ) + .unwrap(); + + let server = SimpleChannelServer::new(channel_dir.path()).await; + let plugins = Gateway::new() + .virtual_package_plugins(&server.channel(), Platform::Linux64) + .await + .unwrap(); + assert!(plugins.is_empty()); + } + + /// A subdir the channel doesn't publish yields an empty map, not an + /// error. noarch matters: the subdir builder propagates its absence as + /// an error instead of `NotFound`. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_gateway_virtual_package_plugins_missing_subdir() { + let channel_dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + channel_dir.path(), + "testpkg", + r#""cuda-detect": ["__cuda"]"#, + ); + + let server = SimpleChannelServer::new(channel_dir.path()).await; + let gateway = Gateway::new(); + for platform in [Platform::Osx64, Platform::NoArch] { + let plugins = gateway + .virtual_package_plugins(&server.channel(), platform) + .await + .unwrap_or_else(|e| panic!("{platform} must return empty, not error: {e}")); + assert!(plugins.is_empty(), "{platform} declared none"); + } + } + // ---------------------------------------------------------------------- // CEP-42 integration tests // ---------------------------------------------------------------------- diff --git a/crates/rattler_repodata_gateway/src/gateway/query.rs b/crates/rattler_repodata_gateway/src/gateway/query.rs index e22af81dc2..7544ab4e6f 100644 --- a/crates/rattler_repodata_gateway/src/gateway/query.rs +++ b/crates/rattler_repodata_gateway/src/gateway/query.rs @@ -5,6 +5,8 @@ use std::{ }; use futures::{FutureExt, StreamExt, select_biased, stream::FuturesUnordered}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ Channel, ChannelUrl, MatchSpec, Matches, PackageName, PackageNameMatcher, Platform, RepoDataRecord, @@ -34,6 +36,29 @@ pub struct RepoDataQueryOutput { /// Non-fatal warnings encountered during the query. Also streamed /// to [`Reporter::on_gateway_warning`] as they are recorded. pub warnings: Vec, + /// Virtual package detection plugins declared by the queried channel + /// subdirs, in resolved channel-priority order. + /// + /// Reported exactly as declared: duplicate claims on the same virtual + /// package are not resolved, and no plugin is fetched or executed. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub virtual_package_plugins: Vec, +} + +/// Plugin registrations declared by a single channel subdir. +/// +/// Kept per subdir rather than per channel because different subdirs of one +/// channel may declare different metadata; collapsing them would silently drop +/// registrations. +#[cfg(feature = "experimental-virtual-package-plugins")] +#[derive(Debug, Clone)] +pub struct SubdirVirtualPackagePlugins { + /// The channel that declared these plugins. + pub channel: ChannelUrl, + /// The subdir the declaration was read from. + pub platform: Platform, + /// Plugin package name mapped to the virtual packages it provides. + pub plugins: VirtualPackagePlugins, } impl std::ops::Deref for RepoDataQueryOutput { @@ -332,6 +357,11 @@ struct QueryExecutor { /// CEP-42 expansion state. expander: ChannelExpander, + + /// Plugin registrations collected from each resolved channel subdir. + /// Emitted in channel-priority order once the final order is known. + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: ahash::HashMap<(ChannelUrl, Platform), VirtualPackagePlugins>, } impl QueryExecutor { @@ -486,6 +516,8 @@ impl QueryExecutor { pending_subdirs, pending_records: FuturesUnordered::new(), expander, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: ahash::HashMap::default(), }) } @@ -825,6 +857,14 @@ impl QueryExecutor { } self.expand_pattern_specs_for_subdir(subdir.as_ref()); if let Some((url, platform)) = kind_url_and_platform { + #[cfg(feature = "experimental-virtual-package-plugins")] + { + let plugins = subdir.virtual_package_plugins(); + if !plugins.is_empty() { + self.virtual_package_plugins + .insert((url.clone(), platform), plugins.clone()); + } + } self.expand_relations_for_subdir(&url, platform, subdir.as_ref())?; } if self.pending_subdirs.is_empty() { @@ -1000,6 +1040,27 @@ impl QueryExecutor { handles = tagged.into_iter().map(|(_, h)| h).collect(); } + // `handles` is already in channel-priority order, so walking it yields + // the registrations in that order too. Custom sources have no channel. + #[cfg(feature = "experimental-virtual-package-plugins")] + let virtual_package_plugins = handles + .iter() + .filter_map(|h| match &h.kind { + SubdirKind::Channel { url, platform } => { + let key = (url.clone(), *platform); + self.virtual_package_plugins.remove(&key).map(|plugins| { + let (channel, platform) = key; + SubdirVirtualPackagePlugins { + channel, + platform, + plugins, + } + }) + } + SubdirKind::Custom => None, + }) + .collect(); + let mut repodata: Vec = Vec::with_capacity(handles.len() + usize::from(direct.is_some())); if let Some(d) = direct { @@ -1014,6 +1075,8 @@ impl QueryExecutor { .into_iter() .map(GatewayWarning::from) .collect(), + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins, }) } } diff --git a/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs b/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs index 1e6ff13885..972b24a196 100644 --- a/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs @@ -1,5 +1,7 @@ use crate::gateway::subdir::{PackageRecords, SubdirClient}; use crate::{GatewayError, Reporter}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ChannelRelations, PackageName, RepodataRevisions}; cfg_if::cfg_if! { @@ -34,4 +36,9 @@ impl SubdirClient for RemoteSubdirClient { fn channel_relations(&self) -> Option<&ChannelRelations> { self.sparse.channel_relations() } + + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + self.sparse.virtual_package_plugins() + } } diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs index 369f43d370..9e1c9d1bf7 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs @@ -183,6 +183,8 @@ mod tests { routing::get, }; use rattler_conda_types::{Channel, RepodataRevisions, ShardedRepodata, ShardedSubdirInfo}; + #[cfg(feature = "experimental-virtual-package-plugins")] + use rattler_conda_types::{PackageName, VirtualPackagePlugins}; use rattler_digest::{Sha256, parse_digest_from_hex}; use std::future::IntoFuture; use std::net::SocketAddr; @@ -223,12 +225,17 @@ mod tests { created_at: Some(jiff::Timestamp::now()), repodata_revisions: RepodataRevisions::default(), channel_relations: None, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: mock_virtual_package_plugins(), }, shards, }; // Encode the index as msgpack and compress with zstd - let index_bytes = rmp_serde::to_vec(&sharded_index).unwrap(); + // Named encoding, matching what the indexer writes for real + // channels; positional encoding misaligns as soon as a skipped + // field precedes a present one. + let index_bytes = rmp_serde::to_vec_named(&sharded_index).unwrap(); let compressed_index = zstd::encode_all(index_bytes.as_slice(), 3).unwrap(); let shard_requests = Arc::new(AtomicUsize::new(0)); @@ -310,6 +317,48 @@ mod tests { Truncated, } + /// Registrations served by the mock index, so the sharded path is covered + /// end to end through msgpack encoding and the HTTP fetch. + #[cfg(feature = "experimental-virtual-package-plugins")] + fn mock_virtual_package_plugins() -> VirtualPackagePlugins { + [( + PackageName::new_unchecked("cuda-detect"), + vec![ + PackageName::new_unchecked("__cuda"), + PackageName::new_unchecked("__cuda_arch"), + ], + )] + .into_iter() + .collect() + } + + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_sharded_index_reports_virtual_package_plugins() { + let server = MockShardedServer::new(MockShardResponse::Empty).await; + let cache_dir = tempfile::tempdir().unwrap(); + + let subdir = ShardedSubdir::new( + server.channel(), + "linux-64".to_string(), + rattler_networking::LazyClient::default(), + cache_dir.path().to_path_buf(), + ShardCachePolicy { + action: CacheAction::NoCache, + missing_shards_are_empty: false, + }, + None, + None, + ) + .await + .unwrap(); + + assert_eq!( + subdir.virtual_package_plugins(), + &mock_virtual_package_plugins() + ); + } + #[tokio::test] async fn test_empty_shard_response_error() { let server = MockShardedServer::new(MockShardResponse::Empty).await; diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs index ed8b219b9a..0b7285cdb6 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs @@ -7,6 +7,8 @@ use std::{ }; use rattler_conda_types::Platform; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use super::{ add_trailing_slash, decode_zst_bytes_async, is_missing_sharded_repodata_status, parse_records, @@ -308,6 +310,11 @@ impl SubdirClient for ShardedSubdir { fn channel_relations(&self) -> Option<&ChannelRelations> { self.sharded_repodata.info.channel_relations.as_ref() } + + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + &self.sharded_repodata.info.virtual_package_plugins + } } /// Atomically writes the shard bytes to the cache. diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs index 4d5a2956df..c9d51d7265 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs @@ -1,6 +1,8 @@ use std::sync::Arc; use futures::future::OptionFuture; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ Channel, ChannelRelations, PackageName, RepodataRevisions, ShardedRepodata, }; @@ -177,4 +179,9 @@ impl SubdirClient for ShardedSubdir { fn channel_relations(&self) -> Option<&ChannelRelations> { self.sharded_repodata.info.channel_relations.as_ref() } + + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + &self.sharded_repodata.info.virtual_package_plugins + } } diff --git a/crates/rattler_repodata_gateway/src/gateway/subdir.rs b/crates/rattler_repodata_gateway/src/gateway/subdir.rs index 3b173bfcfc..fbacd591ca 100644 --- a/crates/rattler_repodata_gateway/src/gateway/subdir.rs +++ b/crates/rattler_repodata_gateway/src/gateway/subdir.rs @@ -1,11 +1,15 @@ use std::sync::Arc; use ahash::HashMap; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ChannelRelations, PackageName, RepoDataRecord, RepodataRevisions}; use super::GatewayError; use crate::Reporter; use crate::sparse::empty_repodata_revisions; +#[cfg(feature = "experimental-virtual-package-plugins")] +use crate::sparse::empty_virtual_package_plugins; use coalesced_map::{CoalescedGetError, CoalescedMap}; /// Records for a single package, with precomputed unique dependency strings @@ -114,6 +118,16 @@ impl Subdir { Subdir::NotFound => None, } } + + /// Virtual package detection plugins registered by this subdir, or empty + /// if none are registered / the subdir was not found. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + match self { + Subdir::Found(subdir) => subdir.virtual_package_plugins(), + Subdir::NotFound => empty_virtual_package_plugins(), + } + } } /// Fetches and caches repodata records by package name for a specific @@ -172,6 +186,12 @@ impl SubdirData { pub fn channel_relations(&self) -> Option<&ChannelRelations> { self.client.channel_relations() } + + /// Virtual package detection plugins registered by this subdir. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + self.client.virtual_package_plugins() + } } /// A client that can be used to fetch repodata for a specific subdirectory. @@ -201,6 +221,13 @@ pub trait SubdirClient: Send + Sync { fn channel_relations(&self) -> Option<&ChannelRelations> { None } + + /// Virtual package detection plugins registered by this subdir. Sources + /// that cannot carry the metadata (e.g. custom) keep the empty default. + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + empty_virtual_package_plugins() + } } #[cfg(test)] diff --git a/crates/rattler_repodata_gateway/src/lib.rs b/crates/rattler_repodata_gateway/src/lib.rs index 40cbae4893..4aeca2226f 100644 --- a/crates/rattler_repodata_gateway/src/lib.rs +++ b/crates/rattler_repodata_gateway/src/lib.rs @@ -73,6 +73,8 @@ pub use reporter::{SUPPORTED_REPODATA_REVISION, UnsupportedRepodataRevision}; #[cfg(feature = "gateway")] mod gateway; +#[cfg(all(feature = "gateway", feature = "experimental-virtual-package-plugins"))] +pub use gateway::SubdirVirtualPackagePlugins; #[cfg(feature = "gateway")] pub use gateway::{ CacheClearMode, ChannelConfig, ChannelRelationsMode, ChannelRelationsWarning, diff --git a/crates/rattler_repodata_gateway/src/sparse/mod.rs b/crates/rattler_repodata_gateway/src/sparse/mod.rs index 5893f625c8..23c1bdfba7 100644 --- a/crates/rattler_repodata_gateway/src/sparse/mod.rs +++ b/crates/rattler_repodata_gateway/src/sparse/mod.rs @@ -14,6 +14,8 @@ use std::{ use bytes::Bytes; use itertools::Itertools; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ Channel, ChannelInfo, ChannelRelations, MatchSpec, Matches, PackageName, PackageRecord, RepoDataRecord, RepodataRevisions, UrlOrPath, WhlPackageRecord, compute_package_url, @@ -37,6 +39,14 @@ pub(crate) fn empty_repodata_revisions() -> &'static RepodataRevisions { &EMPTY } +/// Shared empty plugin registrations, returned by accessors when a subdir +/// registers none. +#[cfg(feature = "experimental-virtual-package-plugins")] +pub(crate) fn empty_virtual_package_plugins() -> &'static VirtualPackagePlugins { + static EMPTY: LazyLock = LazyLock::new(VirtualPackagePlugins::new); + &EMPTY +} + /// Defines how different variants of packages are consolidated. #[derive( Default, @@ -496,6 +506,16 @@ impl SparseRepoData { .channel_relations .as_ref() } + + /// Virtual package detection plugins registered in + /// `info.virtual_package_plugins`. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + match &self.inner.borrow_repo_data().info { + Some(info) => &info.virtual_package_plugins, + None => empty_virtual_package_plugins(), + } + } } /// A serde compatible struct that only sparsely parses a repodata.json file. diff --git a/test-data/channels/virtual-package-plugins/linux-64/repodata.json b/test-data/channels/virtual-package-plugins/linux-64/repodata.json new file mode 100644 index 0000000000..b5a15dad16 --- /dev/null +++ b/test-data/channels/virtual-package-plugins/linux-64/repodata.json @@ -0,0 +1,26 @@ +{ + "info": { + "subdir": "linux-64", + "base_url": "../linux-64", + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"], + "rocm-detect": ["__rocm"] + } + }, + "packages.conda": { + "accelerated-1.0.0-h0000000_0.conda": { + "build": "h0000000_0", + "build_number": 0, + "depends": ["__rocm >=6.0"], + "license": "BSD-3-Clause", + "md5": "00000000000000000000000000000000", + "name": "accelerated", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "size": 1000, + "subdir": "linux-64", + "timestamp": 1700000000000, + "version": "1.0.0" + } + }, + "repodata_version": 2 +} diff --git a/test-data/channels/virtual-package-plugins/noarch/repodata.json b/test-data/channels/virtual-package-plugins/noarch/repodata.json new file mode 100644 index 0000000000..f263ef4f87 --- /dev/null +++ b/test-data/channels/virtual-package-plugins/noarch/repodata.json @@ -0,0 +1,12 @@ +{ + "info": { + "subdir": "noarch", + "base_url": "../noarch", + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"], + "rocm-detect": ["__rocm"] + } + }, + "packages.conda": {}, + "repodata_version": 2 +}