diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index b7c44db220..a16ecaedf0 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -34,6 +34,10 @@ rustls = [ s3 = ["rattler_networking/s3", "rattler_upload/s3"] gcs = ["rattler_networking/gcs"] oauth = ["rattler/oauth"] +experimental-virtual-package-plugins = [ + "rattler_repodata_gateway/experimental-virtual-package-plugins", + "rattler_index/experimental-virtual-package-plugins", +] [dependencies] anyhow = { workspace = true } diff --git a/crates/rattler-bin/src/commands/virtual_packages.rs b/crates/rattler-bin/src/commands/virtual_packages.rs index a1b6fe19a3..f7736316b0 100644 --- a/crates/rattler-bin/src/commands/virtual_packages.rs +++ b/crates/rattler-bin/src/commands/virtual_packages.rs @@ -1,17 +1,111 @@ use miette::IntoDiagnostic; use rattler_conda_types::GenericVirtualPackage; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::Platform; use rattler_virtual_packages::VirtualPackageOverrides; /// Print detected virtual packages. #[derive(Debug, clap::Parser)] -pub struct Opt {} +#[cfg_attr( + feature = "experimental-virtual-package-plugins", + clap(after_help = r#"Examples: + rattler virtual-packages + rattler virtual-packages -c ./test-data/channels/virtual-package-plugins"#) +)] +pub struct Opt { + /// Channels to list registered virtual package plugins for + #[cfg(feature = "experimental-virtual-package-plugins")] + #[clap(short, long)] + channels: Vec, -pub fn virtual_packages(_opt: Opt) -> miette::Result<()> { + /// Platforms to read registrations for [default: current and noarch] + #[cfg(feature = "experimental-virtual-package-plugins")] + #[clap(short, long)] + platforms: Vec, +} + +pub async fn virtual_packages(opt: Opt, offline: bool) -> miette::Result<()> { let virtual_packages = rattler_virtual_packages::VirtualPackage::detect(&VirtualPackageOverrides::from_env()) .into_diagnostic()?; for package in virtual_packages { println!("{}", GenericVirtualPackage::from(package.clone())); } + + #[cfg(feature = "experimental-virtual-package-plugins")] + print_plugins(&opt.channels, &opt.platforms, offline).await?; + + #[cfg(not(feature = "experimental-virtual-package-plugins"))] + let _ = (opt, offline); + + Ok(()) +} + +/// Prints the plugin registrations declared by each `(channel, platform)` +/// subdirectory, in the order the channels were given. +#[cfg(feature = "experimental-virtual-package-plugins")] +async fn print_plugins( + channels: &[String], + platforms: &[Platform], + offline: bool, +) -> miette::Result<()> { + use std::{collections::HashMap, env}; + + use itertools::Itertools; + use rattler_conda_types::{Channel, ChannelConfig, PackageName}; + use rattler_repodata_gateway::{Gateway, SourceConfig}; + + if channels.is_empty() { + return Ok(()); + } + + let channel_config = + ChannelConfig::default_with_root_dir(env::current_dir().into_diagnostic()?); + let channels = channels + .iter() + .map(|channel| Channel::from_str(channel, &channel_config)) + .collect::, _>>() + .into_diagnostic()?; + + let platforms = if platforms.is_empty() { + vec![Platform::current(), Platform::NoArch] + } else { + platforms.to_vec() + }; + + let gateway = Gateway::builder() + .with_client(super::client::create_client_with_middleware(offline)?) + .with_channel_config(rattler_repodata_gateway::ChannelConfig { + default: SourceConfig { + cache_action: super::client::repodata_cache_action(offline), + ..SourceConfig::default() + }, + per_channel: HashMap::new(), + }) + .finish(); + + for channel in &channels { + for platform in &platforms { + let plugins = gateway + .virtual_package_plugins(channel, *platform) + .await + .into_diagnostic()?; + if plugins.is_empty() { + continue; + } + println!( + "\nvirtual package plugins in {} [{platform}]:", + channel.canonical_name() + ); + for (plugin, provided) in &plugins { + println!( + " {} provides {}", + plugin.as_source(), + provided.iter().map(PackageName::as_source).join(", ") + ); + } + } + } + Ok(()) } diff --git a/crates/rattler-bin/src/main.rs b/crates/rattler-bin/src/main.rs index 141db8d627..0deceab30f 100644 --- a/crates/rattler-bin/src/main.rs +++ b/crates/rattler-bin/src/main.rs @@ -124,7 +124,9 @@ async fn async_main() -> miette::Result<()> { Command::Solve(opts) => commands::solve::solve(opts, offline).await, Command::List(opts) => commands::list::list(opts).await, Command::ShellHook(opts) => commands::shell_hook::shell_hook(opts).await, - Command::VirtualPackages(opts) => commands::virtual_packages::virtual_packages(opts), + Command::VirtualPackages(opts) => { + commands::virtual_packages::virtual_packages(opts, offline).await + } Command::InstallMenu(opts) => commands::menu::install_menu(opts).await, Command::RemoveMenu(opts) => commands::menu::remove_menu(opts).await, Command::Run(opts) => commands::run::run(opts).await, diff --git a/crates/rattler_conda_types/Cargo.toml b/crates/rattler_conda_types/Cargo.toml index 923c416982..1042f0e31c 100644 --- a/crates/rattler_conda_types/Cargo.toml +++ b/crates/rattler_conda_types/Cargo.toml @@ -13,6 +13,7 @@ readme.workspace = true [features] default = ["rayon"] +experimental-virtual-package-plugins = [] [package.metadata.docs.rs] all-features = true diff --git a/crates/rattler_conda_types/src/lib.rs b/crates/rattler_conda_types/src/lib.rs index 49b39f503f..57fc49e3bb 100644 --- a/crates/rattler_conda_types/src/lib.rs +++ b/crates/rattler_conda_types/src/lib.rs @@ -64,6 +64,8 @@ pub use platform::{Arch, ParseArchError, ParsePlatformError, Platform}; pub use prefix_data::PrefixData; pub use prefix_record::PrefixRecord; pub use record_traits::HasArtifactIdentificationRefs; +#[cfg(feature = "experimental-virtual-package-plugins")] +pub use repo_data::VirtualPackagePlugins; pub use repo_data::{ ChannelInfo, ChannelRelations, ConvertSubdirError, PackageRecord, RecordFromPath, RepoData, RepodataRevision, RepodataRevisionInfo, RepodataRevisionMetadata, RepodataRevisions, diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 85d81d2156..7e20e9e841 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -107,8 +107,26 @@ pub struct ChannelInfo { /// [CEP-42](https://github.com/conda/ceps/blob/main/cep-0042.md). #[serde(default, skip_serializing_if = "ChannelRelations::is_none_or_empty")] pub channel_relations: Option, + + /// Virtual package detection plugins registered by the channel. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[serde_as( + deserialize_as = "IndexMap>" + )] + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub virtual_package_plugins: VirtualPackagePlugins, } +/// Virtual package detection plugins registered by a channel: the name of the +/// package providing the plugin, mapped to the virtual packages it provides. +/// +/// One plugin may provide several virtual packages, e.g. a `cuda-detect` +/// providing both `__cuda` and `__cuda_arch`. The executable to run is named +/// after the plugin package. Inverting the map is left to the caller: the +/// reverse direction is many-to-many. +#[cfg(feature = "experimental-virtual-package-plugins")] +pub type VirtualPackagePlugins = IndexMap>; + /// Repodata revisions keyed by revision, mirroring the `vN` dictionary of the /// CEP draft . Keying encodes /// uniqueness; insertion order is preserved. @@ -1084,6 +1102,8 @@ mod test { package::DistArchiveIdentifier, repo_data::{compute_package_url, determine_subdir}, }; + #[cfg(feature = "experimental-virtual-package-plugins")] + use crate::{PackageName, repo_data::VirtualPackagePlugins}; // isl-0.12.2-1.tar.bz2 // gmp-5.1.2-6.tar.bz2 @@ -1166,6 +1186,8 @@ mod test { base: Some("../conda-forge".to_string()), overrides: None, }), + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }), packages: IndexMap::default(), conda_packages: IndexMap::default(), @@ -1188,6 +1210,8 @@ mod test { base_url: None, repodata_revisions: IndexMap::default(), channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }), packages: IndexMap::default(), conda_packages: IndexMap::default(), @@ -1199,6 +1223,99 @@ mod test { } } + #[cfg(feature = "experimental-virtual-package-plugins")] + #[test] + fn test_virtual_package_plugins() { + // A single plugin may provide several virtual packages; the order of + // both the plugins and their virtual packages is preserved. + let raw = r#"{ + "info": { + "subdir": "linux-64", + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"], + "rocm-detect": ["__rocm"] + } + }, + "packages": {}, + "packages.conda": {} + }"#; + let repodata: RepoData = serde_json::from_str(raw).unwrap(); + let plugins = &repodata.info.as_ref().unwrap().virtual_package_plugins; + + assert_eq!( + plugins + .keys() + .map(PackageName::as_source) + .collect::>(), + ["cuda-detect", "rocm-detect"] + ); + assert_eq!( + plugins[&PackageName::new_unchecked("cuda-detect")] + .iter() + .map(PackageName::as_source) + .collect::>(), + ["__cuda", "__cuda_arch"] + ); + assert_eq!( + plugins[&PackageName::new_unchecked("rocm-detect")] + .iter() + .map(PackageName::as_source) + .collect::>(), + ["__rocm"] + ); + + let json = serde_json::to_string(&repodata).unwrap(); + assert!(json.contains("\"virtual_package_plugins\"")); + assert_eq!(serde_json::from_str::(&json).unwrap(), repodata); + + let without = RepoData { + version: Some(2), + info: Some(ChannelInfo { + subdir: Some("linux-64".to_string()), + base_url: None, + repodata_revisions: IndexMap::default(), + channel_relations: None, + virtual_package_plugins: VirtualPackagePlugins::default(), + }), + packages: IndexMap::default(), + conda_packages: IndexMap::default(), + v3: V3Packages::default(), + removed: ahash::HashSet::default(), + }; + assert!( + !serde_json::to_string(&without) + .unwrap() + .contains("virtual_package_plugins") + ); + } + + /// Names are deserialized unchecked, so a channel publishing a malformed + /// name does not render the entire repodata unusable. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[test] + fn test_virtual_package_plugins_malformed_name() { + let raw = r#"{ + "info": { + "subdir": "linux-64", + "virtual_package_plugins": { "invalid$plugin": ["__rocm"] } + }, + "packages": {}, + "packages.conda": {} + }"#; + let repodata: RepoData = serde_json::from_str(raw).unwrap(); + assert_eq!( + repodata + .info + .as_ref() + .unwrap() + .virtual_package_plugins + .keys() + .map(PackageName::as_source) + .collect::>(), + ["invalid$plugin"] + ); + } + #[test] fn test_repodata_revisions() { let raw = r#"{ diff --git a/crates/rattler_conda_types/src/repo_data/sharded.rs b/crates/rattler_conda_types/src/repo_data/sharded.rs index e0776c968c..b4af813e6a 100644 --- a/crates/rattler_conda_types/src/repo_data/sharded.rs +++ b/crates/rattler_conda_types/src/repo_data/sharded.rs @@ -2,7 +2,11 @@ use crate::PackageRecord; use crate::package::DistArchiveIdentifier; +#[cfg(feature = "experimental-virtual-package-plugins")] +use crate::repo_data::VirtualPackagePlugins; use crate::repo_data::{ChannelRelations, RepodataRevisions, V3Packages}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use crate::utils::serde::DeserializeFromStrUnchecked; use crate::utils::serde::{sort_index_map_alphabetically, sort_set_alphabetically}; use indexmap::IndexMap; use jiff::Timestamp; @@ -60,6 +64,14 @@ pub struct ShardedSubdirInfo { /// [CEP-42](https://github.com/conda/ceps/blob/main/cep-0042.md). #[serde(default, skip_serializing_if = "ChannelRelations::is_none_or_empty")] pub channel_relations: Option, + + /// Virtual package detection plugins registered by the channel. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[serde_as( + deserialize_as = "IndexMap>" + )] + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub virtual_package_plugins: VirtualPackagePlugins, } #[cfg(test)] @@ -131,11 +143,48 @@ mod tests { created_at: None, repodata_revisions: IndexMap::default(), channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }; let json = serde_json::to_string(&info).unwrap(); assert!(!json.contains("channel_relations")); } } + + #[cfg(feature = "experimental-virtual-package-plugins")] + #[test] + fn test_sharded_subdir_info_virtual_package_plugins() { + let raw = r#"{ + "subdir": "linux-64", + "base_url": "./", + "shards_base_url": "./shards/", + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"] + } + }"#; + let info: ShardedSubdirInfo = serde_json::from_str(raw).unwrap(); + assert_eq!( + info.virtual_package_plugins[&PackageName::new_unchecked("cuda-detect")] + .iter() + .map(PackageName::as_source) + .collect::>(), + ["__cuda", "__cuda_arch"] + ); + + let json = serde_json::to_string(&info).unwrap(); + assert!(json.contains("\"virtual_package_plugins\"")); + + // Omitted entirely when no plugins are registered. + let info = ShardedSubdirInfo { + virtual_package_plugins: VirtualPackagePlugins::default(), + ..info + }; + assert!( + !serde_json::to_string(&info) + .unwrap() + .contains("virtual_package_plugins") + ); + } } /// An individual shard that contains repodata for a single package name. diff --git a/crates/rattler_index/Cargo.toml b/crates/rattler_index/Cargo.toml index fa142ad175..ca71aae2e4 100644 --- a/crates/rattler_index/Cargo.toml +++ b/crates/rattler_index/Cargo.toml @@ -28,6 +28,10 @@ rustls = [ "opendal/reqwest-rustls-tls", ] s3 = ["opendal/services-s3", "dep:rattler_s3"] +experimental-virtual-package-plugins = [ + "rattler_conda_types/experimental-virtual-package-plugins", + "rattler_repodata_gateway/experimental-virtual-package-plugins", +] [[bin]] name = "rattler-index" diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index be5c547bb2..c65d0c03c0 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -28,6 +28,8 @@ use opendal::layers::RetryLayer; #[cfg(feature = "s3")] use opendal::services::S3Config; use opendal::{Configurator, Operator, services::FsConfig}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ ChannelInfo, ChannelRelations, PackageRecord, PatchInstructions, Platform, RepoData, Shard, ShardedRepodata, ShardedSubdirInfo, UrlOrPath, V3Packages, WhlPackageRecord, @@ -931,6 +933,8 @@ async fn index_subdir_inner( base_url: channel_metadata.base_url, repodata_revisions: repodata_revisions_for_packages(&repodata_revisions, &v3), channel_relations: channel_metadata.channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }), packages, conda_packages, @@ -1281,6 +1285,8 @@ pub async fn write_repodata( created_at: Some(jiff::Timestamp::now()), repodata_revisions: sharded_repodata_revisions, channel_relations: sharded_channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }, shards: shards .iter() @@ -1719,6 +1725,8 @@ pub async fn ensure_channel_initialized_with_channel_metadata( base_url: channel_metadata.base_url, repodata_revisions: RepodataRevisions::new(), channel_relations: channel_metadata.channel_relations, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: VirtualPackagePlugins::default(), }), packages: IndexMap::default(), conda_packages: IndexMap::default(), diff --git a/crates/rattler_repodata_gateway/Cargo.toml b/crates/rattler_repodata_gateway/Cargo.toml index 5b0c0bec85..41543b833e 100644 --- a/crates/rattler_repodata_gateway/Cargo.toml +++ b/crates/rattler_repodata_gateway/Cargo.toml @@ -110,6 +110,7 @@ native-tls = ['reqwest/native-tls', 'rattler_networking/native-tls', 'rattler_ca rustls = ['reqwest/rustls', 'rattler_networking/rustls', 'rattler_cache/rustls', 'rattler_redaction/rustls'] sparse = ["rattler_conda_types", "memmap2", "self_cell", "superslice", "itertools", "serde_json/raw_value"] gateway = ["sparse", "http", "http-cache-semantics", "parking_lot", "async-trait", "rattler_package_streaming"] +experimental-virtual-package-plugins = ["rattler_conda_types?/experimental-virtual-package-plugins"] [package.metadata.docs.rs] features = ["sparse", "gateway"] diff --git a/crates/rattler_repodata_gateway/docs/virtual-package-plugins.md b/crates/rattler_repodata_gateway/docs/virtual-package-plugins.md new file mode 100644 index 0000000000..78a7b99fab --- /dev/null +++ b/crates/rattler_repodata_gateway/docs/virtual-package-plugins.md @@ -0,0 +1,289 @@ +# User-Specified Virtual Packages + +## Proposal for Conda Channel-Defined Virtual Package Plugins + +### Status + +The **metadata half** of this proposal is implemented in rattler behind the +`experimental-virtual-package-plugins` cargo feature: the registration is parsed from repodata and +handed to callers. Nothing is visible unless that feature is enabled, and no plugin is fetched or +executed by rattler today. + +| Part | State | +| --- | --- | +| `info.virtual_package_plugins` parsing (`repodata.json` and sharded index) | Implemented | +| `Gateway::virtual_package_plugins(channel, platform)` accessor | Implemented | +| Registrations on `RepoDataQueryOutput`, per channel subdir | Implemented | +| `rattler virtual-packages -c ` for manual inspection | Implemented | +| Conflict resolution across channels | Deliberately not done -- reported as declared, caller decides | +| Plugin protocol types, execution, result caching | Not implemented | +| Solver injection, `CONDA_OVERRIDE_*`, lockfile representation | Not implemented | +| Trust / opt-in model | Open, blocks execution | +| prefix.dev upload validation | Not implemented (server side) | + +`rattler_index` does not yet propagate the field: with the feature off it drops +`info.virtual_package_plugins` on a repodata round-trip, and with the feature on it writes an empty +map. Only a channel server publishing the field directly exercises the path today. + +### Problem + +Today, virtual packages like `__cuda` are hardcoded in the solver client. This made sense when NVIDIA +was the only accelerator that mattered, but the hardware landscape is diversifying fast. AMD ROCm, +Intel oneAPI, and other accelerator stacks each have their own driver versions, runtime libraries, and +capability matrices. Hardcoding detection logic for every new accelerator in every client release +doesn't scale. Channel operators who ship packages targeting these accelerators need a way to define +virtual packages like `__rocm` or `__oneapi` without waiting for upstream client changes. + +### Proposal + +We introduce a plugin-based virtual package system where channel operators on prefix.dev define custom +virtual packages backed by detection plugins. The solver treats them identically to built-in virtual +packages -- packages can depend on `__rocm >= 6.0` or `__oneapi >= 2025.1` the same way they depend on +`__cuda` today. + +The system has two parts: + +1. **Channel-side: plugin registration and validation** +2. **Client-side: plugin execution and caching** + +--- + +### 1. Channel-Side: Plugin Registration + +Channel operators register virtual package plugins as part of their channel configuration on +prefix.dev. Each registration names a conda package containing the detection logic and lists the +virtual packages that plugin provides. + +During package upload, prefix.dev validates that any virtual package dependency declared in a +package's metadata has a corresponding plugin registered in the channel. Uploads referencing undefined +virtual packages are rejected. + +The registration is published in the channel's `repodata.json` under a new `info.virtual_package_plugins` +field, keyed by **plugin package name**: + +```json +{ + "info": { + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"], + "rocm-detect": ["__rocm"] + } + }, + "packages": { ... } +} +``` + +Keying by plugin rather than by virtual package is deliberate. The reverse direction -- +`{"__cuda": "cuda-detect", "__cuda_arch": "cuda-detect"}` -- registers the same detector twice and +gives the client no way to know the two entries are one program doing one piece of work. Keying by +plugin makes "several virtual packages from one plugin" the ordinary case, which is what `__cuda` and +`__cuda_arch` actually need. + +The client resolves the plugin package from the same channel, picking the latest available version. No +version constraint is expressible in the registration. + +The `virtual_package -> plugin` mapping is *derived* by the client if it needs it. That inversion is +many-to-many: two plugins in one channel, or plugins in different channels, may each claim `__rocm`. +Nothing in the metadata prevents it and the client must resolve it. + +The same field is published in the sharded repodata index (`repodata_shards.msgpack.zst`) under +`info`, so sharded channels carry the registration too. + +**Per-subdir, not channel-wide.** `info` lives in each subdir's repodata, so the registration must be +repeated in every subdir of a channel, and different subdirs *may* declare different registrations. +Consumers see one entry per subdir and may union them. A channel-wide location would be better and +needs a CEP. + +**Lenient parsing.** Plugin and virtual package names are parsed without validation, so a channel +publishing a malformed name does not make the whole `repodata.json` unusable. + +### 2. Client-Side: Plugin Execution and Caching + +*Not implemented. This section is the remaining proposal; the metadata above constrains it as noted.* + +When pixi resolves an environment and encounters a dependency on a virtual package provided by a +registered plugin, it: + +1. **Fetches and installs the plugin package** into an isolated, internal environment (separate from + the user's env, cached across solves). + + The plugin environment must be solved using **built-in virtual packages only**. Resolving a + plugin's own dependencies is itself a solve against a channel whose plugin data is not yet + available; restricting that solve to built-ins is what stops the recursion. + +2. **Executes the plugin once**, and the plugin reports on every virtual package it was registered + for. It inspects the local system (checks for driver files, queries `rocm-smi`, reads `/sys/` + entries, etc.) and returns a JSON array: + +```json +{ + "virtual_packages": [ + { "name": "__cuda", "version": "12.4" }, + { "name": "__cuda_arch", "version": "0", "build_string": "sm_89" } + ], + "cache": { + "ttl_seconds": 86400, + "watch_paths": [ + "/opt/rocm/lib/libamdhip64.so", + "/sys/module/amdgpu/version" + ] + } +} +``` + + The array shape follows from one entry point per plugin package: a single `cuda-detect` run has to + be able to report both `__cuda` and `__cuda_arch`. `build_string` is optional and exists because + `__archspec` and `__cuda_arch` carry their information in the build string rather than the version; + without it those cannot be expressed as plugins at all. + +3. **Caches the result** according to the plugin's cache policy: + - **`ttl_seconds`**: how long the cached value is valid. + - **`watch_paths`**: file globs to monitor. If any file's existence or modification time changes, + the cache is invalidated and the plugin re-runs. This handles driver upgrades between solves + without requiring TTL expiry. + + Caches must be keyed on **(channel, plugin package name)**, not the package name alone: names are + unique within a channel, but two channels may each ship a different `cuda-detect`. + +4. **Injects the detected virtual packages** into the solver's virtual package set alongside the + standard ones (`__cuda`, `__glibc`, etc.). A plugin may only inject virtual packages the channel + registered for it; anything else is discarded. Virtual packages the plugin omits are treated as + absent -- the solver simply won't select packages that require them. + +### Plugin Interface + +Plugins are simple executables. **The entry point is the plugin package name**: package `cuda-detect` +ships an executable `cuda-detect`. Package names are unique within a channel and a JSON object cannot +repeat a key, so the entry point needs no separate metadata field, and conda already puts executables +on the environment's `PATH` (`bin/`, `Scripts/`) so no path needs declaring either. + +The contract: + +- **stdin**: empty +- **stdout**: JSON object as shown above +- **stderr**: diagnostic output (logged by pixi at debug level) +- **exit 0**: the plugin ran; `virtual_packages` lists what it detected, and may be empty +- **exit non-zero**: plugin failure (pixi logs a warning, treats all of the plugin's virtual packages + as absent) + +This replaces the draft's earlier three-way exit code contract (`0` present / `1` absent / `2+` +failure). With several virtual packages per plugin, presence is per-entry in the output array and can +no longer be carried by a single exit status: `__cuda` may be present while `__cuda_arch` is not. +**This needs sign-off** -- it is the one part of the interface the implemented metadata forced to +change. + +Plugins can be compiled binaries, shell scripts, or anything else that fits in a conda package. +Keeping the interface this simple means detection for a new accelerator is a single small package with +a shell script that checks a few paths. + +### Gateway Integration + +Implemented. The repodata gateway parses `info.virtual_package_plugins` and reports it; it does not +execute plugins -- it doesn't know what hardware the client has -- and it does not resolve conflicts. + +There are two ways to read the registrations: + +`Gateway::virtual_package_plugins(channel, platform)` returns the map for one subdirectory. It takes +no specs, which is the point: the plugin package names only exist inside the metadata being fetched, +so there is nothing to query for until it has been read. It mirrors `Gateway::channel_relations`, +reusing the internal subdir cache, and yields an empty map for a subdirectory that registers none or +does not exist. + +`RepoDataQueryOutput::virtual_package_plugins` returns one entry per channel subdir that declared a +registration, carrying the channel, the subdir platform, and the plugin-to-virtual-packages map, +ordered by resolved channel priority (including any CEP-42 relation-derived ordering). This is the +view a solve sees, so it also covers channels discovered through CEP-42 that the caller never named. + +Duplicate claims are preserved verbatim in both: two channels each claiming `__rocm`, or two plugins +within one channel each claiming `__rocm`, all come back, and no warning is raised. Deciding which +plugin wins is the caller's job. + +For manual inspection, `rattler virtual-packages -c ` prints the registrations a channel +declares. `test-data/channels/virtual-package-plugins` is a local fixture to point it at, since no +channel publishes the field yet. + +All of this is behind the `experimental-virtual-package-plugins` feature. With the feature off the +gateway's public API and its serialized output are unchanged. + +### Example: Supporting AMD ROCm + +A channel operator shipping packages compiled against ROCm: + +1. Creates a `rocm-detect` conda package containing a shell script named `rocm-detect` that checks for + `/opt/rocm/.info/version` and parses the ROCm version. +2. Registers `rocm-detect -> ["__rocm"]` in their channel config on prefix.dev. +3. Uploads packages with `__rocm >= 6.0` in their run dependencies. +4. When a user with ROCm 6.1.2 installed runs `pixi install`, pixi fetches the plugin, runs it, + discovers ROCm 6.1.2, and the solver selects the appropriate package variants. +5. A user without ROCm gets packages built for CPU fallback (or an unsatisfiable error if no fallback + exists). + +The same pattern works for Intel oneAPI, custom FPGA toolchains, or any other hardware capability that +packages need to select against. + +### Example: One Plugin, Several Virtual Packages + +A `cuda-detect` package registered as `cuda-detect -> ["__cuda", "__cuda_arch"]` queries the driver +once and reports both the driver version and the compute capability: + +```json +{ + "virtual_packages": [ + { "name": "__cuda", "version": "12.4" }, + { "name": "__cuda_arch", "version": "0", "build_string": "sm_89" } + ] +} +``` + +On a machine with no NVIDIA driver the same plugin exits 0 with an empty `virtual_packages` array. +Under the draft's original one-plugin-per-virtual-package scheme this needed two packages, or one +package with two entry points repeating the same driver query. + +### Settled Decisions + +1. **Registration is keyed by plugin package name**, mapping to the list of virtual packages it + provides. +2. **The entry point is the plugin package name.** No entry-point field in the metadata; uniqueness + within a channel comes for free. +3. **No package-record changes.** The registration lives entirely in `info`; `PackageRecord` and + `index.json` are untouched, so a client learns what a plugin provides without fetching the plugin's + record first. +4. **No version constraints in the registration.** Bare package name, latest version. +5. **The gateway reports, it does not decide.** Registrations come back per subdir in channel-priority + order with duplicates intact. +6. **Plugin identity is (channel, package name)** for caching and conflict resolution. +7. **Everything is behind an experimental cargo feature** and invisible when it is off. + +### Open Questions + +1. **Trust and governance.** Execution runs channel-supplied code during solve. Users should have to + opt in, and the shape of that opt-in is unsettled: a single global switch, or a per-channel + allowlist. The nearest precedent in rattler is `run_post_link_scripts`, a two-state setting whose + opt-in value is named `insecure` and which defaults to off. This blocks the executor. +2. **Cache invalidation of the plugin environment.** `ttl_seconds` and `watch_paths` cover the + detection *result*. Setting up the isolated environment is the expensive part, and knowing when + that environment is stale is a separate question. +3. **Cross-installing for another platform.** Detection is inherently host-only. Built-in virtual + packages have `detect_for_platform` with documented cross-compilation defaults; plugins have no + equivalent, and it is not clear what running a host plugin means when solving for a different + target. +4. **Overrides and opt-out.** Users should be able to override or disable a specific virtual package + (e.g. skip detection and assert `__rocm 6.1.2`). Built-ins use `CONDA_OVERRIDE_*`; the naming for + plugin-provided packages, especially with sub-keys like `__cuda_arch` and with several channels + registering the same name, is undecided. +5. **Reproducibility and lockfiles.** Whether the plugin version that produced a detection is recorded + in the lock file, and when plugins are updated. Current leaning: always use the latest available and + do not lock it, but this is unresolved. +6. **Channel-wide storage.** `info` is per-subdir, so the registration is duplicated across subdirs. + A channel-wide metadata location would fix this and needs a CEP. +7. **Channel relations and overriding.** Whether a channel may register a plugin for a virtual package + its base channel already covers (e.g. a private channel overriding `__glibc`), and whether such an + override should affect the base channel. CEP-42 relations already give the gateway a priority + order; the policy question is untouched. +8. **Plugin dependencies.** Detection plugins should be self-contained, but if one needs a shared + library to query a driver API, those deps are resolved from the same channel. Solving the plugin + environment with built-in virtual packages only (see above) breaks the bootstrap recursion; the + remaining risk is ordinary dependency conflict. +9. **Versioning semantics.** Virtual package versions should follow conda version ordering so that + constraints like `__rocm >= 6.0, < 7` work as expected. +10. **wheelnext.** Worth looking at closely -- they are solving essentially the same problem. diff --git a/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs b/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs index ad1d32bdd0..881ef9e269 100644 --- a/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs +++ b/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs @@ -1,5 +1,7 @@ use std::{path::Path, sync::Arc}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{Channel, ChannelRelations, PackageName, RepodataRevisions}; use crate::{ @@ -115,4 +117,9 @@ impl SubdirClient for LocalSubdirClient { fn channel_relations(&self) -> Option<&ChannelRelations> { self.sparse.channel_relations() } + + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + self.sparse.virtual_package_plugins() + } } diff --git a/crates/rattler_repodata_gateway/src/gateway/mod.rs b/crates/rattler_repodata_gateway/src/gateway/mod.rs index c29aa5ab3c..d87540123c 100644 --- a/crates/rattler_repodata_gateway/src/gateway/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/mod.rs @@ -32,9 +32,13 @@ use coalesced_map::{CoalescedGetError, CoalescedMap}; pub use error::GatewayError; #[cfg(feature = "indicatif")] pub use indicatif::{IndicatifReporter, IndicatifReporterBuilder}; +#[cfg(feature = "experimental-virtual-package-plugins")] +pub use query::SubdirVirtualPackagePlugins; pub use query::{NamesQuery, NamesQueryOutput, RepoDataQuery, RepoDataQueryOutput}; #[cfg(not(target_arch = "wasm32"))] use rattler_cache::package_cache::PackageCache; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{Channel, ChannelRelations, MatchSpec, Platform, RepoDataRecord}; use rattler_networking::LazyClient; pub use repo_data::RepoData; @@ -216,6 +220,35 @@ impl Gateway { } } + /// Returns the virtual package detection plugins registered by the + /// given `(channel, platform)` subdirectory, keyed by the name of the + /// package providing the plugin. Empty if the subdirectory registers + /// none or doesn't exist. + /// + /// Needs no specs, unlike [`Gateway::query`], because the plugin + /// packages cannot be named until this metadata has been read. + /// + /// Reuses the internal subdir cache: if the pair has already been + /// fetched by a [`Gateway::query`] this is free. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub async fn virtual_package_plugins( + &self, + channel: &Channel, + platform: Platform, + ) -> Result { + match self + .inner + .get_or_create_subdir(channel, platform, None) + .await + { + Ok(subdir) => Ok(subdir.virtual_package_plugins().clone()), + // As above: noarch reports its absence as an error rather + // than `NotFound`, so an empty map holds for all platforms. + Err(GatewayError::SubdirNotFoundError(_)) => Ok(VirtualPackagePlugins::default()), + Err(err) => Err(err), + } + } + /// Ensure that given repodata records contain `RunExportsJson`. pub async fn ensure_run_exports( &self, @@ -2617,6 +2650,238 @@ mod test { ); } + /// Writes a linux-64 subdir whose `info.virtual_package_plugins` is the + /// given JSON object body, e.g. `"cuda-detect": ["__cuda"]`. + #[cfg(feature = "experimental-virtual-package-plugins")] + fn write_test_subdir_with_plugins(root: &std::path::Path, pkg: &str, plugins: &str) { + let subdir = root.join("linux-64"); + std::fs::create_dir_all(&subdir).unwrap(); + let json = format!( + r#"{{ + "info": {{ + "subdir": "linux-64", + "virtual_package_plugins": {{{plugins}}} + }}, + "packages.conda": {{ + "{pkg}-1.0.0-0.conda": {{ + "build": "0", + "build_number": 0, + "depends": [], + "md5": "00000000000000000000000000000000", + "name": "{pkg}", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "size": 1000, + "subdir": "linux-64", + "timestamp": 1700000000000, + "version": "1.0.0" + }} + }} +}}"# + ); + std::fs::write(subdir.join("repodata.json"), json).unwrap(); + } + + /// One reported registration as + /// `(channel_suffix, platform, [(plugin, [virtual packages])])`. + #[cfg(feature = "experimental-virtual-package-plugins")] + type FlatPlugins = Vec<(String, Platform, Vec<(String, Vec)>)>; + + /// Flattens the reported registrations for order-sensitive comparison. + #[cfg(feature = "experimental-virtual-package-plugins")] + fn flatten_plugins(output: &crate::RepoDataQueryOutput) -> FlatPlugins { + output + .virtual_package_plugins + .iter() + .map(|entry| { + ( + entry.channel.url().path().trim_matches('/').to_string(), + entry.platform, + entry + .plugins + .iter() + .map(|(plugin, provided)| { + ( + plugin.as_source().to_string(), + provided.iter().map(|v| v.as_source().to_string()).collect(), + ) + }) + .collect(), + ) + }) + .collect() + } + + /// A channel registering one plugin that provides several virtual packages + /// is reported with its channel and subdir. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_reported_per_subdir() { + let dir = tempfile::tempdir().unwrap(); + let a = dir.path().join("a"); + write_test_subdir_with_plugins(&a, "shared", r#""cuda-detect": ["__cuda", "__cuda_arch"]"#); + + let server = SimpleChannelServer::new(dir.path()).await; + let output = Gateway::new() + .query( + vec![Channel::from_url(server.url().join("a/").unwrap())], + vec![Platform::Linux64], + vec![MatchSpec::from_str("shared", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + assert_eq!( + flatten_plugins(&output), + vec![( + "a".to_string(), + Platform::Linux64, + vec![( + "cuda-detect".to_string(), + vec!["__cuda".to_string(), "__cuda_arch".to_string()], + )], + )] + ); + assert!(output.warnings.is_empty(), "{:?}", output.warnings); + } + + /// Two channels claiming the same virtual package are both reported, in + /// channel-priority order. Resolving the conflict is the caller's job, so + /// the gateway must not drop either one or warn about it. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_conflicting_channels_both_reported() { + let dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + &dir.path().join("a"), + "shared", + r#""a-detect": ["__rocm"]"#, + ); + write_test_subdir_with_plugins( + &dir.path().join("b"), + "shared", + r#""b-detect": ["__rocm"]"#, + ); + + let server = SimpleChannelServer::new(dir.path()).await; + let a = Channel::from_url(server.url().join("a/").unwrap()); + let b = Channel::from_url(server.url().join("b/").unwrap()); + + let output = Gateway::new() + .query( + vec![a, b], + vec![Platform::Linux64], + vec![MatchSpec::from_str("shared", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + let reported: Vec = flatten_plugins(&output) + .into_iter() + .map(|(channel, _, plugins)| format!("{channel}:{}", plugins[0].0)) + .collect(); + assert_eq!(reported, vec!["a:a-detect", "b:b-detect"]); + assert!(output.warnings.is_empty(), "{:?}", output.warnings); + } + + /// Two plugins in one channel may claim the same virtual package; both + /// survive because inverting the mapping is the caller's job. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_duplicate_claim_within_channel() { + let dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + &dir.path().join("a"), + "shared", + r#""first-detect": ["__rocm"], "second-detect": ["__rocm"]"#, + ); + + let server = SimpleChannelServer::new(dir.path()).await; + let output = Gateway::new() + .query( + vec![Channel::from_url(server.url().join("a/").unwrap())], + vec![Platform::Linux64], + vec![MatchSpec::from_str("shared", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + let plugins = flatten_plugins(&output); + assert_eq!( + plugins[0].2, + vec![ + ("first-detect".to_string(), vec!["__rocm".to_string()]), + ("second-detect".to_string(), vec!["__rocm".to_string()]), + ] + ); + } + + /// Registrations are read off the subdir itself, so a spec that matches no + /// record still reports them. Callers cannot query for the plugin packages + /// by name: the registration is the only place those names come from. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_reported_for_unmatched_spec() { + let dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + &dir.path().join("a"), + "shared", + r#""cuda-detect": ["__cuda"]"#, + ); + + let server = SimpleChannelServer::new(dir.path()).await; + let output = Gateway::new() + .query( + vec![Channel::from_url(server.url().join("a/").unwrap())], + vec![Platform::Linux64], + vec![MatchSpec::from_str("no-such-package", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + assert!( + output.iter().all(RepoData::is_empty), + "the spec must not match any record" + ); + assert_eq!( + flatten_plugins(&output), + vec![( + "a".to_string(), + Platform::Linux64, + vec![("cuda-detect".to_string(), vec!["__cuda".to_string()])], + )] + ); + } + + /// A channel without the metadata reports nothing. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_virtual_package_plugins_absent() { + let dir = tempfile::tempdir().unwrap(); + write_test_subdir(&dir.path().join("a"), "shared", "1.0.0", None, None); + + let server = SimpleChannelServer::new(dir.path()).await; + let output = Gateway::new() + .query( + vec![Channel::from_url(server.url().join("a/").unwrap())], + vec![Platform::Linux64], + vec![MatchSpec::from_str("shared", Strict).unwrap()], + ) + .recursive(false) + .execute() + .await + .unwrap(); + + assert!(output.virtual_package_plugins.is_empty()); + } + /// Repodata with CEP-42 `channel_relations` in `info`. fn make_repodata_with_relations( name: &str, @@ -2740,6 +3005,88 @@ mod test { } } + /// `Gateway::virtual_package_plugins` round-trips the registration + /// without any spec, which is the point: the plugin package names only + /// exist inside the metadata being fetched. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_gateway_virtual_package_plugins_roundtrip() { + let channel_dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + channel_dir.path(), + "testpkg", + r#""cuda-detect": ["__cuda", "__cuda_arch"], "rocm-detect": ["__rocm"]"#, + ); + + let server = SimpleChannelServer::new(channel_dir.path()).await; + let plugins = Gateway::new() + .virtual_package_plugins(&server.channel(), Platform::Linux64) + .await + .unwrap(); + + assert_eq!( + plugins + .iter() + .map(|(plugin, provided)| ( + plugin.as_source(), + provided + .iter() + .map(PackageName::as_source) + .collect::>() + )) + .collect::>(), + vec![ + ("cuda-detect", vec!["__cuda", "__cuda_arch"]), + ("rocm-detect", vec!["__rocm"]), + ] + ); + } + + /// A channel registering no plugins yields an empty map, not an error. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_gateway_virtual_package_plugins_absent() { + let channel_dir = tempfile::tempdir().unwrap(); + let subdir_path = channel_dir.path().join("linux-64"); + std::fs::create_dir_all(&subdir_path).unwrap(); + std::fs::write( + subdir_path.join("repodata.json"), + make_repodata("testpkg", "1.0.0"), + ) + .unwrap(); + + let server = SimpleChannelServer::new(channel_dir.path()).await; + let plugins = Gateway::new() + .virtual_package_plugins(&server.channel(), Platform::Linux64) + .await + .unwrap(); + assert!(plugins.is_empty()); + } + + /// A subdir the channel doesn't publish yields an empty map, not an + /// error. noarch matters: the subdir builder propagates its absence as + /// an error instead of `NotFound`. + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_gateway_virtual_package_plugins_missing_subdir() { + let channel_dir = tempfile::tempdir().unwrap(); + write_test_subdir_with_plugins( + channel_dir.path(), + "testpkg", + r#""cuda-detect": ["__cuda"]"#, + ); + + let server = SimpleChannelServer::new(channel_dir.path()).await; + let gateway = Gateway::new(); + for platform in [Platform::Osx64, Platform::NoArch] { + let plugins = gateway + .virtual_package_plugins(&server.channel(), platform) + .await + .unwrap_or_else(|e| panic!("{platform} must return empty, not error: {e}")); + assert!(plugins.is_empty(), "{platform} declared none"); + } + } + // ---------------------------------------------------------------------- // CEP-42 integration tests // ---------------------------------------------------------------------- diff --git a/crates/rattler_repodata_gateway/src/gateway/query.rs b/crates/rattler_repodata_gateway/src/gateway/query.rs index e22af81dc2..7544ab4e6f 100644 --- a/crates/rattler_repodata_gateway/src/gateway/query.rs +++ b/crates/rattler_repodata_gateway/src/gateway/query.rs @@ -5,6 +5,8 @@ use std::{ }; use futures::{FutureExt, StreamExt, select_biased, stream::FuturesUnordered}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ Channel, ChannelUrl, MatchSpec, Matches, PackageName, PackageNameMatcher, Platform, RepoDataRecord, @@ -34,6 +36,29 @@ pub struct RepoDataQueryOutput { /// Non-fatal warnings encountered during the query. Also streamed /// to [`Reporter::on_gateway_warning`] as they are recorded. pub warnings: Vec, + /// Virtual package detection plugins declared by the queried channel + /// subdirs, in resolved channel-priority order. + /// + /// Reported exactly as declared: duplicate claims on the same virtual + /// package are not resolved, and no plugin is fetched or executed. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub virtual_package_plugins: Vec, +} + +/// Plugin registrations declared by a single channel subdir. +/// +/// Kept per subdir rather than per channel because different subdirs of one +/// channel may declare different metadata; collapsing them would silently drop +/// registrations. +#[cfg(feature = "experimental-virtual-package-plugins")] +#[derive(Debug, Clone)] +pub struct SubdirVirtualPackagePlugins { + /// The channel that declared these plugins. + pub channel: ChannelUrl, + /// The subdir the declaration was read from. + pub platform: Platform, + /// Plugin package name mapped to the virtual packages it provides. + pub plugins: VirtualPackagePlugins, } impl std::ops::Deref for RepoDataQueryOutput { @@ -332,6 +357,11 @@ struct QueryExecutor { /// CEP-42 expansion state. expander: ChannelExpander, + + /// Plugin registrations collected from each resolved channel subdir. + /// Emitted in channel-priority order once the final order is known. + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: ahash::HashMap<(ChannelUrl, Platform), VirtualPackagePlugins>, } impl QueryExecutor { @@ -486,6 +516,8 @@ impl QueryExecutor { pending_subdirs, pending_records: FuturesUnordered::new(), expander, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: ahash::HashMap::default(), }) } @@ -825,6 +857,14 @@ impl QueryExecutor { } self.expand_pattern_specs_for_subdir(subdir.as_ref()); if let Some((url, platform)) = kind_url_and_platform { + #[cfg(feature = "experimental-virtual-package-plugins")] + { + let plugins = subdir.virtual_package_plugins(); + if !plugins.is_empty() { + self.virtual_package_plugins + .insert((url.clone(), platform), plugins.clone()); + } + } self.expand_relations_for_subdir(&url, platform, subdir.as_ref())?; } if self.pending_subdirs.is_empty() { @@ -1000,6 +1040,27 @@ impl QueryExecutor { handles = tagged.into_iter().map(|(_, h)| h).collect(); } + // `handles` is already in channel-priority order, so walking it yields + // the registrations in that order too. Custom sources have no channel. + #[cfg(feature = "experimental-virtual-package-plugins")] + let virtual_package_plugins = handles + .iter() + .filter_map(|h| match &h.kind { + SubdirKind::Channel { url, platform } => { + let key = (url.clone(), *platform); + self.virtual_package_plugins.remove(&key).map(|plugins| { + let (channel, platform) = key; + SubdirVirtualPackagePlugins { + channel, + platform, + plugins, + } + }) + } + SubdirKind::Custom => None, + }) + .collect(); + let mut repodata: Vec = Vec::with_capacity(handles.len() + usize::from(direct.is_some())); if let Some(d) = direct { @@ -1014,6 +1075,8 @@ impl QueryExecutor { .into_iter() .map(GatewayWarning::from) .collect(), + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins, }) } } diff --git a/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs b/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs index 1e6ff13885..972b24a196 100644 --- a/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs @@ -1,5 +1,7 @@ use crate::gateway::subdir::{PackageRecords, SubdirClient}; use crate::{GatewayError, Reporter}; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ChannelRelations, PackageName, RepodataRevisions}; cfg_if::cfg_if! { @@ -34,4 +36,9 @@ impl SubdirClient for RemoteSubdirClient { fn channel_relations(&self) -> Option<&ChannelRelations> { self.sparse.channel_relations() } + + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + self.sparse.virtual_package_plugins() + } } diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs index 369f43d370..9e1c9d1bf7 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs @@ -183,6 +183,8 @@ mod tests { routing::get, }; use rattler_conda_types::{Channel, RepodataRevisions, ShardedRepodata, ShardedSubdirInfo}; + #[cfg(feature = "experimental-virtual-package-plugins")] + use rattler_conda_types::{PackageName, VirtualPackagePlugins}; use rattler_digest::{Sha256, parse_digest_from_hex}; use std::future::IntoFuture; use std::net::SocketAddr; @@ -223,12 +225,17 @@ mod tests { created_at: Some(jiff::Timestamp::now()), repodata_revisions: RepodataRevisions::default(), channel_relations: None, + #[cfg(feature = "experimental-virtual-package-plugins")] + virtual_package_plugins: mock_virtual_package_plugins(), }, shards, }; // Encode the index as msgpack and compress with zstd - let index_bytes = rmp_serde::to_vec(&sharded_index).unwrap(); + // Named encoding, matching what the indexer writes for real + // channels; positional encoding misaligns as soon as a skipped + // field precedes a present one. + let index_bytes = rmp_serde::to_vec_named(&sharded_index).unwrap(); let compressed_index = zstd::encode_all(index_bytes.as_slice(), 3).unwrap(); let shard_requests = Arc::new(AtomicUsize::new(0)); @@ -310,6 +317,48 @@ mod tests { Truncated, } + /// Registrations served by the mock index, so the sharded path is covered + /// end to end through msgpack encoding and the HTTP fetch. + #[cfg(feature = "experimental-virtual-package-plugins")] + fn mock_virtual_package_plugins() -> VirtualPackagePlugins { + [( + PackageName::new_unchecked("cuda-detect"), + vec![ + PackageName::new_unchecked("__cuda"), + PackageName::new_unchecked("__cuda_arch"), + ], + )] + .into_iter() + .collect() + } + + #[cfg(feature = "experimental-virtual-package-plugins")] + #[tokio::test] + async fn test_sharded_index_reports_virtual_package_plugins() { + let server = MockShardedServer::new(MockShardResponse::Empty).await; + let cache_dir = tempfile::tempdir().unwrap(); + + let subdir = ShardedSubdir::new( + server.channel(), + "linux-64".to_string(), + rattler_networking::LazyClient::default(), + cache_dir.path().to_path_buf(), + ShardCachePolicy { + action: CacheAction::NoCache, + missing_shards_are_empty: false, + }, + None, + None, + ) + .await + .unwrap(); + + assert_eq!( + subdir.virtual_package_plugins(), + &mock_virtual_package_plugins() + ); + } + #[tokio::test] async fn test_empty_shard_response_error() { let server = MockShardedServer::new(MockShardResponse::Empty).await; diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs index ed8b219b9a..0b7285cdb6 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs @@ -7,6 +7,8 @@ use std::{ }; use rattler_conda_types::Platform; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use super::{ add_trailing_slash, decode_zst_bytes_async, is_missing_sharded_repodata_status, parse_records, @@ -308,6 +310,11 @@ impl SubdirClient for ShardedSubdir { fn channel_relations(&self) -> Option<&ChannelRelations> { self.sharded_repodata.info.channel_relations.as_ref() } + + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + &self.sharded_repodata.info.virtual_package_plugins + } } /// Atomically writes the shard bytes to the cache. diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs index 4d5a2956df..c9d51d7265 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs @@ -1,6 +1,8 @@ use std::sync::Arc; use futures::future::OptionFuture; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ Channel, ChannelRelations, PackageName, RepodataRevisions, ShardedRepodata, }; @@ -177,4 +179,9 @@ impl SubdirClient for ShardedSubdir { fn channel_relations(&self) -> Option<&ChannelRelations> { self.sharded_repodata.info.channel_relations.as_ref() } + + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + &self.sharded_repodata.info.virtual_package_plugins + } } diff --git a/crates/rattler_repodata_gateway/src/gateway/subdir.rs b/crates/rattler_repodata_gateway/src/gateway/subdir.rs index 3b173bfcfc..fbacd591ca 100644 --- a/crates/rattler_repodata_gateway/src/gateway/subdir.rs +++ b/crates/rattler_repodata_gateway/src/gateway/subdir.rs @@ -1,11 +1,15 @@ use std::sync::Arc; use ahash::HashMap; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ChannelRelations, PackageName, RepoDataRecord, RepodataRevisions}; use super::GatewayError; use crate::Reporter; use crate::sparse::empty_repodata_revisions; +#[cfg(feature = "experimental-virtual-package-plugins")] +use crate::sparse::empty_virtual_package_plugins; use coalesced_map::{CoalescedGetError, CoalescedMap}; /// Records for a single package, with precomputed unique dependency strings @@ -114,6 +118,16 @@ impl Subdir { Subdir::NotFound => None, } } + + /// Virtual package detection plugins registered by this subdir, or empty + /// if none are registered / the subdir was not found. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + match self { + Subdir::Found(subdir) => subdir.virtual_package_plugins(), + Subdir::NotFound => empty_virtual_package_plugins(), + } + } } /// Fetches and caches repodata records by package name for a specific @@ -172,6 +186,12 @@ impl SubdirData { pub fn channel_relations(&self) -> Option<&ChannelRelations> { self.client.channel_relations() } + + /// Virtual package detection plugins registered by this subdir. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + self.client.virtual_package_plugins() + } } /// A client that can be used to fetch repodata for a specific subdirectory. @@ -201,6 +221,13 @@ pub trait SubdirClient: Send + Sync { fn channel_relations(&self) -> Option<&ChannelRelations> { None } + + /// Virtual package detection plugins registered by this subdir. Sources + /// that cannot carry the metadata (e.g. custom) keep the empty default. + #[cfg(feature = "experimental-virtual-package-plugins")] + fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + empty_virtual_package_plugins() + } } #[cfg(test)] diff --git a/crates/rattler_repodata_gateway/src/lib.rs b/crates/rattler_repodata_gateway/src/lib.rs index 40cbae4893..4aeca2226f 100644 --- a/crates/rattler_repodata_gateway/src/lib.rs +++ b/crates/rattler_repodata_gateway/src/lib.rs @@ -73,6 +73,8 @@ pub use reporter::{SUPPORTED_REPODATA_REVISION, UnsupportedRepodataRevision}; #[cfg(feature = "gateway")] mod gateway; +#[cfg(all(feature = "gateway", feature = "experimental-virtual-package-plugins"))] +pub use gateway::SubdirVirtualPackagePlugins; #[cfg(feature = "gateway")] pub use gateway::{ CacheClearMode, ChannelConfig, ChannelRelationsMode, ChannelRelationsWarning, diff --git a/crates/rattler_repodata_gateway/src/sparse/mod.rs b/crates/rattler_repodata_gateway/src/sparse/mod.rs index 5893f625c8..23c1bdfba7 100644 --- a/crates/rattler_repodata_gateway/src/sparse/mod.rs +++ b/crates/rattler_repodata_gateway/src/sparse/mod.rs @@ -14,6 +14,8 @@ use std::{ use bytes::Bytes; use itertools::Itertools; +#[cfg(feature = "experimental-virtual-package-plugins")] +use rattler_conda_types::VirtualPackagePlugins; use rattler_conda_types::{ Channel, ChannelInfo, ChannelRelations, MatchSpec, Matches, PackageName, PackageRecord, RepoDataRecord, RepodataRevisions, UrlOrPath, WhlPackageRecord, compute_package_url, @@ -37,6 +39,14 @@ pub(crate) fn empty_repodata_revisions() -> &'static RepodataRevisions { &EMPTY } +/// Shared empty plugin registrations, returned by accessors when a subdir +/// registers none. +#[cfg(feature = "experimental-virtual-package-plugins")] +pub(crate) fn empty_virtual_package_plugins() -> &'static VirtualPackagePlugins { + static EMPTY: LazyLock = LazyLock::new(VirtualPackagePlugins::new); + &EMPTY +} + /// Defines how different variants of packages are consolidated. #[derive( Default, @@ -496,6 +506,16 @@ impl SparseRepoData { .channel_relations .as_ref() } + + /// Virtual package detection plugins registered in + /// `info.virtual_package_plugins`. + #[cfg(feature = "experimental-virtual-package-plugins")] + pub fn virtual_package_plugins(&self) -> &VirtualPackagePlugins { + match &self.inner.borrow_repo_data().info { + Some(info) => &info.virtual_package_plugins, + None => empty_virtual_package_plugins(), + } + } } /// A serde compatible struct that only sparsely parses a repodata.json file. diff --git a/test-data/channels/virtual-package-plugins/linux-64/repodata.json b/test-data/channels/virtual-package-plugins/linux-64/repodata.json new file mode 100644 index 0000000000..b5a15dad16 --- /dev/null +++ b/test-data/channels/virtual-package-plugins/linux-64/repodata.json @@ -0,0 +1,26 @@ +{ + "info": { + "subdir": "linux-64", + "base_url": "../linux-64", + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"], + "rocm-detect": ["__rocm"] + } + }, + "packages.conda": { + "accelerated-1.0.0-h0000000_0.conda": { + "build": "h0000000_0", + "build_number": 0, + "depends": ["__rocm >=6.0"], + "license": "BSD-3-Clause", + "md5": "00000000000000000000000000000000", + "name": "accelerated", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "size": 1000, + "subdir": "linux-64", + "timestamp": 1700000000000, + "version": "1.0.0" + } + }, + "repodata_version": 2 +} diff --git a/test-data/channels/virtual-package-plugins/noarch/repodata.json b/test-data/channels/virtual-package-plugins/noarch/repodata.json new file mode 100644 index 0000000000..f263ef4f87 --- /dev/null +++ b/test-data/channels/virtual-package-plugins/noarch/repodata.json @@ -0,0 +1,12 @@ +{ + "info": { + "subdir": "noarch", + "base_url": "../noarch", + "virtual_package_plugins": { + "cuda-detect": ["__cuda", "__cuda_arch"], + "rocm-detect": ["__rocm"] + } + }, + "packages.conda": {}, + "repodata_version": 2 +}