diff --git a/crates/rattler_conda_types/src/lib.rs b/crates/rattler_conda_types/src/lib.rs index 01b3536fd8..49b39f503f 100644 --- a/crates/rattler_conda_types/src/lib.rs +++ b/crates/rattler_conda_types/src/lib.rs @@ -66,8 +66,9 @@ pub use prefix_record::PrefixRecord; pub use record_traits::HasArtifactIdentificationRefs; pub use repo_data::{ ChannelInfo, ChannelRelations, ConvertSubdirError, PackageRecord, RecordFromPath, RepoData, - RepodataRevision, RepodataRevisionInfo, SubdirRunExportsJson, UrlOrPath, V3Packages, - ValidatePackageRecordsError, WhlPackageRecord, compute_package_url, + RepodataRevision, RepodataRevisionInfo, RepodataRevisionMetadata, RepodataRevisions, + SubdirRunExportsJson, UrlOrPath, V3Packages, ValidatePackageRecordsError, WhlPackageRecord, + compute_package_url, patches::{PackageRecordPatch, PatchInstructions, RepoDataPatch}, sharded::{Shard, ShardedRepodata, ShardedSubdirInfo}, }; diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 2e077a0bfd..9be6d86c4b 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -18,7 +18,9 @@ use indexmap::IndexMap; use rattler_digest::{Md5Hash, Sha256Hash, serde::SerializableHash}; use rattler_macros::sorted; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use serde_with::{DeserializeFromStr, SerializeDisplay, serde_as, skip_serializing_none}; +use serde_with::{ + DeserializeFromStr, DisplayFromStr, SerializeDisplay, serde_as, skip_serializing_none, +}; use thiserror::Error; use url::Url; @@ -82,6 +84,7 @@ pub struct RepoData { } /// Information about subdirectory of channel in the Conda [`RepoData`] +#[serde_as] #[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Clone)] pub struct ChannelInfo { /// The channel's subdirectory @@ -93,9 +96,11 @@ pub struct ChannelInfo { /// Repodata revisions available in this repodata file. /// - /// See . - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub repodata_revisions: Vec, + /// Serialized as a `vN`-keyed dictionary per the CEP draft + /// . + #[serde_as(as = "IndexMap")] + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub repodata_revisions: RepodataRevisions, /// Optional relationships to other channels as defined in /// [CEP-42](https://github.com/conda/ceps/blob/main/cep-0042.md). @@ -103,12 +108,32 @@ pub struct ChannelInfo { pub channel_relations: Option, } -/// Metadata for a repodata revision advertised in -/// `info.repodata_revisions`. -/// -/// Future repodata revisions are encoded in parallel top-level `vN` maps. This -/// metadata lets older clients tell users that the channel contains newer -/// records that may be invisible to the current client. +/// Repodata revisions keyed by revision, mirroring the `vN` dictionary of the +/// CEP draft . Keying encodes +/// uniqueness; insertion order is preserved. +pub type RepodataRevisions = IndexMap; + +/// Metadata for a single [`RepodataRevisions`] entry; the revision itself is +/// the map key. +#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Clone, Default)] +pub struct RepodataRevisionMetadata { + /// The number of packages available in this revision. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub n_packages: Option, + + /// The Unix timestamp in milliseconds of the oldest record in this + /// revision. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub oldest: Option, + + /// The Unix timestamp in milliseconds of the newest record in this + /// revision. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub newest: Option, +} + +/// A revision paired with its metadata: the flattened form of a +/// [`RepodataRevisions`] entry, used as indexer input and in reporter messages. #[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Clone)] pub struct RepodataRevisionInfo { /// The integer identifying the revision. @@ -130,6 +155,27 @@ pub struct RepodataRevisionInfo { pub newest: Option, } +impl RepodataRevisionInfo { + /// Combines a revision identifier with its metadata. + pub fn from_metadata(revision: RepodataRevision, metadata: RepodataRevisionMetadata) -> Self { + Self { + revision, + n_packages: metadata.n_packages, + oldest: metadata.oldest, + newest: metadata.newest, + } + } + + /// Returns the metadata portion (everything but the revision). + pub fn metadata(&self) -> RepodataRevisionMetadata { + RepodataRevisionMetadata { + n_packages: self.n_packages, + oldest: self.oldest, + newest: self.newest, + } + } +} + /// A repodata revision. /// /// The serialized CEP wire format is an integer. Known variants are exposed as @@ -1121,7 +1167,7 @@ mod test { info: Some(ChannelInfo { subdir: Some("linux-64".to_string()), base_url: None, - repodata_revisions: Vec::new(), + repodata_revisions: IndexMap::default(), channel_relations: Some(ChannelRelations { base: Some("../conda-forge".to_string()), overrides: None, @@ -1146,7 +1192,7 @@ mod test { info: Some(ChannelInfo { subdir: Some("linux-64".to_string()), base_url: None, - repodata_revisions: Vec::new(), + repodata_revisions: IndexMap::default(), channel_relations, }), packages: IndexMap::default(), @@ -1164,36 +1210,38 @@ mod test { let raw = r#"{ "info": { "subdir": "linux-64", - "repodata_revisions": [ - { - "revision": 4, + "repodata_revisions": { + "v4": { "n_packages": 2, "oldest": 1768249989851, "newest": 1773851561010 } - ] + } }, "packages": {}, "packages.conda": {} }"#; let repodata: RepoData = serde_json::from_str(raw).unwrap(); - let revision = &repodata.info.as_ref().unwrap().repodata_revisions[0]; - assert_eq!(revision.revision, RepodataRevision::Unknown(4)); - assert_eq!(revision.n_packages, Some(2)); + let revisions = &repodata.info.as_ref().unwrap().repodata_revisions; + assert_eq!(revisions.len(), 1); + let metadata = &revisions[&RepodataRevision::Unknown(4)]; + assert_eq!(metadata.n_packages, Some(2)); assert_eq!( - revision.oldest.map(|ts| ts.timestamp_millis()), + metadata.oldest.map(|ts| ts.timestamp_millis()), Some(1768249989851) ); assert_eq!( - revision.newest.map(|ts| ts.timestamp_millis()), + metadata.newest.map(|ts| ts.timestamp_millis()), Some(1773851561010) ); let json = serde_json::to_string(&repodata).unwrap(); - assert!(json.contains("\"repodata_revisions\"")); + assert!(json.contains("\"repodata_revisions\":{\"v4\":{")); assert!(json.contains("\"oldest\":1768249989851")); assert!(json.contains("\"newest\":1773851561010")); + // The revision identifier is the map key, not a field of the value. + assert!(!json.contains("\"revision\"")); } #[test] diff --git a/crates/rattler_conda_types/src/repo_data/sharded.rs b/crates/rattler_conda_types/src/repo_data/sharded.rs index 8a59dc9367..bd50c00604 100644 --- a/crates/rattler_conda_types/src/repo_data/sharded.rs +++ b/crates/rattler_conda_types/src/repo_data/sharded.rs @@ -2,11 +2,12 @@ use crate::PackageRecord; use crate::package::DistArchiveIdentifier; -use crate::repo_data::{ChannelRelations, RepodataRevisionInfo, V3Packages}; +use crate::repo_data::{ChannelRelations, RepodataRevisions, V3Packages}; use indexmap::IndexMap; use jiff::Timestamp; use rattler_digest::{Sha256, Sha256Hash, serde::SerializableHash}; use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; /// The sharded repodata holds a hashmap of package name -> shard (hash). /// This index file is stored under @@ -24,6 +25,7 @@ pub struct ShardedRepodata { /// Information about a sharded subdirectory that is stored inside the index /// file. +#[serde_as] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ShardedSubdirInfo { /// The name of the subdirectory @@ -47,9 +49,11 @@ pub struct ShardedSubdirInfo { /// Repodata revisions available through this sharded index. /// - /// See . - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub repodata_revisions: Vec, + /// Serialized as a `vN`-keyed dictionary per the CEP draft + /// . + #[serde_as(as = "IndexMap")] + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub repodata_revisions: RepodataRevisions, /// Optional relationships to other channels as defined in /// [CEP-42](https://github.com/conda/ceps/blob/main/cep-0042.md). @@ -86,7 +90,7 @@ mod tests { base_url: "./".to_string(), shards_base_url: "./shards/".to_string(), created_at: None, - repodata_revisions: Vec::new(), + repodata_revisions: IndexMap::default(), channel_relations, }; let json = serde_json::to_string(&info).unwrap(); diff --git a/crates/rattler_index/src/lib.rs b/crates/rattler_index/src/lib.rs index fad098bd76..ea4ca1332a 100644 --- a/crates/rattler_index/src/lib.rs +++ b/crates/rattler_index/src/lib.rs @@ -36,7 +36,9 @@ use rattler_conda_types::{ RunExportsJson, WheelArchiveType, }, }; -pub use rattler_conda_types::{RepodataRevision, RepodataRevisionInfo}; +pub use rattler_conda_types::{ + RepodataRevision, RepodataRevisionInfo, RepodataRevisionMetadata, RepodataRevisions, +}; pub use rattler_config::config::index::{ IndexChannelConfig, IndexConfig, PackageRevisionAssignment, }; @@ -1081,12 +1083,13 @@ impl RevisionStats { fn repodata_revisions_for_packages( configured: &[RepodataRevisionInfo], v3: &V3Packages, -) -> Vec { +) -> RepodataRevisions { + // `BTreeMap` keeps the result ordered ascending regardless of input order. let mut revisions = configured .iter() - .filter(|revision| revision.revision != RepodataRevision::Legacy) - .map(|revision| (revision.revision, revision.clone())) - .collect::>(); + .filter(|info| info.revision != RepodataRevision::Legacy) + .map(|info| (info.revision, info.metadata())) + .collect::>(); let mut stats = BTreeMap::::new(); for (_, record) in v3.records() { @@ -1094,35 +1097,28 @@ fn repodata_revisions_for_packages( } for (revision, revision_stats) in stats { - let info = revisions - .entry(revision) - .or_insert_with(|| RepodataRevisionInfo { - revision, - n_packages: None, - oldest: None, - newest: None, - }); - if info.n_packages.is_none() { - info.n_packages = Some(revision_stats.n_packages); + let metadata = revisions.entry(revision).or_default(); + if metadata.n_packages.is_none() { + metadata.n_packages = Some(revision_stats.n_packages); } - if info.oldest.is_none() { - info.oldest = revision_stats.oldest; + if metadata.oldest.is_none() { + metadata.oldest = revision_stats.oldest; } - if info.newest.is_none() { - info.newest = revision_stats.newest; + if metadata.newest.is_none() { + metadata.newest = revision_stats.newest; } } // Currently only v3 package maps are supported, but keep configured // revisions with zero packages so clients can still surface channel // capability information. - for revision in revisions.values_mut() { - if revision.n_packages.is_none() { - revision.n_packages = Some(0); + for metadata in revisions.values_mut() { + if metadata.n_packages.is_none() { + metadata.n_packages = Some(0); } } - revisions.into_values().collect() + revisions.into_iter().collect() } /// Write a `repodata.json` for all packages in the given configurator's root. @@ -1699,7 +1695,7 @@ pub async fn ensure_channel_initialized_with_channel_metadata( info: Some(ChannelInfo { subdir: Some(Platform::NoArch.to_string()), base_url: channel_metadata.base_url, - repodata_revisions: Vec::new(), + repodata_revisions: RepodataRevisions::new(), channel_relations: channel_metadata.channel_relations, }), packages: IndexMap::default(), diff --git a/crates/rattler_index/tests/integration/basic_indexing.rs b/crates/rattler_index/tests/integration/basic_indexing.rs index 2118bcf0a2..0f4f8a1d90 100644 --- a/crates/rattler_index/tests/integration/basic_indexing.rs +++ b/crates/rattler_index/tests/integration/basic_indexing.rs @@ -273,8 +273,7 @@ async fn test_index_latest_repodata_revision() { .pointer("/v3/conda/empty-0.1.0-h4616a5c_0") .is_some() ); - let revision = &repodata_json["info"]["repodata_revisions"][0]; - assert_eq!(revision["revision"], 3); + let revision = &repodata_json["info"]["repodata_revisions"]["v3"]; assert_eq!(revision["n_packages"], 1); let shard_index_bytes = fs::read(subdir_path.join("repodata_shards.msgpack.zst")).unwrap(); @@ -282,10 +281,9 @@ async fn test_index_latest_repodata_revision() { let shard_index: ShardedRepodata = rmp_serde::from_slice(&shard_index_bytes).unwrap(); assert_eq!(shard_index.info.repodata_revisions.len(), 1); assert_eq!( - shard_index.info.repodata_revisions[0].revision, - RepodataRevision::V3 + shard_index.info.repodata_revisions[&RepodataRevision::V3].n_packages, + Some(1) ); - assert_eq!(shard_index.info.repodata_revisions[0].n_packages, Some(1)); } #[tokio::test] @@ -374,8 +372,7 @@ async fn test_index_repodata_revision_from_index_json() { .pointer("/v3/tar.bz2/revision-demo-1.0.0-h123_0") .is_some() ); - let revision = &repodata_json["info"]["repodata_revisions"][0]; - assert_eq!(revision["revision"], 3); + let revision = &repodata_json["info"]["repodata_revisions"]["v3"]; assert_eq!(revision["n_packages"], 1); assert_eq!(revision["oldest"], 1710000000000i64); assert_eq!(revision["newest"], 1710000000000i64); @@ -429,11 +426,7 @@ async fn test_index_writes_channel_metadata() { "../fallback" ); assert_eq!( - repodata_json["info"]["repodata_revisions"][0]["revision"], - 3 - ); - assert_eq!( - repodata_json["info"]["repodata_revisions"][0]["n_packages"], + repodata_json["info"]["repodata_revisions"]["v3"]["n_packages"], 0 ); @@ -462,8 +455,7 @@ async fn test_index_writes_channel_metadata() { Some("../fallback") ); assert_eq!( - shard_index.info.repodata_revisions[0].revision, - RepodataRevision::V3 + shard_index.info.repodata_revisions[&RepodataRevision::V3].n_packages, + Some(0) ); - assert_eq!(shard_index.info.repodata_revisions[0].n_packages, Some(0)); } diff --git a/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs b/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs index 76d333c5e3..027ad03ffe 100644 --- a/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs +++ b/crates/rattler_repodata_gateway/src/gateway/local_subdir.rs @@ -1,6 +1,6 @@ use std::{path::Path, sync::Arc}; -use rattler_conda_types::{Channel, PackageName, RepodataRevisionInfo}; +use rattler_conda_types::{Channel, PackageName, RepodataRevisions}; use crate::{ Reporter, @@ -108,7 +108,7 @@ impl SubdirClient for LocalSubdirClient { .collect() } - fn repodata_revisions(&self) -> &[RepodataRevisionInfo] { + fn repodata_revisions(&self) -> &RepodataRevisions { self.sparse.repodata_revisions() } } diff --git a/crates/rattler_repodata_gateway/src/gateway/mod.rs b/crates/rattler_repodata_gateway/src/gateway/mod.rs index 49d8854da7..9902c59882 100644 --- a/crates/rattler_repodata_gateway/src/gateway/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/mod.rs @@ -484,14 +484,13 @@ mod test { "repodata_version": 1, "info": { "subdir": "noarch", - "repodata_revisions": [ - { - "revision": 4, + "repodata_revisions": { + "v4": { "n_packages": 2, "oldest": 1768249989851, "newest": 1773851561010 } - ] + } }, "packages": {}, "packages.conda": { @@ -576,12 +575,11 @@ mod test { "repodata_version": 1, "info": { "subdir": "noarch", - "repodata_revisions": [ - { - "revision": 3, + "repodata_revisions": { + "v3": { "n_packages": 1 } - ] + } }, "packages": {}, "packages.conda": { diff --git a/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs b/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs index 4127dfbafd..b516c3fc53 100644 --- a/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/remote_subdir/mod.rs @@ -1,6 +1,6 @@ use crate::gateway::subdir::{PackageRecords, SubdirClient}; use crate::{GatewayError, Reporter}; -use rattler_conda_types::{PackageName, RepodataRevisionInfo}; +use rattler_conda_types::{PackageName, RepodataRevisions}; cfg_if::cfg_if! { if #[cfg(target_arch = "wasm32")] { @@ -27,7 +27,7 @@ impl SubdirClient for RemoteSubdirClient { self.sparse.package_names() } - fn repodata_revisions(&self) -> &[RepodataRevisionInfo] { + fn repodata_revisions(&self) -> &RepodataRevisions { self.sparse.repodata_revisions() } } diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs index f4c99428f0..a199194fd0 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/mod.rs @@ -182,7 +182,7 @@ mod tests { http::{Response, StatusCode}, routing::get, }; - use rattler_conda_types::{Channel, ShardedRepodata, ShardedSubdirInfo}; + use rattler_conda_types::{Channel, RepodataRevisions, ShardedRepodata, ShardedSubdirInfo}; use rattler_digest::{Sha256, parse_digest_from_hex}; use std::future::IntoFuture; use std::net::SocketAddr; @@ -215,7 +215,7 @@ mod tests { base_url: "./".to_string(), shards_base_url: "./shards/".to_string(), created_at: Some(jiff::Timestamp::now()), - repodata_revisions: Vec::new(), + repodata_revisions: RepodataRevisions::default(), channel_relations: None, }, shards, diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs index 0ea2aad8ea..3a587d6069 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/tokio/mod.rs @@ -23,7 +23,7 @@ use crate::{ use fs_err::tokio as tokio_fs; use futures::future::OptionFuture; use http::{HeaderValue, header::CACHE_CONTROL}; -use rattler_conda_types::{Channel, PackageName, RepodataRevisionInfo, ShardedRepodata}; +use rattler_conda_types::{Channel, PackageName, RepodataRevisions, ShardedRepodata}; use rattler_networking::LazyClient; use simple_spawn_blocking::tokio::run_blocking_task; use url::Url; @@ -276,7 +276,7 @@ impl SubdirClient for ShardedSubdir { self.sharded_repodata.shards.keys().cloned().collect() } - fn repodata_revisions(&self) -> &[RepodataRevisionInfo] { + fn repodata_revisions(&self) -> &RepodataRevisions { &self.sharded_repodata.info.repodata_revisions } } diff --git a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs index 099385dc24..a508373c23 100644 --- a/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs +++ b/crates/rattler_repodata_gateway/src/gateway/sharded_subdir/wasm/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use futures::future::OptionFuture; -use rattler_conda_types::{Channel, PackageName, RepodataRevisionInfo, ShardedRepodata}; +use rattler_conda_types::{Channel, PackageName, RepodataRevisions, ShardedRepodata}; use rattler_networking::LazyClient; use url::Url; @@ -168,7 +168,7 @@ impl SubdirClient for ShardedSubdir { self.sharded_repodata.shards.keys().cloned().collect() } - fn repodata_revisions(&self) -> &[RepodataRevisionInfo] { + fn repodata_revisions(&self) -> &RepodataRevisions { &self.sharded_repodata.info.repodata_revisions } } diff --git a/crates/rattler_repodata_gateway/src/gateway/subdir.rs b/crates/rattler_repodata_gateway/src/gateway/subdir.rs index 312d222548..a94e16fb8c 100644 --- a/crates/rattler_repodata_gateway/src/gateway/subdir.rs +++ b/crates/rattler_repodata_gateway/src/gateway/subdir.rs @@ -1,10 +1,11 @@ use std::sync::Arc; use ahash::HashMap; -use rattler_conda_types::{PackageName, RepoDataRecord, RepodataRevisionInfo}; +use rattler_conda_types::{PackageName, RepoDataRecord, RepodataRevisions}; use super::GatewayError; use crate::Reporter; +use crate::sparse::empty_repodata_revisions; use coalesced_map::{CoalescedGetError, CoalescedMap}; /// Records for a single package, with precomputed unique dependency strings @@ -96,10 +97,10 @@ impl Subdir { } /// Returns repodata revisions advertised by this subdirectory. - pub fn repodata_revisions(&self) -> &[RepodataRevisionInfo] { + pub fn repodata_revisions(&self) -> &RepodataRevisions { match self { Subdir::Found(subdir) => subdir.repodata_revisions(), - Subdir::NotFound => &[], + Subdir::NotFound => empty_repodata_revisions(), } } } @@ -150,7 +151,7 @@ impl SubdirData { self.client.package_names() } - pub fn repodata_revisions(&self) -> &[RepodataRevisionInfo] { + pub fn repodata_revisions(&self) -> &RepodataRevisions { self.client.repodata_revisions() } } @@ -171,8 +172,8 @@ pub trait SubdirClient: Send + Sync { fn package_names(&self) -> Vec; /// Returns repodata revisions advertised by the subdirectory. - fn repodata_revisions(&self) -> &[RepodataRevisionInfo] { - &[] + fn repodata_revisions(&self) -> &RepodataRevisions { + empty_repodata_revisions() } } diff --git a/crates/rattler_repodata_gateway/src/reporter.rs b/crates/rattler_repodata_gateway/src/reporter.rs index 2b588d3cae..4f553c0051 100644 --- a/crates/rattler_repodata_gateway/src/reporter.rs +++ b/crates/rattler_repodata_gateway/src/reporter.rs @@ -3,7 +3,7 @@ use std::future::Future; use bytes::Bytes; use futures::{Stream, TryStreamExt}; #[cfg(feature = "gateway")] -use rattler_conda_types::Channel; +use rattler_conda_types::{Channel, RepodataRevisions}; #[cfg(feature = "sparse")] use rattler_conda_types::{RepodataRevision, RepodataRevisionInfo}; #[cfg(feature = "gateway")] @@ -98,24 +98,24 @@ pub trait Reporter: Send + Sync { } #[cfg(feature = "gateway")] -pub(crate) fn report_unsupported_repodata_revisions<'a>( +pub(crate) fn report_unsupported_repodata_revisions( reporter: Option<&dyn Reporter>, channel: &Channel, subdir: &str, - revisions: impl IntoIterator, + revisions: &RepodataRevisions, ) { let Some(reporter) = reporter else { return; }; let channel = channel.base_url.url().clone().redact().to_string(); - for revision in revisions { - if revision.revision > SUPPORTED_REPODATA_REVISION { + for (&revision, metadata) in revisions { + if revision > SUPPORTED_REPODATA_REVISION { reporter.on_unsupported_repodata_revision(&UnsupportedRepodataRevision { channel: channel.clone(), subdir: subdir.to_string(), supported_revision: SUPPORTED_REPODATA_REVISION, - revision: revision.clone(), + revision: RepodataRevisionInfo::from_metadata(revision, metadata.clone()), }); } } diff --git a/crates/rattler_repodata_gateway/src/sparse/mod.rs b/crates/rattler_repodata_gateway/src/sparse/mod.rs index 40e50a2936..34ba09651a 100644 --- a/crates/rattler_repodata_gateway/src/sparse/mod.rs +++ b/crates/rattler_repodata_gateway/src/sparse/mod.rs @@ -9,13 +9,14 @@ use std::{ fmt, io, marker::PhantomData, path::Path, + sync::LazyLock, }; use bytes::Bytes; use itertools::Itertools; use rattler_conda_types::{ Channel, ChannelInfo, MatchSpec, Matches, PackageName, PackageRecord, RepoDataRecord, - RepodataRevisionInfo, UrlOrPath, WhlPackageRecord, compute_package_url, + RepodataRevisions, UrlOrPath, WhlPackageRecord, compute_package_url, package::{ ArchiveIdentifier, CondaArchiveType, DistArchiveIdentifier, DistArchiveType, WheelArchiveType, @@ -30,6 +31,12 @@ use serde_json::value::RawValue; use superslice::Ext; use thiserror::Error; +/// Shared empty revisions, returned by accessors when none are advertised. +pub(crate) fn empty_repodata_revisions() -> &'static RepodataRevisions { + static EMPTY: LazyLock = LazyLock::new(RepodataRevisions::new); + &EMPTY +} + /// Defines how different variants of packages are consolidated. #[derive( Default, @@ -473,12 +480,11 @@ impl SparseRepoData { } /// Returns the repodata revisions advertised by this repodata file. - pub fn repodata_revisions(&self) -> &[RepodataRevisionInfo] { - self.inner - .borrow_repo_data() - .info - .as_ref() - .map_or(&[], |info| info.repodata_revisions.as_slice()) + pub fn repodata_revisions(&self) -> &RepodataRevisions { + match &self.inner.borrow_repo_data().info { + Some(info) => &info.repodata_revisions, + None => empty_repodata_revisions(), + } } } @@ -1076,6 +1082,7 @@ mod test { use itertools::Itertools; use rattler_conda_types::{ Channel, ChannelConfig, MatchSpec, PackageName, ParseStrictness, RepoData, RepoDataRecord, + RepodataRevision, }; use rstest::rstest; @@ -1414,6 +1421,26 @@ mod test { assert_eq!(count, expected_count); } + #[test] + fn test_repodata_revisions_from_file() { + // The channel advertises a `v3` revision in the CEP `vN`-keyed + // dictionary form; make sure we parse it back into the map. + let (channel, platform, path) = wheel_repo_data(); + let sparse = SparseRepoData::from_file(channel, platform, path, None).unwrap(); + let revisions = sparse.repodata_revisions(); + assert_eq!(revisions.len(), 1); + let metadata = &revisions[&RepodataRevision::V3]; + assert_eq!(metadata.n_packages, Some(2)); + assert_eq!( + metadata.oldest.map(|ts| ts.timestamp_millis()), + Some(1768249989851) + ); + assert_eq!( + metadata.newest.map(|ts| ts.timestamp_millis()), + Some(1773851561010) + ); + } + #[test] fn test_query() { let (channel, platform, path) = dummy_repo_data(); diff --git a/py-rattler/rattler/index/__init__.py b/py-rattler/rattler/index/__init__.py index aca94bacc0..e8caa3cf79 100644 --- a/py-rattler/rattler/index/__init__.py +++ b/py-rattler/rattler/index/__init__.py @@ -1,3 +1,3 @@ -from rattler.index.index import RepodataRevisionInfo, S3Credentials, index_fs, index_s3 +from rattler.index.index import RepodataRevisionMetadata, RepodataRevisions, S3Credentials, index_fs, index_s3 -__all__ = ["index_s3", "index_fs", "S3Credentials", "RepodataRevisionInfo"] +__all__ = ["index_s3", "index_fs", "S3Credentials", "RepodataRevisionMetadata", "RepodataRevisions"] diff --git a/py-rattler/rattler/index/index.py b/py-rattler/rattler/index/index.py index b1ea8fcdf2..3eab449778 100644 --- a/py-rattler/rattler/index/index.py +++ b/py-rattler/rattler/index/index.py @@ -1,9 +1,10 @@ from __future__ import annotations -from dataclasses import asdict, dataclass +import datetime +from dataclasses import dataclass import os import sys -from typing import TYPE_CHECKING, Any, Literal, Mapping, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, Literal, Mapping, Optional, TypedDict if TYPE_CHECKING: if sys.version_info >= (3, 10): @@ -38,38 +39,50 @@ class S3Credentials: addressing_style: Literal["path", "virtual-host"] = "virtual-host" -@dataclass -class RepodataRevisionInfo: - """Metadata for a repodata revision advertised in `info.repodata_revisions`.""" +class RepodataRevisionMetadata(TypedDict, total=False): + """Metadata for a single revision in the `vN`-keyed dictionary form of + `repodata_revisions`. The revision itself is the dictionary key.""" - revision: Literal["v3"] - n_packages: Optional[int] = None - oldest: Optional[int] = None - newest: Optional[int] = None + n_packages: Optional[int] + oldest: Optional[datetime.datetime] + newest: Optional[datetime.datetime] -RepodataRevision: TypeAlias = Union[RepodataRevisionInfo, Mapping[str, Any]] +# Repodata revisions to advertise: a `vN`-keyed dictionary mapping each revision +# to its metadata (the CEP shape), e.g. `{"v3": {"n_packages": 1}}`. +RepodataRevisions: TypeAlias = Mapping[str, Optional[RepodataRevisionMetadata]] -def _revision_to_wire(revision: Any) -> int: +def _revision_to_wire(revision: str) -> int: if revision == "v3": return 3 raise ValueError(f"unsupported repodata revision {revision!r}, expected 'v3'") +def _epoch_ms(timestamp: datetime.datetime) -> int: + # repodata stores timestamps as Unix milliseconds. + return int(round(timestamp.timestamp() * 1000)) + + def _repodata_revisions_to_dicts( - revisions: Optional[Sequence[RepodataRevision]], + revisions: Optional[RepodataRevisions], ) -> Optional[list[dict[str, Any]]]: if revisions is None: return None result = [] - for revision in revisions: - if isinstance(revision, RepodataRevisionInfo): - revision_dict = asdict(revision) - else: - revision_dict = dict(revision) - revision_dict["revision"] = _revision_to_wire(revision_dict.get("revision")) + for revision, metadata in revisions.items(): + revision_dict: dict[str, Any] = {"revision": _revision_to_wire(revision)} + if metadata is not None: + n_packages = metadata.get("n_packages") + if n_packages is not None: + revision_dict["n_packages"] = n_packages + oldest = metadata.get("oldest") + if oldest is not None: + revision_dict["oldest"] = _epoch_ms(oldest) + newest = metadata.get("newest") + if newest is not None: + revision_dict["newest"] = _epoch_ms(newest) result.append(revision_dict) return result @@ -80,7 +93,7 @@ async def index_fs( repodata_patch: Optional[str] = None, write_zst: bool = True, write_shards: bool = True, - repodata_revisions: Optional[Sequence[RepodataRevision]] = None, + repodata_revisions: Optional[RepodataRevisions] = None, package_revision_assignment: Literal["from-index-json", "latest"] = "from-index-json", force: bool = False, max_parallel: int | None = None, @@ -99,7 +112,8 @@ async def index_fs( repodata_patch: The name of the conda package (expected to be in the `noarch` subdir) that should be used for repodata patching. write_zst: Whether to write repodata.json.zst. write_shards: Whether to write sharded repodata. - repodata_revisions: Repodata revisions to advertise, including optional `n_packages`, `oldest`, and `newest` metadata. + repodata_revisions: Repodata revisions to advertise, with optional `n_packages`, `oldest`, and `newest` metadata. + Either a sequence of revisions or a `vN`-keyed dictionary, e.g. `{"v3": {"n_packages": 1}}`. package_revision_assignment: Whether to assign packages to the revision required by their `index.json`, or to the latest advertised revision. force: Whether to forcefully re-index all subdirs. max_parallel: The maximum number of packages to process in-memory simultaneously. @@ -124,7 +138,7 @@ async def index_s3( repodata_patch: Optional[str] = None, write_zst: bool = True, write_shards: bool = True, - repodata_revisions: Optional[Sequence[RepodataRevision]] = None, + repodata_revisions: Optional[RepodataRevisions] = None, package_revision_assignment: Literal["from-index-json", "latest"] = "from-index-json", force: bool = False, max_parallel: int | None = None, @@ -146,7 +160,8 @@ async def index_s3( repodata_patch: The name of the conda package (expected to be in the `noarch` subdir) that should be used for repodata patching. write_zst: Whether to write repodata.json.zst. write_shards: Whether to write sharded repodata. - repodata_revisions: Repodata revisions to advertise, including optional `n_packages`, `oldest`, and `newest` metadata. + repodata_revisions: Repodata revisions to advertise, with optional `n_packages`, `oldest`, and `newest` metadata. + Either a sequence of revisions or a `vN`-keyed dictionary, e.g. `{"v3": {"n_packages": 1}}`. package_revision_assignment: Whether to assign packages to the revision required by their `index.json`, or to the latest advertised revision. force: Whether to forcefully re-index all subdirs. max_parallel: The maximum number of packages to process in-memory simultaneously. diff --git a/py-rattler/tests/unit/test_index.py b/py-rattler/tests/unit/test_index.py index 941f879ddf..e8c9a64620 100644 --- a/py-rattler/tests/unit/test_index.py +++ b/py-rattler/tests/unit/test_index.py @@ -1,4 +1,5 @@ # type: ignore +import datetime import os import json import shutil @@ -11,7 +12,7 @@ import pytest from rattler import Platform -from rattler.index import RepodataRevisionInfo, index_fs, index_s3 +from rattler.index import index_fs, index_s3 from rattler.index.index import S3Credentials @@ -64,18 +65,22 @@ async def test_index_specific_subdir_noarch(package_directory): @pytest.mark.asyncio -async def test_index_repodata_revision_info(package_directory): +async def test_index_repodata_revisions(package_directory): + # Timestamps round-trip through Unix milliseconds, so exercise millisecond + # precision and assert against the exact millisecond values. + epoch = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) + oldest_ms = 1710000000123 + newest_ms = 1773851561010 await index_fs( package_directory, Platform("noarch"), - repodata_revisions=[ - RepodataRevisionInfo( - revision="v3", - n_packages=123, - oldest=1710000000000, - newest=1710000000001, - ) - ], + repodata_revisions={ + "v3": { + "n_packages": 123, + "oldest": epoch + datetime.timedelta(milliseconds=oldest_ms), + "newest": epoch + datetime.timedelta(milliseconds=newest_ms), + } + }, package_revision_assignment="latest", force=True, ) @@ -84,14 +89,13 @@ async def test_index_repodata_revision_info(package_directory): repodata = json.load(f) assert "pytweening-1.0.4-pyhd8ed1ab_0" in repodata["v3"]["tar.bz2"] - assert repodata["info"]["repodata_revisions"] == [ - { - "revision": 3, + assert repodata["info"]["repodata_revisions"] == { + "v3": { "n_packages": 123, - "oldest": 1710000000000, - "newest": 1710000000001, + "oldest": oldest_ms, + "newest": newest_ms, } - ] + } # ---------------------------------------- S3 ---------------------------------------- # diff --git a/test-data/channels/with-wheels/noarch/repodata.json b/test-data/channels/with-wheels/noarch/repodata.json index de1dfddf2d..9621715d22 100644 --- a/test-data/channels/with-wheels/noarch/repodata.json +++ b/test-data/channels/with-wheels/noarch/repodata.json @@ -1,6 +1,13 @@ { "info": { - "subdir": "noarch" + "subdir": "noarch", + "repodata_revisions": { + "v3": { + "n_packages": 2, + "oldest": 1768249989851, + "newest": 1773851561010 + } + } }, "packages": { "six-1.8.0-py3_none_any_0.tar.bz2": {