From e8c93dd734d1e67f7a178cc2bc161c945c8a39c8 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 31 Mar 2026 09:22:08 +0200 Subject: [PATCH 01/18] feat: Filter package records on the environment python minor version Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/api/channel_loader.hpp | 8 +- libmamba/include/mamba/core/shards.hpp | 20 ++- libmamba/src/api/channel_loader.cpp | 117 ++++++++++++++++-- libmamba/src/api/install.cpp | 3 +- libmamba/src/api/update.cpp | 12 +- libmamba/src/api/utils.cpp | 89 ++++++++++++- libmamba/src/api/utils.hpp | 13 +- libmamba/src/core/shards.cpp | 106 ++++++++++++++-- libmamba/src/solver/libsolv/database.cpp | 1 + .../test_sharded_repodata_integration.cpp | 52 +++++++- libmamba/tests/src/core/test_shards.cpp | 79 ++++++++++++ micromamba/tests/test_create.py | 49 ++++---- 12 files changed, 499 insertions(+), 50 deletions(-) diff --git a/libmamba/include/mamba/api/channel_loader.hpp b/libmamba/include/mamba/api/channel_loader.hpp index 89961d4ce5..5b5af70e07 100644 --- a/libmamba/include/mamba/api/channel_loader.hpp +++ b/libmamba/include/mamba/api/channel_loader.hpp @@ -7,11 +7,13 @@ #ifndef MAMBA_API_CHANNEL_LOADER_HPP #define MAMBA_API_CHANNEL_LOADER_HPP +#include #include #include #include #include "mamba/core/error_handling.hpp" +#include "mamba/specs/version.hpp" namespace mamba { @@ -50,7 +52,8 @@ namespace mamba std::vector& subdirs, std::size_t subdir_idx, std::set& loaded_subdirs_with_shards, - const std::vector& priorities + const std::vector& priorities, + std::optional requested_python_minor = std::nullopt ) -> expected_t; class ChannelContext; @@ -86,7 +89,8 @@ namespace mamba ChannelContext& channel_context, solver::libsolv::Database& database, MultiPackageCache& package_caches, - const std::vector& root_packages = {} + const std::vector& root_packages = {}, + std::optional requested_python_minor = std::nullopt ) -> expected_t; /* Brief Creates channels and mirrors objects, diff --git a/libmamba/include/mamba/core/shards.hpp b/libmamba/include/mamba/core/shards.hpp index d69086d497..5e11bd5362 100644 --- a/libmamba/include/mamba/core/shards.hpp +++ b/libmamba/include/mamba/core/shards.hpp @@ -22,6 +22,7 @@ #include "mamba/fs/filesystem.hpp" #include "mamba/specs/authentication_info.hpp" #include "mamba/specs/channel.hpp" +#include "mamba/specs/version.hpp" namespace mamba { @@ -30,6 +31,12 @@ namespace mamba * * This class manages fetching and caching of individual shards from * a sharded repodata index. + * + * **Python minor prefilter:** When constructed with ``requested_python_minor`` (e.g. 3.12), + * parsing a shard msgpack drops package records whose ``depends`` list constrains + * ``python`` to a range that does not contain that minor, reducing work for the solver. + * When that optional is unset, no such filtering is applied and all records in the shard + * are parsed (python compatibility is left to the solver). */ class Shards { @@ -47,6 +54,9 @@ namespace mamba * @param mirrors Optional base mirrors for channel-based downloads. When provided, * extend_mirrors in fetch_shards will be initialized from these before adding * absolute-URL mirrors. + * @param requested_python_minor If set, shard parsing filters out records whose + * ``depends`` python constraints are incompatible with this minor; if unset, + * no python-minor-based record filtering is performed. */ Shards( ShardsIndexDict shards_index, @@ -56,7 +66,8 @@ namespace mamba download::RemoteFetchParams remote_fetch_params, // 0 means: auto; value is normalized with normalize_to_affinity_concurrency(). std::size_t download_threads = 0, - std::optional> mirrors = std::nullopt + std::optional> mirrors = std::nullopt, + std::optional requested_python_minor = std::nullopt ); /** Return the names of all packages available in this shard collection. */ @@ -119,6 +130,13 @@ namespace mamba /** Optional base mirrors for channel-based downloads. */ std::optional> m_mirrors; + /** + * Environment python minor used when parsing shards to prefilter package records + * (see ``record_depends_on_requested_python_minor_version`` in shards.cpp). + * Empty means the prefilter is disabled. + */ + std::optional m_requested_python_minor; + /** Visited shards, keyed by package name. */ std::map m_visited; diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index c4bed328b0..fce5a70a27 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -5,10 +5,13 @@ // The full license is in the file LICENSE, distributed with this software. #include +#include #include #include #include +#include + #include "mamba/api/channel_loader.hpp" #include "mamba/core/channel_context.hpp" #include "mamba/core/context.hpp" @@ -24,11 +27,72 @@ #include "mamba/solver/libsolv/repo_info.hpp" #include "mamba/specs/error.hpp" #include "mamba/specs/package_info.hpp" +#include "mamba/specs/version.hpp" + +#include "utils.hpp" namespace mamba { namespace { + std::optional + installed_python_minor_for_prefix(const fs::u8path& target_prefix) + { + const auto parse_minor = [](std::string_view v) -> std::optional + { + auto maybe_version = specs::Version::parse(std::string(v)); + if (maybe_version.has_value()) + { + return maybe_version.value(); + } + return std::nullopt; + }; + const auto conda_meta = target_prefix / "conda-meta"; + if (!fs::exists(conda_meta) || !fs::is_directory(conda_meta)) + { + return std::nullopt; + } + + for (const auto& entry : fs::directory_iterator(conda_meta)) + { + if (!entry.is_regular_file() || entry.path().extension() != ".json") + { + continue; + } + std::ifstream infile(entry.path().std_path()); + if (!infile.is_open()) + { + continue; + } + nlohmann::json j; + try + { + infile >> j; + } + catch (const std::exception&) + { + continue; + } + if (!j.is_object() || j.value("name", "") != "python") + { + continue; + } + const std::string version = j.value("version", ""); + auto dot = version.find('.'); + if (dot == std::string::npos) + { + continue; + } + auto second_dot = version.find('.', dot + 1); + if (second_dot == std::string::npos) + { + return parse_minor(version); + } + return parse_minor(version.substr(0, second_dot)); + } + return std::nullopt; + } + auto create_repo_from_pkgs_dir( const Context& ctx, ChannelContext& channel_context, @@ -235,7 +299,8 @@ namespace mamba std::size_t subdir_idx, std::set& loaded_subdirs_with_shards, const SubdirDownloadParams& subdir_params, - const std::vector& priorities + const std::vector& priorities, + std::optional requested_python_minor ) { auto& subdir = subdirs[subdir_idx]; @@ -253,7 +318,8 @@ namespace mamba subdirs, subdir_idx, loaded_subdirs_with_shards, - priorities + priorities, + requested_python_minor ); if (!res) @@ -434,7 +500,8 @@ namespace mamba const std::vector& priorities, const SubdirDownloadParams& subdir_params, bool is_retry, - std::vector& error_list + std::vector& error_list, + std::optional requested_python_minor ) { std::set loaded_subdirs_with_shards; @@ -475,7 +542,8 @@ namespace mamba i, loaded_subdirs_with_shards, subdir_params, - priorities + priorities, + requested_python_minor ); if (result) @@ -641,7 +709,8 @@ namespace mamba std::vector& subdirs, std::size_t subdir_idx, std::set& loaded_subdirs_with_shards, - const std::vector& priorities + const std::vector& priorities, + std::optional python_minor_from_specs ) -> expected_t { auto& subdir = subdirs[subdir_idx]; @@ -670,6 +739,19 @@ namespace mamba LOG_DEBUG << "Shard index fetched for " << subdir.name(); const auto& channel = subdir.channel(); std::string current_repodata_url = subdir.repodata_url().str(); + const bool python_minor_from_user_spec = python_minor_from_specs.has_value(); + const auto requested_python_minor = python_minor_from_user_spec + ? std::move(python_minor_from_specs) + : installed_python_minor_for_prefix( + ctx.prefix_params.target_prefix + ); + if (requested_python_minor.has_value()) + { + LOG_DEBUG << "Shard prefilter enabled with python minor " + << requested_python_minor.value().to_string() << " (source=" + << (python_minor_from_user_spec ? "user_spec" : "installed_or_fallback") + << ")"; + } // For all subdirs sharing the same channel URL, fetch their shard indices and build // a Shards instance per subdir; collect them into a RepodataSubset. @@ -702,7 +784,8 @@ namespace mamba ctx.authentication_info(), ctx.remote_fetch_params, normalize_to_affinity_concurrency(static_cast(ctx.repodata_shards_threads)), - std::cref(ctx.mirrors) + std::cref(ctx.mirrors), + requested_python_minor ); url_to_subdir_idx[sdir_url] = j; } @@ -761,7 +844,8 @@ namespace mamba solver::libsolv::Database& database, MultiPackageCache& package_caches, const std::vector& root_packages, - bool is_retry + bool is_retry, + std::optional requested_python_minor ) { std::vector subdirs; @@ -808,7 +892,8 @@ namespace mamba priorities, subdir_params, is_retry, - error_list + error_list, + requested_python_minor ); if (loading_failed) @@ -824,7 +909,8 @@ namespace mamba database, package_caches, root_packages, - retry + retry, + requested_python_minor ); } error_list.emplace_back( @@ -843,11 +929,20 @@ namespace mamba ChannelContext& channel_context, solver::libsolv::Database& database, MultiPackageCache& package_caches, - const std::vector& root_packages + const std::vector& root_packages, + std::optional requested_python_minor ) -> expected_t { bool retry = false; - return load_channels_impl(ctx, channel_context, database, package_caches, root_packages, retry); + return load_channels_impl( + ctx, + channel_context, + database, + package_caches, + root_packages, + retry, + std::move(requested_python_minor) + ); } void init_channels(Context& context, ChannelContext& channel_context) diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index 668be448a0..f2166f93b9 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -5,6 +5,7 @@ // The full license is in the file LICENSE, distributed with this software. #include +#include #include #include @@ -555,7 +556,7 @@ namespace mamba auto& no_env = config.at("no_env").value(); validate_target_prefix_and_channels(ctx, create_env); - auto [db, package_caches] = prepare_solver_context(ctx, channel_context, raw_specs); + auto [db, package_caches] = prepare_solver_context(ctx, channel_context, raw_specs, is_retry); auto prefix_data = load_prefix_data_and_installed(ctx, channel_context, db); diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index 5d391d193d..24a5686b14 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -4,6 +4,11 @@ // // The full license is in the file LICENSE, distributed with this software. +#include + +#include + +#include "mamba/api/channel_loader.hpp" #include "mamba/api/configuration.hpp" #include "mamba/api/install.hpp" #include "mamba/api/update.hpp" @@ -147,7 +152,12 @@ namespace mamba auto& retry_clean_cache = config.at("retry_clean_cache").value(); validate_target_prefix_and_channels(ctx, /* create_env= */ false); - auto [db, package_caches] = prepare_solver_context(ctx, channel_context, raw_update_specs); + auto [db, package_caches] = prepare_solver_context( + ctx, + channel_context, + raw_update_specs, + is_retry + ); auto prefix_data = load_prefix_data_and_installed(ctx, channel_context, db); diff --git a/libmamba/src/api/utils.cpp b/libmamba/src/api/utils.cpp index 4677111dcd..a86696ecd5 100644 --- a/libmamba/src/api/utils.cpp +++ b/libmamba/src/api/utils.cpp @@ -5,6 +5,7 @@ // The full license is in the file LICENSE, distributed with this software. #include +#include #include #include @@ -434,7 +435,8 @@ namespace mamba std::pair prepare_solver_context( Context& ctx, ChannelContext& channel_context, - const std::vector& raw_specs + const std::vector& raw_specs, + bool is_retry ) { populate_context_channels_from_specs(raw_specs, ctx); @@ -444,7 +446,37 @@ namespace mamba auto root_packages = ctx.repodata_use_shards ? build_sharded_root_packages(ctx, channel_context, raw_specs) : std::vector{}; - auto maybe_load = load_channels(ctx, channel_context, db, package_caches, root_packages); + + const auto maybe_explicit_python_minor = extract_requested_python_minor(raw_specs); + const bool has_explicit_python_minor = maybe_explicit_python_minor.has_value(); + const bool use_fallback_python_minor = !has_explicit_python_minor && !is_retry; + const bool dont_prefilter_python_minor = is_retry && !has_explicit_python_minor; + + const auto requested_python_minor = [&]() -> std::optional + { + if (use_fallback_python_minor) + { + LOG_DEBUG << "Applying implicit python minor prefilter for first solve attempt: " + << fallback_python_minor; + return specs::Version::parse(std::string(fallback_python_minor)).value(); + } + if (dont_prefilter_python_minor) + { + LOG_DEBUG << "Explicitly disabling python minor prefilter on retry"; + return std::nullopt; + } + return maybe_explicit_python_minor; + }(); + + auto maybe_load = load_channels( + ctx, + channel_context, + db, + package_caches, + root_packages, + requested_python_minor + ); + if (!maybe_load) { throw maybe_load.error(); @@ -485,6 +517,11 @@ namespace mamba { return false; } + if (!is_retry) + { + retry_fn(); + return true; + } unsolvable->explain_problems_to( db, LOG_ERROR, @@ -599,4 +636,52 @@ namespace mamba execute_other_pkg_managers(other_specs, ctx, update); } } + + std::optional + extract_requested_python_minor(const std::vector& specs) + { + for (const auto& spec : specs) + { + auto maybe_name = specs::MatchSpec::extract_name(spec); + if (!maybe_name.has_value() || maybe_name.value() != "python") + { + continue; + } + for (std::size_t i = 0; (i + 2) < spec.size(); ++i) + { + const unsigned char c0 = static_cast(spec[i]); + const unsigned char c1 = static_cast(spec[i + 1]); + const unsigned char c2 = static_cast(spec[i + 2]); + if (!std::isdigit(c0) || c1 != '.' || !std::isdigit(c2)) + { + continue; + } + std::size_t j = i; + while (j < spec.size() && std::isdigit(static_cast(spec[j]))) + { + ++j; + } + if (j >= spec.size() || spec[j] != '.') + { + continue; + } + std::size_t k = j + 1; + while (k < spec.size() && std::isdigit(static_cast(spec[k]))) + { + ++k; + } + if (k == j + 1) + { + continue; + } + const auto maybe_python_minor = specs::Version::parse(spec.substr(i, k - i)); + if (maybe_python_minor.has_value()) + { + return maybe_python_minor.value(); + } + } + } + return std::nullopt; + } + } diff --git a/libmamba/src/api/utils.hpp b/libmamba/src/api/utils.hpp index 9b04dd423c..319040c2c2 100644 --- a/libmamba/src/api/utils.hpp +++ b/libmamba/src/api/utils.hpp @@ -8,12 +8,14 @@ #define MAMBA_UTILS_HPP #include +#include #include #include #include #include #include "mamba/solver/libsolv/solver.hpp" +#include "mamba/specs/version.hpp" #include "tl/expected.hpp" @@ -52,6 +54,7 @@ namespace mamba } using command_args = std::vector; + inline constexpr std::string_view fallback_python_minor = "3.14"; /** * Build the command-line invocation for a secondary package manager (e.g. pip/uv). @@ -90,6 +93,7 @@ namespace mamba std::vector build_sharded_root_packages(const std::vector& raw_specs); /** + * Print environment activation guidance for the current target prefix. */ void print_activation_message(const Context& ctx); @@ -125,7 +129,8 @@ namespace mamba std::pair prepare_solver_context( Context& ctx, ChannelContext& channel_context, - const std::vector& raw_specs + const std::vector& raw_specs, + bool is_retry ); /** @@ -211,6 +216,12 @@ namespace mamba pip::Update update ); + /** + * Extract an explicit python minor requirement (e.g. "3.12") from specs. + */ + std::optional + extract_requested_python_minor(const std::vector& specs); + } #endif // MAMBA_UTILS_HPP diff --git a/libmamba/src/core/shards.cpp b/libmamba/src/core/shards.cpp index fed42f1c17..3f3fddc657 100644 --- a/libmamba/src/core/shards.cpp +++ b/libmamba/src/core/shards.cpp @@ -25,6 +25,7 @@ #include "mamba/core/util.hpp" #include "mamba/download/downloader.hpp" #include "mamba/fs/filesystem.hpp" +#include "mamba/specs/match_spec.hpp" #include "mamba/specs/version.hpp" #include "mamba/util/cryptography.hpp" #include "mamba/util/encoding.hpp" @@ -383,6 +384,78 @@ namespace mamba return record; } + + auto dependency_matches_requested_python_minor( + const std::string& dependency_spec, + const specs::Version& requested_python_minor + ) -> bool + { + auto maybe_name = specs::MatchSpec::extract_name(dependency_spec); + if (!maybe_name.has_value() || maybe_name.value() != "python") + { + return true; + } + auto maybe_match_spec = specs::MatchSpec::parse(dependency_spec); + if (!maybe_match_spec.has_value()) + { + return true; + } + return maybe_match_spec.value().version().contains(requested_python_minor); + } + + /** + * Whether a raw shard package record's ``depends`` list is compatible with the + * requested environment python minor. + * + * When ``requested_python_minor`` is unset, returns true (no prefilter). + * When set, inspects ``depends`` entries for ``python`` and keeps the record only if + * each such constraint contains that minor (see + * ``dependency_matches_requested_python_minor``). + */ + bool record_depends_on_requested_python_minor_version( + const msgpack_object& raw_record_obj, + const std::optional& requested_python_minor + ) + { + if (!requested_python_minor.has_value()) + { + // No requested python minor version is provided + // so the build is installable in the environment. + return true; + } + if (raw_record_obj.type != MSGPACK_OBJECT_MAP) + { + return true; + } + for (std::uint32_t i = 0; i < raw_record_obj.via.map.size; ++i) + { + const msgpack_object& key_obj = raw_record_obj.via.map.ptr[i].key; + const msgpack_object& val_obj = raw_record_obj.via.map.ptr[i].val; + std::string key; + try + { + key = msgpack_object_to_string(key_obj); + } + catch (const std::exception&) + { + continue; + } + if (key != "depends") + { + continue; + } + const auto depends = msgpack_object_to_string_array(val_obj); + for (const auto& dep : depends) + { + if (!dependency_matches_requested_python_minor(dep, requested_python_minor.value())) + { + return false; + } + } + return true; + } + return true; + } } /****************** @@ -396,7 +469,8 @@ namespace mamba specs::AuthenticationDataBase auth_info, download::RemoteFetchParams remote_fetch_params, std::size_t download_threads, - std::optional> mirrors + std::optional> mirrors, + std::optional requested_python_minor ) : m_shards_index(std::move(shards_index)) , m_url(std::move(url)) @@ -405,6 +479,7 @@ namespace mamba , m_remote_fetch_params(std::move(remote_fetch_params)) , m_download_threads(normalize_to_affinity_concurrency(static_cast(download_threads))) , m_mirrors(std::move(mirrors)) + , m_requested_python_minor(std::move(requested_python_minor)) , m_pkgs_cache_root(fs::u8path(util::user_cache_dir()) / "conda" / "pkgs") , m_shard_cache_dir(m_pkgs_cache_root / "cache" / "shards") { @@ -927,19 +1002,32 @@ namespace mamba const msgpack_object& obj = unpacked.data; ShardDict shard; - auto parse_package_records = [](const msgpack_object& map_obj, - std::map& target_map, - const std::string& map_name) + auto parse_package_records = [this]( + const msgpack_object& map_obj, + std::map& target_map, + const std::string& map_name + ) { for (std::uint32_t k = 0; k < map_obj.via.map.size; ++k) { + const auto& msgpack_record = map_obj.via.map.ptr[k]; + const msgpack_object& val = msgpack_record.val; + const msgpack_object& key = msgpack_record.key; try { - std::string pkg_filename = msgpack_object_to_string(map_obj.via.map.ptr[k].key); - specs::RepoDataPackage record = parse_shard_package_record( - map_obj.via.map.ptr[k].val - ); - target_map[pkg_filename] = record; + // Filter out builds which depend on another python minor version + // than the one in the environment, significantly reducing the number of + // builds to parse and to provide to the solver for dependency resolution. + if (!record_depends_on_requested_python_minor_version( + val, + m_requested_python_minor + )) + { + continue; + } + std::string pkg_filename = msgpack_object_to_string(key); + specs::RepoDataPackage parsed_record = parse_shard_package_record(val); + target_map[pkg_filename] = std::move(parsed_record); } catch (const std::exception& e) { diff --git a/libmamba/src/solver/libsolv/database.cpp b/libmamba/src/solver/libsolv/database.cpp index 33bc7dc8e6..8cf9787b11 100644 --- a/libmamba/src/solver/libsolv/database.cpp +++ b/libmamba/src/solver/libsolv/database.cpp @@ -4,6 +4,7 @@ // // The full license is in the file LICENSE, distributed with this software. +#include #include #include #include diff --git a/libmamba/tests/src/core/test_sharded_repodata_integration.cpp b/libmamba/tests/src/core/test_sharded_repodata_integration.cpp index 94a5820986..f4ed0f7938 100644 --- a/libmamba/tests/src/core/test_sharded_repodata_integration.cpp +++ b/libmamba/tests/src/core/test_sharded_repodata_integration.cpp @@ -502,7 +502,57 @@ TEST_CASE( REQUIRE(found_python); } -TEST_CASE("Sharded repodata - solver results consistency", "[mamba::core][sharded][.integration][!mayfail]") +// Exercises the same sharded path with a large dependency tree: shard index, per-package shards, +// repodata build, and solver. Ensures packages like tensorflow (many python-version-specific +// builds in shards) remain resolvable when `repodata_use_shards` is enabled. +TEST_CASE( + "Sharded repodata - solve tensorflow with conda-forge (anaconda.org)", + "[mamba::core][sharded][.integration]" +) +{ + auto& ctx = mambatests::context(); + const std::vector saved_channels = ctx.channels; + const bool saved_use_shards = ctx.repodata_use_shards; + const bool saved_offline = ctx.offline; + on_scope_exit restore_ctx{ [&] + { + ctx.channels = saved_channels; + ctx.repodata_use_shards = saved_use_shards; + ctx.offline = saved_offline; + } }; + + ctx.channels = { "conda-forge" }; + ctx.repodata_use_shards = true; + ctx.offline = false; + + const TemporaryDirectory tmp_dir; + const fs::u8path cache_dir = tmp_dir.path() / "cache"; + fs::create_directories(cache_dir); + + auto channel_context = ChannelContext::make_conda_compatible(ctx); + init_channels(ctx, channel_context); + + auto solved = solve_environment( + ctx, + channel_context, + std::vector{ "tensorflow" }, + true, + cache_dir + ); + REQUIRE(solved.has_value()); + bool found_tensorflow = false; + for (const auto& pkg : solved.value().packages()) + { + if (pkg.name == "tensorflow") + { + found_tensorflow = true; + break; + } + } + REQUIRE(found_tensorflow); +} + +TEST_CASE("Sharded repodata - solver results consistency", "[mamba::core][sharded][.integration]") { auto& ctx = mambatests::context(); ctx.channels = { "https://prefix.dev/conda-forge" }; diff --git a/libmamba/tests/src/core/test_shards.cpp b/libmamba/tests/src/core/test_shards.cpp index a6eea2f674..5d4cba24ac 100644 --- a/libmamba/tests/src/core/test_shards.cpp +++ b/libmamba/tests/src/core/test_shards.cpp @@ -2770,3 +2770,82 @@ TEST_CASE("Shards - process_downloaded_shard") ); } } + +TEST_CASE("Shards - python minor prefilter") +{ + ShardsIndexDict index; + index.info.base_url = "https://example.com/packages"; + index.info.shards_base_url = "shards"; + index.info.subdir = "linux-64"; + index.version = 1; + index.shards["test-pkg"] = std::vector(32, 0xAB); + + specs::Channel channel = make_simple_channel("https://example.com/conda-forge"); + specs::AuthenticationDataBase auth_info; + download::RemoteFetchParams remote_fetch_params; + + const auto tmp_dir = TemporaryDirectory(); + const auto shard_file = tmp_dir.path() / "test-pkg.msgpack.zst"; + + std::map package_to_cache_path; + package_to_cache_path["test-pkg"] = shard_file; + + auto run_for_dep = [&](const std::string& dep, + std::optional python_minor) -> expected_t + { + auto shard_data = create_shard_with_checksum("test-pkg", "1.0.0", "0", { dep }); + { + std::ofstream file(shard_file.string(), std::ios::binary); + file.write( + reinterpret_cast(shard_data.data()), + static_cast(shard_data.size()) + ); + } + + download::Success success; + success.content = download::Filename{ shard_file.string() }; + success.transfer.downloaded_size = shard_data.size(); + + Shards shards( + index, + "https://example.com/conda-forge/linux-64/repodata.json", + channel, + auth_info, + remote_fetch_params, + 0, + std::nullopt, + std::move(python_minor) + ); + return test_process_downloaded_shard(shards, "test-pkg", success, package_to_cache_path); + }; + + SECTION("mismatching python minor is discarded before record creation") + { + auto result = run_for_dep( + "python >=3.11,<3.12", + specs::Version::parse("3.12").value_or(specs::Version()) + ); + REQUIRE(result.has_value()); + REQUIRE(result->packages.empty()); + REQUIRE(result->conda_packages.empty()); + } + + SECTION("matching python minor is retained") + { + auto result = run_for_dep( + "python >=3.12,<3.13", + specs::Version::parse("3.12").value_or(specs::Version()) + ); + REQUIRE(result.has_value()); + REQUIRE(result->packages.size() == 1); + REQUIRE(result->packages.begin()->second.name == "test-pkg"); + } + + SECTION("no python minor context does not apply prefilter") + { + auto result = run_for_dep("python >=3.11,<3.12", std::nullopt); + REQUIRE(result.has_value()); + REQUIRE(result->packages.size() == 1); + REQUIRE(result->packages.begin()->second.name == "test-pkg"); + } +} diff --git a/micromamba/tests/test_create.py b/micromamba/tests/test_create.py index f1d86e8d82..2179988e07 100644 --- a/micromamba/tests/test_create.py +++ b/micromamba/tests/test_create.py @@ -2529,13 +2529,23 @@ def test_create_from_oci_mirrored_channels(tmp_home, tmp_root_prefix, tmp_path, assert res["success"] packages = helpers.umamba_list("-p", env_prefix, "--json") - assert len(packages) == 1 - pkg = packages[0] - assert pkg["name"] == "pandoc" - if spec == "pandoc=3.1.13": - assert pkg["version"] == "3.1.13" - assert pkg["base_url"] == "oci://ghcr.io/channel-mirrors/conda-forge" - assert pkg["channel"] == "oci://ghcr.io/channel-mirrors/conda-forge" + assert len(packages) >= 1 + + # All resolved packages must come from the mirrored OCI channel. + assert all( + package["base_url"] == "oci://ghcr.io/channel-mirrors/conda-forge" + and package["channel"] == "oci://ghcr.io/channel-mirrors/conda-forge" + for package in packages + ) + + requested_name = spec.split("=")[0] + requested_pkg = next( + (package for package in packages if package["name"] == requested_name), None + ) + assert requested_pkg is not None + if "=" in spec: + requested_version = spec.split("=", 1)[1] + assert requested_pkg["version"] == requested_version @pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True) @@ -2561,18 +2571,13 @@ def test_create_from_oci_mirrored_channels_with_deps(tmp_home, tmp_root_prefix, packages = helpers.umamba_list("-p", env_prefix, "--json") assert len(packages) > 2 - assert any( - package["name"] == "xtensor" - and package["base_url"] == "oci://ghcr.io/channel-mirrors/conda-forge" - and package["channel"] == "oci://ghcr.io/channel-mirrors/conda-forge" - for package in packages - ) - assert any( - package["name"] == "xtl" - and package["base_url"] == "oci://ghcr.io/channel-mirrors/conda-forge" + assert all( + package["base_url"] == "oci://ghcr.io/channel-mirrors/conda-forge" and package["channel"] == "oci://ghcr.io/channel-mirrors/conda-forge" for package in packages ) + assert any(package["name"] == "xtensor" for package in packages) + assert any(package["name"] == "xtl" for package in packages) @pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True) @@ -2601,11 +2606,13 @@ def test_create_from_oci_mirrored_channels_pkg_name_mapping( assert res["success"] packages = helpers.umamba_list("-p", env_prefix, "--json") - assert len(packages) == 1 - pkg = packages[0] - assert pkg["name"] == "_go_select" - assert pkg["base_url"] == "oci://ghcr.io/channel-mirrors/conda-forge" - assert pkg["channel"] == "oci://ghcr.io/channel-mirrors/conda-forge" + assert len(packages) >= 1 + assert all( + package["base_url"] == "oci://ghcr.io/channel-mirrors/conda-forge" + and package["channel"] == "oci://ghcr.io/channel-mirrors/conda-forge" + for package in packages + ) + assert any(package["name"] == "_go_select" for package in packages) @pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True) From 368ec705f12ccf51de09b7cb781416de5fb88904 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 2 Apr 2026 11:52:15 +0200 Subject: [PATCH 02/18] maint: Log information about build for improper fields Such as: ``` warning libmamba Failed to parse field 'noarch' (msgpack type=1) in shard package record for 'tensorboard-2.1.1-py38_0.tar.bz2': Expected STR or BIN type for string conversion. This field will be ignored. ``` Signed-off-by: Julien Jerphanion --- libmamba/src/core/shards.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/libmamba/src/core/shards.cpp b/libmamba/src/core/shards.cpp index 3f3fddc657..d9c51d4899 100644 --- a/libmamba/src/core/shards.cpp +++ b/libmamba/src/core/shards.cpp @@ -184,7 +184,8 @@ namespace mamba * This handles the case where sha256 and md5 can be either strings or bytes * (as per Python TypedDict: NotRequired[str | bytes]). */ - auto parse_shard_package_record(const msgpack_object& obj) -> specs::RepoDataPackage + auto parse_shard_package_record(const msgpack_object& obj, std::string_view package_filename) + -> specs::RepoDataPackage { specs::RepoDataPackage record; @@ -365,8 +366,12 @@ namespace mamba catch (const std::exception& e) { LOG_WARNING << "Failed to parse field '" << key - << "' (type=" << static_cast(val_obj.type) - << ") in shard package record: " << e.what(); + << "' (msgpack type=" << static_cast(val_obj.type) + << ") in shard package record" + << (package_filename.empty() + ? "" + : (" for '" + std::string(package_filename) + "'")) + << ": " << e.what() << ". This field will be ignored."; // Continue parsing other fields } } @@ -1026,7 +1031,10 @@ namespace mamba continue; } std::string pkg_filename = msgpack_object_to_string(key); - specs::RepoDataPackage parsed_record = parse_shard_package_record(val); + specs::RepoDataPackage parsed_record = parse_shard_package_record( + val, + pkg_filename + ); target_map[pkg_filename] = std::move(parsed_record); } catch (const std::exception& e) From e160440619630b67ec393941fcb6450e427131a0 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 2 Apr 2026 14:05:47 +0200 Subject: [PATCH 03/18] Adapt tests Signed-off-by: Julien Jerphanion --- micromamba/tests/test_install.py | 41 +++++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/micromamba/tests/test_install.py b/micromamba/tests/test_install.py index 40720db5b6..681d4c00bc 100644 --- a/micromamba/tests/test_install.py +++ b/micromamba/tests/test_install.py @@ -486,7 +486,9 @@ def test_no_python_pinning(self, existing_cache): keys = {"success", "prefix", "actions", "dry_run"} assert keys.issubset(set(res.keys())) - action_keys = {"LINK", "UNLINK", "PREFIX"} + # LINK and PREFIX are always present; FETCH appears when packages must be + # downloaded; UNLINK may be omitted when nothing is removed. + action_keys = {"LINK", "PREFIX"} assert action_keys.issubset(set(res["actions"].keys())) # When using `--no-py-pin`, it may or may not update the already installed @@ -496,27 +498,27 @@ def test_no_python_pinning(self, existing_cache): link_packages = {pkg["name"] for pkg in res["actions"]["LINK"]} assert expected_link_packages.issubset(link_packages) - unlink_packages = {pkg["name"] for pkg in res["actions"]["UNLINK"]} + unlink_list = res["actions"].get("UNLINK", []) + unlink_packages = {pkg["name"] for pkg in unlink_list} if {"python"}.issubset(link_packages): assert {"python"}.issubset(unlink_packages) py_pkg = [pkg for pkg in res["actions"]["LINK"] if pkg["name"] == "python"][0] assert py_pkg["version"] != ("3.9.19") - py_pkg = [pkg for pkg in res["actions"]["UNLINK"] if pkg["name"] == "python"][0] + py_pkg = [pkg for pkg in unlink_list if pkg["name"] == "python"][0] assert py_pkg["version"] == ("3.9.19") else: - assert len(res["actions"]["LINK"]) == 2 # Should be setuptools and python_abi - - py_abi_pkg = [pkg for pkg in res["actions"]["LINK"] if pkg["name"] == "python_abi"][0] + link_list = res["actions"]["LINK"] + py_abi_pkg = [pkg for pkg in link_list if pkg["name"] == "python_abi"][0] assert py_abi_pkg["version"] == ("3.9") - setuptools_pkg = [pkg for pkg in res["actions"]["LINK"] if pkg["name"] == "setuptools"][ - 0 - ] - assert setuptools_pkg["version"] == ("63.4.3") + if "setuptools" in link_packages: + setuptools_pkg = [pkg for pkg in link_list if pkg["name"] == "setuptools"][0] + assert setuptools_pkg["version"] == ("63.4.3") - assert len(res["actions"]["UNLINK"]) == 1 # Should be setuptools - assert res["actions"]["UNLINK"][0]["name"] == "setuptools" + if unlink_list: + assert len(unlink_list) == 1 # Should be setuptools + assert unlink_list[0]["name"] == "setuptools" @pytest.mark.skipif( helpers.dry_run_tests is helpers.DryRun.ULTRA_DRY, @@ -758,11 +760,16 @@ def test_python_abi_preserved_with_freethreading(tmp_home, tmp_root_prefix): try: helpers.install("-n", env_name, "--json", "matplotlib", no_dry_run=True) except subprocess.CalledProcessError as e: - assert "matplotlib =* * is installable with the potential options" in e.stderr.decode( - "utf-8" - ), ( - "Expected error message about matplotlib being installable with a non-free-threaded python_abi. " - "If this test fails, it might be because matplotlib is now installable with a non-free-threaded python_abi." + # With `--json`, stderr may hold the problem tree and stdout the JSON payload; the + # tree line format can change (e.g. `matplotlib =* *` vs `matplotlib`). JSON + # `solver_problems` uses libsolv strings, which omit the tree phrase. + combined = (e.stderr or b"").decode("utf-8") + (e.stdout or b"").decode("utf-8") + assert "matplotlib" in combined.lower(), combined + tree_explanation = "is installable with the potential options" in combined + mentions_abi = "python_abi" in combined + assert tree_explanation or mentions_abi, ( + "Expected a problem tree or python_abi mention for the matplotlib conflict. Output was:\n" + + combined ) # Verify python_abi is still the same (free-threaded) From b1847720a285e5e23977fc6e04bddc9731ee8293 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 2 Apr 2026 15:46:59 +0200 Subject: [PATCH 04/18] Compare dependencies' `python` `VersionSpec` on their minor version Signed-off-by: Julien Jerphanion --- libmamba/CMakeLists.txt | 1 + .../core/shard_python_minor_prefilter.hpp | 37 +++++ libmamba/src/core/shards.cpp | 76 ++++++--- libmamba/tests/src/core/test_shards.cpp | 155 ++++++++++++++++++ 4 files changed, 250 insertions(+), 19 deletions(-) create mode 100644 libmamba/include/mamba/core/shard_python_minor_prefilter.hpp diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index 15ee98e6ce..ce90037c95 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -401,6 +401,7 @@ set( ${LIBMAMBA_INCLUDE_DIR}/mamba/core/repo_checker_store.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/run.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shell_init.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shard_python_minor_prefilter.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shards.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shard_index_loader.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shard_types.hpp diff --git a/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp b/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp new file mode 100644 index 0000000000..72e6e90a54 --- /dev/null +++ b/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp @@ -0,0 +1,37 @@ +// Copyright (c) 2026, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#ifndef MAMBA_CORE_SHARD_PYTHON_MINOR_PREFILTER_HPP +#define MAMBA_CORE_SHARD_PYTHON_MINOR_PREFILTER_HPP + +#include + +#include "mamba/specs/version.hpp" +#include "mamba/specs/version_spec.hpp" + +namespace mamba +{ + /** + * For a single ``== `` leaf, replace with ``== `` so + * ``VersionSpec::contains`` matches a user ``python=X.Y`` point. Other specs are returned + * unchanged. + */ + [[nodiscard]] auto relax_version_spec_to_minor(const specs::VersionSpec& vs) + -> specs::VersionSpec; + + /** + * Whether a ``depends`` line for ``python`` is compatible with the requested minor. + * Uses ``VersionSpec::contains`` on the parsed version first; if that fails, relaxes exact + * on ``major.minor`` (see ``relax_version_spec_to_minor``) and tests again. + * Non-python dependencies always return true; parse failures return true (no prefilter). + */ + [[nodiscard]] auto dependency_matches_requested_python_minor( + const std::string& dependency_spec, + const specs::Version& requested_python_minor + ) -> bool; +} + +#endif diff --git a/libmamba/src/core/shards.cpp b/libmamba/src/core/shards.cpp index d9c51d4899..907af8c309 100644 --- a/libmamba/src/core/shards.cpp +++ b/libmamba/src/core/shards.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include @@ -19,6 +19,7 @@ #include "mamba/core/logging.hpp" #include "mamba/core/output.hpp" +#include "mamba/core/shard_python_minor_prefilter.hpp" #include "mamba/core/shard_types.hpp" #include "mamba/core/shards.hpp" #include "mamba/core/subdir_index.hpp" @@ -27,6 +28,7 @@ #include "mamba/fs/filesystem.hpp" #include "mamba/specs/match_spec.hpp" #include "mamba/specs/version.hpp" +#include "mamba/specs/version_spec.hpp" #include "mamba/util/cryptography.hpp" #include "mamba/util/encoding.hpp" #include "mamba/util/environment.hpp" @@ -37,6 +39,60 @@ namespace mamba { + auto relax_version_spec_to_minor(const specs::VersionSpec& vs) -> specs::VersionSpec + { + // Only relax a single exact-equality leaf; other shapes keep normal ``contains``. + if (vs.expression_size() != 1) + { + return vs; + } + const std::string vs_str = vs.to_string(); + if (!util::starts_with(vs_str, specs::VersionSpec::equal_str)) + { + return vs; + } + const auto ver_tail = std::string_view(vs_str).substr(specs::VersionSpec::equal_str.size()); + auto maybe_v = specs::Version::parse(util::lstrip(ver_tail)); + if (!maybe_v.has_value()) + { + return vs; + } + const std::string minor_str = maybe_v->to_string(2); + if (auto maybe_minor = specs::Version::parse(minor_str); maybe_minor.has_value()) + { + return specs::VersionSpec::from_predicate( + specs::VersionPredicate::make_equal_to(std::move(maybe_minor).value()) + ); + } + return vs; + } + + auto dependency_matches_requested_python_minor( + const std::string& dependency_spec, + const specs::Version& requested_python_minor + ) -> bool + { + auto maybe_name = specs::MatchSpec::extract_name(dependency_spec); + if (!maybe_name.has_value() || maybe_name.value() != "python") + { + return true; + } + auto maybe_match_spec = specs::MatchSpec::parse(dependency_spec); + if (!maybe_match_spec.has_value()) + { + return true; + } + const auto& ms = maybe_match_spec.value(); + const auto& vs = ms.version(); + if (vs.contains(requested_python_minor)) + { + return true; + } + + // Relax the version spec on the minor version (ignoring the patch version and build string) + return relax_version_spec_to_minor(vs).contains(requested_python_minor); + } + namespace { // Helper functions to extract values from msgpack_object (C API) @@ -390,24 +446,6 @@ namespace mamba return record; } - auto dependency_matches_requested_python_minor( - const std::string& dependency_spec, - const specs::Version& requested_python_minor - ) -> bool - { - auto maybe_name = specs::MatchSpec::extract_name(dependency_spec); - if (!maybe_name.has_value() || maybe_name.value() != "python") - { - return true; - } - auto maybe_match_spec = specs::MatchSpec::parse(dependency_spec); - if (!maybe_match_spec.has_value()) - { - return true; - } - return maybe_match_spec.value().version().contains(requested_python_minor); - } - /** * Whether a raw shard package record's ``depends`` list is compatible with the * requested environment python minor. diff --git a/libmamba/tests/src/core/test_shards.cpp b/libmamba/tests/src/core/test_shards.cpp index 5d4cba24ac..03a8ad248d 100644 --- a/libmamba/tests/src/core/test_shards.cpp +++ b/libmamba/tests/src/core/test_shards.cpp @@ -12,6 +12,7 @@ #include #include "mamba/core/channel_context.hpp" +#include "mamba/core/shard_python_minor_prefilter.hpp" #include "mamba/core/shard_types.hpp" #include "mamba/core/shards.hpp" #include "mamba/core/util.hpp" @@ -23,8 +24,10 @@ #include "mamba/specs/conda_url.hpp" #include "mamba/specs/unresolved_channel.hpp" #include "mamba/specs/version.hpp" +#include "mamba/specs/version_spec.hpp" #include "mamba/util/encoding.hpp" #include "mamba/util/environment.hpp" +#include "mamba/util/string.hpp" #include "mamba/validation/tools.hpp" #include "mambatests.hpp" @@ -2848,4 +2851,156 @@ TEST_CASE("Shards - python minor prefilter") REQUIRE(result->packages.size() == 1); REQUIRE(result->packages.begin()->second.name == "test-pkg"); } + + SECTION("exact python pin matches requested minor (conda three-token depends)") + { + auto result = run_for_dep( + "python 3.7.12 0_73_pypy", + specs::Version::parse("3.7").value_or(specs::Version()) + ); + REQUIRE(result.has_value()); + REQUIRE(result->packages.size() == 1); + REQUIRE(result->packages.begin()->second.name == "test-pkg"); + } +} + +TEST_CASE("relax_version_spec_to_minor") +{ + using specs::Version; + using specs::VersionSpec; + + const auto req = [](std::string_view s) -> Version { return Version::parse(s).value(); }; + + SECTION("bare equality pin relaxes so requested minor is contained") + { + const auto vs = VersionSpec::parse("3.7.12").value(); + const auto relaxed = relax_version_spec_to_minor(vs); + REQUIRE(relaxed.contains(req("3.7"))); + REQUIRE_FALSE(relaxed.contains(req("3.8"))); + } + + SECTION("explicit double-equals string form relaxes") + { + const auto vs = VersionSpec::parse("==3.7.12").value(); + const auto relaxed = relax_version_spec_to_minor(vs); + REQUIRE(relaxed.contains(req("3.7"))); + REQUIRE(util::starts_with(relaxed.to_string(), "==")); + } + + SECTION("four-component pin relaxes to first two components") + { + const auto vs = VersionSpec::parse("1.2.3.4").value(); + const auto relaxed = relax_version_spec_to_minor(vs); + REQUIRE(relaxed.contains(req("1.2"))); + REQUIRE_FALSE(relaxed.contains(req("1.3"))); + } + + SECTION("greater-or-equal is unchanged") + { + const auto vs = VersionSpec::parse(">=3.7").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("less-than is unchanged") + { + const auto vs = VersionSpec::parse("<4").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("compatible-release operator is unchanged") + { + const auto vs = VersionSpec::parse("~=3.7").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("not-equal is unchanged") + { + const auto vs = VersionSpec::parse("!=3.7.12").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("disjunction is unchanged") + { + const auto vs = VersionSpec::parse("==3.7.12|==3.8.0").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("conjunction is unchanged") + { + const auto vs = VersionSpec::parse(">=3.7,<3.8").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("free spec is unchanged") + { + const VersionSpec vs{}; + REQUIRE(relax_version_spec_to_minor(vs).is_explicitly_free()); + } +} + +TEST_CASE("dependency_matches_requested_python_minor") +{ + const auto req = [](std::string_view s) -> specs::Version + { return specs::Version::parse(s).value(); }; + + SECTION("non-python dependency is not filtered") + { + REQUIRE(dependency_matches_requested_python_minor("numpy >=1.0", req("3.12"))); + REQUIRE(dependency_matches_requested_python_minor("libstdcxx-ng >=12", req("3.12"))); + } + + SECTION("name starting with python but not the python package") + { + REQUIRE(dependency_matches_requested_python_minor("python_abi 3.12 1_cp312", req("3.12"))); + } + + SECTION("python version range matches requested minor") + { + REQUIRE(dependency_matches_requested_python_minor("python >=3.12,<3.13", req("3.12"))); + REQUIRE(dependency_matches_requested_python_minor("python >=3.12", req("3.12"))); + } + + SECTION("python version range does not match requested minor") + { + REQUIRE_FALSE(dependency_matches_requested_python_minor("python >=3.12,<3.13", req("3.11"))); + REQUIRE_FALSE(dependency_matches_requested_python_minor("python >=3.12,<3.13", req("3.13"))); + } + + SECTION("exact three-token conda pin matches requested minor") + { + REQUIRE(dependency_matches_requested_python_minor("python 3.7.12 0_73_pypy", req("3.7"))); + } + + SECTION("two-token exact pin matches requested minor") + { + REQUIRE(dependency_matches_requested_python_minor("python 3.7.12", req("3.7"))); + } + + SECTION("exact pin does not match different minor") + { + REQUIRE_FALSE(dependency_matches_requested_python_minor("python 3.8.0", req("3.7"))); + } + + SECTION("leading whitespace on dependency line") + { + REQUIRE(dependency_matches_requested_python_minor(" python >=3.12,<3.13", req("3.12"))); + } + + SECTION("unparsable python dependency does not filter (passes)") + { + REQUIRE(dependency_matches_requested_python_minor("python ,,not-a-valid-spec,,", req("3.12"))); + } + + SECTION("namespaced python pin") + { + REQUIRE(dependency_matches_requested_python_minor( + "conda-forge::python 3.7.12 0_73_pypy", + req("3.7") + )); + } + + SECTION("only python in range with no upper bound") + { + REQUIRE(dependency_matches_requested_python_minor("python", req("3.12"))); + } } From 5da237ba6ebb4186b04f4637359a9bb260f372f5 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 2 Apr 2026 16:52:07 +0200 Subject: [PATCH 05/18] Centralize inference python minor version in `prepare_solver_context` Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/api/channel_loader.hpp | 2 + libmamba/src/api/channel_loader.cpp | 79 ++------------- libmamba/src/api/utils.cpp | 98 ++++++++++++++++--- libmamba/src/api/utils.hpp | 4 + 4 files changed, 99 insertions(+), 84 deletions(-) diff --git a/libmamba/include/mamba/api/channel_loader.hpp b/libmamba/include/mamba/api/channel_loader.hpp index 5b5af70e07..0f94130387 100644 --- a/libmamba/include/mamba/api/channel_loader.hpp +++ b/libmamba/include/mamba/api/channel_loader.hpp @@ -43,6 +43,8 @@ namespace mamba * @param subdir_idx Index of the subdir to load in \p subdirs. * @param loaded_subdirs_with_shards Set of subdir names already loaded via shards (updated). * @param priorities Repo priorities aligned with \p subdirs. + * @param requested_python_minor Optional python minor for shard record prefiltering (from + * \c prepare_solver_context). * @return The repo for the requested subdir, or unexpected mamba_error on failure. */ auto load_subdir_with_shards( diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index fce5a70a27..5e49a2787e 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -5,13 +5,10 @@ // The full license is in the file LICENSE, distributed with this software. #include -#include #include #include #include -#include - #include "mamba/api/channel_loader.hpp" #include "mamba/core/channel_context.hpp" #include "mamba/core/context.hpp" @@ -35,64 +32,6 @@ namespace mamba { namespace { - std::optional - installed_python_minor_for_prefix(const fs::u8path& target_prefix) - { - const auto parse_minor = [](std::string_view v) -> std::optional - { - auto maybe_version = specs::Version::parse(std::string(v)); - if (maybe_version.has_value()) - { - return maybe_version.value(); - } - return std::nullopt; - }; - const auto conda_meta = target_prefix / "conda-meta"; - if (!fs::exists(conda_meta) || !fs::is_directory(conda_meta)) - { - return std::nullopt; - } - - for (const auto& entry : fs::directory_iterator(conda_meta)) - { - if (!entry.is_regular_file() || entry.path().extension() != ".json") - { - continue; - } - std::ifstream infile(entry.path().std_path()); - if (!infile.is_open()) - { - continue; - } - nlohmann::json j; - try - { - infile >> j; - } - catch (const std::exception&) - { - continue; - } - if (!j.is_object() || j.value("name", "") != "python") - { - continue; - } - const std::string version = j.value("version", ""); - auto dot = version.find('.'); - if (dot == std::string::npos) - { - continue; - } - auto second_dot = version.find('.', dot + 1); - if (second_dot == std::string::npos) - { - return parse_minor(version); - } - return parse_minor(version.substr(0, second_dot)); - } - return std::nullopt; - } - auto create_repo_from_pkgs_dir( const Context& ctx, ChannelContext& channel_context, @@ -710,7 +649,7 @@ namespace mamba std::size_t subdir_idx, std::set& loaded_subdirs_with_shards, const std::vector& priorities, - std::optional python_minor_from_specs + std::optional requested_python_minor ) -> expected_t { auto& subdir = subdirs[subdir_idx]; @@ -739,18 +678,14 @@ namespace mamba LOG_DEBUG << "Shard index fetched for " << subdir.name(); const auto& channel = subdir.channel(); std::string current_repodata_url = subdir.repodata_url().str(); - const bool python_minor_from_user_spec = python_minor_from_specs.has_value(); - const auto requested_python_minor = python_minor_from_user_spec - ? std::move(python_minor_from_specs) - : installed_python_minor_for_prefix( - ctx.prefix_params.target_prefix - ); if (requested_python_minor.has_value()) { - LOG_DEBUG << "Shard prefilter enabled with python minor " - << requested_python_minor.value().to_string() << " (source=" - << (python_minor_from_user_spec ? "user_spec" : "installed_or_fallback") - << ")"; + LOG_DEBUG << "Shard prefilter on python minor version enabled with " + << requested_python_minor.value().to_string(); + } + else + { + LOG_DEBUG << "Shard prefilter on python minor version disabled."; } // For all subdirs sharing the same channel URL, fetch their shard indices and build diff --git a/libmamba/src/api/utils.cpp b/libmamba/src/api/utils.cpp index a86696ecd5..7a8e03c27b 100644 --- a/libmamba/src/api/utils.cpp +++ b/libmamba/src/api/utils.cpp @@ -4,6 +4,8 @@ // // The full license is in the file LICENSE, distributed with this software. +#include +#include #include #include @@ -11,6 +13,7 @@ #include #include #include +#include #include #include @@ -115,6 +118,64 @@ namespace mamba ); } } + + std::optional + installed_python_minor_for_prefix(const fs::u8path& target_prefix) + { + const auto parse_minor = [](std::string_view v) -> std::optional + { + auto maybe_version = specs::Version::parse(std::string(v)); + if (maybe_version.has_value()) + { + return maybe_version.value(); + } + return std::nullopt; + }; + const auto conda_meta = target_prefix / "conda-meta"; + if (!fs::exists(conda_meta) || !fs::is_directory(conda_meta)) + { + return std::nullopt; + } + + for (const auto& entry : fs::directory_iterator(conda_meta)) + { + if (!entry.is_regular_file() || entry.path().extension() != ".json") + { + continue; + } + std::ifstream infile(entry.path().std_path()); + if (!infile.is_open()) + { + continue; + } + nlohmann::json j; + try + { + infile >> j; + } + catch (const std::exception&) + { + continue; + } + if (!j.is_object() || j.value("name", "") != "python") + { + continue; + } + const std::string version = j.value("version", ""); + auto dot = version.find('.'); + if (dot == std::string::npos) + { + continue; + } + auto second_dot = version.find('.', dot + 1); + if (second_dot == std::string::npos) + { + return parse_minor(version); + } + return parse_minor(version.substr(0, second_dot)); + } + return std::nullopt; + } } bool reproc_killed(int status) @@ -447,25 +508,38 @@ namespace mamba ? build_sharded_root_packages(ctx, channel_context, raw_specs) : std::vector{}; - const auto maybe_explicit_python_minor = extract_requested_python_minor(raw_specs); - const bool has_explicit_python_minor = maybe_explicit_python_minor.has_value(); - const bool use_fallback_python_minor = !has_explicit_python_minor && !is_retry; - const bool dont_prefilter_python_minor = is_retry && !has_explicit_python_minor; - const auto requested_python_minor = [&]() -> std::optional { - if (use_fallback_python_minor) + const auto maybe_explicit_python_minor = extract_requested_python_minor(raw_specs); + const bool has_explicit_python_minor = maybe_explicit_python_minor.has_value(); + + if (has_explicit_python_minor) { - LOG_DEBUG << "Applying implicit python minor prefilter for first solve attempt: " - << fallback_python_minor; - return specs::Version::parse(std::string(fallback_python_minor)).value(); + LOG_DEBUG << "Pre-filtering shards using explicitly provided python minor version: " + << maybe_explicit_python_minor.value().to_string(); + return maybe_explicit_python_minor.value(); } - if (dont_prefilter_python_minor) + + if (is_retry) { - LOG_DEBUG << "Explicitly disabling python minor prefilter on retry"; + LOG_DEBUG << "Retry without prefiltering on any python minor version."; return std::nullopt; } - return maybe_explicit_python_minor; + + const auto maybe_installed_python_minor = installed_python_minor_for_prefix( + ctx.prefix_params.target_prefix + ); + + if (maybe_installed_python_minor.has_value()) + { + LOG_DEBUG << "Pre-filtering shards using installed python minor version: " + << maybe_installed_python_minor.value().to_string(); + return maybe_installed_python_minor.value(); + } + + LOG_DEBUG << "Pre-filtering shards using fallback python minor version: " + << fallback_python_minor; + return specs::Version::parse(std::string(fallback_python_minor)).value(); }(); auto maybe_load = load_channels( diff --git a/libmamba/src/api/utils.hpp b/libmamba/src/api/utils.hpp index 319040c2c2..85b3b75f50 100644 --- a/libmamba/src/api/utils.hpp +++ b/libmamba/src/api/utils.hpp @@ -125,6 +125,10 @@ namespace mamba /** * Prepare solver state: channels, package cache, database, and root package loading. + * + * Computes ``requested_python_minor`` for sharded repodata: explicit python from specs, + * implicit fallback on the first solve attempt, or the installed prefix minor on retry when + * no explicit python is given. */ std::pair prepare_solver_context( Context& ctx, From 11d3af1ca93c71b8625fc3fb626d449d992f7199 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 2 Apr 2026 17:00:12 +0200 Subject: [PATCH 06/18] Rename `specs::Version` variable used to pre-filter Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/api/channel_loader.hpp | 7 +-- .../core/shard_python_minor_prefilter.hpp | 13 ++--- libmamba/include/mamba/core/shards.hpp | 12 ++--- libmamba/src/api/channel_loader.cpp | 26 +++++----- libmamba/src/api/utils.cpp | 18 ++++--- libmamba/src/api/utils.hpp | 6 +-- libmamba/src/core/shards.cpp | 31 ++++++------ libmamba/tests/src/core/test_shards.cpp | 50 +++++++++++++------ 8 files changed, 95 insertions(+), 68 deletions(-) diff --git a/libmamba/include/mamba/api/channel_loader.hpp b/libmamba/include/mamba/api/channel_loader.hpp index 0f94130387..ee92205c4a 100644 --- a/libmamba/include/mamba/api/channel_loader.hpp +++ b/libmamba/include/mamba/api/channel_loader.hpp @@ -43,7 +43,8 @@ namespace mamba * @param subdir_idx Index of the subdir to load in \p subdirs. * @param loaded_subdirs_with_shards Set of subdir names already loaded via shards (updated). * @param priorities Repo priorities aligned with \p subdirs. - * @param requested_python_minor Optional python minor for shard record prefiltering (from + * @param python_minor_version_for_prefilter Optional python minor for shard record prefiltering + * (from * \c prepare_solver_context). * @return The repo for the requested subdir, or unexpected mamba_error on failure. */ @@ -55,7 +56,7 @@ namespace mamba std::size_t subdir_idx, std::set& loaded_subdirs_with_shards, const std::vector& priorities, - std::optional requested_python_minor = std::nullopt + std::optional python_minor_version_for_prefilter = std::nullopt ) -> expected_t; class ChannelContext; @@ -92,7 +93,7 @@ namespace mamba solver::libsolv::Database& database, MultiPackageCache& package_caches, const std::vector& root_packages = {}, - std::optional requested_python_minor = std::nullopt + std::optional python_minor_version_for_prefilter = std::nullopt ) -> expected_t; /* Brief Creates channels and mirrors objects, diff --git a/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp b/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp index 72e6e90a54..9a0a967612 100644 --- a/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp +++ b/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp @@ -23,14 +23,15 @@ namespace mamba -> specs::VersionSpec; /** - * Whether a ``depends`` line for ``python`` is compatible with the requested minor. - * Uses ``VersionSpec::contains`` on the parsed version first; if that fails, relaxes exact - * on ``major.minor`` (see ``relax_version_spec_to_minor``) and tests again. - * Non-python dependencies always return true; parse failures return true (no prefilter). + * Whether a ``depends`` line for ``python`` is compatible with + * ``python_minor_version_for_prefilter``. Uses ``VersionSpec::contains`` on the parsed version + * first; if that fails, relaxes exact on ``major.minor`` (see ``relax_version_spec_to_minor``) + * and tests again. Non-python dependencies always return true; parse failures return true (no + * prefilter). */ - [[nodiscard]] auto dependency_matches_requested_python_minor( + [[nodiscard]] auto dependency_matches_python_minor_version_for_prefilter( const std::string& dependency_spec, - const specs::Version& requested_python_minor + const specs::Version& python_minor_version_for_prefilter ) -> bool; } diff --git a/libmamba/include/mamba/core/shards.hpp b/libmamba/include/mamba/core/shards.hpp index 5e11bd5362..f933bba12c 100644 --- a/libmamba/include/mamba/core/shards.hpp +++ b/libmamba/include/mamba/core/shards.hpp @@ -32,8 +32,8 @@ namespace mamba * This class manages fetching and caching of individual shards from * a sharded repodata index. * - * **Python minor prefilter:** When constructed with ``requested_python_minor`` (e.g. 3.12), - * parsing a shard msgpack drops package records whose ``depends`` list constrains + * **Python minor prefilter:** When constructed with ``python_minor_version_for_prefilter`` + * (e.g. 3.12), parsing a shard msgpack drops package records whose ``depends`` list constrains * ``python`` to a range that does not contain that minor, reducing work for the solver. * When that optional is unset, no such filtering is applied and all records in the shard * are parsed (python compatibility is left to the solver). @@ -54,7 +54,7 @@ namespace mamba * @param mirrors Optional base mirrors for channel-based downloads. When provided, * extend_mirrors in fetch_shards will be initialized from these before adding * absolute-URL mirrors. - * @param requested_python_minor If set, shard parsing filters out records whose + * @param python_minor_version_for_prefilter If set, shard parsing filters out records whose * ``depends`` python constraints are incompatible with this minor; if unset, * no python-minor-based record filtering is performed. */ @@ -67,7 +67,7 @@ namespace mamba // 0 means: auto; value is normalized with normalize_to_affinity_concurrency(). std::size_t download_threads = 0, std::optional> mirrors = std::nullopt, - std::optional requested_python_minor = std::nullopt + std::optional python_minor_version_for_prefilter = std::nullopt ); /** Return the names of all packages available in this shard collection. */ @@ -132,10 +132,10 @@ namespace mamba /** * Environment python minor used when parsing shards to prefilter package records - * (see ``record_depends_on_requested_python_minor_version`` in shards.cpp). + * (see ``record_depends_on_python_minor_version_for_prefilter`` in shards.cpp). * Empty means the prefilter is disabled. */ - std::optional m_requested_python_minor; + std::optional m_python_minor_version_for_prefilter; /** Visited shards, keyed by package name. */ std::map m_visited; diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index 5e49a2787e..77eb382580 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -239,7 +239,7 @@ namespace mamba std::set& loaded_subdirs_with_shards, const SubdirDownloadParams& subdir_params, const std::vector& priorities, - std::optional requested_python_minor + std::optional python_minor_version_for_prefilter ) { auto& subdir = subdirs[subdir_idx]; @@ -258,7 +258,7 @@ namespace mamba subdir_idx, loaded_subdirs_with_shards, priorities, - requested_python_minor + python_minor_version_for_prefilter ); if (!res) @@ -440,7 +440,7 @@ namespace mamba const SubdirDownloadParams& subdir_params, bool is_retry, std::vector& error_list, - std::optional requested_python_minor + std::optional python_minor_version_for_prefilter ) { std::set loaded_subdirs_with_shards; @@ -482,7 +482,7 @@ namespace mamba loaded_subdirs_with_shards, subdir_params, priorities, - requested_python_minor + python_minor_version_for_prefilter ); if (result) @@ -649,7 +649,7 @@ namespace mamba std::size_t subdir_idx, std::set& loaded_subdirs_with_shards, const std::vector& priorities, - std::optional requested_python_minor + std::optional python_minor_version_for_prefilter ) -> expected_t { auto& subdir = subdirs[subdir_idx]; @@ -678,10 +678,10 @@ namespace mamba LOG_DEBUG << "Shard index fetched for " << subdir.name(); const auto& channel = subdir.channel(); std::string current_repodata_url = subdir.repodata_url().str(); - if (requested_python_minor.has_value()) + if (python_minor_version_for_prefilter.has_value()) { LOG_DEBUG << "Shard prefilter on python minor version enabled with " - << requested_python_minor.value().to_string(); + << python_minor_version_for_prefilter.value().to_string(); } else { @@ -720,7 +720,7 @@ namespace mamba ctx.remote_fetch_params, normalize_to_affinity_concurrency(static_cast(ctx.repodata_shards_threads)), std::cref(ctx.mirrors), - requested_python_minor + python_minor_version_for_prefilter ); url_to_subdir_idx[sdir_url] = j; } @@ -780,7 +780,7 @@ namespace mamba MultiPackageCache& package_caches, const std::vector& root_packages, bool is_retry, - std::optional requested_python_minor + std::optional python_minor_version_for_prefilter ) { std::vector subdirs; @@ -828,7 +828,7 @@ namespace mamba subdir_params, is_retry, error_list, - requested_python_minor + python_minor_version_for_prefilter ); if (loading_failed) @@ -845,7 +845,7 @@ namespace mamba package_caches, root_packages, retry, - requested_python_minor + python_minor_version_for_prefilter ); } error_list.emplace_back( @@ -865,7 +865,7 @@ namespace mamba solver::libsolv::Database& database, MultiPackageCache& package_caches, const std::vector& root_packages, - std::optional requested_python_minor + std::optional python_minor_version_for_prefilter ) -> expected_t { bool retry = false; @@ -876,7 +876,7 @@ namespace mamba package_caches, root_packages, retry, - std::move(requested_python_minor) + std::move(python_minor_version_for_prefilter) ); } diff --git a/libmamba/src/api/utils.cpp b/libmamba/src/api/utils.cpp index 7a8e03c27b..228ca91df9 100644 --- a/libmamba/src/api/utils.cpp +++ b/libmamba/src/api/utils.cpp @@ -508,16 +508,18 @@ namespace mamba ? build_sharded_root_packages(ctx, channel_context, raw_specs) : std::vector{}; - const auto requested_python_minor = [&]() -> std::optional + const std::optional python_minor_version_for_prefilter = + [&]() -> std::optional { - const auto maybe_explicit_python_minor = extract_requested_python_minor(raw_specs); - const bool has_explicit_python_minor = maybe_explicit_python_minor.has_value(); + const auto maybe_explicit_requested_python_minor = extract_requested_python_minor( + raw_specs + ); - if (has_explicit_python_minor) + if (maybe_explicit_requested_python_minor.has_value()) { - LOG_DEBUG << "Pre-filtering shards using explicitly provided python minor version: " - << maybe_explicit_python_minor.value().to_string(); - return maybe_explicit_python_minor.value(); + LOG_DEBUG << "Pre-filtering shards using explicitly requested python minor version: " + << maybe_explicit_requested_python_minor.value().to_string(); + return maybe_explicit_requested_python_minor.value(); } if (is_retry) @@ -548,7 +550,7 @@ namespace mamba db, package_caches, root_packages, - requested_python_minor + python_minor_version_for_prefilter ); if (!maybe_load) diff --git a/libmamba/src/api/utils.hpp b/libmamba/src/api/utils.hpp index 85b3b75f50..899ac271cf 100644 --- a/libmamba/src/api/utils.hpp +++ b/libmamba/src/api/utils.hpp @@ -126,9 +126,9 @@ namespace mamba /** * Prepare solver state: channels, package cache, database, and root package loading. * - * Computes ``requested_python_minor`` for sharded repodata: explicit python from specs, - * implicit fallback on the first solve attempt, or the installed prefix minor on retry when - * no explicit python is given. + * Computes ``python_minor_version_for_prefilter`` for sharded repodata: explicit python from + * specs, implicit fallback on the first solve attempt, or the installed prefix minor on retry + * when no explicit python is given. */ std::pair prepare_solver_context( Context& ctx, diff --git a/libmamba/src/core/shards.cpp b/libmamba/src/core/shards.cpp index 907af8c309..5c8e6715c0 100644 --- a/libmamba/src/core/shards.cpp +++ b/libmamba/src/core/shards.cpp @@ -67,9 +67,9 @@ namespace mamba return vs; } - auto dependency_matches_requested_python_minor( + auto dependency_matches_python_minor_version_for_prefilter( const std::string& dependency_spec, - const specs::Version& requested_python_minor + const specs::Version& python_minor_version_for_prefilter ) -> bool { auto maybe_name = specs::MatchSpec::extract_name(dependency_spec); @@ -84,13 +84,13 @@ namespace mamba } const auto& ms = maybe_match_spec.value(); const auto& vs = ms.version(); - if (vs.contains(requested_python_minor)) + if (vs.contains(python_minor_version_for_prefilter)) { return true; } // Relax the version spec on the minor version (ignoring the patch version and build string) - return relax_version_spec_to_minor(vs).contains(requested_python_minor); + return relax_version_spec_to_minor(vs).contains(python_minor_version_for_prefilter); } namespace @@ -450,17 +450,17 @@ namespace mamba * Whether a raw shard package record's ``depends`` list is compatible with the * requested environment python minor. * - * When ``requested_python_minor`` is unset, returns true (no prefilter). + * When ``python_minor_version_for_prefilter`` is unset, returns true (no prefilter). * When set, inspects ``depends`` entries for ``python`` and keeps the record only if * each such constraint contains that minor (see - * ``dependency_matches_requested_python_minor``). + * ``dependency_matches_python_minor_version_for_prefilter``). */ - bool record_depends_on_requested_python_minor_version( + bool record_depends_on_python_minor_version_for_prefilter( const msgpack_object& raw_record_obj, - const std::optional& requested_python_minor + const std::optional& python_minor_version_for_prefilter ) { - if (!requested_python_minor.has_value()) + if (!python_minor_version_for_prefilter.has_value()) { // No requested python minor version is provided // so the build is installable in the environment. @@ -490,7 +490,10 @@ namespace mamba const auto depends = msgpack_object_to_string_array(val_obj); for (const auto& dep : depends) { - if (!dependency_matches_requested_python_minor(dep, requested_python_minor.value())) + if (!dependency_matches_python_minor_version_for_prefilter( + dep, + python_minor_version_for_prefilter.value() + )) { return false; } @@ -513,7 +516,7 @@ namespace mamba download::RemoteFetchParams remote_fetch_params, std::size_t download_threads, std::optional> mirrors, - std::optional requested_python_minor + std::optional python_minor_version_for_prefilter ) : m_shards_index(std::move(shards_index)) , m_url(std::move(url)) @@ -522,7 +525,7 @@ namespace mamba , m_remote_fetch_params(std::move(remote_fetch_params)) , m_download_threads(normalize_to_affinity_concurrency(static_cast(download_threads))) , m_mirrors(std::move(mirrors)) - , m_requested_python_minor(std::move(requested_python_minor)) + , m_python_minor_version_for_prefilter(std::move(python_minor_version_for_prefilter)) , m_pkgs_cache_root(fs::u8path(util::user_cache_dir()) / "conda" / "pkgs") , m_shard_cache_dir(m_pkgs_cache_root / "cache" / "shards") { @@ -1061,9 +1064,9 @@ namespace mamba // Filter out builds which depend on another python minor version // than the one in the environment, significantly reducing the number of // builds to parse and to provide to the solver for dependency resolution. - if (!record_depends_on_requested_python_minor_version( + if (!record_depends_on_python_minor_version_for_prefilter( val, - m_requested_python_minor + m_python_minor_version_for_prefilter )) { continue; diff --git a/libmamba/tests/src/core/test_shards.cpp b/libmamba/tests/src/core/test_shards.cpp index 03a8ad248d..1bcc72b17a 100644 --- a/libmamba/tests/src/core/test_shards.cpp +++ b/libmamba/tests/src/core/test_shards.cpp @@ -2938,62 +2938,82 @@ TEST_CASE("relax_version_spec_to_minor") } } -TEST_CASE("dependency_matches_requested_python_minor") +TEST_CASE("dependency_matches_python_minor_version_for_prefilter") { const auto req = [](std::string_view s) -> specs::Version { return specs::Version::parse(s).value(); }; SECTION("non-python dependency is not filtered") { - REQUIRE(dependency_matches_requested_python_minor("numpy >=1.0", req("3.12"))); - REQUIRE(dependency_matches_requested_python_minor("libstdcxx-ng >=12", req("3.12"))); + REQUIRE(dependency_matches_python_minor_version_for_prefilter("numpy >=1.0", req("3.12"))); + REQUIRE( + dependency_matches_python_minor_version_for_prefilter("libstdcxx-ng >=12", req("3.12")) + ); } SECTION("name starting with python but not the python package") { - REQUIRE(dependency_matches_requested_python_minor("python_abi 3.12 1_cp312", req("3.12"))); + REQUIRE(dependency_matches_python_minor_version_for_prefilter( + "python_abi 3.12 1_cp312", + req("3.12") + )); } SECTION("python version range matches requested minor") { - REQUIRE(dependency_matches_requested_python_minor("python >=3.12,<3.13", req("3.12"))); - REQUIRE(dependency_matches_requested_python_minor("python >=3.12", req("3.12"))); + REQUIRE( + dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.12")) + ); + REQUIRE(dependency_matches_python_minor_version_for_prefilter("python >=3.12", req("3.12"))); } SECTION("python version range does not match requested minor") { - REQUIRE_FALSE(dependency_matches_requested_python_minor("python >=3.12,<3.13", req("3.11"))); - REQUIRE_FALSE(dependency_matches_requested_python_minor("python >=3.12,<3.13", req("3.13"))); + REQUIRE_FALSE( + dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.11")) + ); + REQUIRE_FALSE( + dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.13")) + ); } SECTION("exact three-token conda pin matches requested minor") { - REQUIRE(dependency_matches_requested_python_minor("python 3.7.12 0_73_pypy", req("3.7"))); + REQUIRE( + dependency_matches_python_minor_version_for_prefilter("python 3.7.12 0_73_pypy", req("3.7")) + ); } SECTION("two-token exact pin matches requested minor") { - REQUIRE(dependency_matches_requested_python_minor("python 3.7.12", req("3.7"))); + REQUIRE(dependency_matches_python_minor_version_for_prefilter("python 3.7.12", req("3.7"))); } SECTION("exact pin does not match different minor") { - REQUIRE_FALSE(dependency_matches_requested_python_minor("python 3.8.0", req("3.7"))); + REQUIRE_FALSE( + dependency_matches_python_minor_version_for_prefilter("python 3.8.0", req("3.7")) + ); } SECTION("leading whitespace on dependency line") { - REQUIRE(dependency_matches_requested_python_minor(" python >=3.12,<3.13", req("3.12"))); + REQUIRE( + dependency_matches_python_minor_version_for_prefilter(" python >=3.12,<3.13", req("3.12")) + ); } SECTION("unparsable python dependency does not filter (passes)") { - REQUIRE(dependency_matches_requested_python_minor("python ,,not-a-valid-spec,,", req("3.12"))); + REQUIRE(dependency_matches_python_minor_version_for_prefilter( + "python ,,not-a-valid-spec,,", + req("3.12") + )); } SECTION("namespaced python pin") { - REQUIRE(dependency_matches_requested_python_minor( + REQUIRE(dependency_matches_python_minor_version_for_prefilter( "conda-forge::python 3.7.12 0_73_pypy", req("3.7") )); @@ -3001,6 +3021,6 @@ TEST_CASE("dependency_matches_requested_python_minor") SECTION("only python in range with no upper bound") { - REQUIRE(dependency_matches_requested_python_minor("python", req("3.12"))); + REQUIRE(dependency_matches_python_minor_version_for_prefilter("python", req("3.12"))); } } From ea75e1df6862c5aeba20fffa1c345dcd2541b814 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 2 Apr 2026 17:35:20 +0200 Subject: [PATCH 07/18] fix: Only show flat repodata cache status when not using shards Signed-off-by: Julien Jerphanion --- libmamba/src/api/channel_loader.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libmamba/src/api/channel_loader.cpp b/libmamba/src/api/channel_loader.cpp index 77eb382580..2a7512a10b 100644 --- a/libmamba/src/api/channel_loader.cpp +++ b/libmamba/src/api/channel_loader.cpp @@ -563,7 +563,11 @@ namespace mamba continue; } SubdirIndexLoader subdir_index_loader = std::move(subdir_index_loader_result).value(); - if (subdir_index_loader.valid_cache_found() && Console::can_report_status()) + + // Only show flat repodata cache status if we're not using shards and we have a + // valid cache + if (!ctx.repodata_use_shards && subdir_index_loader.valid_cache_found() + && Console::can_report_status()) { Console::stream() << fmt::format("{:<50} {:>20}", subdir_index_loader.name(), "Using cache"); From b4408e952f175452d1c81dc1006cdf8547236da6 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Thu, 2 Apr 2026 18:19:31 +0200 Subject: [PATCH 08/18] test: Introduce dedicated test files Signed-off-by: Julien Jerphanion --- libmamba/CMakeLists.txt | 2 +- .../core/shard_python_minor_prefilter.hpp | 7 + libmamba/src/api/utils.cpp | 43 +- libmamba/src/api/utils.hpp | 5 + .../src/core/shard_python_minor_prefilter.cpp | 93 ++++ libmamba/src/core/shards.cpp | 56 --- libmamba/tests/CMakeLists.txt | 1 + .../test_shard_python_minor_prefilter.cpp | 449 ++++++++++++++++++ libmamba/tests/src/core/test_shards.cpp | 252 ---------- 9 files changed, 567 insertions(+), 341 deletions(-) create mode 100644 libmamba/src/core/shard_python_minor_prefilter.cpp create mode 100644 libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index ce90037c95..62a9835e46 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -247,6 +247,7 @@ set( ${LIBMAMBA_SOURCE_DIR}/core/query.cpp ${LIBMAMBA_SOURCE_DIR}/core/repo_checker_store.cpp ${LIBMAMBA_SOURCE_DIR}/core/run.cpp + ${LIBMAMBA_SOURCE_DIR}/core/shard_python_minor_prefilter.cpp ${LIBMAMBA_SOURCE_DIR}/core/shell_init.cpp ${LIBMAMBA_SOURCE_DIR}/core/shards.cpp ${LIBMAMBA_SOURCE_DIR}/core/shard_index_loader.cpp @@ -401,7 +402,6 @@ set( ${LIBMAMBA_INCLUDE_DIR}/mamba/core/repo_checker_store.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/run.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shell_init.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shard_python_minor_prefilter.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shards.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shard_index_loader.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shard_types.hpp diff --git a/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp b/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp index 9a0a967612..c128a2f6b0 100644 --- a/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp +++ b/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp @@ -7,6 +7,7 @@ #ifndef MAMBA_CORE_SHARD_PYTHON_MINOR_PREFILTER_HPP #define MAMBA_CORE_SHARD_PYTHON_MINOR_PREFILTER_HPP +#include #include #include "mamba/specs/version.hpp" @@ -14,6 +15,12 @@ namespace mamba { + /** + * If ``vs`` is a single ``==…`` leaf, return the parsed ``Version``; otherwise ``nullopt``. + */ + [[nodiscard]] auto version_from_single_equality_spec(const specs::VersionSpec& vs) + -> std::optional; + /** * For a single ``== `` leaf, replace with ``== `` so * ``VersionSpec::contains`` matches a user ``python=X.Y`` point. Other specs are returned diff --git a/libmamba/src/api/utils.cpp b/libmamba/src/api/utils.cpp index 228ca91df9..dcee5ea603 100644 --- a/libmamba/src/api/utils.cpp +++ b/libmamba/src/api/utils.cpp @@ -7,7 +7,6 @@ #include #include #include -#include #include #include @@ -28,6 +27,7 @@ #include "mamba/core/package_cache.hpp" #include "mamba/core/package_database_loader.hpp" #include "mamba/core/prefix_data.hpp" +#include "mamba/core/shard_python_minor_prefilter.hpp" #include "mamba/core/transaction.hpp" #include "mamba/core/util.hpp" #include "mamba/core/util_os.hpp" @@ -35,7 +35,9 @@ #include "mamba/solver/libsolv/database.hpp" #include "mamba/solver/request.hpp" #include "mamba/specs/match_spec.hpp" +#include "mamba/specs/version_spec.hpp" #include "mamba/util/environment.hpp" +#include "mamba/util/string.hpp" #include "utils.hpp" @@ -723,38 +725,15 @@ namespace mamba { continue; } - for (std::size_t i = 0; (i + 2) < spec.size(); ++i) + auto maybe_ms = specs::MatchSpec::parse(spec); + if (!maybe_ms.has_value()) { - const unsigned char c0 = static_cast(spec[i]); - const unsigned char c1 = static_cast(spec[i + 1]); - const unsigned char c2 = static_cast(spec[i + 2]); - if (!std::isdigit(c0) || c1 != '.' || !std::isdigit(c2)) - { - continue; - } - std::size_t j = i; - while (j < spec.size() && std::isdigit(static_cast(spec[j]))) - { - ++j; - } - if (j >= spec.size() || spec[j] != '.') - { - continue; - } - std::size_t k = j + 1; - while (k < spec.size() && std::isdigit(static_cast(spec[k]))) - { - ++k; - } - if (k == j + 1) - { - continue; - } - const auto maybe_python_minor = specs::Version::parse(spec.substr(i, k - i)); - if (maybe_python_minor.has_value()) - { - return maybe_python_minor.value(); - } + continue; + } + const specs::VersionSpec relaxed = relax_version_spec_to_minor(maybe_ms.value().version()); + if (auto maybe_v = version_from_single_equality_spec(relaxed)) + { + return maybe_v; } } return std::nullopt; diff --git a/libmamba/src/api/utils.hpp b/libmamba/src/api/utils.hpp index 899ac271cf..66e921f93c 100644 --- a/libmamba/src/api/utils.hpp +++ b/libmamba/src/api/utils.hpp @@ -222,6 +222,11 @@ namespace mamba /** * Extract an explicit python minor requirement (e.g. "3.12") from specs. + * + * Parses each ``python`` ``MatchSpec``, applies ``relax_version_spec_to_minor`` to the + * version, and returns the version if it is a single ``==`` equality (e.g. full pins relax to + * ``major.minor``). Skips specs that do not parse or do not yield such an equality after + * relaxation. */ std::optional extract_requested_python_minor(const std::vector& specs); diff --git a/libmamba/src/core/shard_python_minor_prefilter.cpp b/libmamba/src/core/shard_python_minor_prefilter.cpp new file mode 100644 index 0000000000..8076b2d390 --- /dev/null +++ b/libmamba/src/core/shard_python_minor_prefilter.cpp @@ -0,0 +1,93 @@ +// Copyright (c) 2024, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. +// +// The full license is in the file LICENSE, distributed with this software. + +#include +#include +#include + +#include "mamba/core/shard_python_minor_prefilter.hpp" +#include "mamba/specs/match_spec.hpp" +#include "mamba/specs/version.hpp" +#include "mamba/specs/version_spec.hpp" +#include "mamba/util/string.hpp" + +namespace mamba +{ + auto version_from_single_equality_spec(const specs::VersionSpec& vs) + -> std::optional + { + if (vs.expression_size() != 1) + { + return std::nullopt; + } + const std::string s = vs.to_string(); + if (!util::starts_with(s, specs::VersionSpec::equal_str)) + { + return std::nullopt; + } + const auto tail = std::string_view(s).substr(specs::VersionSpec::equal_str.size()); + auto maybe_v = specs::Version::parse(std::string(util::lstrip(tail))); + if (maybe_v.has_value()) + { + return maybe_v.value(); + } + return std::nullopt; + } + + auto relax_version_spec_to_minor(const specs::VersionSpec& vs) -> specs::VersionSpec + { + // Only relax a single exact-equality leaf; other shapes keep normal ``contains``. + if (vs.expression_size() != 1) + { + return vs; + } + const std::string vs_str = vs.to_string(); + if (!util::starts_with(vs_str, specs::VersionSpec::equal_str)) + { + return vs; + } + const auto ver_tail = std::string_view(vs_str).substr(specs::VersionSpec::equal_str.size()); + auto maybe_v = specs::Version::parse(util::lstrip(ver_tail)); + if (!maybe_v.has_value()) + { + return vs; + } + const std::string minor_str = maybe_v->to_string(2); + if (auto maybe_minor = specs::Version::parse(minor_str); maybe_minor.has_value()) + { + return specs::VersionSpec::from_predicate( + specs::VersionPredicate::make_equal_to(std::move(maybe_minor).value()) + ); + } + return vs; + } + + auto dependency_matches_python_minor_version_for_prefilter( + const std::string& dependency_spec, + const specs::Version& python_minor_version_for_prefilter + ) -> bool + { + auto maybe_name = specs::MatchSpec::extract_name(dependency_spec); + if (!maybe_name.has_value() || maybe_name.value() != "python") + { + return true; + } + auto maybe_match_spec = specs::MatchSpec::parse(dependency_spec); + if (!maybe_match_spec.has_value()) + { + return true; + } + const auto& ms = maybe_match_spec.value(); + const auto& vs = ms.version(); + if (vs.contains(python_minor_version_for_prefilter)) + { + return true; + } + + // Relax the version spec on the minor version (ignoring the patch version and build string) + return relax_version_spec_to_minor(vs).contains(python_minor_version_for_prefilter); + } +} diff --git a/libmamba/src/core/shards.cpp b/libmamba/src/core/shards.cpp index 5c8e6715c0..c18bc14238 100644 --- a/libmamba/src/core/shards.cpp +++ b/libmamba/src/core/shards.cpp @@ -26,9 +26,7 @@ #include "mamba/core/util.hpp" #include "mamba/download/downloader.hpp" #include "mamba/fs/filesystem.hpp" -#include "mamba/specs/match_spec.hpp" #include "mamba/specs/version.hpp" -#include "mamba/specs/version_spec.hpp" #include "mamba/util/cryptography.hpp" #include "mamba/util/encoding.hpp" #include "mamba/util/environment.hpp" @@ -39,60 +37,6 @@ namespace mamba { - auto relax_version_spec_to_minor(const specs::VersionSpec& vs) -> specs::VersionSpec - { - // Only relax a single exact-equality leaf; other shapes keep normal ``contains``. - if (vs.expression_size() != 1) - { - return vs; - } - const std::string vs_str = vs.to_string(); - if (!util::starts_with(vs_str, specs::VersionSpec::equal_str)) - { - return vs; - } - const auto ver_tail = std::string_view(vs_str).substr(specs::VersionSpec::equal_str.size()); - auto maybe_v = specs::Version::parse(util::lstrip(ver_tail)); - if (!maybe_v.has_value()) - { - return vs; - } - const std::string minor_str = maybe_v->to_string(2); - if (auto maybe_minor = specs::Version::parse(minor_str); maybe_minor.has_value()) - { - return specs::VersionSpec::from_predicate( - specs::VersionPredicate::make_equal_to(std::move(maybe_minor).value()) - ); - } - return vs; - } - - auto dependency_matches_python_minor_version_for_prefilter( - const std::string& dependency_spec, - const specs::Version& python_minor_version_for_prefilter - ) -> bool - { - auto maybe_name = specs::MatchSpec::extract_name(dependency_spec); - if (!maybe_name.has_value() || maybe_name.value() != "python") - { - return true; - } - auto maybe_match_spec = specs::MatchSpec::parse(dependency_spec); - if (!maybe_match_spec.has_value()) - { - return true; - } - const auto& ms = maybe_match_spec.value(); - const auto& vs = ms.version(); - if (vs.contains(python_minor_version_for_prefilter)) - { - return true; - } - - // Relax the version spec on the minor version (ignoring the patch version and build string) - return relax_version_spec_to_minor(vs).contains(python_minor_version_for_prefilter); - } - namespace { // Helper functions to extract values from msgpack_object (C API) diff --git a/libmamba/tests/CMakeLists.txt b/libmamba/tests/CMakeLists.txt index 98dca311b1..89019afcf3 100644 --- a/libmamba/tests/CMakeLists.txt +++ b/libmamba/tests/CMakeLists.txt @@ -105,6 +105,7 @@ set( src/core/test_progress_bar.cpp src/core/test_query.cpp src/core/test_shell_init.cpp + src/core/test_shard_python_minor_prefilter.cpp src/core/test_shards.cpp src/core/test_shard_index_loader.cpp src/core/test_shard_traversal.cpp diff --git a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp new file mode 100644 index 0000000000..3e01d71773 --- /dev/null +++ b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp @@ -0,0 +1,449 @@ +// Copyright (c) 2026, QuantStack and Mamba Contributors +// +// Distributed under the terms of the BSD 3-Clause License. + +#include + +#include +#include +#include + +#include "mamba/core/channel_context.hpp" +#include "mamba/core/shard_python_minor_prefilter.hpp" +#include "mamba/core/shard_types.hpp" +#include "mamba/core/shards.hpp" +#include "mamba/core/util.hpp" +#include "mamba/download/mirror.hpp" +#include "mamba/download/parameters.hpp" +#include "mamba/download/request.hpp" +#include "mamba/fs/filesystem.hpp" +#include "mamba/specs/channel.hpp" +#include "mamba/specs/conda_url.hpp" +#include "mamba/specs/unresolved_channel.hpp" +#include "mamba/specs/version.hpp" +#include "mamba/specs/version_spec.hpp" +#include "mamba/util/string.hpp" + +#include "api/utils.hpp" + +#include "mambatests.hpp" +#include "test_shard_utils.hpp" + +using namespace mamba; +using namespace mambatests::shard_test_utils; + +namespace +{ + auto make_simple_channel(std::string_view chan) -> specs::Channel + { + const auto resolve_params = ChannelContext::ChannelResolveParams{ + { "linux-64", "noarch" }, + specs::CondaURL::parse("https://conda.anaconda.org").value() + }; + + return specs::Channel::resolve(specs::UnresolvedChannel::parse(chan).value(), resolve_params) + .value() + .front(); + } + + auto create_shard_with_checksum( + const std::string& package_name, + const std::string& version, + const std::string& build, + const std::vector& depends = {}, + const std::vector& track_features = {} + ) -> std::vector + { + auto package_record = create_shard_package_record_msgpack( + package_name, + version, + build, + 0, + "abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", + std::nullopt, + depends, + {}, + std::nullopt, + HashFormat::String, + HashFormat::String, + track_features + ); + + msgpack_sbuffer sbuf; + msgpack_sbuffer_init(&sbuf); + msgpack_packer pk; + msgpack_packer_init(&pk, &sbuf, msgpack_sbuffer_write); + msgpack_pack_map(&pk, 1); + msgpack_pack_str(&pk, 8); + msgpack_pack_str_body(&pk, "packages", 8); + msgpack_pack_map(&pk, 1); + std::string filename = package_name + "-" + version + "-" + build + ".tar.bz2"; + msgpack_pack_str(&pk, filename.size()); + msgpack_pack_str_body(&pk, filename.c_str(), filename.size()); + msgpack_sbuffer_write( + &sbuf, + reinterpret_cast(package_record.data()), + package_record.size() + ); + std::vector shard_msgpack( + reinterpret_cast(sbuf.data), + reinterpret_cast(sbuf.data + sbuf.size) + ); + msgpack_sbuffer_destroy(&sbuf); + return compress_zstd(shard_msgpack); + } + + auto v(std::string_view s) -> specs::Version + { + return specs::Version::parse(std::string(s)).value(); + } +} + +TEST_CASE("version_from_single_equality_spec") +{ + using specs::VersionSpec; + + SECTION("explicit double-equals") + { + const auto vs = VersionSpec::parse("==3.7.12").value(); + const auto got = version_from_single_equality_spec(vs); + REQUIRE(got.has_value()); + REQUIRE(got.value() == v("3.7.12")); + } + + SECTION("bare equality parses as single ==") + { + const auto vs = VersionSpec::parse("3.12.5").value(); + const auto got = version_from_single_equality_spec(vs); + REQUIRE(got.has_value()); + REQUIRE(got.value() == v("3.12.5")); + } + + SECTION("greater-or-equal is not single equality") + { + const auto vs = VersionSpec::parse(">=3.7").value(); + REQUIRE_FALSE(version_from_single_equality_spec(vs).has_value()); + } + + SECTION("conjunction is not a single leaf") + { + const auto vs = VersionSpec::parse(">=3.12,<3.13").value(); + REQUIRE_FALSE(version_from_single_equality_spec(vs).has_value()); + } + + SECTION("disjunction is not a single leaf") + { + const auto vs = VersionSpec::parse("==3.7.12|==3.8.0").value(); + REQUIRE_FALSE(version_from_single_equality_spec(vs).has_value()); + } + + SECTION("free spec") + { + const VersionSpec vs{}; + REQUIRE_FALSE(version_from_single_equality_spec(vs).has_value()); + } +} + +TEST_CASE("extract_requested_python_minor") +{ + SECTION("empty specs") + { + REQUIRE_FALSE(extract_requested_python_minor({}).has_value()); + } + + SECTION("no python package") + { + REQUIRE_FALSE(extract_requested_python_minor({ "numpy >=1.0", "openssl 3" }).has_value()); + } + + SECTION("two-token exact pin relaxes to minor") + { + const auto got = extract_requested_python_minor({ "python 3.12.5" }); + REQUIRE(got.has_value()); + REQUIRE(got.value() == v("3.12")); + } + + SECTION("three-token conda pin") + { + const auto got = extract_requested_python_minor({ "python 3.7.12 0_73_pypy" }); + REQUIRE(got.has_value()); + REQUIRE(got.value() == v("3.7")); + } + + SECTION("explicit equality operator form") + { + const auto got = extract_requested_python_minor({ "python ==3.11" }); + REQUIRE(got.has_value()); + REQUIRE(got.value() == v("3.11")); + } + + SECTION("range spec yields no minor (not single equality after relax)") + { + REQUIRE_FALSE(extract_requested_python_minor({ "python >=3.12,<3.13" }).has_value()); + } + + SECTION("uses first matching python spec") + { + const auto got = extract_requested_python_minor( + { "numpy 1.0", "python 3.10.0", "python 3.11" } + ); + REQUIRE(got.has_value()); + REQUIRE(got.value() == v("3.10")); + } + + SECTION("bare python name without version constraint") + { + REQUIRE_FALSE(extract_requested_python_minor({ "python" }).has_value()); + } +} + +TEST_CASE("Shards - python minor prefilter") +{ + ShardsIndexDict index; + index.info.base_url = "https://example.com/packages"; + index.info.shards_base_url = "shards"; + index.info.subdir = "linux-64"; + index.version = 1; + index.shards["test-pkg"] = std::vector(32, 0xAB); + + specs::Channel channel = make_simple_channel("https://example.com/conda-forge"); + specs::AuthenticationDataBase auth_info; + download::RemoteFetchParams remote_fetch_params; + + const auto tmp_dir = TemporaryDirectory(); + const auto shard_file = tmp_dir.path() / "test-pkg.msgpack.zst"; + + std::map package_to_cache_path; + package_to_cache_path["test-pkg"] = shard_file; + + auto run_for_dep = [&](const std::string& dep, + std::optional python_minor) -> expected_t + { + auto shard_data = create_shard_with_checksum("test-pkg", "1.0.0", "0", { dep }); + { + std::ofstream file(shard_file.string(), std::ios::binary); + file.write( + reinterpret_cast(shard_data.data()), + static_cast(shard_data.size()) + ); + } + + download::Success success; + success.content = download::Filename{ shard_file.string() }; + success.transfer.downloaded_size = shard_data.size(); + + Shards shards( + index, + "https://example.com/conda-forge/linux-64/repodata.json", + channel, + auth_info, + remote_fetch_params, + 0, + std::nullopt, + std::move(python_minor) + ); + return test_process_downloaded_shard(shards, "test-pkg", success, package_to_cache_path); + }; + + SECTION("mismatching python minor is discarded before record creation") + { + auto result = run_for_dep( + "python >=3.11,<3.12", + specs::Version::parse("3.12").value_or(specs::Version()) + ); + REQUIRE(result.has_value()); + REQUIRE(result->packages.empty()); + REQUIRE(result->conda_packages.empty()); + } + + SECTION("matching python minor is retained") + { + auto result = run_for_dep( + "python >=3.12,<3.13", + specs::Version::parse("3.12").value_or(specs::Version()) + ); + REQUIRE(result.has_value()); + REQUIRE(result->packages.size() == 1); + REQUIRE(result->packages.begin()->second.name == "test-pkg"); + } + + SECTION("no python minor context does not apply prefilter") + { + auto result = run_for_dep("python >=3.11,<3.12", std::nullopt); + REQUIRE(result.has_value()); + REQUIRE(result->packages.size() == 1); + REQUIRE(result->packages.begin()->second.name == "test-pkg"); + } + + SECTION("exact python pin matches requested minor (conda three-token depends)") + { + auto result = run_for_dep( + "python 3.7.12 0_73_pypy", + specs::Version::parse("3.7").value_or(specs::Version()) + ); + REQUIRE(result.has_value()); + REQUIRE(result->packages.size() == 1); + REQUIRE(result->packages.begin()->second.name == "test-pkg"); + } +} + +TEST_CASE("relax_version_spec_to_minor") +{ + using specs::Version; + using specs::VersionSpec; + + const auto req = [](std::string_view s) -> Version { return Version::parse(s).value(); }; + + SECTION("bare equality pin relaxes so requested minor is contained") + { + const auto vs = VersionSpec::parse("3.7.12").value(); + const auto relaxed = relax_version_spec_to_minor(vs); + REQUIRE(relaxed.contains(req("3.7"))); + REQUIRE_FALSE(relaxed.contains(req("3.8"))); + } + + SECTION("explicit double-equals string form relaxes") + { + const auto vs = VersionSpec::parse("==3.7.12").value(); + const auto relaxed = relax_version_spec_to_minor(vs); + REQUIRE(relaxed.contains(req("3.7"))); + REQUIRE(util::starts_with(relaxed.to_string(), "==")); + } + + SECTION("four-component pin relaxes to first two components") + { + const auto vs = VersionSpec::parse("1.2.3.4").value(); + const auto relaxed = relax_version_spec_to_minor(vs); + REQUIRE(relaxed.contains(req("1.2"))); + REQUIRE_FALSE(relaxed.contains(req("1.3"))); + } + + SECTION("greater-or-equal is unchanged") + { + const auto vs = VersionSpec::parse(">=3.7").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("less-than is unchanged") + { + const auto vs = VersionSpec::parse("<4").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("compatible-release operator is unchanged") + { + const auto vs = VersionSpec::parse("~=3.7").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("not-equal is unchanged") + { + const auto vs = VersionSpec::parse("!=3.7.12").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("disjunction is unchanged") + { + const auto vs = VersionSpec::parse("==3.7.12|==3.8.0").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("conjunction is unchanged") + { + const auto vs = VersionSpec::parse(">=3.7,<3.8").value(); + REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); + } + + SECTION("free spec is unchanged") + { + const VersionSpec vs{}; + REQUIRE(relax_version_spec_to_minor(vs).is_explicitly_free()); + } +} + +TEST_CASE("dependency_matches_python_minor_version_for_prefilter") +{ + const auto req = [](std::string_view s) -> specs::Version + { return specs::Version::parse(s).value(); }; + + SECTION("non-python dependency is not filtered") + { + REQUIRE(dependency_matches_python_minor_version_for_prefilter("numpy >=1.0", req("3.12"))); + REQUIRE( + dependency_matches_python_minor_version_for_prefilter("libstdcxx-ng >=12", req("3.12")) + ); + } + + SECTION("name starting with python but not the python package") + { + REQUIRE(dependency_matches_python_minor_version_for_prefilter( + "python_abi 3.12 1_cp312", + req("3.12") + )); + } + + SECTION("python version range matches requested minor") + { + REQUIRE( + dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.12")) + ); + REQUIRE(dependency_matches_python_minor_version_for_prefilter("python >=3.12", req("3.12"))); + } + + SECTION("python version range does not match requested minor") + { + REQUIRE_FALSE( + dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.11")) + ); + REQUIRE_FALSE( + dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.13")) + ); + } + + SECTION("exact three-token conda pin matches requested minor") + { + REQUIRE( + dependency_matches_python_minor_version_for_prefilter("python 3.7.12 0_73_pypy", req("3.7")) + ); + } + + SECTION("two-token exact pin matches requested minor") + { + REQUIRE(dependency_matches_python_minor_version_for_prefilter("python 3.7.12", req("3.7"))); + } + + SECTION("exact pin does not match different minor") + { + REQUIRE_FALSE( + dependency_matches_python_minor_version_for_prefilter("python 3.8.0", req("3.7")) + ); + } + + SECTION("leading whitespace on dependency line") + { + REQUIRE( + dependency_matches_python_minor_version_for_prefilter(" python >=3.12,<3.13", req("3.12")) + ); + } + + SECTION("unparsable python dependency does not filter (passes)") + { + REQUIRE(dependency_matches_python_minor_version_for_prefilter( + "python ,,not-a-valid-spec,,", + req("3.12") + )); + } + + SECTION("namespaced python pin") + { + REQUIRE(dependency_matches_python_minor_version_for_prefilter( + "conda-forge::python 3.7.12 0_73_pypy", + req("3.7") + )); + } + + SECTION("only python in range with no upper bound") + { + REQUIRE(dependency_matches_python_minor_version_for_prefilter("python", req("3.12"))); + } +} diff --git a/libmamba/tests/src/core/test_shards.cpp b/libmamba/tests/src/core/test_shards.cpp index 1bcc72b17a..72e5e181a6 100644 --- a/libmamba/tests/src/core/test_shards.cpp +++ b/libmamba/tests/src/core/test_shards.cpp @@ -12,7 +12,6 @@ #include #include "mamba/core/channel_context.hpp" -#include "mamba/core/shard_python_minor_prefilter.hpp" #include "mamba/core/shard_types.hpp" #include "mamba/core/shards.hpp" #include "mamba/core/util.hpp" @@ -2773,254 +2772,3 @@ TEST_CASE("Shards - process_downloaded_shard") ); } } - -TEST_CASE("Shards - python minor prefilter") -{ - ShardsIndexDict index; - index.info.base_url = "https://example.com/packages"; - index.info.shards_base_url = "shards"; - index.info.subdir = "linux-64"; - index.version = 1; - index.shards["test-pkg"] = std::vector(32, 0xAB); - - specs::Channel channel = make_simple_channel("https://example.com/conda-forge"); - specs::AuthenticationDataBase auth_info; - download::RemoteFetchParams remote_fetch_params; - - const auto tmp_dir = TemporaryDirectory(); - const auto shard_file = tmp_dir.path() / "test-pkg.msgpack.zst"; - - std::map package_to_cache_path; - package_to_cache_path["test-pkg"] = shard_file; - - auto run_for_dep = [&](const std::string& dep, - std::optional python_minor) -> expected_t - { - auto shard_data = create_shard_with_checksum("test-pkg", "1.0.0", "0", { dep }); - { - std::ofstream file(shard_file.string(), std::ios::binary); - file.write( - reinterpret_cast(shard_data.data()), - static_cast(shard_data.size()) - ); - } - - download::Success success; - success.content = download::Filename{ shard_file.string() }; - success.transfer.downloaded_size = shard_data.size(); - - Shards shards( - index, - "https://example.com/conda-forge/linux-64/repodata.json", - channel, - auth_info, - remote_fetch_params, - 0, - std::nullopt, - std::move(python_minor) - ); - return test_process_downloaded_shard(shards, "test-pkg", success, package_to_cache_path); - }; - - SECTION("mismatching python minor is discarded before record creation") - { - auto result = run_for_dep( - "python >=3.11,<3.12", - specs::Version::parse("3.12").value_or(specs::Version()) - ); - REQUIRE(result.has_value()); - REQUIRE(result->packages.empty()); - REQUIRE(result->conda_packages.empty()); - } - - SECTION("matching python minor is retained") - { - auto result = run_for_dep( - "python >=3.12,<3.13", - specs::Version::parse("3.12").value_or(specs::Version()) - ); - REQUIRE(result.has_value()); - REQUIRE(result->packages.size() == 1); - REQUIRE(result->packages.begin()->second.name == "test-pkg"); - } - - SECTION("no python minor context does not apply prefilter") - { - auto result = run_for_dep("python >=3.11,<3.12", std::nullopt); - REQUIRE(result.has_value()); - REQUIRE(result->packages.size() == 1); - REQUIRE(result->packages.begin()->second.name == "test-pkg"); - } - - SECTION("exact python pin matches requested minor (conda three-token depends)") - { - auto result = run_for_dep( - "python 3.7.12 0_73_pypy", - specs::Version::parse("3.7").value_or(specs::Version()) - ); - REQUIRE(result.has_value()); - REQUIRE(result->packages.size() == 1); - REQUIRE(result->packages.begin()->second.name == "test-pkg"); - } -} - -TEST_CASE("relax_version_spec_to_minor") -{ - using specs::Version; - using specs::VersionSpec; - - const auto req = [](std::string_view s) -> Version { return Version::parse(s).value(); }; - - SECTION("bare equality pin relaxes so requested minor is contained") - { - const auto vs = VersionSpec::parse("3.7.12").value(); - const auto relaxed = relax_version_spec_to_minor(vs); - REQUIRE(relaxed.contains(req("3.7"))); - REQUIRE_FALSE(relaxed.contains(req("3.8"))); - } - - SECTION("explicit double-equals string form relaxes") - { - const auto vs = VersionSpec::parse("==3.7.12").value(); - const auto relaxed = relax_version_spec_to_minor(vs); - REQUIRE(relaxed.contains(req("3.7"))); - REQUIRE(util::starts_with(relaxed.to_string(), "==")); - } - - SECTION("four-component pin relaxes to first two components") - { - const auto vs = VersionSpec::parse("1.2.3.4").value(); - const auto relaxed = relax_version_spec_to_minor(vs); - REQUIRE(relaxed.contains(req("1.2"))); - REQUIRE_FALSE(relaxed.contains(req("1.3"))); - } - - SECTION("greater-or-equal is unchanged") - { - const auto vs = VersionSpec::parse(">=3.7").value(); - REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); - } - - SECTION("less-than is unchanged") - { - const auto vs = VersionSpec::parse("<4").value(); - REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); - } - - SECTION("compatible-release operator is unchanged") - { - const auto vs = VersionSpec::parse("~=3.7").value(); - REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); - } - - SECTION("not-equal is unchanged") - { - const auto vs = VersionSpec::parse("!=3.7.12").value(); - REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); - } - - SECTION("disjunction is unchanged") - { - const auto vs = VersionSpec::parse("==3.7.12|==3.8.0").value(); - REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); - } - - SECTION("conjunction is unchanged") - { - const auto vs = VersionSpec::parse(">=3.7,<3.8").value(); - REQUIRE(relax_version_spec_to_minor(vs).to_string() == vs.to_string()); - } - - SECTION("free spec is unchanged") - { - const VersionSpec vs{}; - REQUIRE(relax_version_spec_to_minor(vs).is_explicitly_free()); - } -} - -TEST_CASE("dependency_matches_python_minor_version_for_prefilter") -{ - const auto req = [](std::string_view s) -> specs::Version - { return specs::Version::parse(s).value(); }; - - SECTION("non-python dependency is not filtered") - { - REQUIRE(dependency_matches_python_minor_version_for_prefilter("numpy >=1.0", req("3.12"))); - REQUIRE( - dependency_matches_python_minor_version_for_prefilter("libstdcxx-ng >=12", req("3.12")) - ); - } - - SECTION("name starting with python but not the python package") - { - REQUIRE(dependency_matches_python_minor_version_for_prefilter( - "python_abi 3.12 1_cp312", - req("3.12") - )); - } - - SECTION("python version range matches requested minor") - { - REQUIRE( - dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.12")) - ); - REQUIRE(dependency_matches_python_minor_version_for_prefilter("python >=3.12", req("3.12"))); - } - - SECTION("python version range does not match requested minor") - { - REQUIRE_FALSE( - dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.11")) - ); - REQUIRE_FALSE( - dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.13")) - ); - } - - SECTION("exact three-token conda pin matches requested minor") - { - REQUIRE( - dependency_matches_python_minor_version_for_prefilter("python 3.7.12 0_73_pypy", req("3.7")) - ); - } - - SECTION("two-token exact pin matches requested minor") - { - REQUIRE(dependency_matches_python_minor_version_for_prefilter("python 3.7.12", req("3.7"))); - } - - SECTION("exact pin does not match different minor") - { - REQUIRE_FALSE( - dependency_matches_python_minor_version_for_prefilter("python 3.8.0", req("3.7")) - ); - } - - SECTION("leading whitespace on dependency line") - { - REQUIRE( - dependency_matches_python_minor_version_for_prefilter(" python >=3.12,<3.13", req("3.12")) - ); - } - - SECTION("unparsable python dependency does not filter (passes)") - { - REQUIRE(dependency_matches_python_minor_version_for_prefilter( - "python ,,not-a-valid-spec,,", - req("3.12") - )); - } - - SECTION("namespaced python pin") - { - REQUIRE(dependency_matches_python_minor_version_for_prefilter( - "conda-forge::python 3.7.12 0_73_pypy", - req("3.7") - )); - } - - SECTION("only python in range with no upper bound") - { - REQUIRE(dependency_matches_python_minor_version_for_prefilter("python", req("3.12"))); - } -} From 6bbc561126ee7eb8b468cd7efe95fd2f85077f78 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 3 Apr 2026 09:03:46 +0200 Subject: [PATCH 09/18] Disable prefilter when `--no-py-pin` is used Signed-off-by: Julien Jerphanion --- libmamba/include/mamba/api/channel_loader.hpp | 3 +-- libmamba/src/api/install.cpp | 8 +++++++- libmamba/src/api/update.cpp | 3 ++- libmamba/src/api/utils.cpp | 9 ++++++++- libmamba/src/api/utils.hpp | 7 ++++--- libmamba/src/core/shard_python_minor_prefilter.cpp | 2 +- 6 files changed, 23 insertions(+), 9 deletions(-) diff --git a/libmamba/include/mamba/api/channel_loader.hpp b/libmamba/include/mamba/api/channel_loader.hpp index ee92205c4a..0b0d350a3c 100644 --- a/libmamba/include/mamba/api/channel_loader.hpp +++ b/libmamba/include/mamba/api/channel_loader.hpp @@ -44,8 +44,7 @@ namespace mamba * @param loaded_subdirs_with_shards Set of subdir names already loaded via shards (updated). * @param priorities Repo priorities aligned with \p subdirs. * @param python_minor_version_for_prefilter Optional python minor for shard record prefiltering - * (from - * \c prepare_solver_context). + * (from \c prepare_solver_context). * @return The repo for the requested subdir, or unexpected mamba_error on failure. */ auto load_subdir_with_shards( diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index f2166f93b9..dbeb85e33e 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -556,7 +556,13 @@ namespace mamba auto& no_env = config.at("no_env").value(); validate_target_prefix_and_channels(ctx, create_env); - auto [db, package_caches] = prepare_solver_context(ctx, channel_context, raw_specs, is_retry); + auto [db, package_caches] = prepare_solver_context( + ctx, + channel_context, + raw_specs, + is_retry, + no_py_pin + ); auto prefix_data = load_prefix_data_and_installed(ctx, channel_context, db); diff --git a/libmamba/src/api/update.cpp b/libmamba/src/api/update.cpp index 24a5686b14..28f12610cb 100644 --- a/libmamba/src/api/update.cpp +++ b/libmamba/src/api/update.cpp @@ -156,7 +156,8 @@ namespace mamba ctx, channel_context, raw_update_specs, - is_retry + is_retry, + no_py_pin ); auto prefix_data = load_prefix_data_and_installed(ctx, channel_context, db); diff --git a/libmamba/src/api/utils.cpp b/libmamba/src/api/utils.cpp index dcee5ea603..77a6049741 100644 --- a/libmamba/src/api/utils.cpp +++ b/libmamba/src/api/utils.cpp @@ -499,7 +499,8 @@ namespace mamba Context& ctx, ChannelContext& channel_context, const std::vector& raw_specs, - bool is_retry + bool is_retry, + bool no_py_pin ) { populate_context_channels_from_specs(raw_specs, ctx); @@ -513,6 +514,12 @@ namespace mamba const std::optional python_minor_version_for_prefilter = [&]() -> std::optional { + if (no_py_pin) + { + LOG_DEBUG << "Shard python minor prefilter disabled (--no-py-pin)."; + return std::nullopt; + } + const auto maybe_explicit_requested_python_minor = extract_requested_python_minor( raw_specs ); diff --git a/libmamba/src/api/utils.hpp b/libmamba/src/api/utils.hpp index 66e921f93c..ad291591ae 100644 --- a/libmamba/src/api/utils.hpp +++ b/libmamba/src/api/utils.hpp @@ -125,16 +125,17 @@ namespace mamba /** * Prepare solver state: channels, package cache, database, and root package loading. - * * Computes ``python_minor_version_for_prefilter`` for sharded repodata: explicit python from * specs, implicit fallback on the first solve attempt, or the installed prefix minor on retry - * when no explicit python is given. + * when no explicit python is given. When ``no_py_pin`` is true (``--no-py-pin``), no python + * minor is used for shard prefiltering. */ std::pair prepare_solver_context( Context& ctx, ChannelContext& channel_context, const std::vector& raw_specs, - bool is_retry + bool is_retry, + bool no_py_pin ); /** diff --git a/libmamba/src/core/shard_python_minor_prefilter.cpp b/libmamba/src/core/shard_python_minor_prefilter.cpp index 8076b2d390..a356d34f1d 100644 --- a/libmamba/src/core/shard_python_minor_prefilter.cpp +++ b/libmamba/src/core/shard_python_minor_prefilter.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2024, QuantStack and Mamba Contributors +// Copyright (c) 2026, QuantStack and Mamba Contributors // // Distributed under the terms of the BSD 3-Clause License. // From d393ec1b38b75bde89545592974340ed4487dddb Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 3 Apr 2026 09:29:59 +0200 Subject: [PATCH 10/18] Explicitly install pip with python Signed-off-by: Julien Jerphanion --- libmamba/src/api/install.cpp | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp index dbeb85e33e..643411fb32 100644 --- a/libmamba/src/api/install.cpp +++ b/libmamba/src/api/install.cpp @@ -5,7 +5,6 @@ // The full license is in the file LICENSE, distributed with this software. #include -#include #include #include @@ -377,6 +376,33 @@ namespace mamba { using Request = solver::Request; + // When the user explicitly asks for ``python`` in the requested specs, also inject a + // plain ``pip`` request unless it is already present. This complements + // ``add_pip_as_python_dependency`` at the repo level and makes sure that the Request + // is in phase with the root packages including both ``python`` and ``pip`` when requested. + bool wants_python = false; + bool wants_pip = false; + for (const auto& s : specs) + { + const auto maybe_name = specs::MatchSpec::extract_name(s); + if (!maybe_name.has_value()) + { + continue; + } + if (maybe_name.value() == "python") + { + wants_python = true; + } + else if (maybe_name.value() == "pip") + { + wants_pip = true; + } + } + if (wants_python && !wants_pip) + { + specs.emplace_back("pip"); + } + const auto& prefix_pkgs = prefix_data.records(); auto request = Request(); From ac17770fb707ed2bac6d885208e37314a925c0d6 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 3 Apr 2026 09:31:21 +0200 Subject: [PATCH 11/18] Revert changes made to `test_no_python_pinning` Signed-off-by: Julien Jerphanion --- micromamba/tests/test_install.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/micromamba/tests/test_install.py b/micromamba/tests/test_install.py index 681d4c00bc..263477a37e 100644 --- a/micromamba/tests/test_install.py +++ b/micromamba/tests/test_install.py @@ -486,9 +486,7 @@ def test_no_python_pinning(self, existing_cache): keys = {"success", "prefix", "actions", "dry_run"} assert keys.issubset(set(res.keys())) - # LINK and PREFIX are always present; FETCH appears when packages must be - # downloaded; UNLINK may be omitted when nothing is removed. - action_keys = {"LINK", "PREFIX"} + action_keys = {"LINK", "UNLINK", "PREFIX"} assert action_keys.issubset(set(res["actions"].keys())) # When using `--no-py-pin`, it may or may not update the already installed @@ -498,27 +496,27 @@ def test_no_python_pinning(self, existing_cache): link_packages = {pkg["name"] for pkg in res["actions"]["LINK"]} assert expected_link_packages.issubset(link_packages) - unlink_list = res["actions"].get("UNLINK", []) - unlink_packages = {pkg["name"] for pkg in unlink_list} + unlink_packages = {pkg["name"] for pkg in res["actions"]["UNLINK"]} if {"python"}.issubset(link_packages): assert {"python"}.issubset(unlink_packages) py_pkg = [pkg for pkg in res["actions"]["LINK"] if pkg["name"] == "python"][0] assert py_pkg["version"] != ("3.9.19") - py_pkg = [pkg for pkg in unlink_list if pkg["name"] == "python"][0] + py_pkg = [pkg for pkg in res["actions"]["UNLINK"] if pkg["name"] == "python"][0] assert py_pkg["version"] == ("3.9.19") else: - link_list = res["actions"]["LINK"] - py_abi_pkg = [pkg for pkg in link_list if pkg["name"] == "python_abi"][0] + assert len(res["actions"]["LINK"]) == 2 # Should be setuptools and python_abi + + py_abi_pkg = [pkg for pkg in res["actions"]["LINK"] if pkg["name"] == "python_abi"][0] assert py_abi_pkg["version"] == ("3.9") - if "setuptools" in link_packages: - setuptools_pkg = [pkg for pkg in link_list if pkg["name"] == "setuptools"][0] - assert setuptools_pkg["version"] == ("63.4.3") + setuptools_pkg = [pkg for pkg in res["actions"]["LINK"] if pkg["name"] == "setuptools"][ + 0 + ] + assert setuptools_pkg["version"] == ("63.4.3") - if unlink_list: - assert len(unlink_list) == 1 # Should be setuptools - assert unlink_list[0]["name"] == "setuptools" + assert len(res["actions"]["UNLINK"]) == 1 # Should be setuptools + assert res["actions"]["UNLINK"][0]["name"] == "setuptools" @pytest.mark.skipif( helpers.dry_run_tests is helpers.DryRun.ULTRA_DRY, From 1cec40d70b13a84b7759324dcaf32749788540d7 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 3 Apr 2026 09:32:15 +0200 Subject: [PATCH 12/18] Revert changes made to `libmamba/tests/src/core/test_shards.cpp` Signed-off-by: Julien Jerphanion --- libmamba/tests/src/core/test_shards.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libmamba/tests/src/core/test_shards.cpp b/libmamba/tests/src/core/test_shards.cpp index 72e5e181a6..a6eea2f674 100644 --- a/libmamba/tests/src/core/test_shards.cpp +++ b/libmamba/tests/src/core/test_shards.cpp @@ -23,10 +23,8 @@ #include "mamba/specs/conda_url.hpp" #include "mamba/specs/unresolved_channel.hpp" #include "mamba/specs/version.hpp" -#include "mamba/specs/version_spec.hpp" #include "mamba/util/encoding.hpp" #include "mamba/util/environment.hpp" -#include "mamba/util/string.hpp" #include "mamba/validation/tools.hpp" #include "mambatests.hpp" From a4e042538d7838fa5f3cf29c96f81f72acabb449 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Fri, 3 Apr 2026 09:32:39 +0200 Subject: [PATCH 13/18] Revert changes made to `libmamba/src/solver/libsolv/database.cpp` Signed-off-by: Julien Jerphanion --- libmamba/src/solver/libsolv/database.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libmamba/src/solver/libsolv/database.cpp b/libmamba/src/solver/libsolv/database.cpp index 8cf9787b11..33bc7dc8e6 100644 --- a/libmamba/src/solver/libsolv/database.cpp +++ b/libmamba/src/solver/libsolv/database.cpp @@ -4,7 +4,6 @@ // // The full license is in the file LICENSE, distributed with this software. -#include #include #include #include From d68aa3aa2a978edea62f70075c8730b1b2a96000 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 7 Apr 2026 10:42:46 +0200 Subject: [PATCH 14/18] docs: Adapt comments Signed-off-by: Julien Jerphanion --- libmamba/src/api/utils.hpp | 1 - libmamba/src/core/shards.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/libmamba/src/api/utils.hpp b/libmamba/src/api/utils.hpp index ad291591ae..18067b483d 100644 --- a/libmamba/src/api/utils.hpp +++ b/libmamba/src/api/utils.hpp @@ -93,7 +93,6 @@ namespace mamba std::vector build_sharded_root_packages(const std::vector& raw_specs); /** - * Print environment activation guidance for the current target prefix. */ void print_activation_message(const Context& ctx); diff --git a/libmamba/src/core/shards.cpp b/libmamba/src/core/shards.cpp index c18bc14238..661f0f1e5a 100644 --- a/libmamba/src/core/shards.cpp +++ b/libmamba/src/core/shards.cpp @@ -1006,7 +1006,7 @@ namespace mamba try { // Filter out builds which depend on another python minor version - // than the one in the environment, significantly reducing the number of + // than the one requested. This significantly reduces the number of // builds to parse and to provide to the solver for dependency resolution. if (!record_depends_on_python_minor_version_for_prefilter( val, From dda234d53ceaa6b9938a5632c031253c96d1a06c Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 7 Apr 2026 11:07:39 +0200 Subject: [PATCH 15/18] Move `shard_python_minor_prefilter.hpp` to `src` Signed-off-by: Julien Jerphanion Co-authored-by: Johan Mabille --- libmamba/CMakeLists.txt | 1 + libmamba/src/api/utils.cpp | 3 ++- libmamba/src/core/shard_python_minor_prefilter.cpp | 2 +- .../mamba => src}/core/shard_python_minor_prefilter.hpp | 0 libmamba/src/core/shards.cpp | 3 ++- libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) rename libmamba/{include/mamba => src}/core/shard_python_minor_prefilter.hpp (100%) diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index 62a9835e46..ef53a772f6 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -247,6 +247,7 @@ set( ${LIBMAMBA_SOURCE_DIR}/core/query.cpp ${LIBMAMBA_SOURCE_DIR}/core/repo_checker_store.cpp ${LIBMAMBA_SOURCE_DIR}/core/run.cpp + ${LIBMAMBA_SOURCE_DIR}/core/shard_python_minor_prefilter.hpp ${LIBMAMBA_SOURCE_DIR}/core/shard_python_minor_prefilter.cpp ${LIBMAMBA_SOURCE_DIR}/core/shell_init.cpp ${LIBMAMBA_SOURCE_DIR}/core/shards.cpp diff --git a/libmamba/src/api/utils.cpp b/libmamba/src/api/utils.cpp index 77a6049741..05f9837198 100644 --- a/libmamba/src/api/utils.cpp +++ b/libmamba/src/api/utils.cpp @@ -27,7 +27,6 @@ #include "mamba/core/package_cache.hpp" #include "mamba/core/package_database_loader.hpp" #include "mamba/core/prefix_data.hpp" -#include "mamba/core/shard_python_minor_prefilter.hpp" #include "mamba/core/transaction.hpp" #include "mamba/core/util.hpp" #include "mamba/core/util_os.hpp" @@ -39,6 +38,8 @@ #include "mamba/util/environment.hpp" #include "mamba/util/string.hpp" +#include "core/shard_python_minor_prefilter.hpp" + #include "utils.hpp" namespace mamba diff --git a/libmamba/src/core/shard_python_minor_prefilter.cpp b/libmamba/src/core/shard_python_minor_prefilter.cpp index a356d34f1d..4f388bb73c 100644 --- a/libmamba/src/core/shard_python_minor_prefilter.cpp +++ b/libmamba/src/core/shard_python_minor_prefilter.cpp @@ -8,7 +8,7 @@ #include #include -#include "mamba/core/shard_python_minor_prefilter.hpp" +#include "core/shard_python_minor_prefilter.hpp" #include "mamba/specs/match_spec.hpp" #include "mamba/specs/version.hpp" #include "mamba/specs/version_spec.hpp" diff --git a/libmamba/include/mamba/core/shard_python_minor_prefilter.hpp b/libmamba/src/core/shard_python_minor_prefilter.hpp similarity index 100% rename from libmamba/include/mamba/core/shard_python_minor_prefilter.hpp rename to libmamba/src/core/shard_python_minor_prefilter.hpp diff --git a/libmamba/src/core/shards.cpp b/libmamba/src/core/shards.cpp index 661f0f1e5a..216c378753 100644 --- a/libmamba/src/core/shards.cpp +++ b/libmamba/src/core/shards.cpp @@ -19,7 +19,6 @@ #include "mamba/core/logging.hpp" #include "mamba/core/output.hpp" -#include "mamba/core/shard_python_minor_prefilter.hpp" #include "mamba/core/shard_types.hpp" #include "mamba/core/shards.hpp" #include "mamba/core/subdir_index.hpp" @@ -35,6 +34,8 @@ #include "mamba/util/url_manip.hpp" #include "mamba/validation/tools.hpp" +#include "core/shard_python_minor_prefilter.hpp" + namespace mamba { namespace diff --git a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp index 3e01d71773..74a0f1723b 100644 --- a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp +++ b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp @@ -9,7 +9,6 @@ #include #include "mamba/core/channel_context.hpp" -#include "mamba/core/shard_python_minor_prefilter.hpp" #include "mamba/core/shard_types.hpp" #include "mamba/core/shards.hpp" #include "mamba/core/util.hpp" @@ -25,6 +24,7 @@ #include "mamba/util/string.hpp" #include "api/utils.hpp" +#include "core/shard_python_minor_prefilter.hpp" #include "mambatests.hpp" #include "test_shard_utils.hpp" From a91dd6a3610758b64e5c06fb2c52651b8ceedb94 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 7 Apr 2026 11:08:50 +0200 Subject: [PATCH 16/18] fix: Rename `dependency_matches_python_minor_version_for_prefilter` to `matches_python_minor` Signed-off-by: Julien Jerphanion Co-authored-by: Johan Mabille --- .../src/core/shard_python_minor_prefilter.cpp | 2 +- .../src/core/shard_python_minor_prefilter.hpp | 2 +- libmamba/src/core/shards.cpp | 7 +-- .../test_shard_python_minor_prefilter.cpp | 53 ++++++------------- 4 files changed, 19 insertions(+), 45 deletions(-) diff --git a/libmamba/src/core/shard_python_minor_prefilter.cpp b/libmamba/src/core/shard_python_minor_prefilter.cpp index 4f388bb73c..c3f7fb5a68 100644 --- a/libmamba/src/core/shard_python_minor_prefilter.cpp +++ b/libmamba/src/core/shard_python_minor_prefilter.cpp @@ -65,7 +65,7 @@ namespace mamba return vs; } - auto dependency_matches_python_minor_version_for_prefilter( + auto matches_python_minor( const std::string& dependency_spec, const specs::Version& python_minor_version_for_prefilter ) -> bool diff --git a/libmamba/src/core/shard_python_minor_prefilter.hpp b/libmamba/src/core/shard_python_minor_prefilter.hpp index c128a2f6b0..1f5634e04b 100644 --- a/libmamba/src/core/shard_python_minor_prefilter.hpp +++ b/libmamba/src/core/shard_python_minor_prefilter.hpp @@ -36,7 +36,7 @@ namespace mamba * and tests again. Non-python dependencies always return true; parse failures return true (no * prefilter). */ - [[nodiscard]] auto dependency_matches_python_minor_version_for_prefilter( + [[nodiscard]] auto matches_python_minor( const std::string& dependency_spec, const specs::Version& python_minor_version_for_prefilter ) -> bool; diff --git a/libmamba/src/core/shards.cpp b/libmamba/src/core/shards.cpp index 216c378753..7e70782a3b 100644 --- a/libmamba/src/core/shards.cpp +++ b/libmamba/src/core/shards.cpp @@ -398,7 +398,7 @@ namespace mamba * When ``python_minor_version_for_prefilter`` is unset, returns true (no prefilter). * When set, inspects ``depends`` entries for ``python`` and keeps the record only if * each such constraint contains that minor (see - * ``dependency_matches_python_minor_version_for_prefilter``). + * ``matches_python_minor``). */ bool record_depends_on_python_minor_version_for_prefilter( const msgpack_object& raw_record_obj, @@ -435,10 +435,7 @@ namespace mamba const auto depends = msgpack_object_to_string_array(val_obj); for (const auto& dep : depends) { - if (!dependency_matches_python_minor_version_for_prefilter( - dep, - python_minor_version_for_prefilter.value() - )) + if (!matches_python_minor(dep, python_minor_version_for_prefilter.value())) { return false; } diff --git a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp index 74a0f1723b..6920ca59ee 100644 --- a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp +++ b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp @@ -361,89 +361,66 @@ TEST_CASE("relax_version_spec_to_minor") } } -TEST_CASE("dependency_matches_python_minor_version_for_prefilter") +TEST_CASE("matches_python_minor") { const auto req = [](std::string_view s) -> specs::Version { return specs::Version::parse(s).value(); }; SECTION("non-python dependency is not filtered") { - REQUIRE(dependency_matches_python_minor_version_for_prefilter("numpy >=1.0", req("3.12"))); - REQUIRE( - dependency_matches_python_minor_version_for_prefilter("libstdcxx-ng >=12", req("3.12")) - ); + REQUIRE(matches_python_minor("numpy >=1.0", req("3.12"))); + REQUIRE(matches_python_minor("libstdcxx-ng >=12", req("3.12"))); } SECTION("name starting with python but not the python package") { - REQUIRE(dependency_matches_python_minor_version_for_prefilter( - "python_abi 3.12 1_cp312", - req("3.12") - )); + REQUIRE(matches_python_minor("python_abi 3.12 1_cp312", req("3.12"))); } SECTION("python version range matches requested minor") { - REQUIRE( - dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.12")) - ); - REQUIRE(dependency_matches_python_minor_version_for_prefilter("python >=3.12", req("3.12"))); + REQUIRE(matches_python_minor("python >=3.12,<3.13", req("3.12"))); + REQUIRE(matches_python_minor("python >=3.12", req("3.12"))); } SECTION("python version range does not match requested minor") { - REQUIRE_FALSE( - dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.11")) - ); - REQUIRE_FALSE( - dependency_matches_python_minor_version_for_prefilter("python >=3.12,<3.13", req("3.13")) - ); + REQUIRE_FALSE(matches_python_minor("python >=3.12,<3.13", req("3.11"))); + REQUIRE_FALSE(matches_python_minor("python >=3.12,<3.13", req("3.13"))); } SECTION("exact three-token conda pin matches requested minor") { - REQUIRE( - dependency_matches_python_minor_version_for_prefilter("python 3.7.12 0_73_pypy", req("3.7")) - ); + REQUIRE(matches_python_minor("python 3.7.12 0_73_pypy", req("3.7"))); } SECTION("two-token exact pin matches requested minor") { - REQUIRE(dependency_matches_python_minor_version_for_prefilter("python 3.7.12", req("3.7"))); + REQUIRE(matches_python_minor("python 3.7.12", req("3.7"))); } SECTION("exact pin does not match different minor") { - REQUIRE_FALSE( - dependency_matches_python_minor_version_for_prefilter("python 3.8.0", req("3.7")) - ); + REQUIRE_FALSE(matches_python_minor("python 3.8.0", req("3.7"))); } SECTION("leading whitespace on dependency line") { - REQUIRE( - dependency_matches_python_minor_version_for_prefilter(" python >=3.12,<3.13", req("3.12")) - ); + REQUIRE(matches_python_minor(" python >=3.12,<3.13", req("3.12"))); } SECTION("unparsable python dependency does not filter (passes)") { - REQUIRE(dependency_matches_python_minor_version_for_prefilter( - "python ,,not-a-valid-spec,,", - req("3.12") - )); + REQUIRE(matches_python_minor("python ,,not-a-valid-spec,,", req("3.12"))); } SECTION("namespaced python pin") { - REQUIRE(dependency_matches_python_minor_version_for_prefilter( - "conda-forge::python 3.7.12 0_73_pypy", - req("3.7") - )); + REQUIRE(matches_python_minor("conda-forge::python 3.7.12 0_73_pypy", req("3.7"))); } SECTION("only python in range with no upper bound") { - REQUIRE(dependency_matches_python_minor_version_for_prefilter("python", req("3.12"))); + REQUIRE(matches_python_minor("python", req("3.12"))); } } From d62b30fd68bfc95e2a636abcd295478cffff19de Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 7 Apr 2026 11:40:17 +0200 Subject: [PATCH 17/18] fix: Support single equals conda pin form Signed-off-by: Julien Jerphanion --- .../src/core/shard_python_minor_prefilter.cpp | 29 +++++++++++++++---- .../test_shard_python_minor_prefilter.cpp | 7 +++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/libmamba/src/core/shard_python_minor_prefilter.cpp b/libmamba/src/core/shard_python_minor_prefilter.cpp index c3f7fb5a68..0fe49daaaf 100644 --- a/libmamba/src/core/shard_python_minor_prefilter.cpp +++ b/libmamba/src/core/shard_python_minor_prefilter.cpp @@ -16,6 +16,23 @@ namespace mamba { + namespace + { + // Allows supporting both form of pin, e.g. ``python=3.13`` and ``python ==3.13``. + auto equality_tail(std::string_view spec_str) -> std::optional + { + if (util::starts_with(spec_str, specs::VersionSpec::equal_str)) + { + return spec_str.substr(specs::VersionSpec::equal_str.size()); + } + if (util::starts_with(spec_str, "=")) + { + return spec_str.substr(1); + } + return std::nullopt; + } + } + auto version_from_single_equality_spec(const specs::VersionSpec& vs) -> std::optional { @@ -24,12 +41,12 @@ namespace mamba return std::nullopt; } const std::string s = vs.to_string(); - if (!util::starts_with(s, specs::VersionSpec::equal_str)) + const auto maybe_tail = equality_tail(s); + if (!maybe_tail.has_value()) { return std::nullopt; } - const auto tail = std::string_view(s).substr(specs::VersionSpec::equal_str.size()); - auto maybe_v = specs::Version::parse(std::string(util::lstrip(tail))); + auto maybe_v = specs::Version::parse(std::string(util::lstrip(maybe_tail.value()))); if (maybe_v.has_value()) { return maybe_v.value(); @@ -45,12 +62,12 @@ namespace mamba return vs; } const std::string vs_str = vs.to_string(); - if (!util::starts_with(vs_str, specs::VersionSpec::equal_str)) + const auto maybe_tail = equality_tail(vs_str); + if (!maybe_tail.has_value()) { return vs; } - const auto ver_tail = std::string_view(vs_str).substr(specs::VersionSpec::equal_str.size()); - auto maybe_v = specs::Version::parse(util::lstrip(ver_tail)); + auto maybe_v = specs::Version::parse(util::lstrip(maybe_tail.value())); if (!maybe_v.has_value()) { return vs; diff --git a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp index 6920ca59ee..3f84d48e46 100644 --- a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp +++ b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp @@ -177,6 +177,13 @@ TEST_CASE("extract_requested_python_minor") REQUIRE(got.value() == v("3.11")); } + SECTION("single equals conda pin form") + { + const auto got = extract_requested_python_minor({ "python=3.13" }); + REQUIRE(got.has_value()); + REQUIRE(got.value() == v("3.13")); + } + SECTION("range spec yields no minor (not single equality after relax)") { REQUIRE_FALSE(extract_requested_python_minor({ "python >=3.12,<3.13" }).has_value()); From 3ff817cca44a8657c47560f93d87a616425d2db3 Mon Sep 17 00:00:00 2001 From: Julien Jerphanion Date: Tue, 7 Apr 2026 13:24:47 +0200 Subject: [PATCH 18/18] fix: Do not prefilter if major version only are provided Signed-off-by: Julien Jerphanion --- libmamba/src/api/utils.cpp | 13 ++++++++++++- .../src/core/test_shard_python_minor_prefilter.cpp | 6 ++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/libmamba/src/api/utils.cpp b/libmamba/src/api/utils.cpp index 05f9837198..a51c446e5d 100644 --- a/libmamba/src/api/utils.cpp +++ b/libmamba/src/api/utils.cpp @@ -738,7 +738,18 @@ namespace mamba { continue; } - const specs::VersionSpec relaxed = relax_version_spec_to_minor(maybe_ms.value().version()); + const auto& raw_version_spec = maybe_ms.value().version(); + // Pins like ``python=2`` or ``python=3`` specify only the major version. Relaxing those + // to ``2.0`` / ``3.0`` for shard prefiltering would drop packages whose ``depends`` + // require a real minor (e.g. ``python >=2.7``). Skip the prefilter for such specs. + if (auto maybe_single_v = version_from_single_equality_spec(raw_version_spec)) + { + if (maybe_single_v->version().size() <= std::size_t{ 1 }) + { + return std::nullopt; + } + } + const specs::VersionSpec relaxed = relax_version_spec_to_minor(raw_version_spec); if (auto maybe_v = version_from_single_equality_spec(relaxed)) { return maybe_v; diff --git a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp index 3f84d48e46..8698517ad7 100644 --- a/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp +++ b/libmamba/tests/src/core/test_shard_python_minor_prefilter.cpp @@ -184,6 +184,12 @@ TEST_CASE("extract_requested_python_minor") REQUIRE(got.value() == v("3.13")); } + SECTION("major-only pin yields no minor (avoids bogus 2.0/3.0 shard prefilter)") + { + REQUIRE_FALSE(extract_requested_python_minor({ "python=2" }).has_value()); + REQUIRE_FALSE(extract_requested_python_minor({ "python=3" }).has_value()); + } + SECTION("range spec yields no minor (not single equality after relax)") { REQUIRE_FALSE(extract_requested_python_minor({ "python >=3.12,<3.13" }).has_value());