diff --git a/README.md b/README.md index cb8f30a02..b26c90dc3 100644 --- a/README.md +++ b/README.md @@ -244,11 +244,21 @@ conda-lock install --auth-file auth.json conda-linux-64.lock ### --virtual-package-spec -Conda makes use of [virtual packages](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html) that are available at -runtime to gate dependency on system features. Due to these not generally existing on your local execution platform conda-lock will inject -them into the solution environment with a reasonable guess at what a default system configuration should be. +Conda uses [virtual packages](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html) to describe system-level +features like the glibc version, macOS version, or CUDA version. Since conda-lock generates +lockfiles independently of a target system, it assumes a default set of virtual package versions +that represent a reasonable minimum system configuration. -If you want to override which virtual packages are injected you can create a file like +Each virtual package version acts as an upper bound on which packages are considered by +the solver (packages requiring a newer version are excluded) and as a lower bound on +compatible target systems (the lockfile is installable on systems that meet or exceed +these versions). + +There is a tradeoff: increasing a version widens the set of candidate packages but narrows +the set of compatible target systems. If a package you need is being filtered out, increase the +relevant version. If the lockfile is incompatible with your target system, decrease it. + +To override the defaults, create a `virtual-packages.yml` file like ```yaml # virtual-packages.yml diff --git a/conda_lock/default-virtual-packages.yaml b/conda_lock/default-virtual-packages.yaml index dcc05b0d1..b31add736 100644 --- a/conda_lock/default-virtual-packages.yaml +++ b/conda_lock/default-virtual-packages.yaml @@ -1,10 +1,25 @@ +# Default virtual packages used when no --virtual-package-spec is provided. +# +# Each version represents the assumed minimum system capability. The solver +# filters out any package that requires a newer version, so these act as upper +# bounds on which packages are considered. Conversely, they act as lower bounds +# on the systems where the resulting lockfile can be installed. +# +# For example, with __glibc: "2.28", packages requiring glibc > 2.28 are +# excluded from the lockfile, and the lockfile is installable on any system +# with glibc >= 2.28. +# +# Tradeoff: increasing a version widens the set of candidate packages but +# narrows the set of compatible target systems. If a package you need is being +# filtered out, increase the version. If the lockfile is incompatible with your +# target system, decrease the version. To override these defaults, create a +# virtual-packages.yml file (see README.md). subdirs: linux-64: packages: __unix: "0" __linux: "5.10" __archspec: "1 x86_64" - # NOTE: Keep this in sync with the MANYLINUX_TAGS maximum in pypi_solver.py __glibc: "2.28" __cuda: "11.4" linux-aarch64: @@ -12,7 +27,6 @@ subdirs: __unix: "0" __linux: "5.10" __archspec: "1 aarch64" - # NOTE: Keep this in sync with the MANYLINUX_TAGS maximum in pypi_solver.py __glibc: "2.28" __cuda: "11.4" linux-ppc64le: @@ -20,7 +34,6 @@ subdirs: __unix: "0" __linux: "5.10" __archspec: "1 ppc64le" - # NOTE: Keep this in sync with the MANYLINUX_TAGS maximum in pypi_solver.py __glibc: "2.28" __cuda: "11.4" osx-64: diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index 846b2d28c..be58ba4a6 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -58,8 +58,6 @@ # NB: in principle these depend on the glibc on the machine creating the conda env. # We use tags supported by manylinux Docker images, which are likely the most common # in practice, see https://github.com/pypa/manylinux/blob/main/README.rst#docker-images. -# NOTE: -# Keep the max in sync with the default value used in default-virtual-packages.yaml. MANYLINUX_TAGS = ["1", "2010", "2014", "_2_17", "_2_18", "_2_24", "_2_28"] # This needs to be updated periodically as new macOS versions are released. diff --git a/docs/flags.md b/docs/flags.md index 0fd816bc3..e0d17807b 100644 --- a/docs/flags.md +++ b/docs/flags.md @@ -135,11 +135,21 @@ When the input_hash of the input files, channels match those present in a given ## --virtual-package-spec -Conda makes use of [virtual packages](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html) that are available at -runtime to gate dependency on system features. Due to these not generally existing on your local execution platform conda-lock will inject -them into the solution environment with a reasonable guess at what a default system configuration should be. +Conda uses [virtual packages](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-virtual.html) to describe system-level +features like the glibc version, macOS version, or CUDA version. Since conda-lock generates +lockfiles independently of a target system, it assumes a default set of virtual package versions +that represent a reasonable minimum system configuration. -If you want to override which virtual packages are injected you can create a virtual package spec file +Each virtual package version acts as an upper bound on which packages are considered by +the solver (packages requiring a newer version are excluded) and as a lower bound on +compatible target systems (the lockfile is installable on systems that meet or exceed +these versions). + +There is a tradeoff: increasing a version widens the set of candidate packages but narrows +the set of compatible target systems. If a package you need is being filtered out, increase the +relevant version. If the lockfile is incompatible with your target system, decrease it. + +To override the defaults, create a virtual package spec file ```{.yaml title="virtual-packages.yml"} subdirs: diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index d4f053c8e..cbf99e49e 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -3258,15 +3258,17 @@ def test_manylinux_tags(): assert versions[0] == Version("2.17") assert versions == sorted(versions) - # Verify that the default repodata uses the highest glibc version + # Verify that the default glibc versions are covered by MANYLINUX_TAGS. + # (The default may be lower than the max tag; higher tags are available + # via --virtual-package-spec.) default_repodata = default_virtual_package_repodata() glibc_versions_in_default_repodata: set[Version] = { Version(package.version) for package in default_repodata.packages_by_subdir if package.name == "__glibc" } - max_glibc_version_from_manylinux_tags = versions[-1] - assert glibc_versions_in_default_repodata == {max_glibc_version_from_manylinux_tags} + manylinux_glibc_versions = set(versions) + assert glibc_versions_in_default_repodata <= manylinux_glibc_versions def test_pip_respects_glibc_version(