From f8a76f95a949e73450cf3fb70ee06a556af88a10 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 24 Jan 2024 14:21:54 +0100 Subject: [PATCH 1/8] Add .DS_Store to .gitignore Co-authored-by: Srinivas Lade --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9914ad0e6..528ffc72b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +**.DS_Store *.egg-info *.eggs *.pyc From 3c1b69a484c519e2720e702ebc2fda723223e6fd Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 24 Jan 2024 14:24:13 +0100 Subject: [PATCH 2/8] =?UTF-8?q?Rename=20variable=20categories=20=E2=86=92?= =?UTF-8?q?=20categories=5Fto=5Finstall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Srinivas Lade --- conda_lock/conda_lock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index f9edd810b..f7c9fb35d 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -602,7 +602,7 @@ def render_lockfile_for_platform( # noqa: C901 f"# input_hash: {lockfile.metadata.content_hash.get(platform)}\n", ] - categories = { + categories_to_install: Set[str] = { "main", *(extras or []), *(["dev"] if include_dev_dependencies else []), @@ -620,7 +620,7 @@ def render_lockfile_for_platform( # noqa: C901 lockfile.filter_virtual_packages_inplace() for p in lockfile.package: - if p.platform == platform and p.category in categories: + if p.platform == platform and p.category in categories_to_install: if p.manager == "pip": pip_deps.append(p) elif p.manager == "conda": From 5e4bb1f0f7e8f18b64b04301411d851ea19f82ea Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 24 Jan 2024 14:28:25 +0100 Subject: [PATCH 3/8] Move .category from BaseLockedDependency into subclass defs Co-authored-by: Srinivas Lade --- conda_lock/lockfile/v1/models.py | 2 +- conda_lock/lockfile/v2prelim/models.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_lock/lockfile/v1/models.py b/conda_lock/lockfile/v1/models.py index e431025ab..66f95c263 100644 --- a/conda_lock/lockfile/v1/models.py +++ b/conda_lock/lockfile/v1/models.py @@ -54,7 +54,6 @@ class BaseLockedDependency(StrictModel): dependencies: Dict[str, str] = {} url: str hash: HashModel - category: str = "main" source: Optional[DependencySource] = None build: Optional[str] = None @@ -69,6 +68,7 @@ def validate_hash(cls, v: HashModel, values: Dict[str, typing.Any]) -> HashModel class LockedDependency(BaseLockedDependency): + category: str = "main" optional: bool diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index e322c803c..61686e7a6 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -17,6 +17,8 @@ class LockedDependency(BaseLockedDependency): + category: str = "main" + def to_v1(self) -> LockedDependencyV1: return LockedDependencyV1( name=self.name, From 0045df06f052fcff50a9a8f7261c55b0f843b862 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 24 Jan 2024 14:35:35 +0100 Subject: [PATCH 4/8] Rewrite containment in terms of sets Co-authored-by: Srinivas Lade --- conda_lock/conda_lock.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index f7c9fb35d..e7184788d 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -620,7 +620,7 @@ def render_lockfile_for_platform( # noqa: C901 lockfile.filter_virtual_packages_inplace() for p in lockfile.package: - if p.platform == platform and p.category in categories_to_install: + if p.platform == platform and len({p.category} & categories_to_install) > 0: if p.manager == "pip": pip_deps.append(p) elif p.manager == "conda": From 347aa5c3e8a387adb40d4d29a6a523664d335ff1 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 24 Jan 2024 14:41:56 +0100 Subject: [PATCH 5/8] Rewrite union in terms of singleton sets Co-authored-by: Srinivas Lade --- conda_lock/lockfile/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index cd1060842..a572fab09 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -163,7 +163,7 @@ def write_conda_lock_file( content.filter_virtual_packages_inplace() with path.open("w") as f: if include_help_text: - categories = set(p.category for p in content.package) + categories: Set[str] = set().union(*({p.category} for p in content.package)) def write_section(text: str) -> None: lines = dedent(text).split("\n") From ac1ab47207efb060e7a451fab4dea10264823cfb Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 24 Jan 2024 15:13:10 +0100 Subject: [PATCH 6/8] Convert v2 locked deps into list of v1 deps Co-authored-by: Srinivas Lade --- conda_lock/lockfile/v2prelim/models.py | 47 ++++++++++++++++++-------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index 61686e7a6..0c1a49f71 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -19,20 +19,30 @@ class LockedDependency(BaseLockedDependency): category: str = "main" - def to_v1(self) -> LockedDependencyV1: - return LockedDependencyV1( - name=self.name, - version=self.version, - manager=self.manager, - platform=self.platform, - dependencies=self.dependencies, - url=self.url, - hash=self.hash, - category=self.category, - source=self.source, - build=self.build, - optional=self.category != "main", - ) + def to_v1(self) -> List[LockedDependencyV1]: + """Convert a v2 dependency into a list of v1 dependencies. + + In case a v2 dependency might contain multiple categories, but a v1 dependency + can only contain a single category, we represent multiple categories as a list + of v1 dependencies that are identical except for the `category` field. The + `category` field runs over all categories.""" + package_entries_per_category = [ + LockedDependencyV1( + name=self.name, + version=self.version, + manager=self.manager, + platform=self.platform, + dependencies=self.dependencies, + url=self.url, + hash=self.hash, + category=category, + source=self.source, + build=self.build, + optional=category != "main", + ) + for category in sorted({self.category}) + ] + return package_entries_per_category class Lockfile(StrictModel): @@ -129,8 +139,15 @@ def _toposort(package: List[LockedDependency]) -> List[LockedDependency]: return final_package def to_v1(self) -> LockfileV1: + # Each v2 package gives a list of v1 packages. + # Flatten these into a single list of v1 packages. + v1_packages = [ + package_entry_per_category + for p in self.package + for package_entry_per_category in p.to_v1() + ] return LockfileV1( - package=[p.to_v1() for p in self.package], + package=v1_packages, metadata=self.metadata, ) From 9fdf5f60b70a612488126195b09f07ea233801f8 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 24 Jan 2024 15:45:21 +0100 Subject: [PATCH 7/8] Handle multiple categories when parsing v1 lockfiles Co-authored-by: Srinivas Lade --- conda_lock/lockfile/v2prelim/models.py | 63 ++++++++++++++++++++------ 1 file changed, 48 insertions(+), 15 deletions(-) diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index 0c1a49f71..f2f510f0f 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import ClassVar, Dict, List, Optional +from typing import ClassVar, Dict, List, Optional, Set from conda_lock.lockfile.v1.models import ( BaseLockedDependency, @@ -7,6 +7,7 @@ GitMeta, HashModel, InputMeta, + LockKey, LockMeta, MetadataOption, TimeMeta, @@ -152,29 +153,61 @@ def to_v1(self) -> LockfileV1: ) -def _locked_dependency_v1_to_v2(dep: LockedDependencyV1) -> LockedDependency: +def _locked_dependency_v1_to_v2( + package_entries_per_category: List[LockedDependencyV1], +) -> LockedDependency: """Convert a LockedDependency from v1 to v2. - * Remove the optional field (it is always equal to category != "main") + This is an inverse to `LockedDependency.to_v1()`. """ + # Dependencies are parsed from a v1 lockfile, so there will always be + # at least one entry corresponding to what was parsed. + assert len(package_entries_per_category) > 0 + # All the package entries should share the same key. + assert all( + d.key() == package_entries_per_category[0].key() + for d in package_entries_per_category + ) + + categories = {d.category for d in package_entries_per_category} + + # Each entry should correspond to a distinct category + assert len(categories) == len(package_entries_per_category) + + # Until we allow multiple categories in v2 we need this workaround + single_category = package_entries_per_category[0].category + return LockedDependency( - name=dep.name, - version=dep.version, - manager=dep.manager, - platform=dep.platform, - dependencies=dep.dependencies, - url=dep.url, - hash=dep.hash, - category=dep.category, - source=dep.source, - build=dep.build, + name=package_entries_per_category[0].name, + version=package_entries_per_category[0].version, + manager=package_entries_per_category[0].manager, + platform=package_entries_per_category[0].platform, + dependencies=package_entries_per_category[0].dependencies, + url=package_entries_per_category[0].url, + hash=package_entries_per_category[0].hash, + category=single_category, + source=package_entries_per_category[0].source, + build=package_entries_per_category[0].build, ) def lockfile_v1_to_v2(lockfile_v1: LockfileV1) -> Lockfile: - """Convert a Lockfile from v1 to v2.""" + """Convert a Lockfile from v1 to v2. + + Entries may share the same key if they represent a dependency + belonging to multiple categories. They must be collected here. + """ + dependencies_for_key: Dict[LockKey, List[LockedDependencyV1]] = defaultdict(list) + for dep in lockfile_v1.package: + dependencies_for_key[dep.key()].append(dep) + + v2_packages = [ + _locked_dependency_v1_to_v2(package_entries_per_category) + for package_entries_per_category in dependencies_for_key.values() + ] + return Lockfile( - package=[_locked_dependency_v1_to_v2(p) for p in lockfile_v1.package], + package=v2_packages, metadata=lockfile_v1.metadata, ) From b18fc3d55b7bccc566587ff9fb2924575b837854 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 24 Jan 2024 16:57:04 +0100 Subject: [PATCH 8/8] =?UTF-8?q?Change=20category=20=E2=86=92=20categories?= =?UTF-8?q?=20in=20LockedDependencyV2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Srinivas Lade --- conda_lock/conda_lock.py | 2 +- conda_lock/lockfile/__init__.py | 6 ++++-- conda_lock/lockfile/v2prelim/models.py | 9 +++------ tests/test_conda_lock.py | 5 ++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/conda_lock/conda_lock.py b/conda_lock/conda_lock.py index e7184788d..48b1d7171 100644 --- a/conda_lock/conda_lock.py +++ b/conda_lock/conda_lock.py @@ -620,7 +620,7 @@ def render_lockfile_for_platform( # noqa: C901 lockfile.filter_virtual_packages_inplace() for p in lockfile.package: - if p.platform == platform and len({p.category} & categories_to_install) > 0: + if p.platform == platform and len(p.categories & categories_to_install) > 0: if p.manager == "pip": pip_deps.append(p) elif p.manager == "conda": diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index a572fab09..12e89ac0c 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -131,7 +131,7 @@ def dep_name(manager: str, dep: str) -> str: if not isinstance(targets, list): targets = [targets] for target in targets: - target.category = source.category + target.categories = {source.category} def parse_conda_lock_file(path: pathlib.Path) -> Lockfile: @@ -163,7 +163,9 @@ def write_conda_lock_file( content.filter_virtual_packages_inplace() with path.open("w") as f: if include_help_text: - categories: Set[str] = set().union(*({p.category} for p in content.package)) + categories: Set[str] = { + category for p in content.package for category in p.categories + } def write_section(text: str) -> None: lines = dedent(text).split("\n") diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index f2f510f0f..3138d20e6 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -18,7 +18,7 @@ class LockedDependency(BaseLockedDependency): - category: str = "main" + categories: Set[str] = {"main"} def to_v1(self) -> List[LockedDependencyV1]: """Convert a v2 dependency into a list of v1 dependencies. @@ -41,7 +41,7 @@ def to_v1(self) -> List[LockedDependencyV1]: build=self.build, optional=category != "main", ) - for category in sorted({self.category}) + for category in sorted(self.categories) ] return package_entries_per_category @@ -174,9 +174,6 @@ def _locked_dependency_v1_to_v2( # Each entry should correspond to a distinct category assert len(categories) == len(package_entries_per_category) - # Until we allow multiple categories in v2 we need this workaround - single_category = package_entries_per_category[0].category - return LockedDependency( name=package_entries_per_category[0].name, version=package_entries_per_category[0].version, @@ -185,7 +182,7 @@ def _locked_dependency_v1_to_v2( dependencies=package_entries_per_category[0].dependencies, url=package_entries_per_category[0].url, hash=package_entries_per_category[0].hash, - category=single_category, + categories=categories, source=package_entries_per_category[0].source, build=package_entries_per_category[0].build, ) diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 858aef4c7..c611296e4 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -12,7 +12,6 @@ import tempfile import typing import uuid -import warnings from glob import glob from pathlib import Path @@ -394,10 +393,10 @@ def test_lock_poetry_ibis( ) lockfile = parse_conda_lock_file(pyproject.parent / DEFAULT_LOCKFILE_NAME) - all_categories = set() + all_categories: Set[str] = set() for pkg in lockfile.package: - all_categories.add(pkg.category) + all_categories.update(pkg.categories) for desired_category in extra_categories: assert (