From cefa098cc163f4d7ed6e1c5b06f4d0dc4e06997e Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 31 Jul 2025 11:30:15 +0200 Subject: [PATCH 1/8] Raise error if conda dependencdies are missing --- conda_lock/lockfile/__init__.py | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index 35830200d..3e62e420f 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -26,6 +26,57 @@ class UnknownLockfileVersion(ValueError): pass +class InconsistentCondaDependencies(ValueError): + """Raised when conda dependencies in the lockfile are inconsistent.""" + + pass + + +def _verify_no_missing_conda_packages(content: Lockfile) -> None: + """ + Ensure all subdependencies of conda packages are also present as conda packages. + + This does not check version constraints. + + Raises: + InconsistentCondaDependencies: If any conda dependency is missing + """ + # Build a mapping of (name, platform) -> LockedDependency for conda packages + conda_packages: dict[tuple[str, str], LockedDependency] = {} + for package in content.package: + if package.manager == "conda": + conda_packages[(package.name, package.platform)] = package + + # Iterate through the mapping while checking for missing dependencies + missing_dependencies: set[tuple[str, str]] = set() + for (_primary_dep_name, platform), dependency in conda_packages.items(): + subdependencies = dependency.dependencies + for subdep_name in subdependencies: + if subdep_name.startswith("__"): + # Virtual packages like __linux are not real packages so not present + continue + if (subdep_name, platform) not in conda_packages: + missing_dependencies.add((subdep_name, platform)) + + if missing_dependencies: + error_msg = ( + "Conda dependency consistency check failed. The following conda " + "subdependencies are missing from the lockfile:\n\n" + ) + for current_platform in content.metadata.platforms: + missing_on_platform = [ + (name, dep_platform) + for name, dep_platform in missing_dependencies + if dep_platform == current_platform + ] + if missing_on_platform: + error_msg += f" {current_platform}:\n" + for subdep_name, _subdep_platform in sorted(missing_on_platform): + error_msg += f" - {subdep_name}\n" + error_msg += "\n\nThis indicates that the conda dependency graph is incomplete." + raise InconsistentCondaDependencies(error_msg) + + def _seperator_munge_get( d: Mapping[str, Union[list[LockedDependency], LockedDependency]], key: str ) -> Union[list[LockedDependency], LockedDependency]: @@ -193,6 +244,10 @@ def write_conda_lock_file( ) -> None: content.alphasort_inplace() content.filter_virtual_packages_inplace() + + # Validate conda dependency consistency before writing + _verify_no_missing_conda_packages(content) + with path.open("w") as f: if include_help_text: categories: set[str] = { From d101e28fa1f4d445306cf86887b0139cdde0dcc3 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 31 Jul 2025 20:14:10 +0200 Subject: [PATCH 2/8] Print all satisfied dependencies --- conda_lock/lockfile/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index 3e62e420f..7df0f4909 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -41,6 +41,7 @@ def _verify_no_missing_conda_packages(content: Lockfile) -> None: Raises: InconsistentCondaDependencies: If any conda dependency is missing """ + print("------------------VERIFYING CONDA DEPENDENCY CONSISTENCY------------------") # Build a mapping of (name, platform) -> LockedDependency for conda packages conda_packages: dict[tuple[str, str], LockedDependency] = {} for package in content.package: @@ -51,12 +52,17 @@ def _verify_no_missing_conda_packages(content: Lockfile) -> None: missing_dependencies: set[tuple[str, str]] = set() for (_primary_dep_name, platform), dependency in conda_packages.items(): subdependencies = dependency.dependencies + satisfied_deps = [] for subdep_name in subdependencies: if subdep_name.startswith("__"): # Virtual packages like __linux are not real packages so not present continue if (subdep_name, platform) not in conda_packages: missing_dependencies.add((subdep_name, platform)) + else: + satisfied_deps.append(subdep_name) + if len(satisfied_deps) > 0: + print(f"Satisfied dependencies for {_primary_dep_name}: {satisfied_deps}") if missing_dependencies: error_msg = ( @@ -75,6 +81,7 @@ def _verify_no_missing_conda_packages(content: Lockfile) -> None: error_msg += f" - {subdep_name}\n" error_msg += "\n\nThis indicates that the conda dependency graph is incomplete." raise InconsistentCondaDependencies(error_msg) + print("------------------CONDA DEPENDENCY CONSISTENCY VERIFIED-------------------") def _seperator_munge_get( From 0429678d2fff75a5c39d35d65dcbc63dab63a3ce Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 31 Jul 2025 20:44:13 +0200 Subject: [PATCH 3/8] Output v2 and v1 intermediate lockfiles --- conda_lock/lockfile/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index 7df0f4909..ac1e6e1d3 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -309,5 +309,8 @@ def write_section(text: str) -> None: conda-lock {metadata_flags}{" ".join("-f " + path for path in content.metadata.sources)} --lockfile {path.name} """ ) - output = content.to_v1().dict_for_output() + pathlib.Path("outputv2.json").write_text(content.model_dump_json(indent=2)) + content_v1 = content.to_v1() + pathlib.Path("outputv1.json").write_text(content_v1.model_dump_json(indent=2)) + output = content_v1.dict_for_output() yaml.dump(output, stream=f, sort_keys=False) From a9788f4d667209b16a8e30875908d685759b503a Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Thu, 31 Jul 2025 20:54:21 +0200 Subject: [PATCH 4/8] Do another round-trip check --- conda_lock/lockfile/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index ac1e6e1d3..ee3a34c69 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -314,3 +314,7 @@ def write_section(text: str) -> None: pathlib.Path("outputv1.json").write_text(content_v1.model_dump_json(indent=2)) output = content_v1.dict_for_output() yaml.dump(output, stream=f, sort_keys=False) + + # Verify round-trip consistency by reading back the lockfile and checking again + parsed_lockfile = parse_conda_lock_file(path) + _verify_no_missing_conda_packages(parsed_lockfile) From b8a33486e58c32a1e869002e096d441f57c2c929 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Mon, 4 Aug 2025 23:39:47 +0200 Subject: [PATCH 5/8] Assert that all packages from parsed lockfiles have nonempty categories --- conda_lock/lockfile/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_lock/lockfile/__init__.py b/conda_lock/lockfile/__init__.py index ee3a34c69..bde819432 100644 --- a/conda_lock/lockfile/__init__.py +++ b/conda_lock/lockfile/__init__.py @@ -240,6 +240,8 @@ def parse_conda_lock_file(path: pathlib.Path) -> Lockfile: else: raise UnknownLockfileVersion(f"{path} has unknown version {version}") lockfile.toposort_inplace() + for p in lockfile.package: + assert len(p.categories) > 0, f"Package {p.name} has no categories" return lockfile From 3ed90c93f66cea63790a2ba8b09733cf05114d0c Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Tue, 5 Aug 2025 19:23:15 +0200 Subject: [PATCH 6/8] Add more debugging --- conda_lock/conda_solver.py | 4 ++++ conda_lock/lockfile/v1/models.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/conda_lock/conda_solver.py b/conda_lock/conda_solver.py index e1bae2332..14825e79e 100644 --- a/conda_lock/conda_solver.py +++ b/conda_lock/conda_solver.py @@ -259,6 +259,10 @@ def _reconstruct_fetch_actions( else: raise ValueError(f"Unable to extract the dist_name from {link_action}.") repodata = _get_repodata_record(pkgs_dirs, dist_name) + if link_pkg_name == "pyzmq": + print( + f"In _reconstruct_fetch_actions for {link_pkg_name}, repodata: {repodata}" + ) if repodata is None: raise FileNotFoundError( f"Distribution '{dist_name}' not found in pkgs_dirs {pkgs_dirs}" diff --git a/conda_lock/lockfile/v1/models.py b/conda_lock/lockfile/v1/models.py index c69845fe3..51eef8e48 100644 --- a/conda_lock/lockfile/v1/models.py +++ b/conda_lock/lockfile/v1/models.py @@ -130,6 +130,10 @@ def to_fetch_action(self) -> FetchAction: channel_url = f"{base_url}/{self.platform}" # e.g. "https://user:pass@conda.anaconda.org/conda-forge/linux-64" + depends = [f"{k} {v}".strip() for k, v in self.dependencies.items()] + if self.name == "pyzmq": + print(f"In to_fetch_action for {self.name}, depends: {depends}") + fetch_action = FetchAction( name=self.name, version=self.version, @@ -138,7 +142,7 @@ def to_fetch_action(self) -> FetchAction: fn=filename_with_extension, md5=self.hash.md5, sha256=self.hash.sha256, - depends=[f"{k} {v}".strip() for k, v in self.dependencies.items()], + depends=depends, constrains=[], subdir=self.platform, timestamp=0, From b63aa9cbc47f8f5df898f1211f52a540eaf6f93a Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Tue, 5 Aug 2025 19:36:32 +0200 Subject: [PATCH 7/8] Recover when categories are empty --- conda_lock/lockfile/v2prelim/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda_lock/lockfile/v2prelim/models.py b/conda_lock/lockfile/v2prelim/models.py index b684e98cb..7f884d743 100644 --- a/conda_lock/lockfile/v2prelim/models.py +++ b/conda_lock/lockfile/v2prelim/models.py @@ -27,6 +27,9 @@ def to_v1(self) -> list[LockedDependencyV1]: can only contain a single category, we represent multiple categories as a list of v1 dependencies that are identical except for the `category` field. The `category` field runs over all categories.""" + if len(self.categories) == 0: + print(f"In to_v1, no categories for {self.name}, adding to main") + self.categories = {"main"} package_entries_per_category = [ LockedDependencyV1( name=self.name, From 3ef7e4e7a63dc423c26f957d1790076b361a135d Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Wed, 13 Aug 2025 23:36:04 +0200 Subject: [PATCH 8/8] Fail if depends is empty --- conda_lock/conda_solver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_lock/conda_solver.py b/conda_lock/conda_solver.py index 14825e79e..161f9c97e 100644 --- a/conda_lock/conda_solver.py +++ b/conda_lock/conda_solver.py @@ -129,6 +129,8 @@ def solve_conda( planned = {} for action in dry_run_install["actions"]["FETCH"]: dependencies = {} + if action.get("depends") is None: + raise ValueError(f"No depends found for FETCH action {action}") for dep in action.get("depends") or []: matchspec = MatchSpec(dep) # pyright: ignore[reportArgumentType] name = matchspec.name