From af7baaa32cb4e1c49857d1667ca8e42b8b0b0e5f Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Wed, 11 Mar 2026 17:38:20 -0400 Subject: [PATCH 01/32] update typing --- conda_index/api.py | 6 ++-- conda_index/index/cache.py | 64 ++++++++++++++++++++++++-------------- conda_index/index/fs.py | 5 ++- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/conda_index/api.py b/conda_index/api.py index a2b31b19..f85cf0a9 100644 --- a/conda_index/api.py +++ b/conda_index/api.py @@ -28,9 +28,9 @@ def update_index( # we basically expect there to be one path now dir_paths = [os.path.abspath(path) for path in ensure_list(dir_paths)] - assert ( - output_dir is None or len(dir_paths) == 1 - ), "Cannot combine output_dir with multiple paths" + assert output_dir is None or len(dir_paths) == 1, ( + "Cannot combine output_dir with multiple paths" + ) if isinstance(current_index_versions, str): with open(current_index_versions) as f: diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index 03a1efc2..bc4ca2b1 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -9,16 +9,20 @@ import fnmatch import json import logging -from numbers import Number from pathlib import Path -from typing import Any, Iterator, TypedDict +from typing import TYPE_CHECKING, TypedDict from zipfile import BadZipFile from conda_package_streaming import package_streaming from .. import yaml from ..utils import CONDA_PACKAGE_EXTENSIONS, _checksum -from .fs import FileInfo, MinimalFS +from .fs import MinimalFS + +if TYPE_CHECKING: + from typing import IO, Any, Iterator + + from .fs import FileInfo log = logging.getLogger(__name__) @@ -78,8 +82,8 @@ def __get__(self, inst, objtype=None) -> Any: class ChangedPackage(TypedDict): path: str - mtime: Number - size: Number + mtime: int + size: int class BaseCondaIndexCache(metaclass=abc.ABCMeta): @@ -119,12 +123,12 @@ def __init__( self.cache_is_brand_new = False @abc.abstractmethod - def convert(self): + def convert(self) -> None: """ Convert filesystem cache to database. """ - def close(self): + def close(self) -> None: """ Remove and close any database connections. """ @@ -136,19 +140,19 @@ def database_prefix(self) -> str: """ return "" - def database_path(self, fn) -> str: + def database_path(self, fn: str) -> str: """ Return filename with database prefix added. """ return f"{self.database_prefix}{fn}" - def plain_path(self, path): + def plain_path(self, path: str) -> str: """ Return filename with any database-specfic prefix stripped off. """ return path.rsplit("/", 1)[-1] - def open(self, fn: str): + def open(self, fn: str) -> IO[bytes]: """ Given a base package name "somepackage.conda", return an open, seekable file object from our channel_url/subdir/fn suitable for reading that @@ -157,7 +161,9 @@ def open(self, fn: str): abs_fn = self.fs.join(self.channel_url, self.subdir, fn) return self.fs.open(abs_fn) - def extract_to_cache_info_object(self, channel_root, subdir, fn_info: FileInfo): + def extract_to_cache_info_object( + self, channel_root: Path | str, subdir: str, fn_info: FileInfo + ) -> tuple[str, int, int, dict[str, Any] | None]: """ fn_info: avoid having to call stat() a second time on package file. """ @@ -165,7 +171,13 @@ def extract_to_cache_info_object(self, channel_root, subdir, fn_info: FileInfo): channel_root, subdir, fn_info.fn, stat_result=fn_info ) - def _extract_to_cache(self, channel_root, subdir, fn, stat_result=None): + def _extract_to_cache( + self, + channel_root: Path | str, + subdir: str, + fn: str, + stat_result: FileInfo | None = None, + ) -> tuple[str, int, int, dict[str, Any] | None]: if stat_result is None: # this code path is deprecated abs_fn = self.fs.join(self.subdir_path, fn) @@ -196,7 +208,9 @@ def _extract_to_cache(self, channel_root, subdir, fn, stat_result=None): log.exception("Error extracting %s", fn) return retval - def extract_to_cache_unconditional(self, fn, abs_fn, size, mtime): + def extract_to_cache_unconditional( + self, fn: str, abs_fn: str, size: int, mtime: int + ) -> dict[str, Any]: """ Add or replace fn into cache, disregarding whether it is already cached. @@ -244,7 +258,7 @@ def extract_to_cache_unconditional(self, fn, abs_fn, size, mtime): # XXX if we are reindexing a channel, provide a way to assert that # checksums match the upstream stage. - def checksums(): + def checksums() -> Iterator[str]: """ Use utility function that accepts open file instead of filename. """ @@ -299,21 +313,21 @@ def store( self, fn: str, size: int, - mtime, + mtime: int, members: dict[str, str | bytes], - index_json: dict, - ): + index_json: dict[str, Any], + ) -> None: """ Write a single package's index data to database. """ @abc.abstractmethod - def load_all_from_cache(self, fn) -> dict: + def load_all_from_cache(self, fn: str) -> dict[str, Any]: """ Load package data merged into a single dict for channeldata. """ - def save_fs_state(self, subdir_path: str | Path | None = None): + def save_fs_state(self, subdir_path: str | Path | None = None) -> None: """ stat all files in subdir_path to compare against cached repodata. @@ -326,7 +340,7 @@ def save_fs_state(self, subdir_path: str | Path | None = None): # Put filesystem 'ground truth' into stat table. Will we eventually stat # everything on fs, or can we shortcut for new files? - def listdir_stat(): + def listdir_stat() -> Iterator[dict[str, Any]]: # Gather conda package filenames in subdir for entry in self.fs.listdir(subdir_url): if not entry["name"].endswith(CONDA_PACKAGE_EXTENSIONS): @@ -364,7 +378,9 @@ def indexed_packages(self) -> tuple[dict, dict]: """ @abc.abstractmethod - def indexed_shards(self, desired: set | None = None): + def indexed_shards( + self, desired: set[str] | None = None + ) -> Iterator[tuple[str, Any]]: """ Yield (package name, all packages with that name) from database ordered by name, path i.o.w. filename. @@ -383,7 +399,7 @@ def run_exports(self) -> Iterator[tuple[str, dict]]: """ -def _cache_post_install_details(paths_json_str): +def _cache_post_install_details(paths_json_str: str | bytes) -> str: post_install_details_json = { "binary_prefix": False, "text_prefix": False, @@ -419,7 +435,7 @@ def _cache_post_install_details(paths_json_str): return json.dumps(post_install_details_json) -def _cache_recipe(recipe_reader): +def _cache_recipe(recipe_reader: str | bytes) -> str: recipe_json = yaml.determined_load(recipe_reader) try: @@ -431,7 +447,7 @@ def _cache_recipe(recipe_reader): return recipe_json_str -def clear_newline_chars(record, field_name): +def clear_newline_chars(record: dict[str, Any], field_name: str) -> None: if field_name in record: try: record[field_name] = record[field_name].strip().replace("\n", " ") diff --git a/conda_index/index/fs.py b/conda_index/index/fs.py index 7ee46b25..516432b9 100644 --- a/conda_index/index/fs.py +++ b/conda_index/index/fs.py @@ -11,7 +11,6 @@ import os.path import typing from dataclasses import dataclass -from numbers import Number from pathlib import Path if typing.TYPE_CHECKING: # pragma: no cover @@ -28,8 +27,8 @@ class FileInfo: """ fn: str - st_mtime: Number - st_size: Number + st_mtime: int + st_size: int class MinimalFS: From dce47ed8e94f35a7d0036765c96b46d9e4e4d077 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Wed, 11 Mar 2026 18:11:10 -0400 Subject: [PATCH 02/32] ability to generate "packages.whl" section --- conda_index/index/__init__.py | 9 ++++--- conda_index/index/cache.py | 38 ++++++++++++++++++++++++--- conda_index/index/sqlitecache.py | 44 ++++++++++++++++++++------------ conda_index/postgres/cache.py | 27 +++++++++++++++----- tests/test_cache.py | 10 +++++--- tests/test_psql.py | 37 ++++++++++++++++++++++++--- 6 files changed, 128 insertions(+), 37 deletions(-) diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index dedf6756..2fb5661a 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -734,11 +734,11 @@ def index_subdir(self, subdir, verbose=False, progress=False): log.debug("Building repodata for %s/%s", self.channel_name, subdir) - new_repodata_packages, new_repodata_conda_packages = cache.indexed_packages() + indexed_packages = cache.indexed_packages() new_repodata = { - "packages": new_repodata_packages, - "packages.conda": new_repodata_conda_packages, + "packages": indexed_packages.packages, + "packages.conda": indexed_packages.packages_conda, "info": { "subdir": subdir, }, @@ -746,6 +746,9 @@ def index_subdir(self, subdir, verbose=False, progress=False): "removed": [], # can be added by patch/hotfix process } + if indexed_packages.packages_whl: + new_repodata["packages.whl"] = indexed_packages.packages_whl + if self.base_url: # per https://github.com/conda-incubator/ceps/blob/main/cep-15.md new_repodata["info"]["base_url"] = f"{self.base_url.rstrip('/')}/{subdir}/" diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index bc4ca2b1..0f4fc491 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -9,6 +9,8 @@ import fnmatch import json import logging +import re +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, TypedDict from zipfile import BadZipFile @@ -86,6 +88,13 @@ class ChangedPackage(TypedDict): size: int +@dataclass +class IndexedPackages: + packages: dict[str, dict[str, Any]] + packages_conda: dict[str, dict[str, Any]] + packages_whl: dict[str, dict[str, Any]] + + class BaseCondaIndexCache(metaclass=abc.ABCMeta): def __init__( self, @@ -95,6 +104,7 @@ def __init__( fs: MinimalFS | None = None, channel_url: str | None = None, upstream_stage: str = "fs", + package_extensions: tuple[str, ...] = CONDA_PACKAGE_EXTENSIONS, update_only: bool = False, ): """ @@ -111,6 +121,7 @@ def __init__( self.subdir_path = Path(channel_root, subdir) self.cache_dir = Path(channel_root, subdir, ".cache") self.upstream_stage = upstream_stage + self.package_extensions = package_extensions self.update_only = update_only self.fs = fs or MinimalFS() @@ -152,6 +163,25 @@ def plain_path(self, path: str) -> str: """ return path.rsplit("/", 1)[-1] + @cacher + def _package_section_re(self): + extension_pattern = "|".join( + re.escape(extension) + for extension in sorted(self.package_extensions, key=len, reverse=True) + ) + return re.compile(f"({extension_pattern})$") + + def package_section_for_path(self, path: str) -> str | None: + package_sections = { + ".tar.bz2": "packages", + ".conda": "packages.conda", + ".whl": "packages.whl", + } + match = self._package_section_re.search(path) + if match is None: + return None + return package_sections.get(match.group(1)) + def open(self, fn: str) -> IO[bytes]: """ Given a base package name "somepackage.conda", return an open, seekable @@ -343,7 +373,7 @@ def save_fs_state(self, subdir_path: str | Path | None = None) -> None: def listdir_stat() -> Iterator[dict[str, Any]]: # Gather conda package filenames in subdir for entry in self.fs.listdir(subdir_url): - if not entry["name"].endswith(CONDA_PACKAGE_EXTENSIONS): + if not entry["name"].endswith(self.package_extensions): continue if "mtime" not in entry or "size" not in entry: entry.update(self.fs.stat(entry["name"])) @@ -371,10 +401,10 @@ def changed_packages(self) -> list[ChangedPackage]: """ @abc.abstractmethod - def indexed_packages(self) -> tuple[dict, dict]: + def indexed_packages(self) -> IndexedPackages: """ - Return "packages" and "packages.conda" values from the cache for - "monolithic repodata.json" query. + Return package sections from the cache for "monolithic repodata.json" + query. """ @abc.abstractmethod diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index c55c1e98..8c1101c7 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -15,9 +15,8 @@ import msgpack -from ..utils import CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2 from . import common, convert_cache -from .cache import BaseCondaIndexCache, ChangedPackage, cacher +from .cache import BaseCondaIndexCache, ChangedPackage, IndexedPackages, cacher from .cache import clear_newline_chars as _clear_newline_chars from .fs import MinimalFS @@ -333,12 +332,15 @@ def changed_packages(self) -> list[ChangedPackage]: return query - def indexed_packages(self): + def indexed_packages(self) -> IndexedPackages: """ - Return "packages" and "packages.conda" values from the cache. + Return package sections from the cache. """ - new_repodata_packages = {} - new_repodata_conda_packages = {} + new_packages = { + "packages": {}, + "packages.conda": {}, + "packages.whl": {}, + } # load cached packages for row in self.db.execute( @@ -351,14 +353,21 @@ def indexed_packages(self): ): path, index_json = row index_json = json.loads(index_json) - if path.endswith(CONDA_PACKAGE_EXTENSION_V1): - new_repodata_packages[path] = index_json - elif path.endswith(CONDA_PACKAGE_EXTENSION_V2): - new_repodata_conda_packages[path] = index_json - else: + if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) - - return new_repodata_packages, new_repodata_conda_packages + continue + + section = self.package_section_for_path(path) + if section is None: + log.warning("%s has unsupported package extension", path) + continue + new_packages[section][path] = index_json + + return IndexedPackages( + packages=new_packages["packages"], + packages_conda=new_packages["packages.conda"], + packages_whl=new_packages["packages.whl"], + ) def indexed_shards(self, desired: set | None = None): """ @@ -379,14 +388,17 @@ def indexed_shards(self, desired: set | None = None): shard = {"packages": {}, "packages.conda": {}} for row in rows: name, path, index_json = row - if not path.endswith((".tar.bz2", ".conda")): + if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) continue record = json.loads(index_json) - key = "packages" if path.endswith(".tar.bz2") else "packages.conda" + key = self.package_section_for_path(path) + if key is None: + log.warning("%s has unsupported package extension", path) + continue # we may have to pack later for patch functions that look for # hex hashes - shard[key][path] = pack_record(record) + shard.setdefault(key, {})[path] = pack_record(record) if not desired or name in desired: yield (name, shard) diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index b96dd343..3a8315a1 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -17,7 +17,11 @@ from sqlalchemy import Connection, cte, join, or_, select from sqlalchemy.dialects.postgresql import insert -from conda_index.index.cache import BaseCondaIndexCache, clear_newline_chars +from conda_index.index.cache import ( + BaseCondaIndexCache, + IndexedPackages, + clear_newline_chars, +) from conda_index.index.fs import MinimalFS from conda_index.index.sqlitecache import ( ICON_PATH, @@ -307,23 +311,27 @@ def indexed_shards(self, desired: set | None = None, *, pack_record=pack_record) for row in rows: name, path, record = row path = self.plain_path(path) - if not path.endswith((".tar.bz2", ".conda")): + if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) continue - key = "packages" if path.endswith(".tar.bz2") else "packages.conda" + key = self.package_section_for_path(path) + if key is None: + log.warning("%s has unsupported package extension", path) + continue # This will be passed to the patch function, which we hope # does not look for hex hash values. - shard[key][path] = pack_record(record) + shard.setdefault(key, {})[path] = pack_record(record) if not desired or name in desired: yield (name, shard) - def indexed_packages(self): + def indexed_packages(self) -> IndexedPackages: """ - Return "packages" and "packages.conda" values from the cache. + Return package sections from the cache. """ packages = {} packages_conda = {} + packages_whl = {} def nopack_record(record): return record @@ -331,8 +339,13 @@ def nopack_record(record): for _, shard in self.indexed_shards(pack_record=nopack_record): packages.update(shard["packages"]) packages_conda.update(shard["packages.conda"]) + packages_whl.update(shard.get("packages.whl", {})) - return packages, packages_conda + return IndexedPackages( + packages=packages, + packages_conda=packages_conda, + packages_whl=packages_whl, + ) def load_all_from_cache(self, fn: str): """ diff --git a/tests/test_cache.py b/tests/test_cache.py index 96c9ef91..ca6fa182 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -4,7 +4,7 @@ from __future__ import annotations -from typing import Any +from typing import Any, Iterator from conda_index.index import cache @@ -33,19 +33,19 @@ def store( def load_all_from_cache(self, fn): raise NotImplementedError - def store_fs_state(self, listdir_stat: cache.Iterator[dict[str, Any]]): + def store_fs_state(self, listdir_stat: Iterator[dict[str, Any]]): raise NotImplementedError def changed_packages(self) -> list[cache.ChangedPackage]: raise NotImplementedError - def indexed_packages(self) -> tuple[dict, dict]: + def indexed_packages(self) -> cache.IndexedPackages: raise NotImplementedError def indexed_shards(self, desired: set | None = None): raise NotImplementedError - def run_exports(self) -> cache.Iterator[tuple[str, dict]]: + def run_exports(self) -> Iterator[tuple[str, dict]]: raise NotImplementedError @@ -63,3 +63,5 @@ def test_cache(tmp_path): c.convert() c.close() + + assert c.package_section_for_path("file.whl") == "packages.whl" diff --git a/tests/test_psql.py b/tests/test_psql.py index 045d8d94..5976fccd 100644 --- a/tests/test_psql.py +++ b/tests/test_psql.py @@ -11,6 +11,7 @@ from conda_index.index import ChannelIndex from conda_index.index.sqlitecache import ICON_PATH +from conda_index.utils import CONDA_PACKAGE_EXTENSIONS try: from conda_index.postgres import model @@ -286,9 +287,39 @@ class DummyResult(NamedTuple): assert len(data["packages"]) == 1 assert len(data["packages.conda"]) == 1 - packages, packages_conda = cache.indexed_packages() - assert len(packages) == 1 - assert len(packages_conda) == 1 + indexed_packages = cache.indexed_packages() + assert len(indexed_packages.packages) == 1 + assert len(indexed_packages.packages_conda) == 1 + assert indexed_packages.packages_whl == {} + + +def test_psql_include_wheel_extension(tmp_path: Path): + assert PsqlCache + cache = PsqlCache( + tmp_path, + "noarch", + db_url="postgresql://example", + package_extensions=CONDA_PACKAGE_EXTENSIONS + (".whl",), + ) + connection = MockConnection() + cache.engine = MockEngine(connection) # type: ignore + + class DummyResult(NamedTuple): + name: str + path: str + record: object + + connection.results_factory = lambda: [ + DummyResult("package", "package.whl", {}), + DummyResult("package", "package.conda", {}), + ] + shards = list(cache.indexed_shards()) + _, data = shards[0] + assert len(data["packages.whl"]) == 1 + assert len(data["packages.conda"]) == 1 + + indexed_packages = cache.indexed_packages() + assert len(indexed_packages.packages_whl) == 1 def test_psql_run_exports(tmp_path: Path): From 4f19f29b7077796b6cfda2a7d52f97acea927401 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 12 Mar 2026 16:42:37 -0400 Subject: [PATCH 03/32] ignore some files for ruff; remove bare except: in conftest --- pyproject.toml | 6 ++++++ tests/conftest.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a096026c..6f114596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,12 @@ source = ["conda_index"] [tool.isort] profile = "black" +[tool.ruff] +exclude = [ + "tests/split_repo.py", + "conda_index/utils_build.py", +] + [tool.hatch.build] include = ["conda_index"] diff --git a/tests/conftest.py b/tests/conftest.py index afbe5a6a..6bb2667c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -71,11 +71,11 @@ def testing_homedir(tmpdir, request): try: shutil.rmtree(new_dir) - except: + except OSError: pass try: os.makedirs(new_dir) - except: + except OSError: print(f"Failed to create {new_dir}") return None os.chdir(new_dir) From d0ae2081cdf80034ddcbdaf60e115bf1bca4a14b Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 12 Mar 2026 16:46:24 -0400 Subject: [PATCH 04/32] draft repodata v3 implementation --- conda_index/api.py | 2 ++ conda_index/cli/__init__.py | 11 ++++++ conda_index/index/__init__.py | 26 ++++++++++++-- conda_index/index/cache.py | 20 +++++++++-- conda_index/index/sqlitecache.py | 62 ++++++++++++++++++++++++-------- conda_index/postgres/cache.py | 62 ++++++++++++++++++++++++-------- tests/test_cache.py | 11 ++++-- tests/test_index.py | 26 ++++++++++++++ tests/test_psql.py | 14 ++++++++ 9 files changed, 198 insertions(+), 36 deletions(-) diff --git a/conda_index/api.py b/conda_index/api.py index f85cf0a9..738bb3e7 100644 --- a/conda_index/api.py +++ b/conda_index/api.py @@ -17,6 +17,7 @@ def update_index( progress=False, current_index_versions=None, write_run_exports=False, + repodata_v3=False, ): import os @@ -49,4 +50,5 @@ def update_index( subdirs=ensure_list(subdir), current_index_versions=current_index_versions, write_run_exports=write_run_exports, + repodata_v3=repodata_v3, ) diff --git a/conda_index/cli/__init__.py b/conda_index/cli/__init__.py index a33f265a..73db7fae 100644 --- a/conda_index/cli/__init__.py +++ b/conda_index/cli/__init__.py @@ -185,6 +185,15 @@ default=False, show_default=True, ) +@click.option( + "--repodata-v3/--no-repodata-v3", + help=""" + Write CEP-XXXX v3 repodata layout with all package records under a + top-level v3 key. + """, + default=False, + show_default=True, +) def cli( dir, patch_generator=None, @@ -210,6 +219,7 @@ def cli( db_url="", html_dependencies=False, update_only=False, + repodata_v3=False, ): logutil.configure() if verbose: @@ -269,6 +279,7 @@ def cli( cache_kwargs=cache_kwargs, html_dependencies=html_dependencies, update_only=update_only, + repodata_v3=repodata_v3, ) if update_cache is False: diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index 2fb5661a..258ddaea 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -111,6 +111,7 @@ def update_index( write_zst=False, write_run_exports=False, html_dependencies=False, + repodata_v3=False, ): """ High-level interface to ``ChannelIndex``. Index all subdirs under @@ -147,6 +148,7 @@ def update_index( write_zst=write_zst, write_run_exports=write_run_exports, html_dependencies=html_dependencies, + repodata_v3=repodata_v3, ) channel_index.index( @@ -177,6 +179,7 @@ def _make_seconds(timestamp): ) CHANNELDATA_VERSION = 1 RUN_EXPORTS_VERSION = 1 +REPODATA_REVISION_V3 = 3 REPODATA_JSON_FN = "repodata.json" REPODATA_FROM_PKGS_JSON_FN = "repodata_from_packages.json" REPODATA_SHARDS_FN = "repodata_shards.msgpack.zst" @@ -397,6 +400,7 @@ def __init__( upstream_stage: str = "fs", cache_kwargs=None, update_only=False, + repodata_v3=False, ): if threads is None: threads = MAX_THREADS_DEFAULT @@ -430,6 +434,7 @@ def __init__( self.write_current_repodata = write_current_repodata self.upstream_stage = upstream_stage self.update_only = update_only + self.repodata_v3 = repodata_v3 self.cache_kwargs = cache_kwargs @@ -713,7 +718,7 @@ def index_subdir_shards(self, subdir, verbose=False, progress=False): (self.output_root / subdir).mkdir(parents=True, exist_ok=True) - for name, shard in cache.indexed_shards(): + for name, shard in cache.indexed_shards(v3=self.repodata_v3): shard_data = compressor.compress(sqlitecache.packb_typed(shard)) shard_hash = hashlib.sha256(shard_data).digest() output_path = self.output_root / subdir / f"{shard_hash.hex()}.msgpack.zst" @@ -721,6 +726,14 @@ def index_subdir_shards(self, subdir, verbose=False, progress=False): output_path.write_bytes(shard_data) shards[name] = shard_hash + if self.repodata_v3: + shards_index["info"]["repodata_revisions"] = [ + { + "revision": REPODATA_REVISION_V3, + "migrated_at": 0, + } + ] + return shards_index def index_subdir(self, subdir, verbose=False, progress=False): @@ -734,7 +747,7 @@ def index_subdir(self, subdir, verbose=False, progress=False): log.debug("Building repodata for %s/%s", self.channel_name, subdir) - indexed_packages = cache.indexed_packages() + indexed_packages = cache.indexed_packages(v3=self.repodata_v3) new_repodata = { "packages": indexed_packages.packages, @@ -749,6 +762,15 @@ def index_subdir(self, subdir, verbose=False, progress=False): if indexed_packages.packages_whl: new_repodata["packages.whl"] = indexed_packages.packages_whl + if indexed_packages.v3 is not None: + new_repodata["v3"] = indexed_packages.v3 + new_repodata["info"]["repodata_revisions"] = [ + { + "revision": REPODATA_REVISION_V3, + "migrated_at": 0, + } + ] + if self.base_url: # per https://github.com/conda-incubator/ceps/blob/main/cep-15.md new_repodata["info"]["base_url"] = f"{self.base_url.rstrip('/')}/{subdir}/" diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index 0f4fc491..717acc87 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -93,6 +93,7 @@ class IndexedPackages: packages: dict[str, dict[str, Any]] packages_conda: dict[str, dict[str, Any]] packages_whl: dict[str, dict[str, Any]] + v3: dict[str, dict[str, Any]] | None = None class BaseCondaIndexCache(metaclass=abc.ABCMeta): @@ -182,6 +183,21 @@ def package_section_for_path(self, path: str) -> str | None: return None return package_sections.get(match.group(1)) + def v3_section_and_key_for_path(self, path: str) -> tuple[str, str] | None: + package_sections = { + ".tar.bz2": "tar.bz2", + ".conda": "conda", + ".whl": "whl", + } + match = self._package_section_re.search(path) + if match is None: + return None + extension = match.group(1) + section = package_sections.get(extension) + if section is None: + return None + return section, path[: -len(extension)] + def open(self, fn: str) -> IO[bytes]: """ Given a base package name "somepackage.conda", return an open, seekable @@ -401,7 +417,7 @@ def changed_packages(self) -> list[ChangedPackage]: """ @abc.abstractmethod - def indexed_packages(self) -> IndexedPackages: + def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: """ Return package sections from the cache for "monolithic repodata.json" query. @@ -409,7 +425,7 @@ def indexed_packages(self) -> IndexedPackages: @abc.abstractmethod def indexed_shards( - self, desired: set[str] | None = None + self, desired: set[str] | None = None, *, v3: bool = False ) -> Iterator[tuple[str, Any]]: """ Yield (package name, all packages with that name) from database ordered diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index 8c1101c7..13e3023b 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -332,7 +332,7 @@ def changed_packages(self) -> list[ChangedPackage]: return query - def indexed_packages(self) -> IndexedPackages: + def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: """ Return package sections from the cache. """ @@ -341,6 +341,11 @@ def indexed_packages(self) -> IndexedPackages: "packages.conda": {}, "packages.whl": {}, } + new_v3_packages = { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } # load cached packages for row in self.db.execute( @@ -357,19 +362,28 @@ def indexed_packages(self) -> IndexedPackages: log.warning("%s doesn't look like a conda package", path) continue - section = self.package_section_for_path(path) - if section is None: - log.warning("%s has unsupported package extension", path) - continue - new_packages[section][path] = index_json + if v3: + section_and_key = self.v3_section_and_key_for_path(path) + if section_and_key is None: + log.warning("%s has unsupported package extension", path) + continue + section, key = section_and_key + new_v3_packages[section][key] = index_json + else: + section = self.package_section_for_path(path) + if section is None: + log.warning("%s has unsupported package extension", path) + continue + new_packages[section][path] = index_json return IndexedPackages( packages=new_packages["packages"], packages_conda=new_packages["packages.conda"], packages_whl=new_packages["packages.whl"], + v3=new_v3_packages if v3 else None, ) - def indexed_shards(self, desired: set | None = None): + def indexed_shards(self, desired: set[str] | None = None, *, v3: bool = False): """ Yield (package name, all packages with that name) from database ordered by name, path i.o.w. filename. @@ -385,20 +399,38 @@ def indexed_shards(self, desired: set | None = None): ), lambda k: k[0], ): - shard = {"packages": {}, "packages.conda": {}} + shard = ( + { + "v3": { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } + } + if v3 + else {"packages": {}, "packages.conda": {}} + ) for row in rows: name, path, index_json = row if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) continue record = json.loads(index_json) - key = self.package_section_for_path(path) - if key is None: - log.warning("%s has unsupported package extension", path) - continue - # we may have to pack later for patch functions that look for - # hex hashes - shard.setdefault(key, {})[path] = pack_record(record) + if v3: + section_and_key = self.v3_section_and_key_for_path(path) + if section_and_key is None: + log.warning("%s has unsupported package extension", path) + continue + key, v3_path = section_and_key + shard["v3"][key][v3_path] = pack_record(record) + else: + key = self.package_section_for_path(path) + if key is None: + log.warning("%s has unsupported package extension", path) + continue + # we may have to pack later for patch functions that look for + # hex hashes + shard.setdefault(key, {})[path] = pack_record(record) if not desired or name in desired: yield (name, shard) diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 3a8315a1..ee0a076b 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -267,7 +267,13 @@ def changed_packages(self) -> list[ChangedPackage]: # XXX or FileInfo dataclass for row in connection.execute(query) ] # type: ignore - def indexed_shards(self, desired: set | None = None, *, pack_record=pack_record): + def indexed_shards( + self, + desired: set[str] | None = None, + *, + v3: bool = False, + pack_record=pack_record, + ): """ Yield (package name, all packages with that name) from database ordered by name, path i.o.w. filename. @@ -307,44 +313,72 @@ def indexed_shards(self, desired: set | None = None, *, pack_record=pack_record) connection.execute(query), lambda k: k.name, ): - shard = {"packages": {}, "packages.conda": {}} + shard = ( + { + "v3": { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } + } + if v3 + else {"packages": {}, "packages.conda": {}} + ) for row in rows: name, path, record = row path = self.plain_path(path) if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) continue - key = self.package_section_for_path(path) - if key is None: - log.warning("%s has unsupported package extension", path) - continue - # This will be passed to the patch function, which we hope - # does not look for hex hash values. - shard.setdefault(key, {})[path] = pack_record(record) + if v3: + section_and_key = self.v3_section_and_key_for_path(path) + if section_and_key is None: + log.warning("%s has unsupported package extension", path) + continue + key, v3_path = section_and_key + shard["v3"][key][v3_path] = pack_record(record) + else: + key = self.package_section_for_path(path) + if key is None: + log.warning("%s has unsupported package extension", path) + continue + # This will be passed to the patch function, which we hope + # does not look for hex hash values. + shard.setdefault(key, {})[path] = pack_record(record) if not desired or name in desired: yield (name, shard) - def indexed_packages(self) -> IndexedPackages: + def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: """ Return package sections from the cache. """ packages = {} packages_conda = {} packages_whl = {} + v3_packages = { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } def nopack_record(record): return record - for _, shard in self.indexed_shards(pack_record=nopack_record): - packages.update(shard["packages"]) - packages_conda.update(shard["packages.conda"]) - packages_whl.update(shard.get("packages.whl", {})) + for _, shard in self.indexed_shards(v3=v3, pack_record=nopack_record): + if v3: + for section, records in shard["v3"].items(): + v3_packages[section].update(records) + else: + packages.update(shard["packages"]) + packages_conda.update(shard["packages.conda"]) + packages_whl.update(shard.get("packages.whl", {})) return IndexedPackages( packages=packages, packages_conda=packages_conda, packages_whl=packages_whl, + v3=v3_packages if v3 else None, ) def load_all_from_cache(self, fn: str): diff --git a/tests/test_cache.py b/tests/test_cache.py index ca6fa182..091d11e5 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -7,6 +7,7 @@ from typing import Any, Iterator from conda_index.index import cache +from conda_index.utils import CONDA_PACKAGE_EXTENSIONS class DummyCache(cache.BaseCondaIndexCache): @@ -39,10 +40,10 @@ def store_fs_state(self, listdir_stat: Iterator[dict[str, Any]]): def changed_packages(self) -> list[cache.ChangedPackage]: raise NotImplementedError - def indexed_packages(self) -> cache.IndexedPackages: + def indexed_packages(self, *, v3: bool = False) -> cache.IndexedPackages: raise NotImplementedError - def indexed_shards(self, desired: set | None = None): + def indexed_shards(self, desired: set[str] | None = None, *, v3: bool = False): raise NotImplementedError def run_exports(self) -> Iterator[tuple[str, dict]]: @@ -54,7 +55,11 @@ def test_cache(tmp_path): Code coverage. """ - c = DummyCache(str(tmp_path), "linux-64") + c = DummyCache( + str(tmp_path), + "linux-64", + package_extensions=CONDA_PACKAGE_EXTENSIONS + (".whl",), + ) package = "foo.conda" db_path = c.database_path(package) diff --git a/tests/test_index.py b/tests/test_index.py index 05f98d5a..2ba44352 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1360,6 +1360,32 @@ def _patch_repodata_shards( assert package_url == "https://example.org/somechannel/osx-64/package-1.0.conda" +def test_repodata_v3(index_data): + pkg_dir = Path(index_data, "packages") + + channel_index = conda_index.index.ChannelIndex( + pkg_dir, + None, + write_bz2=False, + write_zst=False, + compact_json=True, + threads=1, + repodata_v3=True, + ) + + channel_index.index(None) + + noarch = json.loads((pkg_dir / "noarch" / "repodata.json").read_text()) + + assert noarch["info"]["repodata_revisions"] == [{"revision": 3, "migrated_at": 0}] + assert "v3" in noarch + assert set(noarch["v3"]) == {"tar.bz2", "conda", "whl"} + assert noarch["packages"] == {} + assert noarch["packages.conda"] == {} + assert "packages.whl" not in noarch + assert noarch["v3"]["tar.bz2"] or noarch["v3"]["conda"] + + def test_write_current_repodata(index_data): """ Test that we can skip current_repodata, and that it deletes the old one. diff --git a/tests/test_psql.py b/tests/test_psql.py index 5976fccd..36133aea 100644 --- a/tests/test_psql.py +++ b/tests/test_psql.py @@ -321,6 +321,20 @@ class DummyResult(NamedTuple): indexed_packages = cache.indexed_packages() assert len(indexed_packages.packages_whl) == 1 + v3_shards = list(cache.indexed_shards(v3=True)) + _, v3_data = v3_shards[0] + assert set(v3_data) == {"v3"} + assert len(v3_data["v3"]["whl"]) == 1 + assert len(v3_data["v3"]["conda"]) == 1 + + indexed_packages_v3 = cache.indexed_packages(v3=True) + assert indexed_packages_v3.packages == {} + assert indexed_packages_v3.packages_conda == {} + assert indexed_packages_v3.packages_whl == {} + assert indexed_packages_v3.v3 is not None + assert len(indexed_packages_v3.v3["whl"]) == 1 + assert len(indexed_packages_v3.v3["conda"]) == 1 + def test_psql_run_exports(tmp_path: Path): # XXX this should be tested end-to-end From 5a12faf5f268f246703bca78e78c804f1e914847 Mon Sep 17 00:00:00 2001 From: Mahe Iram Khan <65779580+ForgottenProgramme@users.noreply.github.com> Date: Fri, 27 Mar 2026 16:20:44 +0100 Subject: [PATCH 05/32] fix naming issue (#272) --- conda_index/index/sqlitecache.py | 4 ++-- conda_index/postgres/cache.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index 13e3023b..b6f25ee2 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -421,8 +421,8 @@ def indexed_shards(self, desired: set[str] | None = None, *, v3: bool = False): if section_and_key is None: log.warning("%s has unsupported package extension", path) continue - key, v3_path = section_and_key - shard["v3"][key][v3_path] = pack_record(record) + section, key = section_and_key + shard["v3"][section][key] = pack_record(record) else: key = self.package_section_for_path(path) if key is None: diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index ee0a076b..73979817 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -335,8 +335,8 @@ def indexed_shards( if section_and_key is None: log.warning("%s has unsupported package extension", path) continue - key, v3_path = section_and_key - shard["v3"][key][v3_path] = pack_record(record) + section, key = section_and_key + shard["v3"][section][key] = pack_record(record) else: key = self.package_section_for_path(path) if key is None: From a083728551a615581850e3e1f9edbe32a9ae4b22 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Sun, 29 Mar 2026 18:40:05 -0400 Subject: [PATCH 06/32] Update conda_index/cli/__init__.py Co-authored-by: jaimergp --- conda_index/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_index/cli/__init__.py b/conda_index/cli/__init__.py index 73db7fae..a35011f6 100644 --- a/conda_index/cli/__init__.py +++ b/conda_index/cli/__init__.py @@ -188,8 +188,8 @@ @click.option( "--repodata-v3/--no-repodata-v3", help=""" - Write CEP-XXXX v3 repodata layout with all package records under a - top-level v3 key. + EXPERIMENTAL. Write CEP-XXXX v3 repodata layout with all package + records under a top-level v3 key. """, default=False, show_default=True, From 9dec51e36353d010b91cc8747fbd0fb2e64b828f Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 31 Mar 2026 15:13:51 -0400 Subject: [PATCH 07/32] type hint --- conda_index/index/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index d0bb860b..f7ccfd2b 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -166,7 +166,7 @@ def plain_path(self, path: str) -> str: return path.rsplit("/", 1)[-1] @cacher - def _package_section_re(self): + def _package_section_re(self) -> re.Pattern[str]: extension_pattern = "|".join( re.escape(extension) for extension in sorted(self.package_extensions, key=len, reverse=True) From 237264dd4d447813f87cb6724e765f4587b52356 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 31 Mar 2026 16:46:08 -0400 Subject: [PATCH 08/32] adjust for no top-level "packages.whl" key --- conda_index/index/__init__.py | 3 --- conda_index/index/cache.py | 1 - conda_index/index/sqlitecache.py | 3 ++- conda_index/postgres/cache.py | 3 --- tests/test_psql.py | 5 ++--- 5 files changed, 4 insertions(+), 11 deletions(-) diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index dfb6dbab..fc991385 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -759,9 +759,6 @@ def index_subdir(self, subdir, verbose=False, progress=False): "removed": [], # can be added by patch/hotfix process } - if indexed_packages.packages_whl: - new_repodata["packages.whl"] = indexed_packages.packages_whl - if indexed_packages.v3 is not None: new_repodata["v3"] = indexed_packages.v3 new_repodata["info"]["repodata_revisions"] = [ diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index f7ccfd2b..69421984 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -93,7 +93,6 @@ class ChangedPackage(TypedDict): class IndexedPackages: packages: dict[str, dict[str, Any]] packages_conda: dict[str, dict[str, Any]] - packages_whl: dict[str, dict[str, Any]] v3: dict[str, dict[str, Any]] | None = None diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index b72b2bbb..d67a3b9a 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -373,12 +373,13 @@ def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: if section is None: log.warning("%s has unsupported package extension", path) continue + if section == "packages.whl": + continue new_packages[section][path] = index_json return IndexedPackages( packages=new_packages["packages"], packages_conda=new_packages["packages.conda"], - packages_whl=new_packages["packages.whl"], v3=new_v3_packages if v3 else None, ) diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 73979817..99d07f00 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -355,7 +355,6 @@ def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: """ packages = {} packages_conda = {} - packages_whl = {} v3_packages = { "tar.bz2": {}, "conda": {}, @@ -372,12 +371,10 @@ def nopack_record(record): else: packages.update(shard["packages"]) packages_conda.update(shard["packages.conda"]) - packages_whl.update(shard.get("packages.whl", {})) return IndexedPackages( packages=packages, packages_conda=packages_conda, - packages_whl=packages_whl, v3=v3_packages if v3 else None, ) diff --git a/tests/test_psql.py b/tests/test_psql.py index 36133aea..3f0438af 100644 --- a/tests/test_psql.py +++ b/tests/test_psql.py @@ -290,7 +290,6 @@ class DummyResult(NamedTuple): indexed_packages = cache.indexed_packages() assert len(indexed_packages.packages) == 1 assert len(indexed_packages.packages_conda) == 1 - assert indexed_packages.packages_whl == {} def test_psql_include_wheel_extension(tmp_path: Path): @@ -319,7 +318,8 @@ class DummyResult(NamedTuple): assert len(data["packages.conda"]) == 1 indexed_packages = cache.indexed_packages() - assert len(indexed_packages.packages_whl) == 1 + assert indexed_packages.packages == {} + assert len(indexed_packages.packages_conda) == 1 v3_shards = list(cache.indexed_shards(v3=True)) _, v3_data = v3_shards[0] @@ -330,7 +330,6 @@ class DummyResult(NamedTuple): indexed_packages_v3 = cache.indexed_packages(v3=True) assert indexed_packages_v3.packages == {} assert indexed_packages_v3.packages_conda == {} - assert indexed_packages_v3.packages_whl == {} assert indexed_packages_v3.v3 is not None assert len(indexed_packages_v3.v3["whl"]) == 1 assert len(indexed_packages_v3.v3["conda"]) == 1 From f054c617f25cf101ed2064eaca0c291108572194 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 31 Mar 2026 16:51:46 -0400 Subject: [PATCH 09/32] add news --- news/262-wheel | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 news/262-wheel diff --git a/news/262-wheel b/news/262-wheel new file mode 100644 index 00000000..b1b9b2d5 --- /dev/null +++ b/news/262-wheel @@ -0,0 +1,4 @@ +### Enhancements + +* Support experimental "repodata v3" (package data in `["v3"][]` + dict under the top level) to support conda wheel experiments. (#262) \ No newline at end of file From 5331b50e1a47c45956bd543c30de711aa7635112 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 10:35:30 -0400 Subject: [PATCH 10/32] Apply suggestion from @dholth --- conda_index/cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_index/cli/__init__.py b/conda_index/cli/__init__.py index a35011f6..e8a0ebeb 100644 --- a/conda_index/cli/__init__.py +++ b/conda_index/cli/__init__.py @@ -186,7 +186,7 @@ show_default=True, ) @click.option( - "--repodata-v3/--no-repodata-v3", + "--repodata-next/--no-repodata-next", help=""" EXPERIMENTAL. Write CEP-XXXX v3 repodata layout with all package records under a top-level v3 key. From 602963b50e7475c43873a491910b99ab7f4e446e Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 10:54:28 -0400 Subject: [PATCH 11/32] add typed index_shards_2 method --- conda_index/index/cache.py | 30 ++++++++++++++++++----- conda_index/postgres/cache.py | 45 +++++++++++++++++++---------------- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index 23283f86..eacdb0f2 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -82,12 +82,15 @@ def __get__(self, inst, objtype=None) -> Any: return value return self + class ChangedPackage(TypedDict): path: str mtime: Number size: int + if TYPE_CHECKING: + class HasChecksumsAndSize(TypedDict, extra_items=Any): """ Enforce keys accessed in conda-index store() @@ -102,7 +105,16 @@ class HasChecksumsAndSize(TypedDict, extra_items=Any): class IndexedPackages: packages: dict[str, dict[str, Any]] packages_conda: dict[str, dict[str, Any]] - v3: dict[str, dict[str, Any]] | None = None + packages_whl: dict[str, dict[str, Any]] + + +@dataclass +class IndexedShard(IndexedPackages): + """ + IndexedPackages for a single package name. + """ + + name: str class BaseCondaIndexCache(metaclass=abc.ABCMeta): @@ -430,17 +442,15 @@ def changed_packages(self) -> list[ChangedPackage]: Return packages in upstream that are changed or missing compared to 'indexed'. """ - # XXX make v3 a class property to be more friendly to CondaIndexCache subclasses? @abc.abstractmethod - def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: + def indexed_packages(self) -> IndexedPackages: """ - Return package sections from the cache for "monolithic repodata.json" - query. + Return all data for "monolithic repodata.json" query. """ @abc.abstractmethod def indexed_shards( - self, desired: set[str] | None = None, *, v3: bool = False + self, desired: set[str] | None = None ) -> Iterator[tuple[str, Any]]: """ Yield (package name, all packages with that name) from database ordered @@ -449,6 +459,14 @@ def indexed_shards( :desired: If not None, set of desired package names. """ + @abc.abstractmethod + def indexed_shards_2( + self, desired: set[str] | None = None + ) -> Iterator[IndexedShard]: + """ + indexed_shards with dataclass instead of dict. + """ + @abc.abstractmethod def run_exports(self) -> Iterator[tuple[str, dict]]: """ diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 6e869f4e..eeff15e4 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -21,6 +21,7 @@ BaseCondaIndexCache, ChangedPackage, IndexedPackages, + IndexedShard, clear_newline_chars, ) from conda_index.index.fs import MinimalFS @@ -282,9 +283,21 @@ def indexed_shards( self, desired: set[str] | None = None, *, - v3: bool = False, pack_record=pack_record, ): + """ " + Yield (package name, all packages with that name as dict) from database ordered + by name, path i.o.w. filename. + """ + for shard in self.indexed_shards_2(desired, pack_record=pack_record): + yield ( + shard.name, + {"packages": shard.packages, "packages.conda": shard.packages_conda}, + ) + + def indexed_shards_2( + self, desired: set[str] | None = None, *, pack_record=pack_record + ) -> Iterator[IndexedShard]: """ Yield (package name, all packages with that name) from database ordered by name, path i.o.w. filename. @@ -297,6 +310,8 @@ def indexed_shards( index_json_table = model.Base.metadata.tables["index_json"] stat_table = model.Base.metadata.tables["stat"] + # not optimized for "desired" partial shards case but that's not + # currently used. query = ( select( index_json_table.c.name, @@ -324,16 +339,12 @@ def indexed_shards( connection.execute(query), lambda k: k.name, ): - shard = ( - { - "v3": { - "tar.bz2": {}, - "conda": {}, - "whl": {}, - } - } - if v3 - else {"packages": {}, "packages.conda": {}} + shard_dict = {"packages": {}, "packages.conda": {}, "packages.whl": {}} + shard = IndexedShard( + name=name, + packages=shard_dict["packages"], + packages_conda=shard_dict["packages.conda"], + packages_whl=shard_dict["packages.whl"], ) for row in rows: name, path, record = row @@ -341,13 +352,7 @@ def indexed_shards( if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) continue - if v3: - section_and_key = self.v3_section_and_key_for_path(path) - if section_and_key is None: - log.warning("%s has unsupported package extension", path) - continue - section, key = section_and_key - shard["v3"][section][key] = pack_record(record) + else: key = self.package_section_for_path(path) if key is None: @@ -355,10 +360,10 @@ def indexed_shards( continue # This will be passed to the patch function, which we hope # does not look for hex hash values. - shard.setdefault(key, {})[path] = pack_record(record) + shard_dict[key][path] = pack_record(record) if not desired or name in desired: - yield (name, shard) + yield shard def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: """ From 96f0b03823b5d220845be32afa3a7b8362468560 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 11:02:30 -0400 Subject: [PATCH 12/32] add indexed_shards_2 to sqlitecache --- conda_index/index/cache.py | 11 +++ conda_index/index/sqlitecache.py | 118 +++++++++++++++---------------- conda_index/postgres/cache.py | 2 +- 3 files changed, 69 insertions(+), 62 deletions(-) diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index eacdb0f2..f07e2664 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -539,3 +539,14 @@ def clear_newline_chars(record: dict[str, Any], field_name: str) -> None: except TypeError: log.warning("Could not _clear_newline_chars from field %s", field_name) + + +def pack_record(record): + """ + Convert hex checksums to bytes. + """ + if sha256 := record.get("sha256"): + record["sha256"] = bytes.fromhex(sha256) + if md5 := record.get("md5"): + record["md5"] = bytes.fromhex(md5) + return record diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index 6db7ad9e..506e4327 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -16,7 +16,13 @@ import msgpack from . import common, convert_cache -from .cache import BaseCondaIndexCache, IndexedPackages, cacher +from .cache import ( + BaseCondaIndexCache, + IndexedPackages, + IndexedShard, + cacher, + pack_record, +) from .cache import clear_newline_chars as _clear_newline_chars from .fs import MinimalFS @@ -346,11 +352,6 @@ def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: "packages": {}, "packages.conda": {}, } - new_v3_packages = { - "tar.bz2": {}, - "conda": {}, - "whl": {}, - } # load cached packages for row in self.db.execute( @@ -367,35 +368,51 @@ def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: log.warning("%s doesn't look like a conda package", path) continue - if v3: - section_and_key = self.v3_section_and_key_for_path(path) - if section_and_key is None: - log.warning("%s has unsupported package extension", path) - continue - section, key = section_and_key - new_v3_packages[section][key] = index_json - else: - section = self.package_section_for_path(path) - if section is None: - log.warning("%s has unsupported package extension", path) - continue - if section == "packages.whl": - continue - new_packages[section][path] = index_json + section = self.package_section_for_path(path) + if section is None: + log.warning("%s has unsupported package extension", path) + continue + new_packages[section][path] = index_json return IndexedPackages( packages=new_packages["packages"], packages_conda=new_packages["packages.conda"], - v3=new_v3_packages if v3 else None, + packages_whl=new_packages.get("packages.whl"), ) - def indexed_shards(self, desired: set[str] | None = None, *, v3: bool = False): + def indexed_shards( + self, + desired: set[str] | None = None, + *, + v3: bool = False, + pack_record=pack_record, + ): """ Yield (package name, all packages with that name) from database ordered by name, path i.o.w. filename. :desired: If not None, set of desired package names. """ + + for shard in self.indexed_shards_2(desired=desired, pack_record=pack_record): + shard_dict = { + "packages": shard.packages, + "packages.conda": shard.packages_conda, + } + if shard.packages_whl: + shard_dict["packages.whl"] = shard.packages_whl + + yield (shard.name, shard_dict) + + def indexed_shards_2( + self, desired: set[str] | None = None, *, pack_record=pack_record + ) -> Iterator[IndexedShard]: + """ + Yield package shards as IndexedShard records. + + :desired: If not None, set of desired package names. + """ + for name, rows in itertools.groupby( self.db.execute( """SELECT index_json.name, path, index_json @@ -405,41 +422,31 @@ def indexed_shards(self, desired: set[str] | None = None, *, v3: bool = False): ), lambda k: k[0], ): - shard = ( - { - "v3": { - "tar.bz2": {}, - "conda": {}, - "whl": {}, - } - } - if v3 - else {"packages": {}, "packages.conda": {}} + shard_dict = { + "packages": {}, + "packages.conda": {}, + "packages.whl": {}, + } + shard = IndexedShard( + name=name, + packages=shard_dict["packages"], + packages_conda=shard_dict["packages.conda"], + packages_whl=shard_dict["packages.whl"], ) for row in rows: - name, path, index_json = row + _, path, index_json = row if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) continue record = json.loads(index_json) - if v3: - section_and_key = self.v3_section_and_key_for_path(path) - if section_and_key is None: - log.warning("%s has unsupported package extension", path) - continue - section, key = section_and_key - shard["v3"][section][key] = pack_record(record) - else: - key = self.package_section_for_path(path) - if key is None: - log.warning("%s has unsupported package extension", path) - continue - # we may have to pack later for patch functions that look for - # hex hashes - shard.setdefault(key, {})[path] = pack_record(record) + key = self.package_section_for_path(path) + if key is None: + log.warning("%s has unsupported package extension", path) + continue + shard_dict[key][path] = pack_record(record) if not desired or name in desired: - yield (name, shard) + yield shard def store_index_json_stat( self, database_path, mtime, size, index_json: HasChecksumsAndSize @@ -467,17 +474,6 @@ def run_exports(self) -> Iterator[tuple[str, dict]]: yield (path, json.loads(run_exports or "{}")) -def pack_record(record): - """ - Convert hex checksums to bytes. - """ - if sha256 := record.get("sha256"): - record["sha256"] = bytes.fromhex(sha256) - if md5 := record.get("md5"): - record["md5"] = bytes.fromhex(md5) - return record - - def packb_typed(o: Any) -> bytes: """ Sidestep lack of typing in msgpack. diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index eeff15e4..4d18f69c 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -23,6 +23,7 @@ IndexedPackages, IndexedShard, clear_newline_chars, + pack_record, ) from conda_index.index.fs import MinimalFS from conda_index.index.sqlitecache import ( @@ -30,7 +31,6 @@ PATH_TO_TABLE, TABLE_NO_CACHE, cacher, - pack_record, ) if TYPE_CHECKING: From 9cfec13879f05883fb9236af3b22a87be209db4d Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 11:35:18 -0400 Subject: [PATCH 13/32] consolidate v3 handling in ChannelIndex --- conda_index/index/__init__.py | 140 ++++++++++++++++++++++++++----- conda_index/index/sqlitecache.py | 6 +- conda_index/postgres/cache.py | 34 ++++---- tests/test_cache.py | 10 ++- tests/test_index.py | 5 +- tests/test_psql.py | 16 +--- 6 files changed, 153 insertions(+), 58 deletions(-) diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index fc991385..b6491314 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -15,7 +15,7 @@ from datetime import datetime, timezone from os.path import basename, getmtime, getsize, isfile, join from pathlib import Path -from typing import Iterable +from typing import TYPE_CHECKING, Iterable from uuid import uuid4 import msgpack @@ -36,6 +36,9 @@ from .current_repodata import build_current_repodata from .fs import FileInfo, MinimalFS +if TYPE_CHECKING: + from .cache import IndexedPackages, IndexedShard + log = logging.getLogger(__name__) # zstd -T0 -b15 -e17 repodata.json @@ -255,15 +258,17 @@ def _apply_instructions(subdir, repodata, instructions, new_pkg_fixes=None): for key in ("packages", "packages.conda"): if key == "packages.conda" and fn.endswith(CONDA_PACKAGE_EXTENSION_V1): fn = fn.replace(CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2) - if fn in repodata[key]: - repodata[key][fn]["revoked"] = True - repodata[key][fn]["depends"].append("package_has_been_revoked") + records = repodata.get(key, {}) + if fn in records: + records[fn]["revoked"] = True + records[fn]["depends"].append("package_has_been_revoked") for fn in instructions.get("remove", ()): for key in ("packages", "packages.conda"): if key == "packages.conda" and fn.endswith(CONDA_PACKAGE_EXTENSION_V1): fn = fn.replace(CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2) - popped = repodata[key].pop(fn, None) + records = repodata.get(key, {}) + popped = records.pop(fn, None) if popped: repodata["removed"].append(fn) repodata["removed"].sort() @@ -718,20 +723,28 @@ def index_subdir_shards(self, subdir, verbose=False, progress=False): (self.output_root / subdir).mkdir(parents=True, exist_ok=True) - for name, shard in cache.indexed_shards(v3=self.repodata_v3): - shard_data = compressor.compress(sqlitecache.packb_typed(shard)) - shard_hash = hashlib.sha256(shard_data).digest() + v3_data = { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } + + for shard in cache.indexed_shards_2(): + repodata_shard = self._indexed_shard_to_repodata(shard) + shard_bytes = compressor.compress(sqlitecache.packb_typed(repodata_shard)) + shard_hash = hashlib.sha256(shard_bytes).digest() output_path = self.output_root / subdir / f"{shard_hash.hex()}.msgpack.zst" if not output_path.exists(): - output_path.write_bytes(shard_data) - shards[name] = shard_hash + output_path.write_bytes(shard_bytes) + shards[shard.name] = shard_hash + + if self.repodata_v3: + for section, records in repodata_shard["v3"].items(): + v3_data[section].update(records) if self.repodata_v3: shards_index["info"]["repodata_revisions"] = [ - { - "revision": REPODATA_REVISION_V3, - "migrated_at": 0, - } + self._make_repodata_revision_data(v3_data) ] return shards_index @@ -747,7 +760,7 @@ def index_subdir(self, subdir, verbose=False, progress=False): log.debug("Building repodata for %s/%s", self.channel_name, subdir) - indexed_packages = cache.indexed_packages(v3=self.repodata_v3) + indexed_packages = cache.indexed_packages() new_repodata = { "packages": indexed_packages.packages, @@ -759,13 +772,13 @@ def index_subdir(self, subdir, verbose=False, progress=False): "removed": [], # can be added by patch/hotfix process } - if indexed_packages.v3 is not None: - new_repodata["v3"] = indexed_packages.v3 + if self.repodata_v3: + v3_packages = self._extract_indexed_packages_v3(indexed_packages) + new_repodata["v3"] = v3_packages + new_repodata["packages"] = {} + new_repodata["packages.conda"] = {} new_repodata["info"]["repodata_revisions"] = [ - { - "revision": REPODATA_REVISION_V3, - "migrated_at": 0, - } + self._make_repodata_revision_data(v3_packages) ] if self.base_url: @@ -775,6 +788,91 @@ def index_subdir(self, subdir, verbose=False, progress=False): return new_repodata + def _extract_indexed_packages_v3( + self, indexed_packages: IndexedPackages + ) -> dict[str, dict[str, dict[str, object]]]: + v3 = { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } + for section, records in ( + ("tar.bz2", indexed_packages.packages), + ("conda", indexed_packages.packages_conda), + ("whl", indexed_packages.packages_whl), + ): + for filename, record in records.items(): + key = self._v3_key_for_path(filename) + if key is None: + log.warning("%s has unsupported package extension", filename) + continue + v3[section][key] = record + + return v3 + + def _indexed_shard_to_repodata(self, indexed_shard: IndexedShard) -> dict: + if self.repodata_v3: + return { + "v3": self._v3_section_data_from_indexed_shard(indexed_shard), + } + + shard_data = { + "packages": indexed_shard.packages, + "packages.conda": indexed_shard.packages_conda, + } + if indexed_shard.packages_whl: + shard_data["packages.whl"] = indexed_shard.packages_whl + return shard_data + + @staticmethod + def _v3_key_for_path(path: str) -> str | None: + for extension in (".tar.bz2", ".conda", ".whl"): + if path.endswith(extension): + return path[: -len(extension)] + return None + + @staticmethod + def _make_repodata_revision_data( + revision_data: dict[str, dict[str, dict]], + ) -> dict[str, int | None]: + timestamps = [] + n_packages = 0 + for section_records in revision_data.values(): + n_packages += len(section_records) + for record in section_records.values(): + timestamp = record.get("timestamp") + if isinstance(timestamp, (int, float)): + timestamps.append(int(timestamp)) + + return { + "revision": REPODATA_REVISION_V3, + "n_packages": n_packages, + "oldest": min(timestamps) if timestamps else None, + "newest": max(timestamps) if timestamps else None, + } + + @staticmethod + def _v3_section_data_from_indexed_shard( + indexed_shard: IndexedShard, + ) -> dict[str, dict[str, dict]]: + v3 = { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } + for section, records in ( + ("tar.bz2", indexed_shard.packages), + ("conda", indexed_shard.packages_conda), + ("whl", indexed_shard.packages_whl), + ): + for path, record in records.items(): + key = ChannelIndex._v3_key_for_path(path) + if key is None: + log.warning("%s has unsupported package extension", path) + continue + v3[section][key] = record + return v3 + def extract_subdir_to_cache( self, subdir: str, diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index 506e4327..57d654af 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -344,13 +344,14 @@ def changed_packages(self) -> list[ChangedPackage]: return query - def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: + def indexed_packages(self) -> IndexedPackages: """ Return package sections from the cache. """ new_packages = { "packages": {}, "packages.conda": {}, + "packages.whl": {}, } # load cached packages @@ -377,14 +378,13 @@ def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: return IndexedPackages( packages=new_packages["packages"], packages_conda=new_packages["packages.conda"], - packages_whl=new_packages.get("packages.whl"), + packages_whl=new_packages["packages.whl"], ) def indexed_shards( self, desired: set[str] | None = None, *, - v3: bool = False, pack_record=pack_record, ): """ diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 4d18f69c..172d0037 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -285,15 +285,18 @@ def indexed_shards( *, pack_record=pack_record, ): - """ " + """ Yield (package name, all packages with that name as dict) from database ordered by name, path i.o.w. filename. """ for shard in self.indexed_shards_2(desired, pack_record=pack_record): - yield ( - shard.name, - {"packages": shard.packages, "packages.conda": shard.packages_conda}, - ) + shard_data = { + "packages": shard.packages, + "packages.conda": shard.packages_conda, + } + if shard.packages_whl: + shard_data["packages.whl"] = shard.packages_whl + yield (shard.name, shard_data) def indexed_shards_2( self, desired: set[str] | None = None, *, pack_record=pack_record @@ -365,33 +368,26 @@ def indexed_shards_2( if not desired or name in desired: yield shard - def indexed_packages(self, *, v3: bool = False) -> IndexedPackages: + def indexed_packages(self) -> IndexedPackages: """ Return package sections from the cache. """ packages = {} packages_conda = {} - v3_packages = { - "tar.bz2": {}, - "conda": {}, - "whl": {}, - } + packages_whl = {} def nopack_record(record): return record - for _, shard in self.indexed_shards(v3=v3, pack_record=nopack_record): - if v3: - for section, records in shard["v3"].items(): - v3_packages[section].update(records) - else: - packages.update(shard["packages"]) - packages_conda.update(shard["packages.conda"]) + for shard in self.indexed_shards_2(pack_record=nopack_record): + packages.update(shard.packages) + packages_conda.update(shard.packages_conda) + packages_whl.update(shard.packages_whl) return IndexedPackages( packages=packages, packages_conda=packages_conda, - v3=v3_packages if v3 else None, + packages_whl=packages_whl, ) def load_all_from_cache(self, fn: str): diff --git a/tests/test_cache.py b/tests/test_cache.py index 091d11e5..4e09258d 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -40,10 +40,16 @@ def store_fs_state(self, listdir_stat: Iterator[dict[str, Any]]): def changed_packages(self) -> list[cache.ChangedPackage]: raise NotImplementedError - def indexed_packages(self, *, v3: bool = False) -> cache.IndexedPackages: + def indexed_packages(self) -> cache.IndexedPackages: raise NotImplementedError - def indexed_shards(self, desired: set[str] | None = None, *, v3: bool = False): + def indexed_shards(self, desired: set[str] | None = None): + raise NotImplementedError + + def indexed_shards_2( + self, + desired: set[str] | None = None, + ) -> Iterator[cache.IndexedShard]: raise NotImplementedError def run_exports(self) -> Iterator[tuple[str, dict]]: diff --git a/tests/test_index.py b/tests/test_index.py index 2ba44352..58d8fcb7 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1377,7 +1377,10 @@ def test_repodata_v3(index_data): noarch = json.loads((pkg_dir / "noarch" / "repodata.json").read_text()) - assert noarch["info"]["repodata_revisions"] == [{"revision": 3, "migrated_at": 0}] + assert set(noarch["info"]["repodata_revisions"][0].keys()) == set( + ("revision", "n_packages", "oldest", "newest") + ) + assert "v3" in noarch assert set(noarch["v3"]) == {"tar.bz2", "conda", "whl"} assert noarch["packages"] == {} diff --git a/tests/test_psql.py b/tests/test_psql.py index cbb96005..fc81c90f 100644 --- a/tests/test_psql.py +++ b/tests/test_psql.py @@ -356,18 +356,10 @@ class DummyResult(NamedTuple): assert indexed_packages.packages == {} assert len(indexed_packages.packages_conda) == 1 - v3_shards = list(cache.indexed_shards(v3=True)) - _, v3_data = v3_shards[0] - assert set(v3_data) == {"v3"} - assert len(v3_data["v3"]["whl"]) == 1 - assert len(v3_data["v3"]["conda"]) == 1 - - indexed_packages_v3 = cache.indexed_packages(v3=True) - assert indexed_packages_v3.packages == {} - assert indexed_packages_v3.packages_conda == {} - assert indexed_packages_v3.v3 is not None - assert len(indexed_packages_v3.v3["whl"]) == 1 - assert len(indexed_packages_v3.v3["conda"]) == 1 + shards_2 = list(cache.indexed_shards_2()) + assert len(shards_2) == 1 + assert len(shards_2[0].packages_whl) == 1 + assert len(shards_2[0].packages_conda) == 1 def test_psql_run_exports(tmp_path: Path): From 9426ae32663d0f2273a33c4c195c6a362a13eaf8 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 11:45:42 -0400 Subject: [PATCH 14/32] rename cli option --- conda_index/cli/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_index/cli/__init__.py b/conda_index/cli/__init__.py index e8a0ebeb..9a1165d9 100644 --- a/conda_index/cli/__init__.py +++ b/conda_index/cli/__init__.py @@ -219,7 +219,7 @@ def cli( db_url="", html_dependencies=False, update_only=False, - repodata_v3=False, + repodata_next=False, ): logutil.configure() if verbose: @@ -279,7 +279,7 @@ def cli( cache_kwargs=cache_kwargs, html_dependencies=html_dependencies, update_only=update_only, - repodata_v3=repodata_v3, + repodata_v3=repodata_next, ) if update_cache is False: From 58550727822da77b9eb620849f4a7d9bdcc73cd2 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 12:06:39 -0400 Subject: [PATCH 15/32] move indexed_shards() to superclass --- conda_index/index/cache.py | 47 ++++++++++++++++++++------------ conda_index/index/sqlitecache.py | 23 ---------------- conda_index/postgres/cache.py | 19 ------------- tests/test_cache.py | 5 ++-- 4 files changed, 32 insertions(+), 62 deletions(-) diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index f07e2664..a6274d74 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -117,6 +117,17 @@ class IndexedShard(IndexedPackages): name: str +def pack_record(record): + """ + Convert hex checksums to bytes. + """ + if sha256 := record.get("sha256"): + record["sha256"] = bytes.fromhex(sha256) + if md5 := record.get("md5"): + record["md5"] = bytes.fromhex(md5) + return record + + class BaseCondaIndexCache(metaclass=abc.ABCMeta): def __init__( self, @@ -448,20 +459,33 @@ def indexed_packages(self) -> IndexedPackages: Return all data for "monolithic repodata.json" query. """ - @abc.abstractmethod def indexed_shards( - self, desired: set[str] | None = None - ) -> Iterator[tuple[str, Any]]: + self, + desired: set[str] | None = None, + *, + pack_record=pack_record, + ): """ - Yield (package name, all packages with that name) from database ordered - by name, path i.o.w. filename. + Yield (package name, all packages with that name as dict) from database + ordered by name, path i.o.w. filename. :desired: If not None, set of desired package names. """ + for shard in self.indexed_shards_2(desired, pack_record=pack_record): + shard_data = { + "packages": shard.packages, + "packages.conda": shard.packages_conda, + } + if shard.packages_whl: + shard_data["packages.whl"] = shard.packages_whl + yield (shard.name, shard_data) @abc.abstractmethod def indexed_shards_2( - self, desired: set[str] | None = None + self, + desired: set[str] | None = None, + *, + pack_record=pack_record, ) -> Iterator[IndexedShard]: """ indexed_shards with dataclass instead of dict. @@ -539,14 +563,3 @@ def clear_newline_chars(record: dict[str, Any], field_name: str) -> None: except TypeError: log.warning("Could not _clear_newline_chars from field %s", field_name) - - -def pack_record(record): - """ - Convert hex checksums to bytes. - """ - if sha256 := record.get("sha256"): - record["sha256"] = bytes.fromhex(sha256) - if md5 := record.get("md5"): - record["md5"] = bytes.fromhex(md5) - return record diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index 57d654af..b01b4422 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -381,29 +381,6 @@ def indexed_packages(self) -> IndexedPackages: packages_whl=new_packages["packages.whl"], ) - def indexed_shards( - self, - desired: set[str] | None = None, - *, - pack_record=pack_record, - ): - """ - Yield (package name, all packages with that name) from database ordered - by name, path i.o.w. filename. - - :desired: If not None, set of desired package names. - """ - - for shard in self.indexed_shards_2(desired=desired, pack_record=pack_record): - shard_dict = { - "packages": shard.packages, - "packages.conda": shard.packages_conda, - } - if shard.packages_whl: - shard_dict["packages.whl"] = shard.packages_whl - - yield (shard.name, shard_dict) - def indexed_shards_2( self, desired: set[str] | None = None, *, pack_record=pack_record ) -> Iterator[IndexedShard]: diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 172d0037..c5a83da5 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -279,25 +279,6 @@ def changed_packages(self) -> list[ChangedPackage]: # XXX or FileInfo dataclass for row in connection.execute(query) ] # type: ignore - def indexed_shards( - self, - desired: set[str] | None = None, - *, - pack_record=pack_record, - ): - """ - Yield (package name, all packages with that name as dict) from database ordered - by name, path i.o.w. filename. - """ - for shard in self.indexed_shards_2(desired, pack_record=pack_record): - shard_data = { - "packages": shard.packages, - "packages.conda": shard.packages_conda, - } - if shard.packages_whl: - shard_data["packages.whl"] = shard.packages_whl - yield (shard.name, shard_data) - def indexed_shards_2( self, desired: set[str] | None = None, *, pack_record=pack_record ) -> Iterator[IndexedShard]: diff --git a/tests/test_cache.py b/tests/test_cache.py index 4e09258d..81064ca4 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -43,12 +43,11 @@ def changed_packages(self) -> list[cache.ChangedPackage]: def indexed_packages(self) -> cache.IndexedPackages: raise NotImplementedError - def indexed_shards(self, desired: set[str] | None = None): - raise NotImplementedError - def indexed_shards_2( self, desired: set[str] | None = None, + *, + pack_record=None, ) -> Iterator[cache.IndexedShard]: raise NotImplementedError From d9e6b727a2c011bd2c30a17e6cd21e7e1d622cde Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 12:08:00 -0400 Subject: [PATCH 16/32] update news --- news/262-wheel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/news/262-wheel b/news/262-wheel index b1b9b2d5..1a140a7d 100644 --- a/news/262-wheel +++ b/news/262-wheel @@ -1,4 +1,5 @@ ### Enhancements * Support experimental "repodata v3" (package data in `["v3"][]` - dict under the top level) to support conda wheel experiments. (#262) \ No newline at end of file + dict under the top level) to support conda wheel experiments. A new + `--repodata-next` command line flag places package data under `v3` key. (#262) \ No newline at end of file From 8f37fcd98a34038ddd784fa56e7facc74386dab2 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 14:11:35 -0400 Subject: [PATCH 17/32] remove unused import --- conda_index/index/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index cb60a676..a4169835 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -12,7 +12,6 @@ import sys import time from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor -from contextlib import nullcontext from datetime import datetime, timezone from os.path import basename, getmtime, getsize, isfile, join from pathlib import Path From 5cfe9d7f4542ec6fe0826b3101aabb5c0390f146 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 14:37:02 -0400 Subject: [PATCH 18/32] tidy v3 logic and add typing --- conda_index/index/__init__.py | 89 ++++++++++++++++++----------------- conda_index/index/cache.py | 2 +- conda_index/index/fs.py | 2 +- 3 files changed, 49 insertions(+), 44 deletions(-) diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index a4169835..c4686ddc 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -36,8 +36,26 @@ from .fs import FileInfo, MinimalFS if TYPE_CHECKING: + from typing import Any, NotRequired, TypedDict + from .cache import IndexedPackages, IndexedShard + V3Section = TypedDict( + "V3Section", + {"tar.bz2": dict, "conda": dict, "whl": dict}, + ) + + # in this style because "packages.conda" is not a Python identifier + ShardDict = TypedDict( + "ShardDict", + { + "packages": dict[str, dict[str, Any]], + "packages.conda": dict[str, dict[str, Any]], + "v3": NotRequired[V3Section], + }, + ) + + log = logging.getLogger(__name__) # zstd -T0 -b15 -e17 repodata.json @@ -791,10 +809,20 @@ def index_subdir(self, subdir, verbose=False, progress=False): return new_repodata + @staticmethod + def _v3_key_for_path(path: str) -> str | None: + for extension in (".tar.bz2", ".conda", ".whl"): + if path.endswith(extension): + return path[: -len(extension)] + return None + def _extract_indexed_packages_v3( self, indexed_packages: IndexedPackages - ) -> dict[str, dict[str, dict[str, object]]]: - v3 = { + ) -> V3Section: + """ + Return all packages from IndexedPackages as the "v3": {...} section. + """ + v3: V3Section = { "tar.bz2": {}, "conda": {}, "whl": {}, @@ -813,31 +841,28 @@ def _extract_indexed_packages_v3( return v3 - def _indexed_shard_to_repodata(self, indexed_shard: IndexedShard) -> dict: + def _indexed_shard_to_repodata(self, indexed_shard: IndexedShard) -> ShardDict: if self.repodata_v3: - return { - "v3": self._v3_section_data_from_indexed_shard(indexed_shard), + shard_data: ShardDict = { + "packages": {}, + "packages.conda": {}, + "v3": self._extract_indexed_packages_v3(indexed_shard), + } + else: + shard_data = { + "packages": indexed_shard.packages, + "packages.conda": indexed_shard.packages_conda, } - - shard_data = { - "packages": indexed_shard.packages, - "packages.conda": indexed_shard.packages_conda, - } - if indexed_shard.packages_whl: - shard_data["packages.whl"] = indexed_shard.packages_whl return shard_data - @staticmethod - def _v3_key_for_path(path: str) -> str | None: - for extension in (".tar.bz2", ".conda", ".whl"): - if path.endswith(extension): - return path[: -len(extension)] - return None - @staticmethod def _make_repodata_revision_data( revision_data: dict[str, dict[str, dict]], ) -> dict[str, int | None]: + """ + Return { "revision": 3, ... } dict with package statistics derived from + revision_data, which is similar to monolithic repodata. + """ timestamps = [] n_packages = 0 for section_records in revision_data.values(): @@ -854,28 +879,6 @@ def _make_repodata_revision_data( "newest": max(timestamps) if timestamps else None, } - @staticmethod - def _v3_section_data_from_indexed_shard( - indexed_shard: IndexedShard, - ) -> dict[str, dict[str, dict]]: - v3 = { - "tar.bz2": {}, - "conda": {}, - "whl": {}, - } - for section, records in ( - ("tar.bz2", indexed_shard.packages), - ("conda", indexed_shard.packages_conda), - ("whl", indexed_shard.packages_whl), - ): - for path, record in records.items(): - key = ChannelIndex._v3_key_for_path(path) - if key is None: - log.warning("%s has unsupported package extension", path) - continue - v3[section][key] = record - return v3 - def extract_subdir_to_cache( self, subdir: str, @@ -945,7 +948,7 @@ def extract_subdir_to_cache( return subdir - #### + # region: channeldata def channeldata_path(self): channeldata_file = os.path.join(self.output_root, "channeldata.json") @@ -988,6 +991,8 @@ def update_channeldata(self, rss=False): log.debug("write channeldata") self._write_channeldata(channel_data) + # endregion + def detect_subdirs(self): if not self._subdirs: detected_subdirs = { diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index a6274d74..d244082a 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -85,7 +85,7 @@ def __get__(self, inst, objtype=None) -> Any: class ChangedPackage(TypedDict): path: str - mtime: Number + mtime: float | int size: int diff --git a/conda_index/index/fs.py b/conda_index/index/fs.py index 516432b9..c42bce80 100644 --- a/conda_index/index/fs.py +++ b/conda_index/index/fs.py @@ -27,7 +27,7 @@ class FileInfo: """ fn: str - st_mtime: int + st_mtime: float | int st_size: int From 58bd3f2a8a7c0f0bd6853bf169aca8105abc5dee Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 14:40:50 -0400 Subject: [PATCH 19/32] remove unused import --- conda_index/index/cache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index d244082a..76a2ed3e 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -22,7 +22,6 @@ from .fs import MinimalFS if TYPE_CHECKING: - from numbers import Number from typing import IO, Any, Iterator from .fs import FileInfo From d148cb4b15402f63e8ddd3a114c6c8b549fb6ee1 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 14:46:58 -0400 Subject: [PATCH 20/32] no .whl for indexed_shards() --- conda_index/index/cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index 76a2ed3e..218a59a0 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -24,6 +24,8 @@ if TYPE_CHECKING: from typing import IO, Any, Iterator + from conda_index.index import ShardDict + from .fs import FileInfo log = logging.getLogger(__name__) @@ -463,7 +465,7 @@ def indexed_shards( desired: set[str] | None = None, *, pack_record=pack_record, - ): + ) -> Iterator[tuple[str, ShardDict]]: """ Yield (package name, all packages with that name as dict) from database ordered by name, path i.o.w. filename. @@ -471,12 +473,10 @@ def indexed_shards( :desired: If not None, set of desired package names. """ for shard in self.indexed_shards_2(desired, pack_record=pack_record): - shard_data = { + shard_data: ShardDict = { "packages": shard.packages, "packages.conda": shard.packages_conda, } - if shard.packages_whl: - shard_data["packages.whl"] = shard.packages_whl yield (shard.name, shard_data) @abc.abstractmethod From 160196eb1fcfce5e41c868f0550c7b16b624da7b Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Thu, 2 Apr 2026 15:52:31 -0400 Subject: [PATCH 21/32] fix test --- tests/test_psql.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_psql.py b/tests/test_psql.py index fc81c90f..cab596c3 100644 --- a/tests/test_psql.py +++ b/tests/test_psql.py @@ -347,10 +347,9 @@ class DummyResult(NamedTuple): DummyResult("package", "package.whl", {}), DummyResult("package", "package.conda", {}), ] - shards = list(cache.indexed_shards()) - _, data = shards[0] - assert len(data["packages.whl"]) == 1 - assert len(data["packages.conda"]) == 1 + shards = list(cache.indexed_shards_2()) + data = shards[0] + assert len(data.packages_conda) == 1 indexed_packages = cache.indexed_packages() assert indexed_packages.packages == {} From 1f0b44d0dfac41f4b3b998b14285d10b66f95716 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Mon, 6 Apr 2026 16:21:39 -0400 Subject: [PATCH 22/32] Update news Mention conda-pypi --- news/262-wheel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/news/262-wheel b/news/262-wheel index 1a140a7d..eb30e411 100644 --- a/news/262-wheel +++ b/news/262-wheel @@ -1,5 +1,5 @@ ### Enhancements * Support experimental "repodata v3" (package data in `["v3"][]` - dict under the top level) to support conda wheel experiments. A new - `--repodata-next` command line flag places package data under `v3` key. (#262) \ No newline at end of file + dict under the top level) to support conda-pypi wheel repodata generation. A new + `--repodata-next` command line flag places package data under `v3` key. (#262) From 136b07c40691c0190f92e82e9866b29059203f42 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Mon, 6 Apr 2026 18:48:05 -0400 Subject: [PATCH 23/32] show repodata-with-wheels API --- tests/demonstrate_wheel.json | 168 ++++++++++++++++++++++++++++++++ tests/test_demonstrate_wheel.py | 87 +++++++++++++++++ 2 files changed, 255 insertions(+) create mode 100644 tests/demonstrate_wheel.json create mode 100644 tests/test_demonstrate_wheel.py diff --git a/tests/demonstrate_wheel.json b/tests/demonstrate_wheel.json new file mode 100644 index 00000000..ac55ae17 --- /dev/null +++ b/tests/demonstrate_wheel.json @@ -0,0 +1,168 @@ +{ + "info": { + "subdir": "noarch" + }, + "packages": {}, + "packages.conda": {}, + "removed": [], + "repodata_version": 3, + "signatures": {}, + "v3": { + "whl": { + "fastapi-0.115.6-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/52/b3/7e4df40e585df024fac2f80d1a2d579c854ac37109675db2b0cc22c0bb9e/fastapi-0.115.6-py3-none-any.whl", + "name": "fastapi", + "version": "0.115.6", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "starlette<0.42.0,>=0.40.0", + "pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4", + "typing_extensions>=4.8.0", + "python >=3.8" + ], + "extra_depends": { + "standard": [ + "fastapi-cli[standard]>=0.0.5", + "httpx>=0.23.0", + "jinja2>=2.11.2", + "python-multipart>=0.0.7", + "email-validator>=2.0.0", + "uvicorn[standard]>=0.12.0" + ], + "all": [ + "fastapi-cli[standard]>=0.0.5", + "httpx>=0.23.0", + "jinja2>=2.11.2", + "python-multipart>=0.0.7", + "itsdangerous>=1.1.0", + "pyyaml>=5.3.1", + "ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1", + "orjson>=3.2.1", + "email-validator>=2.0.0", + "uvicorn[standard]>=0.12.0", + "pydantic-settings>=2.0.0", + "pydantic-extra-types>=2.0.0" + ] + }, + "fn": "fastapi-0.115.6-py3-none-any.whl", + "sha256": "e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305", + "size": 94843, + "subdir": "noarch", + "noarch": "python" + }, + "httpx-0.28.1-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", + "name": "httpx", + "version": "0.28.1", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "anyio", + "certifi", + "httpcore==1.*", + "idna", + "python >=3.8" + ], + "extra_depends": { + "brotli": [ + "brotli", + "brotlicffi" + ], + "cli": [ + "click==8.*", + "pygments==2.*", + "rich<14,>=10" + ], + "http2": [ + "h2<5,>=3" + ], + "socks": [ + "socksio==1.*" + ], + "zstd": [ + "zstandard>=0.18.0" + ] + }, + "fn": "httpx-0.28.1-py3-none-any.whl", + "sha256": "d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", + "size": 73517, + "subdir": "noarch", + "noarch": "python" + }, + "pydantic-2.10.6-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/f4/3c/8cc1cc84deffa6e25d2d0c688ebb80635dfdbf1dbea3e30c541c8cf4d860/pydantic-2.10.6-py3-none-any.whl", + "name": "pydantic", + "version": "2.10.6", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "annotated-types>=0.6.0", + "pydantic-core==2.27.2", + "typing_extensions>=4.12.2", + "python >=3.8" + ], + "extra_depends": { + "email": [ + "email-validator>=2.0.0" + ], + "timezone": [ + "python-tzdata[when=\"(python>=3.9 and __win)\"]" + ] + }, + "fn": "pydantic-2.10.6-py3-none-any.whl", + "sha256": "427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584", + "size": 431696, + "subdir": "noarch", + "noarch": "python" + }, + "python-dotenv-1.0.1-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", + "name": "python-dotenv", + "version": "1.0.1", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "python >=3.8" + ], + "extra_depends": { + "cli": [ + "click>=5.0" + ] + }, + "fn": "python-dotenv-1.0.1-py3-none-any.whl", + "sha256": "f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", + "size": 19863, + "subdir": "noarch", + "noarch": "python" + }, + "requests-2.32.3-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", + "name": "requests", + "version": "2.32.3", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1", + "certifi>=2017.4.17", + "python >=3.8" + ], + "extra_depends": { + "socks": [ + "pysocks!=1.5.7,>=1.5.6" + ], + "use-chardet-on-py3": [ + "chardet<6,>=3.0.2" + ] + }, + "fn": "requests-2.32.3-py3-none-any.whl", + "sha256": "70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", + "size": 64928, + "subdir": "noarch", + "noarch": "python" + } + } + } +} \ No newline at end of file diff --git a/tests/test_demonstrate_wheel.py b/tests/test_demonstrate_wheel.py new file mode 100644 index 00000000..737f8838 --- /dev/null +++ b/tests/test_demonstrate_wheel.py @@ -0,0 +1,87 @@ +""" +Show how an API user might use ChannelIndex() to generate .whl repodata. +""" + +import json +from pathlib import Path + +from conda_index.index import ChannelIndex +from conda_index.utils import CONDA_PACKAGE_EXTENSIONS + +HERE = Path(__file__).parent + + +def test_demonstrate_wheel(tmp_path: Path): + """ + Write v3 draft format .whl repodata to tmp_path. + """ + channel_index = ChannelIndex( + tmp_path, + "haswheels", # channel name if different than last segment of tmp_path + repodata_v3=True, + update_only=True, + save_fs_state=False, + write_current_repodata=False, + cache_kwargs={"package_extensions": CONDA_PACKAGE_EXTENSIONS + (".whl",)}, + ) + cache = channel_index.cache_for_subdir("noarch") + + input = json.loads((HERE / "demonstrate_wheel.json").read_text()) + wheels = { + f"{path}.whl": repodata for (path, repodata) in input["v3"]["whl"].items() + } + + # Define the set all packages that will be included in repodata.json, or add + # packages and leave existing packages if ChannelIndex.update_only == True. + # This updates the list of packages in the "upstream" state in the Stat() + # table. Cached package metadata (stat table where state = 'indexed', + # index_json table, etc.) is retained even if those package names are no + # longer included in the repodata.json output. + + def listdir_like(): + for path, repodata in wheels.items(): + yield { + "path": cache.database_path(path), + "size": repodata["size"], + "mtime": repodata.get( + "timestamp", 1 + ), # timestamp missing from generate.py wheel repodata + } + + cache.store_fs_state(listdir_like()) + + # Has to be in stat JOIN index_json to appear in repodat + for path, repodata in wheels.items(): + # must contain sha256 and md5 keys but values may be None + assert "sha256" in repodata + if "md5" not in repodata: + repodata["md5"] = None + # pretend we have a package with index.json but no other info/ files + cache.store( + cache.database_path(path), + repodata["size"], + repodata.get("timestamp", 1), + {}, + repodata, + ) + + # packages from database + packages = cache.indexed_packages() + + assert len(packages.packages_whl) == len(wheels) + + # repodata.json without repodata patches applied. Saved to + # repodata_from_packages in full index() method, but in this case there are + # no patches. + repodata_json = channel_index.index_subdir("noarch") + assert "v3" in repodata_json + + # Write complete repodata to output path for all detected subdirs. Normal + # conda-index API users would only call this method. Since we passed + # update_only=True, save_fs_state=False to ChannelIndex, this skips + # extracting packages from channel_root and only outputs metadata from the + # database. + channel_index.index(None) + + assert list(p.name for p in tmp_path.iterdir()) == ["noarch"] + assert (tmp_path / "noarch" / "repodata.json").exists() From 9c98ee65c0ea7acff63a39418c4fe888782db8de Mon Sep 17 00:00:00 2001 From: Dan Yeaw Date: Tue, 7 Apr 2026 14:18:48 -0400 Subject: [PATCH 24/32] Fix key format for wheel records --- conda_index/index/__init__.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index c4686ddc..55540e38 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -833,10 +833,23 @@ def _extract_indexed_packages_v3( ("whl", indexed_packages.packages_whl), ): for filename, record in records.items(): - key = self._v3_key_for_path(filename) - if key is None: - log.warning("%s has unsupported package extension", filename) - continue + if section == "whl": + name = record.get("name") + version = record.get("version") + build = record.get("build") + if name is None or version is None or build is None: + log.warning( + "%s: v3 whl records require name, version, and build; skipping", + filename, + ) + continue + key = f"{name}-{version}-{build}" + else: + key = self._v3_key_for_path(filename) + if key is None: + log.warning("%s has unsupported package extension", filename) + continue + v3[section][key] = record return v3 From e0d9c183b3911559d7ab710364a61fa14a0220b5 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 7 Apr 2026 14:50:39 -0400 Subject: [PATCH 25/32] Update conda_index/index/__init__.py with comment --- conda_index/index/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index 55540e38..8284228b 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -833,6 +833,7 @@ def _extract_indexed_packages_v3( ("whl", indexed_packages.packages_whl), ): for filename, record in records.items(): + # Per draft wheel-in-conda work, key is conda-like so that some conda-like parsing can occur on the key only. So we derive the key here. `record["fn"]` contains the filename or URL. if section == "whl": name = record.get("name") version = record.get("version") From ea7a884c6b115acd9f68e955fde3c246f4133961 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 7 Apr 2026 14:52:07 -0400 Subject: [PATCH 26/32] wrap lines --- conda_index/index/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index 8284228b..60cc70ff 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -833,7 +833,9 @@ def _extract_indexed_packages_v3( ("whl", indexed_packages.packages_whl), ): for filename, record in records.items(): - # Per draft wheel-in-conda work, key is conda-like so that some conda-like parsing can occur on the key only. So we derive the key here. `record["fn"]` contains the filename or URL. + # Per draft wheel-in-conda work, key is conda-like so that some + # conda-like parsing can occur on the key only. So we derive the + # key here. `record["fn"]` contains the filename or URL. if section == "whl": name = record.get("name") version = record.get("version") From e2ca1e8df9b24ad11ff0f266ca53a05a3da2db52 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 7 Apr 2026 14:54:12 -0400 Subject: [PATCH 27/32] Update conda_index/index/cache.py Co-authored-by: Dan Yeaw --- conda_index/index/cache.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index 218a59a0..55bf6d42 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -216,21 +216,6 @@ def package_section_for_path(self, path: str) -> str | None: return None return package_sections.get(match.group(1)) - def v3_section_and_key_for_path(self, path: str) -> tuple[str, str] | None: - package_sections = { - ".tar.bz2": "tar.bz2", - ".conda": "conda", - ".whl": "whl", - } - match = self._package_section_re.search(path) - if match is None: - return None - extension = match.group(1) - section = package_sections.get(extension) - if section is None: - return None - return section, path[: -len(extension)] - def open(self, fn: str) -> IO[bytes]: """ Given a base package name "somepackage.conda", return an open, seekable From cc28205a27a0d2ee0218021713653789f06458f1 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 7 Apr 2026 15:18:50 -0400 Subject: [PATCH 28/32] Apply suggestion from @danyeaw Co-authored-by: Dan Yeaw --- conda_index/postgres/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index c5a83da5..2aa3d07f 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -331,7 +331,7 @@ def indexed_shards_2( packages_whl=shard_dict["packages.whl"], ) for row in rows: - name, path, record = row + _, path, record = row path = self.plain_path(path) if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) From 40d0054701aeca5fdd8328ac2f5f431b5b814bab Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 7 Apr 2026 15:19:40 -0400 Subject: [PATCH 29/32] Apply suggestion from @danyeaw Co-authored-by: Dan Yeaw --- conda_index/postgres/cache.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 2aa3d07f..1be6c96c 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -337,14 +337,13 @@ def indexed_shards_2( log.warning("%s doesn't look like a conda package", path) continue - else: - key = self.package_section_for_path(path) - if key is None: - log.warning("%s has unsupported package extension", path) - continue - # This will be passed to the patch function, which we hope - # does not look for hex hash values. - shard_dict[key][path] = pack_record(record) + key = self.package_section_for_path(path) + if key is None: + log.warning("%s has unsupported package extension", path) + continue + # This will be passed to the patch function, which we hope + # does not look for hex hash values. + shard_dict[key][path] = pack_record(record) if not desired or name in desired: yield shard From 722d6c56cfdc2caba213a955b93a73103086fb3e Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 7 Apr 2026 15:22:36 -0400 Subject: [PATCH 30/32] package_section_for_path() replaces endswith() --- conda_index/index/sqlitecache.py | 3 --- conda_index/postgres/cache.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index b01b4422..b4a38d17 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -365,9 +365,6 @@ def indexed_packages(self) -> IndexedPackages: ): path, index_json = row index_json = json.loads(index_json) - if not path.endswith(self.package_extensions): - log.warning("%s doesn't look like a conda package", path) - continue section = self.package_section_for_path(path) if section is None: diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 1be6c96c..6bad8140 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -333,9 +333,6 @@ def indexed_shards_2( for row in rows: _, path, record = row path = self.plain_path(path) - if not path.endswith(self.package_extensions): - log.warning("%s doesn't look like a conda package", path) - continue key = self.package_section_for_path(path) if key is None: From 1db53cdbf9009949cb3a6cba6cba8ad252d84e94 Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Tue, 7 Apr 2026 16:13:19 -0400 Subject: [PATCH 31/32] compare whl section from output --- tests/test_demonstrate_wheel.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_demonstrate_wheel.py b/tests/test_demonstrate_wheel.py index 737f8838..c1f591cf 100644 --- a/tests/test_demonstrate_wheel.py +++ b/tests/test_demonstrate_wheel.py @@ -85,3 +85,8 @@ def listdir_like(): assert list(p.name for p in tmp_path.iterdir()) == ["noarch"] assert (tmp_path / "noarch" / "repodata.json").exists() + + output = json.loads((tmp_path / "noarch" / "repodata.json").read_text()) + + # other details differ + assert output["v3"]["whl"] == input["v3"]["whl"] From f51a29f815408c9ab6436e40838fa11f684cbb2a Mon Sep 17 00:00:00 2001 From: Daniel Holth Date: Wed, 8 Apr 2026 11:45:31 -0400 Subject: [PATCH 32/32] Support CEP 21 run exports in shards (#275) * cep-21 run exports in shards * update typing --- conda_index/index/cache.py | 17 ++++++++--------- conda_index/index/sqlitecache.py | 10 +++++++--- conda_index/postgres/cache.py | 16 ++++++++++++---- news/232-cep-21-run-exports | 3 +++ tests/test_psql.py | 21 ++++++++++++++------- 5 files changed, 44 insertions(+), 23 deletions(-) create mode 100644 news/232-cep-21-run-exports diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index 55bf6d42..388f2322 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -84,14 +84,13 @@ def __get__(self, inst, objtype=None) -> Any: return self -class ChangedPackage(TypedDict): - path: str - mtime: float | int - size: int - - if TYPE_CHECKING: + class ChangedPackage(TypedDict): + path: str + mtime: float | int + size: int + class HasChecksumsAndSize(TypedDict, extra_items=Any): """ Enforce keys accessed in conda-index store() @@ -227,7 +226,7 @@ def open(self, fn: str) -> IO[bytes]: def extract_to_cache_info_object( self, channel_root: Path | str, subdir: str, fn_info: FileInfo - ) -> tuple[str, int, int, dict[str, Any] | None]: + ) -> tuple[str, int, int, HasChecksumsAndSize | None]: """ fn_info: avoid having to call stat() a second time on package file. """ @@ -241,7 +240,7 @@ def _extract_to_cache( subdir: str, fn: str, stat_result: FileInfo | None = None, - ) -> tuple[str, int, int, dict[str, Any] | None]: + ) -> tuple[str, int, int, HasChecksumsAndSize | None]: if stat_result is None: # this code path is deprecated abs_fn = self.fs.join(self.subdir_path, fn) @@ -274,7 +273,7 @@ def _extract_to_cache( def extract_to_cache_unconditional( self, fn: str, abs_fn: str, size: int, mtime: int - ) -> dict[str, Any]: + ) -> HasChecksumsAndSize: """ Add or replace fn into cache, disregarding whether it is already cached. diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index b4a38d17..a5a640c0 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -389,8 +389,11 @@ def indexed_shards_2( for name, rows in itertools.groupby( self.db.execute( - """SELECT index_json.name, path, index_json - FROM stat JOIN index_json USING (path) WHERE stat.stage = ? + """SELECT index_json.name, index_json.path, index_json.index_json, run_exports.run_exports + FROM stat + JOIN index_json USING (path) + LEFT JOIN run_exports USING (path) + WHERE stat.stage = ? ORDER BY index_json.name, index_json.path""", (self.upstream_stage,), ), @@ -408,11 +411,12 @@ def indexed_shards_2( packages_whl=shard_dict["packages.whl"], ) for row in rows: - _, path, index_json = row + _, path, index_json, run_exports = row if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) continue record = json.loads(index_json) + record["run_exports"] = json.loads(run_exports or "{}") key = self.package_section_for_path(path) if key is None: log.warning("%s has unsupported package extension", path) diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 6bad8140..09ecaf21 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -293,6 +293,7 @@ def indexed_shards_2( """ index_json_table = model.Base.metadata.tables["index_json"] stat_table = model.Base.metadata.tables["stat"] + run_exports_table = model.Base.metadata.tables["run_exports"] # not optimized for "desired" partial shards case but that's not # currently used. @@ -301,12 +302,18 @@ def indexed_shards_2( index_json_table.c.name, index_json_table.c.path, index_json_table.c.index_json, + run_exports_table.c.run_exports, ) .select_from( join( - index_json_table, - stat_table, - index_json_table.c.path == stat_table.c.path, + join( + index_json_table, + stat_table, + index_json_table.c.path == stat_table.c.path, + ), + run_exports_table, + index_json_table.c.path == run_exports_table.c.path, + isouter=True, ) ) .where(stat_table.c.stage == self.upstream_stage) @@ -331,7 +338,8 @@ def indexed_shards_2( packages_whl=shard_dict["packages.whl"], ) for row in rows: - _, path, record = row + _, path, record, run_exports = row + record["run_exports"] = run_exports or {} path = self.plain_path(path) key = self.package_section_for_path(path) diff --git a/news/232-cep-21-run-exports b/news/232-cep-21-run-exports new file mode 100644 index 00000000..afa4c0c3 --- /dev/null +++ b/news/232-cep-21-run-exports @@ -0,0 +1,3 @@ +### Enhancements + +* Support CEP 21 "run_exports in shards" (#232) diff --git a/tests/test_psql.py b/tests/test_psql.py index cab596c3..4fdc61aa 100644 --- a/tests/test_psql.py +++ b/tests/test_psql.py @@ -307,12 +307,14 @@ class DummyResult(NamedTuple): name: str path: str record: object + run_exports: object # no index.json validation at this step, empty {} as record is passed on. connection.results_factory = lambda: [ - DummyResult("package", "package.notconda", {}), - DummyResult("package", "package.conda", {}), - DummyResult("package", "package.tar.bz2", {}), + DummyResult("package", "package.notconda", {}, {}), + DummyResult("package", "package-1.0.notconda", {}, {}), + DummyResult("package", "package-1.0.conda", {}, {"weak": ["zlib"]}), + DummyResult("package", "package-1.0.tar.bz2", {}, {}), ] shards = list(cache.indexed_shards()) assert len(shards) == 1 @@ -321,6 +323,9 @@ class DummyResult(NamedTuple): assert name == "package" assert len(data["packages"]) == 1 assert len(data["packages.conda"]) == 1 + assert data["packages.conda"]["package-1.0.conda"]["run_exports"] == { + "weak": ["zlib"] + } indexed_packages = cache.indexed_packages() assert len(indexed_packages.packages) == 1 @@ -342,14 +347,16 @@ class DummyResult(NamedTuple): name: str path: str record: object + run_exports: object connection.results_factory = lambda: [ - DummyResult("package", "package.whl", {}), - DummyResult("package", "package.conda", {}), + DummyResult("package", "package.whl", {}, {}), + DummyResult("package", "package.conda", {}, {}), ] shards = list(cache.indexed_shards_2()) - data = shards[0] - assert len(data.packages_conda) == 1 + assert len(shards) == 1 + assert len(shards[0].packages_whl) == 1 + assert len(shards[0].packages_conda) == 1 indexed_packages = cache.indexed_packages() assert indexed_packages.packages == {}