diff --git a/conda_index/api.py b/conda_index/api.py index f85cf0a..738bb3e 100644 --- a/conda_index/api.py +++ b/conda_index/api.py @@ -17,6 +17,7 @@ def update_index( progress=False, current_index_versions=None, write_run_exports=False, + repodata_v3=False, ): import os @@ -49,4 +50,5 @@ def update_index( subdirs=ensure_list(subdir), current_index_versions=current_index_versions, write_run_exports=write_run_exports, + repodata_v3=repodata_v3, ) diff --git a/conda_index/cli/__init__.py b/conda_index/cli/__init__.py index a33f265..9a1165d 100644 --- a/conda_index/cli/__init__.py +++ b/conda_index/cli/__init__.py @@ -185,6 +185,15 @@ default=False, show_default=True, ) +@click.option( + "--repodata-next/--no-repodata-next", + help=""" + EXPERIMENTAL. Write CEP-XXXX v3 repodata layout with all package + records under a top-level v3 key. + """, + default=False, + show_default=True, +) def cli( dir, patch_generator=None, @@ -210,6 +219,7 @@ def cli( db_url="", html_dependencies=False, update_only=False, + repodata_next=False, ): logutil.configure() if verbose: @@ -269,6 +279,7 @@ def cli( cache_kwargs=cache_kwargs, html_dependencies=html_dependencies, update_only=update_only, + repodata_v3=repodata_next, ) if update_cache is False: diff --git a/conda_index/index/__init__.py b/conda_index/index/__init__.py index 8017d79..60cc70f 100644 --- a/conda_index/index/__init__.py +++ b/conda_index/index/__init__.py @@ -12,11 +12,10 @@ import sys import time from concurrent.futures import Executor, ProcessPoolExecutor, ThreadPoolExecutor -from contextlib import nullcontext from datetime import datetime, timezone from os.path import basename, getmtime, getsize, isfile, join from pathlib import Path -from typing import Iterable +from typing import TYPE_CHECKING, Iterable from uuid import uuid4 import msgpack @@ -36,6 +35,27 @@ from . import rss, sqlitecache from .fs import FileInfo, MinimalFS +if TYPE_CHECKING: + from typing import Any, NotRequired, TypedDict + + from .cache import IndexedPackages, IndexedShard + + V3Section = TypedDict( + "V3Section", + {"tar.bz2": dict, "conda": dict, "whl": dict}, + ) + + # in this style because "packages.conda" is not a Python identifier + ShardDict = TypedDict( + "ShardDict", + { + "packages": dict[str, dict[str, Any]], + "packages.conda": dict[str, dict[str, Any]], + "v3": NotRequired[V3Section], + }, + ) + + log = logging.getLogger(__name__) # zstd -T0 -b15 -e17 repodata.json @@ -111,6 +131,7 @@ def update_index( write_zst=False, write_run_exports=False, html_dependencies=False, + repodata_v3=False, ): """ High-level interface to ``ChannelIndex``. Index all subdirs under @@ -147,6 +168,7 @@ def update_index( write_zst=write_zst, write_run_exports=write_run_exports, html_dependencies=html_dependencies, + repodata_v3=repodata_v3, ) channel_index.index( @@ -177,6 +199,7 @@ def _make_seconds(timestamp): ) CHANNELDATA_VERSION = 1 RUN_EXPORTS_VERSION = 1 +REPODATA_REVISION_V3 = 3 REPODATA_JSON_FN = "repodata.json" REPODATA_FROM_PKGS_JSON_FN = "repodata_from_packages.json" REPODATA_SHARDS_FN = "repodata_shards.msgpack.zst" @@ -252,15 +275,17 @@ def _apply_instructions(subdir, repodata, instructions, new_pkg_fixes=None): for key in ("packages", "packages.conda"): if key == "packages.conda" and fn.endswith(CONDA_PACKAGE_EXTENSION_V1): fn = fn.replace(CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2) - if fn in repodata[key]: - repodata[key][fn]["revoked"] = True - repodata[key][fn]["depends"].append("package_has_been_revoked") + records = repodata.get(key, {}) + if fn in records: + records[fn]["revoked"] = True + records[fn]["depends"].append("package_has_been_revoked") for fn in instructions.get("remove", ()): for key in ("packages", "packages.conda"): if key == "packages.conda" and fn.endswith(CONDA_PACKAGE_EXTENSION_V1): fn = fn.replace(CONDA_PACKAGE_EXTENSION_V1, CONDA_PACKAGE_EXTENSION_V2) - popped = repodata[key].pop(fn, None) + records = repodata.get(key, {}) + popped = records.pop(fn, None) if popped: repodata["removed"].append(fn) repodata["removed"].sort() @@ -397,6 +422,7 @@ def __init__( upstream_stage: str = "fs", cache_kwargs=None, update_only=False, + repodata_v3=False, ): if threads is None: threads = MAX_THREADS_DEFAULT @@ -430,6 +456,7 @@ def __init__( self.write_current_repodata = write_current_repodata self.upstream_stage = upstream_stage self.update_only = update_only + self.repodata_v3 = repodata_v3 self.cache_kwargs = cache_kwargs @@ -717,13 +744,29 @@ def index_subdir_shards(self, subdir, verbose=False, progress=False): (self.output_root / subdir).mkdir(parents=True, exist_ok=True) - for name, shard in cache.indexed_shards(): - shard_data = compressor.compress(sqlitecache.packb_typed(shard)) - shard_hash = hashlib.sha256(shard_data).digest() + v3_data = { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } + + for shard in cache.indexed_shards_2(): + repodata_shard = self._indexed_shard_to_repodata(shard) + shard_bytes = compressor.compress(sqlitecache.packb_typed(repodata_shard)) + shard_hash = hashlib.sha256(shard_bytes).digest() output_path = self.output_root / subdir / f"{shard_hash.hex()}.msgpack.zst" if not output_path.exists(): - output_path.write_bytes(shard_data) - shards[name] = shard_hash + output_path.write_bytes(shard_bytes) + shards[shard.name] = shard_hash + + if self.repodata_v3: + for section, records in repodata_shard["v3"].items(): + v3_data[section].update(records) + + if self.repodata_v3: + shards_index["info"]["repodata_revisions"] = [ + self._make_repodata_revision_data(v3_data) + ] return shards_index @@ -750,6 +793,15 @@ def index_subdir(self, subdir, verbose=False, progress=False): "removed": [], # can be added by patch/hotfix process } + if self.repodata_v3: + v3_packages = self._extract_indexed_packages_v3(indexed_packages) + new_repodata["v3"] = v3_packages + new_repodata["packages"] = {} + new_repodata["packages.conda"] = {} + new_repodata["info"]["repodata_revisions"] = [ + self._make_repodata_revision_data(v3_packages) + ] + if self.base_url: # per https://github.com/conda-incubator/ceps/blob/main/cep-15.md new_repodata["info"]["base_url"] = f"{self.base_url.rstrip('/')}/{subdir}/" @@ -757,6 +809,92 @@ def index_subdir(self, subdir, verbose=False, progress=False): return new_repodata + @staticmethod + def _v3_key_for_path(path: str) -> str | None: + for extension in (".tar.bz2", ".conda", ".whl"): + if path.endswith(extension): + return path[: -len(extension)] + return None + + def _extract_indexed_packages_v3( + self, indexed_packages: IndexedPackages + ) -> V3Section: + """ + Return all packages from IndexedPackages as the "v3": {...} section. + """ + v3: V3Section = { + "tar.bz2": {}, + "conda": {}, + "whl": {}, + } + for section, records in ( + ("tar.bz2", indexed_packages.packages), + ("conda", indexed_packages.packages_conda), + ("whl", indexed_packages.packages_whl), + ): + for filename, record in records.items(): + # Per draft wheel-in-conda work, key is conda-like so that some + # conda-like parsing can occur on the key only. So we derive the + # key here. `record["fn"]` contains the filename or URL. + if section == "whl": + name = record.get("name") + version = record.get("version") + build = record.get("build") + if name is None or version is None or build is None: + log.warning( + "%s: v3 whl records require name, version, and build; skipping", + filename, + ) + continue + key = f"{name}-{version}-{build}" + else: + key = self._v3_key_for_path(filename) + if key is None: + log.warning("%s has unsupported package extension", filename) + continue + + v3[section][key] = record + + return v3 + + def _indexed_shard_to_repodata(self, indexed_shard: IndexedShard) -> ShardDict: + if self.repodata_v3: + shard_data: ShardDict = { + "packages": {}, + "packages.conda": {}, + "v3": self._extract_indexed_packages_v3(indexed_shard), + } + else: + shard_data = { + "packages": indexed_shard.packages, + "packages.conda": indexed_shard.packages_conda, + } + return shard_data + + @staticmethod + def _make_repodata_revision_data( + revision_data: dict[str, dict[str, dict]], + ) -> dict[str, int | None]: + """ + Return { "revision": 3, ... } dict with package statistics derived from + revision_data, which is similar to monolithic repodata. + """ + timestamps = [] + n_packages = 0 + for section_records in revision_data.values(): + n_packages += len(section_records) + for record in section_records.values(): + timestamp = record.get("timestamp") + if isinstance(timestamp, (int, float)): + timestamps.append(int(timestamp)) + + return { + "revision": REPODATA_REVISION_V3, + "n_packages": n_packages, + "oldest": min(timestamps) if timestamps else None, + "newest": max(timestamps) if timestamps else None, + } + def extract_subdir_to_cache( self, subdir: str, @@ -826,7 +964,7 @@ def extract_subdir_to_cache( return subdir - #### + # region: channeldata def channeldata_path(self): channeldata_file = os.path.join(self.output_root, "channeldata.json") @@ -869,6 +1007,8 @@ def update_channeldata(self, rss=False): log.debug("write channeldata") self._write_channeldata(channel_data) + # endregion + def detect_subdirs(self): if not self._subdirs: detected_subdirs = { diff --git a/conda_index/index/cache.py b/conda_index/index/cache.py index 002d463..388f232 100644 --- a/conda_index/index/cache.py +++ b/conda_index/index/cache.py @@ -9,6 +9,7 @@ import fnmatch import json import logging +import re from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, TypedDict @@ -21,8 +22,9 @@ from .fs import MinimalFS if TYPE_CHECKING: - from numbers import Number - from typing import Any, Iterator + from typing import IO, Any, Iterator + + from conda_index.index import ShardDict from .fs import FileInfo @@ -81,12 +83,14 @@ def __get__(self, inst, objtype=None) -> Any: return value return self -class ChangedPackage(TypedDict): - path: str - mtime: Number - size: Number if TYPE_CHECKING: + + class ChangedPackage(TypedDict): + path: str + mtime: float | int + size: int + class HasChecksumsAndSize(TypedDict, extra_items=Any): """ Enforce keys accessed in conda-index store() @@ -101,6 +105,27 @@ class HasChecksumsAndSize(TypedDict, extra_items=Any): class IndexedPackages: packages: dict[str, dict[str, Any]] packages_conda: dict[str, dict[str, Any]] + packages_whl: dict[str, dict[str, Any]] + + +@dataclass +class IndexedShard(IndexedPackages): + """ + IndexedPackages for a single package name. + """ + + name: str + + +def pack_record(record): + """ + Convert hex checksums to bytes. + """ + if sha256 := record.get("sha256"): + record["sha256"] = bytes.fromhex(sha256) + if md5 := record.get("md5"): + record["md5"] = bytes.fromhex(md5) + return record class BaseCondaIndexCache(metaclass=abc.ABCMeta): @@ -171,11 +196,26 @@ def plain_path(self, path: str) -> str: """ return path.rsplit("/", 1)[-1] - def package_section_for_path(self, path: str) -> str: - key = "packages" if path.endswith(".tar.bz2") else "packages.conda" - return key + @cacher + def _package_section_re(self) -> re.Pattern[str]: + extension_pattern = "|".join( + re.escape(extension) + for extension in sorted(self.package_extensions, key=len, reverse=True) + ) + return re.compile(f"({extension_pattern})$") + + def package_section_for_path(self, path: str) -> str | None: + package_sections = { + ".tar.bz2": "packages", + ".conda": "packages.conda", + ".whl": "packages.whl", + } + match = self._package_section_re.search(path) + if match is None: + return None + return package_sections.get(match.group(1)) - def open(self, fn: str): + def open(self, fn: str) -> IO[bytes]: """ Given a base package name "somepackage.conda", return an open, seekable file object from our channel_url/subdir/fn suitable for reading that @@ -186,7 +226,7 @@ def open(self, fn: str): def extract_to_cache_info_object( self, channel_root: Path | str, subdir: str, fn_info: FileInfo - ) -> tuple[str, int, int, dict[str, Any] | None]: + ) -> tuple[str, int, int, HasChecksumsAndSize | None]: """ fn_info: avoid having to call stat() a second time on package file. """ @@ -200,7 +240,7 @@ def _extract_to_cache( subdir: str, fn: str, stat_result: FileInfo | None = None, - ) -> tuple[str, int, int, dict[str, Any] | None]: + ) -> tuple[str, int, int, HasChecksumsAndSize | None]: if stat_result is None: # this code path is deprecated abs_fn = self.fs.join(self.subdir_path, fn) @@ -233,7 +273,7 @@ def _extract_to_cache( def extract_to_cache_unconditional( self, fn: str, abs_fn: str, size: int, mtime: int - ) -> dict[str, Any]: + ) -> HasChecksumsAndSize: """ Add or replace fn into cache, disregarding whether it is already cached. @@ -401,20 +441,38 @@ def changed_packages(self) -> list[ChangedPackage]: @abc.abstractmethod def indexed_packages(self) -> IndexedPackages: """ - Return package sections from the cache for "monolithic repodata.json" - query. + Return all data for "monolithic repodata.json" query. """ - @abc.abstractmethod def indexed_shards( - self, desired: set[str] | None = None - ) -> Iterator[tuple[str, Any]]: + self, + desired: set[str] | None = None, + *, + pack_record=pack_record, + ) -> Iterator[tuple[str, ShardDict]]: """ - Yield (package name, all packages with that name) from database ordered - by name, path i.o.w. filename. + Yield (package name, all packages with that name as dict) from database + ordered by name, path i.o.w. filename. :desired: If not None, set of desired package names. """ + for shard in self.indexed_shards_2(desired, pack_record=pack_record): + shard_data: ShardDict = { + "packages": shard.packages, + "packages.conda": shard.packages_conda, + } + yield (shard.name, shard_data) + + @abc.abstractmethod + def indexed_shards_2( + self, + desired: set[str] | None = None, + *, + pack_record=pack_record, + ) -> Iterator[IndexedShard]: + """ + indexed_shards with dataclass instead of dict. + """ @abc.abstractmethod def run_exports(self) -> Iterator[tuple[str, dict]]: diff --git a/conda_index/index/fs.py b/conda_index/index/fs.py index 516432b..c42bce8 100644 --- a/conda_index/index/fs.py +++ b/conda_index/index/fs.py @@ -27,7 +27,7 @@ class FileInfo: """ fn: str - st_mtime: int + st_mtime: float | int st_size: int diff --git a/conda_index/index/sqlitecache.py b/conda_index/index/sqlitecache.py index 0465d4e..a5a640c 100644 --- a/conda_index/index/sqlitecache.py +++ b/conda_index/index/sqlitecache.py @@ -16,7 +16,13 @@ import msgpack from . import common, convert_cache -from .cache import BaseCondaIndexCache, IndexedPackages, cacher +from .cache import ( + BaseCondaIndexCache, + IndexedPackages, + IndexedShard, + cacher, + pack_record, +) from .cache import clear_newline_chars as _clear_newline_chars from .fs import MinimalFS @@ -345,6 +351,7 @@ def indexed_packages(self) -> IndexedPackages: new_packages = { "packages": {}, "packages.conda": {}, + "packages.whl": {}, } # load cached packages @@ -358,48 +365,66 @@ def indexed_packages(self) -> IndexedPackages: ): path, index_json = row index_json = json.loads(index_json) - if not path.endswith(self.package_extensions): - log.warning("%s doesn't look like a conda package", path) - continue section = self.package_section_for_path(path) + if section is None: + log.warning("%s has unsupported package extension", path) + continue new_packages[section][path] = index_json return IndexedPackages( packages=new_packages["packages"], packages_conda=new_packages["packages.conda"], + packages_whl=new_packages["packages.whl"], ) - def indexed_shards(self, desired: set[str] | None = None): + def indexed_shards_2( + self, desired: set[str] | None = None, *, pack_record=pack_record + ) -> Iterator[IndexedShard]: """ - Yield (package name, all packages with that name) from database ordered - by name, path i.o.w. filename. + Yield package shards as IndexedShard records. :desired: If not None, set of desired package names. """ + for name, rows in itertools.groupby( self.db.execute( - """SELECT index_json.name, path, index_json - FROM stat JOIN index_json USING (path) WHERE stat.stage = ? + """SELECT index_json.name, index_json.path, index_json.index_json, run_exports.run_exports + FROM stat + JOIN index_json USING (path) + LEFT JOIN run_exports USING (path) + WHERE stat.stage = ? ORDER BY index_json.name, index_json.path""", (self.upstream_stage,), ), lambda k: k[0], ): - shard = {"packages": {}, "packages.conda": {}} + shard_dict = { + "packages": {}, + "packages.conda": {}, + "packages.whl": {}, + } + shard = IndexedShard( + name=name, + packages=shard_dict["packages"], + packages_conda=shard_dict["packages.conda"], + packages_whl=shard_dict["packages.whl"], + ) for row in rows: - name, path, index_json = row + _, path, index_json, run_exports = row if not path.endswith(self.package_extensions): log.warning("%s doesn't look like a conda package", path) continue record = json.loads(index_json) + record["run_exports"] = json.loads(run_exports or "{}") key = self.package_section_for_path(path) - # we may have to pack later for patch functions that look for - # hex hashes - shard.setdefault(key, {})[path] = pack_record(record) + if key is None: + log.warning("%s has unsupported package extension", path) + continue + shard_dict[key][path] = pack_record(record) if not desired or name in desired: - yield (name, shard) + yield shard def store_index_json_stat( self, database_path, mtime, size, index_json: HasChecksumsAndSize @@ -427,17 +452,6 @@ def run_exports(self) -> Iterator[tuple[str, dict]]: yield (path, json.loads(run_exports or "{}")) -def pack_record(record): - """ - Convert hex checksums to bytes. - """ - if sha256 := record.get("sha256"): - record["sha256"] = bytes.fromhex(sha256) - if md5 := record.get("md5"): - record["md5"] = bytes.fromhex(md5) - return record - - def packb_typed(o: Any) -> bytes: """ Sidestep lack of typing in msgpack. diff --git a/conda_index/postgres/cache.py b/conda_index/postgres/cache.py index 16ea104..09ecaf2 100644 --- a/conda_index/postgres/cache.py +++ b/conda_index/postgres/cache.py @@ -21,7 +21,9 @@ BaseCondaIndexCache, ChangedPackage, IndexedPackages, + IndexedShard, clear_newline_chars, + pack_record, ) from conda_index.index.fs import MinimalFS from conda_index.index.sqlitecache import ( @@ -29,7 +31,6 @@ PATH_TO_TABLE, TABLE_NO_CACHE, cacher, - pack_record, ) if TYPE_CHECKING: @@ -278,12 +279,9 @@ def changed_packages(self) -> list[ChangedPackage]: # XXX or FileInfo dataclass for row in connection.execute(query) ] # type: ignore - def indexed_shards( - self, - desired: set[str] | None = None, - *, - pack_record=pack_record, - ): + def indexed_shards_2( + self, desired: set[str] | None = None, *, pack_record=pack_record + ) -> Iterator[IndexedShard]: """ Yield (package name, all packages with that name) from database ordered by name, path i.o.w. filename. @@ -295,18 +293,27 @@ def indexed_shards( """ index_json_table = model.Base.metadata.tables["index_json"] stat_table = model.Base.metadata.tables["stat"] + run_exports_table = model.Base.metadata.tables["run_exports"] + # not optimized for "desired" partial shards case but that's not + # currently used. query = ( select( index_json_table.c.name, index_json_table.c.path, index_json_table.c.index_json, + run_exports_table.c.run_exports, ) .select_from( join( - index_json_table, - stat_table, - index_json_table.c.path == stat_table.c.path, + join( + index_json_table, + stat_table, + index_json_table.c.path == stat_table.c.path, + ), + run_exports_table, + index_json_table.c.path == run_exports_table.c.path, + isouter=True, ) ) .where(stat_table.c.stage == self.upstream_stage) @@ -323,20 +330,28 @@ def indexed_shards( connection.execute(query), lambda k: k.name, ): - shard = {"packages": {}, "packages.conda": {}} + shard_dict = {"packages": {}, "packages.conda": {}, "packages.whl": {}} + shard = IndexedShard( + name=name, + packages=shard_dict["packages"], + packages_conda=shard_dict["packages.conda"], + packages_whl=shard_dict["packages.whl"], + ) for row in rows: - name, path, record = row + _, path, record, run_exports = row + record["run_exports"] = run_exports or {} path = self.plain_path(path) - if not path.endswith(self.package_extensions): - log.warning("%s doesn't look like a conda package", path) - continue + key = self.package_section_for_path(path) + if key is None: + log.warning("%s has unsupported package extension", path) + continue # This will be passed to the patch function, which we hope # does not look for hex hash values. - shard.setdefault(key, {})[path] = pack_record(record) + shard_dict[key][path] = pack_record(record) if not desired or name in desired: - yield (name, shard) + yield shard def indexed_packages(self) -> IndexedPackages: """ @@ -344,17 +359,20 @@ def indexed_packages(self) -> IndexedPackages: """ packages = {} packages_conda = {} + packages_whl = {} def nopack_record(record): return record - for _, shard in self.indexed_shards(pack_record=nopack_record): - packages.update(shard["packages"]) - packages_conda.update(shard["packages.conda"]) + for shard in self.indexed_shards_2(pack_record=nopack_record): + packages.update(shard.packages) + packages_conda.update(shard.packages_conda) + packages_whl.update(shard.packages_whl) return IndexedPackages( packages=packages, packages_conda=packages_conda, + packages_whl=packages_whl, ) def load_all_from_cache(self, fn: str): diff --git a/news/232-cep-21-run-exports b/news/232-cep-21-run-exports new file mode 100644 index 0000000..afa4c0c --- /dev/null +++ b/news/232-cep-21-run-exports @@ -0,0 +1,3 @@ +### Enhancements + +* Support CEP 21 "run_exports in shards" (#232) diff --git a/news/262-wheel b/news/262-wheel new file mode 100644 index 0000000..eb30e41 --- /dev/null +++ b/news/262-wheel @@ -0,0 +1,5 @@ +### Enhancements + +* Support experimental "repodata v3" (package data in `["v3"][]` + dict under the top level) to support conda-pypi wheel repodata generation. A new + `--repodata-next` command line flag places package data under `v3` key. (#262) diff --git a/tests/demonstrate_wheel.json b/tests/demonstrate_wheel.json new file mode 100644 index 0000000..ac55ae1 --- /dev/null +++ b/tests/demonstrate_wheel.json @@ -0,0 +1,168 @@ +{ + "info": { + "subdir": "noarch" + }, + "packages": {}, + "packages.conda": {}, + "removed": [], + "repodata_version": 3, + "signatures": {}, + "v3": { + "whl": { + "fastapi-0.115.6-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/52/b3/7e4df40e585df024fac2f80d1a2d579c854ac37109675db2b0cc22c0bb9e/fastapi-0.115.6-py3-none-any.whl", + "name": "fastapi", + "version": "0.115.6", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "starlette<0.42.0,>=0.40.0", + "pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4", + "typing_extensions>=4.8.0", + "python >=3.8" + ], + "extra_depends": { + "standard": [ + "fastapi-cli[standard]>=0.0.5", + "httpx>=0.23.0", + "jinja2>=2.11.2", + "python-multipart>=0.0.7", + "email-validator>=2.0.0", + "uvicorn[standard]>=0.12.0" + ], + "all": [ + "fastapi-cli[standard]>=0.0.5", + "httpx>=0.23.0", + "jinja2>=2.11.2", + "python-multipart>=0.0.7", + "itsdangerous>=1.1.0", + "pyyaml>=5.3.1", + "ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1", + "orjson>=3.2.1", + "email-validator>=2.0.0", + "uvicorn[standard]>=0.12.0", + "pydantic-settings>=2.0.0", + "pydantic-extra-types>=2.0.0" + ] + }, + "fn": "fastapi-0.115.6-py3-none-any.whl", + "sha256": "e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305", + "size": 94843, + "subdir": "noarch", + "noarch": "python" + }, + "httpx-0.28.1-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", + "name": "httpx", + "version": "0.28.1", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "anyio", + "certifi", + "httpcore==1.*", + "idna", + "python >=3.8" + ], + "extra_depends": { + "brotli": [ + "brotli", + "brotlicffi" + ], + "cli": [ + "click==8.*", + "pygments==2.*", + "rich<14,>=10" + ], + "http2": [ + "h2<5,>=3" + ], + "socks": [ + "socksio==1.*" + ], + "zstd": [ + "zstandard>=0.18.0" + ] + }, + "fn": "httpx-0.28.1-py3-none-any.whl", + "sha256": "d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", + "size": 73517, + "subdir": "noarch", + "noarch": "python" + }, + "pydantic-2.10.6-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/f4/3c/8cc1cc84deffa6e25d2d0c688ebb80635dfdbf1dbea3e30c541c8cf4d860/pydantic-2.10.6-py3-none-any.whl", + "name": "pydantic", + "version": "2.10.6", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "annotated-types>=0.6.0", + "pydantic-core==2.27.2", + "typing_extensions>=4.12.2", + "python >=3.8" + ], + "extra_depends": { + "email": [ + "email-validator>=2.0.0" + ], + "timezone": [ + "python-tzdata[when=\"(python>=3.9 and __win)\"]" + ] + }, + "fn": "pydantic-2.10.6-py3-none-any.whl", + "sha256": "427d664bf0b8a2b34ff5dd0f5a18df00591adcee7198fbd71981054cef37b584", + "size": 431696, + "subdir": "noarch", + "noarch": "python" + }, + "python-dotenv-1.0.1-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/6a/3e/b68c118422ec867fa7ab88444e1274aa40681c606d59ac27de5a5588f082/python_dotenv-1.0.1-py3-none-any.whl", + "name": "python-dotenv", + "version": "1.0.1", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "python >=3.8" + ], + "extra_depends": { + "cli": [ + "click>=5.0" + ] + }, + "fn": "python-dotenv-1.0.1-py3-none-any.whl", + "sha256": "f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a", + "size": 19863, + "subdir": "noarch", + "noarch": "python" + }, + "requests-2.32.3-py3_none_any_0": { + "url": "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", + "name": "requests", + "version": "2.32.3", + "build": "py3_none_any_0", + "build_number": 0, + "depends": [ + "charset-normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1", + "certifi>=2017.4.17", + "python >=3.8" + ], + "extra_depends": { + "socks": [ + "pysocks!=1.5.7,>=1.5.6" + ], + "use-chardet-on-py3": [ + "chardet<6,>=3.0.2" + ] + }, + "fn": "requests-2.32.3-py3-none-any.whl", + "sha256": "70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", + "size": 64928, + "subdir": "noarch", + "noarch": "python" + } + } + } +} \ No newline at end of file diff --git a/tests/test_cache.py b/tests/test_cache.py index a8c20c9..81064ca 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -43,7 +43,12 @@ def changed_packages(self) -> list[cache.ChangedPackage]: def indexed_packages(self) -> cache.IndexedPackages: raise NotImplementedError - def indexed_shards(self, desired: set[str] | None = None): + def indexed_shards_2( + self, + desired: set[str] | None = None, + *, + pack_record=None, + ) -> Iterator[cache.IndexedShard]: raise NotImplementedError def run_exports(self) -> Iterator[tuple[str, dict]]: @@ -56,7 +61,9 @@ def test_cache(tmp_path): """ c = DummyCache( - str(tmp_path), "linux-64", package_extensions=CONDA_PACKAGE_EXTENSIONS + str(tmp_path), + "linux-64", + package_extensions=CONDA_PACKAGE_EXTENSIONS + (".whl",), ) package = "foo.conda" @@ -66,3 +73,5 @@ def test_cache(tmp_path): c.convert() c.close() + + assert c.package_section_for_path("file.whl") == "packages.whl" diff --git a/tests/test_demonstrate_wheel.py b/tests/test_demonstrate_wheel.py new file mode 100644 index 0000000..c1f591c --- /dev/null +++ b/tests/test_demonstrate_wheel.py @@ -0,0 +1,92 @@ +""" +Show how an API user might use ChannelIndex() to generate .whl repodata. +""" + +import json +from pathlib import Path + +from conda_index.index import ChannelIndex +from conda_index.utils import CONDA_PACKAGE_EXTENSIONS + +HERE = Path(__file__).parent + + +def test_demonstrate_wheel(tmp_path: Path): + """ + Write v3 draft format .whl repodata to tmp_path. + """ + channel_index = ChannelIndex( + tmp_path, + "haswheels", # channel name if different than last segment of tmp_path + repodata_v3=True, + update_only=True, + save_fs_state=False, + write_current_repodata=False, + cache_kwargs={"package_extensions": CONDA_PACKAGE_EXTENSIONS + (".whl",)}, + ) + cache = channel_index.cache_for_subdir("noarch") + + input = json.loads((HERE / "demonstrate_wheel.json").read_text()) + wheels = { + f"{path}.whl": repodata for (path, repodata) in input["v3"]["whl"].items() + } + + # Define the set all packages that will be included in repodata.json, or add + # packages and leave existing packages if ChannelIndex.update_only == True. + # This updates the list of packages in the "upstream" state in the Stat() + # table. Cached package metadata (stat table where state = 'indexed', + # index_json table, etc.) is retained even if those package names are no + # longer included in the repodata.json output. + + def listdir_like(): + for path, repodata in wheels.items(): + yield { + "path": cache.database_path(path), + "size": repodata["size"], + "mtime": repodata.get( + "timestamp", 1 + ), # timestamp missing from generate.py wheel repodata + } + + cache.store_fs_state(listdir_like()) + + # Has to be in stat JOIN index_json to appear in repodat + for path, repodata in wheels.items(): + # must contain sha256 and md5 keys but values may be None + assert "sha256" in repodata + if "md5" not in repodata: + repodata["md5"] = None + # pretend we have a package with index.json but no other info/ files + cache.store( + cache.database_path(path), + repodata["size"], + repodata.get("timestamp", 1), + {}, + repodata, + ) + + # packages from database + packages = cache.indexed_packages() + + assert len(packages.packages_whl) == len(wheels) + + # repodata.json without repodata patches applied. Saved to + # repodata_from_packages in full index() method, but in this case there are + # no patches. + repodata_json = channel_index.index_subdir("noarch") + assert "v3" in repodata_json + + # Write complete repodata to output path for all detected subdirs. Normal + # conda-index API users would only call this method. Since we passed + # update_only=True, save_fs_state=False to ChannelIndex, this skips + # extracting packages from channel_root and only outputs metadata from the + # database. + channel_index.index(None) + + assert list(p.name for p in tmp_path.iterdir()) == ["noarch"] + assert (tmp_path / "noarch" / "repodata.json").exists() + + output = json.loads((tmp_path / "noarch" / "repodata.json").read_text()) + + # other details differ + assert output["v3"]["whl"] == input["v3"]["whl"] diff --git a/tests/test_index.py b/tests/test_index.py index 683fcc4..89b26a6 100644 --- a/tests/test_index.py +++ b/tests/test_index.py @@ -1360,6 +1360,35 @@ def _patch_repodata_shards( assert package_url == "https://example.org/somechannel/osx-64/package-1.0.conda" +def test_repodata_v3(index_data): + pkg_dir = Path(index_data, "packages") + + channel_index = conda_index.index.ChannelIndex( + pkg_dir, + None, + write_bz2=False, + write_zst=False, + compact_json=True, + threads=1, + repodata_v3=True, + ) + + channel_index.index(None) + + noarch = json.loads((pkg_dir / "noarch" / "repodata.json").read_text()) + + assert set(noarch["info"]["repodata_revisions"][0].keys()) == set( + ("revision", "n_packages", "oldest", "newest") + ) + + assert "v3" in noarch + assert set(noarch["v3"]) == {"tar.bz2", "conda", "whl"} + assert noarch["packages"] == {} + assert noarch["packages.conda"] == {} + assert "packages.whl" not in noarch + assert noarch["v3"]["tar.bz2"] or noarch["v3"]["conda"] + + def test_write_current_repodata(index_data): """ Test that we can skip current_repodata, and that it deletes the old one. diff --git a/tests/test_psql.py b/tests/test_psql.py index 237c72e..4fdc61a 100644 --- a/tests/test_psql.py +++ b/tests/test_psql.py @@ -307,12 +307,14 @@ class DummyResult(NamedTuple): name: str path: str record: object + run_exports: object # no index.json validation at this step, empty {} as record is passed on. connection.results_factory = lambda: [ - DummyResult("package", "package.notconda", {}), - DummyResult("package", "package.conda", {}), - DummyResult("package", "package.tar.bz2", {}), + DummyResult("package", "package.notconda", {}, {}), + DummyResult("package", "package-1.0.notconda", {}, {}), + DummyResult("package", "package-1.0.conda", {}, {"weak": ["zlib"]}), + DummyResult("package", "package-1.0.tar.bz2", {}, {}), ] shards = list(cache.indexed_shards()) assert len(shards) == 1 @@ -321,12 +323,49 @@ class DummyResult(NamedTuple): assert name == "package" assert len(data["packages"]) == 1 assert len(data["packages.conda"]) == 1 + assert data["packages.conda"]["package-1.0.conda"]["run_exports"] == { + "weak": ["zlib"] + } + + indexed_packages = cache.indexed_packages() + assert len(indexed_packages.packages) == 1 + assert len(indexed_packages.packages_conda) == 1 + + +def test_psql_include_wheel_extension(tmp_path: Path): + assert PsqlCache + cache = PsqlCache( + tmp_path, + "noarch", + db_url="postgresql://example", + package_extensions=CONDA_PACKAGE_EXTENSIONS + (".whl",), + ) + connection = MockConnection() + cache.engine = MockEngine(connection) # type: ignore + + class DummyResult(NamedTuple): + name: str + path: str + record: object + run_exports: object + + connection.results_factory = lambda: [ + DummyResult("package", "package.whl", {}, {}), + DummyResult("package", "package.conda", {}, {}), + ] + shards = list(cache.indexed_shards_2()) + assert len(shards) == 1 + assert len(shards[0].packages_whl) == 1 + assert len(shards[0].packages_conda) == 1 indexed_packages = cache.indexed_packages() - packages = indexed_packages.packages - packages_conda = indexed_packages.packages_conda - assert len(packages) == 1 - assert len(packages_conda) == 1 + assert indexed_packages.packages == {} + assert len(indexed_packages.packages_conda) == 1 + + shards_2 = list(cache.indexed_shards_2()) + assert len(shards_2) == 1 + assert len(shards_2[0].packages_whl) == 1 + assert len(shards_2[0].packages_conda) == 1 def test_psql_run_exports(tmp_path: Path):