diff --git a/dclimate_client_py/dclimate_client.py b/dclimate_client_py/dclimate_client.py index 4474c79..88cc27b 100644 --- a/dclimate_client_py/dclimate_client.py +++ b/dclimate_client_py/dclimate_client.py @@ -136,6 +136,7 @@ async def load_dataset( cid: typing.Optional[str] = None, return_xarray: bool = False, zarr_group: typing.Optional[str] = None, + shard_read_mode: typing.Literal["full", "sparse"] = "sparse", ) -> typing.Union[ typing.Tuple[GeotemporalData, DatasetMetadata], typing.Tuple[xr.Dataset, DatasetMetadata], @@ -171,6 +172,10 @@ async def load_dataset( Explicit Zarr group to open for grouped/pyramid sharded stores. If omitted, py-hamt v2 stores with multiple top-level groups default to group "0" when available. + shard_read_mode : {"full", "sparse"}, optional + Sharded Zarr shard-index read strategy. Defaults to ``"sparse"`` + and decodes only the requested shard slot on read-only cache + misses. ``"full"`` preserves the decoded-shard cache behavior. Returns ------- @@ -235,6 +240,7 @@ async def load_dataset( ipfs_cid=cid, kubo_cas=self._kubo_cas, zarr_group=zarr_group, + shard_read_mode=shard_read_mode, ) # Build metadata for direct CID case @@ -313,6 +319,7 @@ async def load_dataset( ipfs_cid=final_cid, kubo_cas=self._kubo_cas, zarr_group=zarr_group, + shard_read_mode=shard_read_mode, ) # Build metadata for STAC case diff --git a/dclimate_client_py/ipfs_retrieval.py b/dclimate_client_py/ipfs_retrieval.py index 34b35dc..5c4298c 100644 --- a/dclimate_client_py/ipfs_retrieval.py +++ b/dclimate_client_py/ipfs_retrieval.py @@ -16,6 +16,7 @@ from py_hamt import ( HAMT, KuboCAS, + ShardReadMode, ShardedZarrStore, ShardedZarrV1DeprecationWarning, ZarrHAMTStore, @@ -226,12 +227,16 @@ async def _open_sharded_zarr_store( *, ipfs_cid: str, kubo_cas: KuboCAS, + shard_read_mode: ShardReadMode = "sparse", ) -> ShardedZarrStore: """Open a py-hamt sharded store without surfacing legacy v1 read warnings.""" with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ShardedZarrV1DeprecationWarning) return await ShardedZarrStore.open( - root_cid=ipfs_cid, cas=kubo_cas, read_only=True + root_cid=ipfs_cid, + cas=kubo_cas, + read_only=True, + shard_read_mode=shard_read_mode, ) @@ -242,6 +247,7 @@ async def _load_dataset_from_ipfs_cid( ipfs_cid: str, kubo_cas: KuboCAS, zarr_group: str | None = None, + shard_read_mode: ShardReadMode = "sparse", ) -> xr.Dataset: """ Internal function to load a Zarr dataset from IPFS using a provided KuboCAS instance. @@ -315,6 +321,7 @@ async def _load_dataset_from_ipfs_cid( sharded_store = await _open_sharded_zarr_store( ipfs_cid=ipfs_cid, kubo_cas=kubo_cas, + shard_read_mode=shard_read_mode, ) sharded_store_opened = True ds, opened_zarr_group = _open_zarr_from_store( diff --git a/pyproject.toml b/pyproject.toml index ad9159c..fc0d750 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "pdm.backend" [project] name = "dclimate-client-py" -version = "0.5.10" # Set a static version or handle it in versioning strategy +version = "0.5.11" # Set a static version or handle it in versioning strategy description = "Python client library for accessing dClimate weather and climate data" readme = "README.md" license = {text = "MIT"} @@ -34,7 +34,7 @@ dependencies = [ "rioxarray", "zarr>=3.0.8", "numpy>=2.1.3", - "py_hamt>=3.4.0", + "py_hamt>=3.4.1", "requests", "pyarrow", "geopandas", diff --git a/tests/test_ipfs_retrieval.py b/tests/test_ipfs_retrieval.py index c6e489c..1197fd0 100644 --- a/tests/test_ipfs_retrieval.py +++ b/tests/test_ipfs_retrieval.py @@ -51,7 +51,7 @@ def test_is_connection_error_classifies_gateway_failures(message): @pytest.mark.asyncio async def test_sharded_connection_error_skips_hamt_fallback(monkeypatch): - async def sharded_open(*, root_cid, cas, read_only): + async def sharded_open(**kwargs): raise TimeoutError("timed out opening sharded store") async def hamt_build(**kwargs): @@ -69,7 +69,10 @@ async def hamt_build(**kwargs): @pytest.mark.asyncio async def test_multigroup_sharded_store_defaults_to_group_zero(monkeypatch): - async def sharded_open(*, root_cid, cas, read_only): + open_kwargs = [] + + async def sharded_open(**kwargs): + open_kwargs.append(kwargs) return DummyGroupedStore() opened_groups = [] @@ -87,13 +90,14 @@ def open_zarr(*, store, group=None): ) assert opened_groups == ["0"] + assert open_kwargs[0]["shard_read_mode"] == "sparse" assert ds.attrs["_ipfs_store_type"] == "ShardedZarrStore" assert ds.attrs["_ipfs_zarr_group"] == "0" @pytest.mark.asyncio async def test_explicit_zarr_group_is_passed_to_open_zarr(monkeypatch): - async def sharded_open(*, root_cid, cas, read_only): + async def sharded_open(**kwargs): return DummyStore() opened_groups = [] @@ -117,7 +121,7 @@ def open_zarr(*, store, group=None): @pytest.mark.asyncio async def test_zarr_group_error_after_sharded_open_does_not_fallback(monkeypatch): - async def sharded_open(*, root_cid, cas, read_only): + async def sharded_open(**kwargs): return DummyStore() async def hamt_build(**kwargs): @@ -139,7 +143,7 @@ def open_zarr(*, store, group=None): @pytest.mark.asyncio async def test_sharded_v1_warning_is_suppressed_in_client_loader(monkeypatch): - async def sharded_open(*, root_cid, cas, read_only): + async def sharded_open(**kwargs): warnings.warn( "sharded_zarr_v1 is deprecated", ipfs_retrieval.ShardedZarrV1DeprecationWarning, @@ -172,7 +176,7 @@ async def sharded_open(*, root_cid, cas, read_only): @pytest.mark.asyncio async def test_hamt_fallback_preserves_explicit_zarr_group(monkeypatch): - async def sharded_open(*, root_cid, cas, read_only): + async def sharded_open(**kwargs): raise ValueError("not a sharded zarr store") async def hamt_build(**kwargs): @@ -206,9 +210,12 @@ def open_zarr(*, store, group=None): @pytest.mark.asyncio async def test_client_direct_cid_passes_zarr_group_and_records_metadata(monkeypatch): - async def load_dataset_from_ipfs_cid(*, ipfs_cid, kubo_cas, zarr_group): + async def load_dataset_from_ipfs_cid( + *, ipfs_cid, kubo_cas, zarr_group, shard_read_mode + ): assert ipfs_cid == VALID_CID assert zarr_group == "2" + assert shard_read_mode == "sparse" return xr.Dataset(attrs={"_ipfs_zarr_group": "2"}) monkeypatch.setattr( @@ -225,6 +232,7 @@ async def load_dataset_from_ipfs_cid(*, ipfs_cid, kubo_cas, zarr_group): cid=VALID_CID, return_xarray=True, zarr_group="2", + shard_read_mode="sparse", ) assert isinstance(ds, xr.Dataset) diff --git a/uv.lock b/uv.lock index aa3e5b9..aead06f 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 3 +revision = 2 requires-python = ">=3.12" [[package]] @@ -632,7 +632,7 @@ wheels = [ [[package]] name = "dclimate-client-py" -version = "0.5.10" +version = "0.5.11" source = { editable = "." } dependencies = [ { name = "aiobotocore" }, @@ -677,7 +677,7 @@ requires-dist = [ { name = "opentelemetry-api", specifier = ">=1.30.0" }, { name = "pandas" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=4.1.0" }, - { name = "py-hamt", specifier = ">=3.4.0" }, + { name = "py-hamt", specifier = ">=3.4.1" }, { name = "pyarrow" }, { name = "pycryptodome", specifier = ">=3.21.0" }, { name = "pystac", specifier = ">=1.10.0" }, @@ -1455,7 +1455,7 @@ wheels = [ [[package]] name = "py-hamt" -version = "3.4.0" +version = "3.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dag-cbor" }, @@ -1466,9 +1466,9 @@ dependencies = [ { name = "pycryptodome" }, { name = "zarr" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/58/b563c568fde2dca469dd5f1aeb90362393ba1b0f658dbfd5f3bb671c19df/py_hamt-3.4.0.tar.gz", hash = "sha256:13095e96cefca2d1a9bbc36e571b627f7c1ada2c3253771da5c549d3e7984aa2", size = 226710, upload-time = "2026-07-07T18:03:55.703Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/dd/26a3653588dfe9992d6c25d23aa0e2d38e2db1e026dfd4d7b781412bd846/py_hamt-3.4.1.tar.gz", hash = "sha256:4563bd875dfae95cda4a87a052a7765e485fce5347db0f9d178002a1c1c55442", size = 228893, upload-time = "2026-07-10T17:46:53.195Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/0f/241808122e1be49b00ed4db6e6edff7a230f19dcee12bd9657d1f7226b6d/py_hamt-3.4.0-py3-none-any.whl", hash = "sha256:ee2f82a4de475e7693276f10298af3f6b9f643507b5eacd81af6ad9f68c05696", size = 58714, upload-time = "2026-07-07T18:03:57.368Z" }, + { url = "https://files.pythonhosted.org/packages/12/eb/3808e239135727accb07eeca9e4c049967e1ca4606838db94afaf38432e0/py_hamt-3.4.1-py3-none-any.whl", hash = "sha256:f5ef312844609ce14c56b32c0fe2fc388a57f1e639406dfb7a2c1aa7acf7c036", size = 59805, upload-time = "2026-07-10T17:46:52.09Z" }, ] [[package]]