perf: add sparse decode option

dclimate_client_py/dclimate_client.py

@@ -136,6 +136,7 @@ async def load_dataset(
         cid: typing.Optional[str] = None,
         return_xarray: bool = False,
         zarr_group: typing.Optional[str] = None,
+        shard_read_mode: typing.Literal["full", "sparse"] = "sparse",
     ) -> typing.Union[
         typing.Tuple[GeotemporalData, DatasetMetadata],
         typing.Tuple[xr.Dataset, DatasetMetadata],
@@ -171,6 +172,10 @@ async def load_dataset(
             Explicit Zarr group to open for grouped/pyramid sharded stores. If
             omitted, py-hamt v2 stores with multiple top-level groups default
             to group "0" when available.
+        shard_read_mode : {"full", "sparse"}, optional
+            Sharded Zarr shard-index read strategy. Defaults to ``"sparse"``
+            and decodes only the requested shard slot on read-only cache
+            misses. ``"full"`` preserves the decoded-shard cache behavior.
 
         Returns
         -------
@@ -235,6 +240,7 @@ async def load_dataset(
                 ipfs_cid=cid,
                 kubo_cas=self._kubo_cas,
                 zarr_group=zarr_group,
+                shard_read_mode=shard_read_mode,
             )
 
             # Build metadata for direct CID case
@@ -313,6 +319,7 @@ async def load_dataset(
             ipfs_cid=final_cid,
             kubo_cas=self._kubo_cas,
             zarr_group=zarr_group,
+            shard_read_mode=shard_read_mode,
         )
 
         # Build metadata for STAC case

dclimate_client_py/ipfs_retrieval.py

@@ -16,6 +16,7 @@
 from py_hamt import (
     HAMT,
     KuboCAS,
+    ShardReadMode,
     ShardedZarrStore,
     ShardedZarrV1DeprecationWarning,
     ZarrHAMTStore,
@@ -226,12 +227,16 @@ async def _open_sharded_zarr_store(
     *,
     ipfs_cid: str,
     kubo_cas: KuboCAS,
+    shard_read_mode: ShardReadMode = "sparse",
 ) -> ShardedZarrStore:
     """Open a py-hamt sharded store without surfacing legacy v1 read warnings."""
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", category=ShardedZarrV1DeprecationWarning)
         return await ShardedZarrStore.open(
-            root_cid=ipfs_cid, cas=kubo_cas, read_only=True
+            root_cid=ipfs_cid,
+            cas=kubo_cas,
+            read_only=True,
+            shard_read_mode=shard_read_mode,
         )
 
 
@@ -242,6 +247,7 @@ async def _load_dataset_from_ipfs_cid(
     ipfs_cid: str,
     kubo_cas: KuboCAS,
     zarr_group: str | None = None,
+    shard_read_mode: ShardReadMode = "sparse",
 ) -> xr.Dataset:
     """
     Internal function to load a Zarr dataset from IPFS using a provided KuboCAS instance.
@@ -315,6 +321,7 @@ async def _load_dataset_from_ipfs_cid(
                         sharded_store = await _open_sharded_zarr_store(
                             ipfs_cid=ipfs_cid,
                             kubo_cas=kubo_cas,
+                            shard_read_mode=shard_read_mode,
                         )
                         sharded_store_opened = True
                         ds, opened_zarr_group = _open_zarr_from_store(

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "dclimate-client-py"
-version = "0.5.10" # Set a static version or handle it in versioning strategy
+version = "0.5.11" # Set a static version or handle it in versioning strategy
 description = "Python client library for accessing dClimate weather and climate data"
 readme = "README.md"
 license = {text = "MIT"}
@@ -34,7 +34,7 @@ dependencies = [
     "rioxarray",
     "zarr>=3.0.8",
     "numpy>=2.1.3",
-    "py_hamt>=3.4.0",
+    "py_hamt>=3.4.1",
     "requests",
     "pyarrow",
     "geopandas",

tests/test_ipfs_retrieval.py

@@ -51,7 +51,7 @@ def test_is_connection_error_classifies_gateway_failures(message):
 
 @pytest.mark.asyncio
 async def test_sharded_connection_error_skips_hamt_fallback(monkeypatch):
-    async def sharded_open(*, root_cid, cas, read_only):
+    async def sharded_open(**kwargs):
         raise TimeoutError("timed out opening sharded store")
 
     async def hamt_build(**kwargs):
@@ -69,7 +69,10 @@ async def hamt_build(**kwargs):
 
 @pytest.mark.asyncio
 async def test_multigroup_sharded_store_defaults_to_group_zero(monkeypatch):
-    async def sharded_open(*, root_cid, cas, read_only):
+    open_kwargs = []
+
+    async def sharded_open(**kwargs):
+        open_kwargs.append(kwargs)
         return DummyGroupedStore()
 
     opened_groups = []
@@ -87,13 +90,14 @@ def open_zarr(*, store, group=None):
     )
 
     assert opened_groups == ["0"]
+    assert open_kwargs[0]["shard_read_mode"] == "sparse"
     assert ds.attrs["_ipfs_store_type"] == "ShardedZarrStore"
     assert ds.attrs["_ipfs_zarr_group"] == "0"
 
 
 @pytest.mark.asyncio
 async def test_explicit_zarr_group_is_passed_to_open_zarr(monkeypatch):
-    async def sharded_open(*, root_cid, cas, read_only):
+    async def sharded_open(**kwargs):
         return DummyStore()
 
     opened_groups = []
@@ -117,7 +121,7 @@ def open_zarr(*, store, group=None):
 
 @pytest.mark.asyncio
 async def test_zarr_group_error_after_sharded_open_does_not_fallback(monkeypatch):
-    async def sharded_open(*, root_cid, cas, read_only):
+    async def sharded_open(**kwargs):
         return DummyStore()
 
     async def hamt_build(**kwargs):
@@ -139,7 +143,7 @@ def open_zarr(*, store, group=None):
 
 @pytest.mark.asyncio
 async def test_sharded_v1_warning_is_suppressed_in_client_loader(monkeypatch):
-    async def sharded_open(*, root_cid, cas, read_only):
+    async def sharded_open(**kwargs):
         warnings.warn(
             "sharded_zarr_v1 is deprecated",
             ipfs_retrieval.ShardedZarrV1DeprecationWarning,
@@ -172,7 +176,7 @@ async def sharded_open(*, root_cid, cas, read_only):
 
 @pytest.mark.asyncio
 async def test_hamt_fallback_preserves_explicit_zarr_group(monkeypatch):
-    async def sharded_open(*, root_cid, cas, read_only):
+    async def sharded_open(**kwargs):
         raise ValueError("not a sharded zarr store")
 
     async def hamt_build(**kwargs):
@@ -206,9 +210,12 @@ def open_zarr(*, store, group=None):
 
 @pytest.mark.asyncio
 async def test_client_direct_cid_passes_zarr_group_and_records_metadata(monkeypatch):
-    async def load_dataset_from_ipfs_cid(*, ipfs_cid, kubo_cas, zarr_group):
+    async def load_dataset_from_ipfs_cid(
+        *, ipfs_cid, kubo_cas, zarr_group, shard_read_mode
+    ):
         assert ipfs_cid == VALID_CID
         assert zarr_group == "2"
+        assert shard_read_mode == "sparse"
         return xr.Dataset(attrs={"_ipfs_zarr_group": "2"})
 
     monkeypatch.setattr(
@@ -225,6 +232,7 @@ async def load_dataset_from_ipfs_cid(*, ipfs_cid, kubo_cas, zarr_group):
         cid=VALID_CID,
         return_xarray=True,
         zarr_group="2",
+        shard_read_mode="sparse",
     )
 
     assert isinstance(ds, xr.Dataset)