Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 0 additions & 21 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,24 +77,3 @@ You can also read individual chunks by their grid index:
```py
data = array.retrieve_chunk([0, 0])
```

## Development

Requires a Rust toolchain and Python 3.11+. We use
[uv](https://docs.astral.sh/uv/) and [maturin](https://www.maturin.rs/).

```bash
# Create a dev environment and install the dev dependencies
uv sync --no-install-package zarrista

# Build the Rust extension and install it into the environment (debug build)
uv run --no-project maturin develop --uv

# Or, in release mode:
uv run --no-project maturin develop --uv --release

# Run the tests
uv run --no-project pytest
```

The `--no-project` is annoying but unavoidable in our current setup. Otherwise `uv` will try to build the rust library _in release mode, as a dependency of the project_ before reaching `uv sync` or `uv run`.
6 changes: 6 additions & 0 deletions python/zarrista/_array.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,9 @@ class Array:
The bytes are returned verbatim, without running the codec pipeline.
Returns `None` if the subchunk is absent from the store.
"""
@property
def store(self) -> FilesystemStore | MemoryStore:
"""Retrieve the store backing this array."""
def store_chunk(
self,
chunk_indices: list[int],
Expand Down Expand Up @@ -369,6 +372,9 @@ class AsyncArray:
The bytes are returned verbatim, without running the codec pipeline.
Returns `None` if the subchunk is absent from the store.
"""
@property
def store(self) -> AsyncStore:
"""Retrieve the store backing this array."""
async def store_chunk(
self,
chunk_indices: list[int],
Expand Down
10 changes: 8 additions & 2 deletions python/zarrista/_group.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,15 @@ class Group:
"""Return the full paths of the group's direct child arrays."""
def child_group_paths(self) -> list[str]:
"""Return the full paths of the group's direct child groups."""
def store_metadata(self) -> None:
"""Write the group metadata to the store."""
def erase_metadata(self) -> None:
"""Erase the group metadata from the store. Succeeds if it does not exist."""
def child(self, name: str) -> Array | Group:
"""Open a direct child array or group by name."""
def store_metadata(self) -> None:
"""Write the group metadata to the store."""
@property
def store(self) -> FilesystemStore | MemoryStore:
"""Retrieve the store backing this group."""
def __getitem__(self, name: str) -> Array | Group:
"""Open a direct child array or group by name."""

Expand Down Expand Up @@ -89,3 +92,6 @@ class AsyncGroup:
"""Erase the group metadata from the store. Succeeds if it does not exist."""
async def open_child_async(self, name: str) -> AsyncArray | AsyncGroup:
"""Open a direct child array or group by name."""
@property
def store(self) -> AsyncStore:
"""Retrieve the store backing this group."""
40 changes: 19 additions & 21 deletions src/array/async.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,15 @@ use zarrs::storage::AsyncReadableWritableListableStorageTraits;
#[pyclass(module = "zarrista", frozen, name = "AsyncArray", from_py_object)]
pub struct PyAsyncArray {
pub(crate) inner: Arc<Array<dyn AsyncReadableWritableListableStorageTraits>>,
store: PyAsyncStorage,
}

impl PyAsyncArray {
pub(crate) fn new(inner: Arc<Array<dyn AsyncReadableWritableListableStorageTraits>>) -> Self {
Self { inner }
pub(crate) fn new(
inner: Arc<Array<dyn AsyncReadableWritableListableStorageTraits>>,
store: PyAsyncStorage,
) -> Self {
Self { inner, store }
}

pub fn inner(&self) -> &Arc<Array<dyn AsyncReadableWritableListableStorageTraits>> {
Expand Down Expand Up @@ -70,9 +74,8 @@ impl PyAsyncArray {
store: PyAsyncStorage,
path: PyNodePath,
) -> ZarristaResult<Self> {
let inner =
Array::new_with_metadata(store.into_inner(), path.as_str(), metadata.into_inner())?;
Ok(Self::new(Arc::new(inner)))
let inner = Array::new_with_metadata(store.inner(), path.as_str(), metadata.into_inner())?;
Ok(Self::new(Arc::new(inner), store))
}

/// Open the array stored at `path` in `store`.
Expand All @@ -86,12 +89,11 @@ impl PyAsyncArray {
store: PyAsyncStorage,
path: PyNodePath,
) -> PyResult<Bound<'py, PyAny>> {
let storage = store.into();
future_into_py(py, async move {
let inner = Array::async_open(storage, path.as_str())
let inner = Array::async_open(store.inner(), path.as_str())
.await
.map_err(ZarristaError::from)?;
Ok(Self::new(Arc::new(inner)))
Ok(Self::new(Arc::new(inner), store))
})
}

Expand Down Expand Up @@ -135,7 +137,10 @@ impl PyAsyncArray {
fn read_only(&self) -> Self {
let read_list_storage = self.inner.storage().readable_listable();
let storage = Arc::new(AsyncReadOnlyStorageAdapter::new(read_list_storage));
Self::new(Arc::new(self.inner.with_storage(storage)))
Self::new(
Arc::new(self.inner.with_storage(storage)),
self.store.clone(),
)
}

fn erase_metadata<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
Expand Down Expand Up @@ -249,6 +254,11 @@ impl PyAsyncArray {
})
}

#[getter]
fn store(&self) -> &PyAsyncStorage {
&self.store
}

#[pyo3(signature = (chunk_indices, decoded_chunk, **codec_options))]
fn store_chunk<'py>(
&self,
Expand Down Expand Up @@ -295,15 +305,3 @@ impl PyAsyncArray {
})
}
}

impl From<Array<dyn AsyncReadableWritableListableStorageTraits>> for PyAsyncArray {
fn from(inner: Array<dyn AsyncReadableWritableListableStorageTraits>) -> Self {
Self::new(Arc::new(inner))
}
}

impl From<Arc<Array<dyn AsyncReadableWritableListableStorageTraits>>> for PyAsyncArray {
fn from(inner: Arc<Array<dyn AsyncReadableWritableListableStorageTraits>>) -> Self {
Self::new(inner)
}
}
8 changes: 4 additions & 4 deletions src/array/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,9 +81,9 @@ impl PyArrayBuilder {
}

fn create(&self, store: PySyncStorage, path: &str) -> ZarristaResult<PyArray> {
let array = self.0.build_arc(store.into_inner(), path)?;
let array = self.0.build_arc(store.inner(), path)?;
array.store_metadata()?;
Ok(array.into())
Ok(PyArray::new(array, store))
}

#[cfg(feature = "async")]
Expand All @@ -97,15 +97,15 @@ impl PyArrayBuilder {

let array = self
.0
.build_arc(store.into_inner(), path)
.build_arc(store.inner(), path)
.map_err(ZarristaError::from)?;

pyo3_async_runtimes::tokio::future_into_py(py, async move {
array
.async_store_metadata()
.await
.map_err(ZarristaError::from)?;
Ok(crate::array::PyAsyncArray::from(array))
Ok(crate::array::PyAsyncArray::new(array, store))
})
}

Expand Down
39 changes: 19 additions & 20 deletions src/array/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,17 @@ use zarrs::storage::ReadableWritableListableStorageTraits;
#[pyclass(module = "zarrista", frozen, name = "Array")]
pub struct PyArray {
pub(crate) inner: Arc<Array<dyn ReadableWritableListableStorageTraits>>,
pub(crate) store: PySyncStorage,
}

crate::wasm_send_sync!(PyArray);

impl PyArray {
pub(crate) fn new(inner: Arc<Array<dyn ReadableWritableListableStorageTraits>>) -> Self {
Self { inner }
pub(crate) fn new(
inner: Arc<Array<dyn ReadableWritableListableStorageTraits>>,
store: PySyncStorage,
) -> Self {
Self { inner, store }
}

pub fn inner(&self) -> &Arc<Array<dyn ReadableWritableListableStorageTraits>> {
Expand Down Expand Up @@ -66,9 +70,8 @@ impl PyArray {
store: PySyncStorage,
path: PyNodePath,
) -> ZarristaResult<Self> {
let inner =
Array::new_with_metadata(store.into_inner(), path.as_str(), metadata.into_inner())?;
Ok(Self::new(Arc::new(inner)))
let inner = Array::new_with_metadata(store.inner(), path.as_str(), metadata.into_inner())?;
Ok(Self::new(Arc::new(inner), store))
}

/// Open the array stored at `path` in `store`.
Expand All @@ -78,8 +81,8 @@ impl PyArray {
text_signature = "(store, path='/')"
)]
fn open(store: PySyncStorage, path: PyNodePath) -> ZarristaResult<Self> {
let inner = Array::open(store.into(), path.as_str())?;
Ok(Self::new(Arc::new(inner)))
let inner = Array::open(store.inner(), path.as_str())?;
Ok(Self::new(Arc::new(inner), store))
}

#[pyo3(signature = (chunk_indices, **codec_options))]
Expand Down Expand Up @@ -107,7 +110,10 @@ impl PyArray {
fn read_only(&self) -> Self {
let read_list_storage = self.inner.storage().readable_listable();
let storage = Arc::new(ReadOnlyStorageAdapter::new(read_list_storage));
Self::new(Arc::new(self.inner.with_storage(storage)))
Self::new(
Arc::new(self.inner.with_storage(storage)),
self.store.clone(),
)
}

/// Read a region of the array, using numpy-style basic indexing.
Expand Down Expand Up @@ -172,6 +178,11 @@ impl PyArray {
.retrieve_subchunk_opt(&subchunk_cache, &subchunk_indices, &codec_options)?)
}

#[getter]
fn store(&self) -> PySyncStorage {
self.store.clone()
}

#[pyo3(signature = (chunk_indices, decoded_chunk, **codec_options))]
fn store_chunk(
&self,
Expand Down Expand Up @@ -204,15 +215,3 @@ impl PyArray {
Ok(())
}
}

impl From<Array<dyn ReadableWritableListableStorageTraits>> for PyArray {
fn from(inner: Array<dyn ReadableWritableListableStorageTraits>) -> Self {
Self::new(Arc::new(inner))
}
}

impl From<Arc<Array<dyn ReadableWritableListableStorageTraits>>> for PyArray {
fn from(inner: Arc<Array<dyn ReadableWritableListableStorageTraits>>) -> Self {
Self::new(inner)
}
}
Loading
Loading